diff --git a/.gitignore b/.gitignore
index e257658..96caa21 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,11 @@
 # ---> C++
 # Prerequisites
 *.d
+/test
+/.vscode
+/build
+build.sh
+run.sh
 
 # Compiled Object files
 *.slo
diff --git a/CMakeLists.txt b/CMakeLists.txt
new file mode 100644
index 0000000..11472b5
--- /dev/null
+++ b/CMakeLists.txt
@@ -0,0 +1,10 @@
+cmake_minimum_required(VERSION 3.0)
+project(FastSort)
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread")
+# set(CMAKE_BUILD_TYPE Debug)
+# set(CMAKE_BUILD_TYPE Release)
+add_definitions(-DSHOW_PERF=1)
+add_definitions(-DSPDLOG_COMPILED_LIB)
+add_subdirectory(src)
\ No newline at end of file
diff --git a/ext/argparse/argparse.hpp b/ext/argparse/argparse.hpp
new file mode 100644
index 0000000..cb4465b
--- /dev/null
+++ b/ext/argparse/argparse.hpp
@@ -0,0 +1,2555 @@
+/*
+  __ _ _ __ __ _ _ __   __ _ _ __ ___  ___
+ / _` | '__/ _` | '_ \ / _` | '__/ __|/ _ \ Argument Parser for Modern C++
+| (_| | | | (_| | |_) | (_| | |  \__ \  __/ http://github.com/p-ranav/argparse
+ \__,_|_|  \__, | .__/ \__,_|_|  |___/\___|
+           |___/|_|
+
+Licensed under the MIT License <http://opensource.org/licenses/MIT>.
+SPDX-License-Identifier: MIT
+Copyright (c) 2019-2022 Pranav Srinivas Kumar <pranav.srinivas.kumar@gmail.com>
+and other contributors.
+
+Permission is hereby  granted, free of charge, to any  person obtaining a copy
+of this software and associated  documentation files (the "Software"), to deal
+in the Software  without restriction, including without  limitation the rights
+to  use, copy,  modify, merge,  publish, distribute,  sublicense, and/or  sell
+copies  of  the Software,  and  to  permit persons  to  whom  the Software  is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE  IS PROVIDED "AS  IS", WITHOUT WARRANTY  OF ANY KIND,  EXPRESS OR
+IMPLIED,  INCLUDING BUT  NOT  LIMITED TO  THE  WARRANTIES OF  MERCHANTABILITY,
+FITNESS FOR  A PARTICULAR PURPOSE AND  NONINFRINGEMENT. IN NO EVENT  SHALL THE
+AUTHORS  OR COPYRIGHT  HOLDERS  BE  LIABLE FOR  ANY  CLAIM,  DAMAGES OR  OTHER
+LIABILITY, WHETHER IN AN ACTION OF  CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE  OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+#pragma once
+
+#include <cerrno>
+
+#ifndef ARGPARSE_MODULE_USE_STD_MODULE
+#include <algorithm>
+#include <any>
+#include <array>
+#include <set>
+#include <charconv>
+#include <cstdlib>
+#include <functional>
+#include <iomanip>
+#include <iostream>
+#include <iterator>
+#include <limits>
+#include <list>
+#include <map>
+#include <numeric>
+#include <optional>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <string_view>
+#include <tuple>
+#include <type_traits>
+#include <utility>
+#include <variant>
+#include <vector>
+#endif
+
+#ifndef ARGPARSE_CUSTOM_STRTOF
+#define ARGPARSE_CUSTOM_STRTOF strtof
+#endif
+
+#ifndef ARGPARSE_CUSTOM_STRTOD
+#define ARGPARSE_CUSTOM_STRTOD strtod
+#endif
+
+#ifndef ARGPARSE_CUSTOM_STRTOLD
+#define ARGPARSE_CUSTOM_STRTOLD strtold
+#endif
+
+namespace argparse {
+
+namespace details { // namespace for helper methods
+
+template <typename T, typename = void>
+struct HasContainerTraits : std::false_type {};
+
+template <> struct HasContainerTraits<std::string> : std::false_type {};
+
+template <> struct HasContainerTraits<std::string_view> : std::false_type {};
+
+template <typename T>
+struct HasContainerTraits<
+    T, std::void_t<typename T::value_type, decltype(std::declval<T>().begin()),
+                   decltype(std::declval<T>().end()),
+                   decltype(std::declval<T>().size())>> : std::true_type {};
+
+template <typename T>
+inline constexpr bool IsContainer = HasContainerTraits<T>::value;
+
+template <typename T, typename = void>
+struct HasStreamableTraits : std::false_type {};
+
+template <typename T>
+struct HasStreamableTraits<
+    T,
+    std::void_t<decltype(std::declval<std::ostream &>() << std::declval<T>())>>
+    : std::true_type {};
+
+template <typename T>
+inline constexpr bool IsStreamable = HasStreamableTraits<T>::value;
+
+constexpr std::size_t repr_max_container_size = 5;
+
+template <typename T> std::string repr(T const &val) {
+  if constexpr (std::is_same_v<T, bool>) {
+    return val ? "true" : "false";
+  } else if constexpr (std::is_convertible_v<T, std::string_view>) {
+    return '"' + std::string{std::string_view{val}} + '"';
+  } else if constexpr (IsContainer<T>) {
+    std::stringstream out;
+    out << "{";
+    const auto size = val.size();
+    if (size > 1) {
+      out << repr(*val.begin());
+      std::for_each(
+          std::next(val.begin()),
+          std::next(
+              val.begin(),
+              static_cast<typename T::iterator::difference_type>(
+                  std::min<std::size_t>(size, repr_max_container_size) - 1)),
+          [&out](const auto &v) { out << " " << repr(v); });
+      if (size <= repr_max_container_size) {
+        out << " ";
+      } else {
+        out << "...";
+      }
+    }
+    if (size > 0) {
+      out << repr(*std::prev(val.end()));
+    }
+    out << "}";
+    return out.str();
+  } else if constexpr (IsStreamable<T>) {
+    std::stringstream out;
+    out << val;
+    return out.str();
+  } else {
+    return "<not representable>";
+  }
+}
+
+namespace {
+
+template <typename T> constexpr bool standard_signed_integer = false;
+template <> constexpr bool standard_signed_integer<signed char> = true;
+template <> constexpr bool standard_signed_integer<short int> = true;
+template <> constexpr bool standard_signed_integer<int> = true;
+template <> constexpr bool standard_signed_integer<long int> = true;
+template <> constexpr bool standard_signed_integer<long long int> = true;
+
+template <typename T> constexpr bool standard_unsigned_integer = false;
+template <> constexpr bool standard_unsigned_integer<unsigned char> = true;
+template <> constexpr bool standard_unsigned_integer<unsigned short int> = true;
+template <> constexpr bool standard_unsigned_integer<unsigned int> = true;
+template <> constexpr bool standard_unsigned_integer<unsigned long int> = true;
+template <>
+constexpr bool standard_unsigned_integer<unsigned long long int> = true;
+
+} // namespace
+
+constexpr int radix_2 = 2;
+constexpr int radix_8 = 8;
+constexpr int radix_10 = 10;
+constexpr int radix_16 = 16;
+
+template <typename T>
+constexpr bool standard_integer =
+    standard_signed_integer<T> || standard_unsigned_integer<T>;
+
+template <class F, class Tuple, class Extra, std::size_t... I>
+constexpr decltype(auto)
+apply_plus_one_impl(F &&f, Tuple &&t, Extra &&x,
+                    std::index_sequence<I...> /*unused*/) {
+  return std::invoke(std::forward<F>(f), std::get<I>(std::forward<Tuple>(t))...,
+                     std::forward<Extra>(x));
+}
+
+template <class F, class Tuple, class Extra>
+constexpr decltype(auto) apply_plus_one(F &&f, Tuple &&t, Extra &&x) {
+  return details::apply_plus_one_impl(
+      std::forward<F>(f), std::forward<Tuple>(t), std::forward<Extra>(x),
+      std::make_index_sequence<
+          std::tuple_size_v<std::remove_reference_t<Tuple>>>{});
+}
+
+constexpr auto pointer_range(std::string_view s) noexcept {
+  return std::tuple(s.data(), s.data() + s.size());
+}
+
+template <class CharT, class Traits>
+constexpr bool starts_with(std::basic_string_view<CharT, Traits> prefix,
+                           std::basic_string_view<CharT, Traits> s) noexcept {
+  return s.substr(0, prefix.size()) == prefix;
+}
+
+enum class chars_format {
+  scientific = 0xf1,
+  fixed = 0xf2,
+  hex = 0xf4,
+  binary = 0xf8,
+  general = fixed | scientific
+};
+
+struct ConsumeBinaryPrefixResult {
+  bool is_binary;
+  std::string_view rest;
+};
+
+constexpr auto consume_binary_prefix(std::string_view s)
+    -> ConsumeBinaryPrefixResult {
+  if (starts_with(std::string_view{"0b"}, s) ||
+      starts_with(std::string_view{"0B"}, s)) {
+    s.remove_prefix(2);
+    return {true, s};
+  }
+  return {false, s};
+}
+
+struct ConsumeHexPrefixResult {
+  bool is_hexadecimal;
+  std::string_view rest;
+};
+
+using namespace std::literals;
+
+constexpr auto consume_hex_prefix(std::string_view s)
+    -> ConsumeHexPrefixResult {
+  if (starts_with("0x"sv, s) || starts_with("0X"sv, s)) {
+    s.remove_prefix(2);
+    return {true, s};
+  }
+  return {false, s};
+}
+
+template <class T, auto Param>
+inline auto do_from_chars(std::string_view s) -> T {
+  T x{0};
+  auto [first, last] = pointer_range(s);
+  auto [ptr, ec] = std::from_chars(first, last, x, Param);
+  if (ec == std::errc()) {
+    if (ptr == last) {
+      return x;
+    }
+    throw std::invalid_argument{"pattern '" + std::string(s) +
+                                "' does not match to the end"};
+  }
+  if (ec == std::errc::invalid_argument) {
+    throw std::invalid_argument{"pattern '" + std::string(s) + "' not found"};
+  }
+  if (ec == std::errc::result_out_of_range) {
+    throw std::range_error{"'" + std::string(s) + "' not representable"};
+  }
+  return x; // unreachable
+}
+
+template <class T, auto Param = 0> struct parse_number {
+  auto operator()(std::string_view s) -> T {
+    return do_from_chars<T, Param>(s);
+  }
+};
+
+template <class T> struct parse_number<T, radix_2> {
+  auto operator()(std::string_view s) -> T {
+    if (auto [ok, rest] = consume_binary_prefix(s); ok) {
+      return do_from_chars<T, radix_2>(rest);
+    }
+    throw std::invalid_argument{"pattern not found"};
+  }
+};
+
+template <class T> struct parse_number<T, radix_16> {
+  auto operator()(std::string_view s) -> T {
+    if (starts_with("0x"sv, s) || starts_with("0X"sv, s)) {
+      if (auto [ok, rest] = consume_hex_prefix(s); ok) {
+        try {
+          return do_from_chars<T, radix_16>(rest);
+        } catch (const std::invalid_argument &err) {
+          throw std::invalid_argument("Failed to parse '" + std::string(s) +
+                                      "' as hexadecimal: " + err.what());
+        } catch (const std::range_error &err) {
+          throw std::range_error("Failed to parse '" + std::string(s) +
+                                 "' as hexadecimal: " + err.what());
+        }
+      }
+    } else {
+      // Allow passing hex numbers without prefix
+      // Shape 'x' already has to be specified
+      try {
+        return do_from_chars<T, radix_16>(s);
+      } catch (const std::invalid_argument &err) {
+        throw std::invalid_argument("Failed to parse '" + std::string(s) +
+                                    "' as hexadecimal: " + err.what());
+      } catch (const std::range_error &err) {
+        throw std::range_error("Failed to parse '" + std::string(s) +
+                               "' as hexadecimal: " + err.what());
+      }
+    }
+
+    throw std::invalid_argument{"pattern '" + std::string(s) +
+                                "' not identified as hexadecimal"};
+  }
+};
+
+template <class T> struct parse_number<T> {
+  auto operator()(std::string_view s) -> T {
+    auto [ok, rest] = consume_hex_prefix(s);
+    if (ok) {
+      try {
+        return do_from_chars<T, radix_16>(rest);
+      } catch (const std::invalid_argument &err) {
+        throw std::invalid_argument("Failed to parse '" + std::string(s) +
+                                    "' as hexadecimal: " + err.what());
+      } catch (const std::range_error &err) {
+        throw std::range_error("Failed to parse '" + std::string(s) +
+                               "' as hexadecimal: " + err.what());
+      }
+    }
+
+    auto [ok_binary, rest_binary] = consume_binary_prefix(s);
+    if (ok_binary) {
+      try {
+        return do_from_chars<T, radix_2>(rest_binary);
+      } catch (const std::invalid_argument &err) {
+        throw std::invalid_argument("Failed to parse '" + std::string(s) +
+                                    "' as binary: " + err.what());
+      } catch (const std::range_error &err) {
+        throw std::range_error("Failed to parse '" + std::string(s) +
+                               "' as binary: " + err.what());
+      }
+    }
+
+    if (starts_with("0"sv, s)) {
+      try {
+        return do_from_chars<T, radix_8>(rest);
+      } catch (const std::invalid_argument &err) {
+        throw std::invalid_argument("Failed to parse '" + std::string(s) +
+                                    "' as octal: " + err.what());
+      } catch (const std::range_error &err) {
+        throw std::range_error("Failed to parse '" + std::string(s) +
+                               "' as octal: " + err.what());
+      }
+    }
+
+    try {
+      return do_from_chars<T, radix_10>(rest);
+    } catch (const std::invalid_argument &err) {
+      throw std::invalid_argument("Failed to parse '" + std::string(s) +
+                                  "' as decimal integer: " + err.what());
+    } catch (const std::range_error &err) {
+      throw std::range_error("Failed to parse '" + std::string(s) +
+                             "' as decimal integer: " + err.what());
+    }
+  }
+};
+
+namespace {
+
+template <class T> inline const auto generic_strtod = nullptr;
+template <> inline const auto generic_strtod<float> = ARGPARSE_CUSTOM_STRTOF;
+template <> inline const auto generic_strtod<double> = ARGPARSE_CUSTOM_STRTOD;
+template <>
+inline const auto generic_strtod<long double> = ARGPARSE_CUSTOM_STRTOLD;
+
+} // namespace
+
+template <class T> inline auto do_strtod(std::string const &s) -> T {
+  if (isspace(static_cast<unsigned char>(s[0])) || s[0] == '+') {
+    throw std::invalid_argument{"pattern '" + s + "' not found"};
+  }
+
+  auto [first, last] = pointer_range(s);
+  char *ptr;
+
+  errno = 0;
+  auto x = generic_strtod<T>(first, &ptr);
+  if (errno == 0) {
+    if (ptr == last) {
+      return x;
+    }
+    throw std::invalid_argument{"pattern '" + s +
+                                "' does not match to the end"};
+  }
+  if (errno == ERANGE) {
+    throw std::range_error{"'" + s + "' not representable"};
+  }
+  return x; // unreachable
+}
+
+template <class T> struct parse_number<T, chars_format::general> {
+  auto operator()(std::string const &s) -> T {
+    if (auto r = consume_hex_prefix(s); r.is_hexadecimal) {
+      throw std::invalid_argument{
+          "chars_format::general does not parse hexfloat"};
+    }
+    if (auto r = consume_binary_prefix(s); r.is_binary) {
+      throw std::invalid_argument{
+          "chars_format::general does not parse binfloat"};
+    }
+
+    try {
+      return do_strtod<T>(s);
+    } catch (const std::invalid_argument &err) {
+      throw std::invalid_argument("Failed to parse '" + s +
+                                  "' as number: " + err.what());
+    } catch (const std::range_error &err) {
+      throw std::range_error("Failed to parse '" + s +
+                             "' as number: " + err.what());
+    }
+  }
+};
+
+template <class T> struct parse_number<T, chars_format::hex> {
+  auto operator()(std::string const &s) -> T {
+    if (auto r = consume_hex_prefix(s); !r.is_hexadecimal) {
+      throw std::invalid_argument{"chars_format::hex parses hexfloat"};
+    }
+    if (auto r = consume_binary_prefix(s); r.is_binary) {
+      throw std::invalid_argument{"chars_format::hex does not parse binfloat"};
+    }
+
+    try {
+      return do_strtod<T>(s);
+    } catch (const std::invalid_argument &err) {
+      throw std::invalid_argument("Failed to parse '" + s +
+                                  "' as hexadecimal: " + err.what());
+    } catch (const std::range_error &err) {
+      throw std::range_error("Failed to parse '" + s +
+                             "' as hexadecimal: " + err.what());
+    }
+  }
+};
+
+template <class T> struct parse_number<T, chars_format::binary> {
+  auto operator()(std::string const &s) -> T {
+    if (auto r = consume_hex_prefix(s); r.is_hexadecimal) {
+      throw std::invalid_argument{
+          "chars_format::binary does not parse hexfloat"};
+    }
+    if (auto r = consume_binary_prefix(s); !r.is_binary) {
+      throw std::invalid_argument{"chars_format::binary parses binfloat"};
+    }
+
+    return do_strtod<T>(s);
+  }
+};
+
+template <class T> struct parse_number<T, chars_format::scientific> {
+  auto operator()(std::string const &s) -> T {
+    if (auto r = consume_hex_prefix(s); r.is_hexadecimal) {
+      throw std::invalid_argument{
+          "chars_format::scientific does not parse hexfloat"};
+    }
+    if (auto r = consume_binary_prefix(s); r.is_binary) {
+      throw std::invalid_argument{
+          "chars_format::scientific does not parse binfloat"};
+    }
+    if (s.find_first_of("eE") == std::string::npos) {
+      throw std::invalid_argument{
+          "chars_format::scientific requires exponent part"};
+    }
+
+    try {
+      return do_strtod<T>(s);
+    } catch (const std::invalid_argument &err) {
+      throw std::invalid_argument("Failed to parse '" + s +
+                                  "' as scientific notation: " + err.what());
+    } catch (const std::range_error &err) {
+      throw std::range_error("Failed to parse '" + s +
+                             "' as scientific notation: " + err.what());
+    }
+  }
+};
+
+template <class T> struct parse_number<T, chars_format::fixed> {
+  auto operator()(std::string const &s) -> T {
+    if (auto r = consume_hex_prefix(s); r.is_hexadecimal) {
+      throw std::invalid_argument{
+          "chars_format::fixed does not parse hexfloat"};
+    }
+    if (auto r = consume_binary_prefix(s); r.is_binary) {
+      throw std::invalid_argument{
+          "chars_format::fixed does not parse binfloat"};
+    }
+    if (s.find_first_of("eE") != std::string::npos) {
+      throw std::invalid_argument{
+          "chars_format::fixed does not parse exponent part"};
+    }
+
+    try {
+      return do_strtod<T>(s);
+    } catch (const std::invalid_argument &err) {
+      throw std::invalid_argument("Failed to parse '" + s +
+                                  "' as fixed notation: " + err.what());
+    } catch (const std::range_error &err) {
+      throw std::range_error("Failed to parse '" + s +
+                             "' as fixed notation: " + err.what());
+    }
+  }
+};
+
+template <typename StrIt>
+std::string join(StrIt first, StrIt last, const std::string &separator) {
+  if (first == last) {
+    return "";
+  }
+  std::stringstream value;
+  value << *first;
+  ++first;
+  while (first != last) {
+    value << separator << *first;
+    ++first;
+  }
+  return value.str();
+}
+
+template <typename T> struct can_invoke_to_string {
+  template <typename U>
+  static auto test(int)
+      -> decltype(std::to_string(std::declval<U>()), std::true_type{});
+
+  template <typename U> static auto test(...) -> std::false_type;
+
+  static constexpr bool value = decltype(test<T>(0))::value;
+};
+
+template <typename T> struct IsChoiceTypeSupported {
+  using CleanType = typename std::decay<T>::type;
+  static const bool value = std::is_integral<CleanType>::value ||
+                            std::is_same<CleanType, std::string>::value ||
+                            std::is_same<CleanType, std::string_view>::value ||
+                            std::is_same<CleanType, const char *>::value;
+};
+
+template <typename StringType>
+std::size_t get_levenshtein_distance(const StringType &s1,
+                                     const StringType &s2) {
+  std::vector<std::vector<std::size_t>> dp(
+      s1.size() + 1, std::vector<std::size_t>(s2.size() + 1, 0));
+
+  for (std::size_t i = 0; i <= s1.size(); ++i) {
+    for (std::size_t j = 0; j <= s2.size(); ++j) {
+      if (i == 0) {
+        dp[i][j] = j;
+      } else if (j == 0) {
+        dp[i][j] = i;
+      } else if (s1[i - 1] == s2[j - 1]) {
+        dp[i][j] = dp[i - 1][j - 1];
+      } else {
+        dp[i][j] = 1 + std::min<std::size_t>({dp[i - 1][j], dp[i][j - 1], dp[i - 1][j - 1]});
+      }
+    }
+  }
+
+  return dp[s1.size()][s2.size()];
+}
+
+template <typename ValueType>
+std::string get_most_similar_string(const std::map<std::string, ValueType> &map,
+                                    const std::string &input) {
+  std::string most_similar{};
+  std::size_t min_distance = (std::numeric_limits<std::size_t>::max)();
+
+  for (const auto &entry : map) {
+    std::size_t distance = get_levenshtein_distance(entry.first, input);
+    if (distance < min_distance) {
+      min_distance = distance;
+      most_similar = entry.first;
+    }
+  }
+
+  return most_similar;
+}
+
+} // namespace details
+
+enum class nargs_pattern { optional, any, at_least_one };
+
+enum class default_arguments : unsigned int {
+  none = 0,
+  help = 1,
+  version = 2,
+  all = help | version,
+};
+
+inline default_arguments operator&(const default_arguments &a,
+                                   const default_arguments &b) {
+  return static_cast<default_arguments>(
+      static_cast<std::underlying_type<default_arguments>::type>(a) &
+      static_cast<std::underlying_type<default_arguments>::type>(b));
+}
+
+class ArgumentParser;
+
+class Argument {
+  friend class ArgumentParser;
+  friend auto operator<<(std::ostream &stream, const ArgumentParser &parser)
+      -> std::ostream &;
+
+  template <std::size_t N, std::size_t... I>
+  explicit Argument(std::string_view prefix_chars,
+                    std::array<std::string_view, N> &&a,
+                    std::index_sequence<I...> /*unused*/)
+      : m_accepts_optional_like_value(false),
+        m_is_optional((is_optional(a[I], prefix_chars) || ...)),
+        m_is_required(false), m_is_repeatable(false), m_is_used(false),
+        m_is_hidden(false), m_prefix_chars(prefix_chars) {
+    ((void)m_names.emplace_back(a[I]), ...);
+    std::sort(
+        m_names.begin(), m_names.end(), [](const auto &lhs, const auto &rhs) {
+          return lhs.size() == rhs.size() ? lhs < rhs : lhs.size() < rhs.size();
+        });
+  }
+
+public:
+  template <std::size_t N>
+  explicit Argument(std::string_view prefix_chars,
+                    std::array<std::string_view, N> &&a)
+      : Argument(prefix_chars, std::move(a), std::make_index_sequence<N>{}) {}
+
+  Argument &help(std::string help_text) {
+    m_help = std::move(help_text);
+    return *this;
+  }
+
+  Argument &metavar(std::string metavar) {
+    m_metavar = std::move(metavar);
+    return *this;
+  }
+
+  template <typename T> Argument &default_value(T &&value) {
+    m_num_args_range = NArgsRange{0, m_num_args_range.get_max()};
+    m_default_value_repr = details::repr(value);
+
+    if constexpr (std::is_convertible_v<T, std::string_view>) {
+      m_default_value_str = std::string{std::string_view{value}};
+    } else if constexpr (details::can_invoke_to_string<T>::value) {
+      m_default_value_str = std::to_string(value);
+    }
+
+    m_default_value = std::forward<T>(value);
+    return *this;
+  }
+
+  Argument &default_value(const char *value) {
+    return default_value(std::string(value));
+  }
+
+  Argument &required() {
+    m_is_required = true;
+    return *this;
+  }
+
+  Argument &implicit_value(std::any value) {
+    m_implicit_value = std::move(value);
+    m_num_args_range = NArgsRange{0, 0};
+    return *this;
+  }
+
+  // This is shorthand for:
+  //   program.add_argument("foo")
+  //     .default_value(false)
+  //     .implicit_value(true)
+  Argument &flag() {
+    default_value(false);
+    implicit_value(true);
+    return *this;
+  }
+
+  template <class F, class... Args>
+  auto action(F &&callable, Args &&... bound_args)
+      -> std::enable_if_t<std::is_invocable_v<F, Args..., std::string const>,
+                          Argument &> {
+    using action_type = std::conditional_t<
+        std::is_void_v<std::invoke_result_t<F, Args..., std::string const>>,
+        void_action, valued_action>;
+    if constexpr (sizeof...(Args) == 0) {
+      m_action.emplace<action_type>(std::forward<F>(callable));
+    } else {
+      m_action.emplace<action_type>(
+          [f = std::forward<F>(callable),
+           tup = std::make_tuple(std::forward<Args>(bound_args)...)](
+              std::string const &opt) mutable {
+            return details::apply_plus_one(f, tup, opt);
+          });
+    }
+    return *this;
+  }
+
+  auto &store_into(bool &var) {
+    if ((!m_default_value.has_value()) && (!m_implicit_value.has_value())) {
+      flag();
+    }
+    if (m_default_value.has_value()) {
+      var = std::any_cast<bool>(m_default_value);
+    }
+    action([&var](const auto & /*unused*/) { var = true; });
+    return *this;
+  }
+
+  template <typename T, typename std::enable_if<std::is_integral<T>::value>::type * = nullptr>
+  auto &store_into(T &var) {
+    if (m_default_value.has_value()) {
+      var = std::any_cast<T>(m_default_value);
+    }
+    action([&var](const auto &s) {
+      var = details::parse_number<T, details::radix_10>()(s);
+    });
+    return *this;
+  }
+
+  auto &store_into(double &var) {
+    if (m_default_value.has_value()) {
+      var = std::any_cast<double>(m_default_value);
+    }
+    action([&var](const auto &s) {
+      var = details::parse_number<double, details::chars_format::general>()(s);
+    });
+    return *this;
+  }
+
+  auto &store_into(std::string &var) {
+    if (m_default_value.has_value()) {
+      var = std::any_cast<std::string>(m_default_value);
+    }
+    action([&var](const std::string &s) { var = s; });
+    return *this;
+  }
+
+  auto &store_into(std::vector<std::string> &var) {
+    if (m_default_value.has_value()) {
+      var = std::any_cast<std::vector<std::string>>(m_default_value);
+    }
+    action([this, &var](const std::string &s) {
+      if (!m_is_used) {
+        var.clear();
+      }
+      m_is_used = true;
+      var.push_back(s);
+    });
+    return *this;
+  }
+
+  auto &store_into(std::vector<int> &var) {
+    if (m_default_value.has_value()) {
+      var = std::any_cast<std::vector<int>>(m_default_value);
+    }
+    action([this, &var](const std::string &s) {
+      if (!m_is_used) {
+        var.clear();
+      }
+      m_is_used = true;
+      var.push_back(details::parse_number<int, details::radix_10>()(s));
+    });
+    return *this;
+  }
+
+  auto &store_into(std::set<std::string> &var) {
+    if (m_default_value.has_value()) {
+      var = std::any_cast<std::set<std::string>>(m_default_value);
+    }
+    action([this, &var](const std::string &s) {
+      if (!m_is_used) {
+        var.clear();
+      }
+      m_is_used = true;
+      var.insert(s);
+    });
+    return *this;
+  }
+
+  auto &store_into(std::set<int> &var) {
+    if (m_default_value.has_value()) {
+      var = std::any_cast<std::set<int>>(m_default_value);
+    }
+    action([this, &var](const std::string &s) {
+      if (!m_is_used) {
+        var.clear();
+      }
+      m_is_used = true;
+      var.insert(details::parse_number<int, details::radix_10>()(s));
+    });
+    return *this;
+  }
+
+  auto &append() {
+    m_is_repeatable = true;
+    return *this;
+  }
+
+  // Cause the argument to be invisible in usage and help
+  auto &hidden() {
+    m_is_hidden = true;
+    return *this;
+  }
+
+  template <char Shape, typename T>
+  auto scan() -> std::enable_if_t<std::is_arithmetic_v<T>, Argument &> {
+    static_assert(!(std::is_const_v<T> || std::is_volatile_v<T>),
+                  "T should not be cv-qualified");
+    auto is_one_of = [](char c, auto... x) constexpr {
+      return ((c == x) || ...);
+    };
+
+    if constexpr (is_one_of(Shape, 'd') && details::standard_integer<T>) {
+      action(details::parse_number<T, details::radix_10>());
+    } else if constexpr (is_one_of(Shape, 'i') &&
+                         details::standard_integer<T>) {
+      action(details::parse_number<T>());
+    } else if constexpr (is_one_of(Shape, 'u') &&
+                         details::standard_unsigned_integer<T>) {
+      action(details::parse_number<T, details::radix_10>());
+    } else if constexpr (is_one_of(Shape, 'b') &&
+                         details::standard_unsigned_integer<T>) {
+      action(details::parse_number<T, details::radix_2>());
+    } else if constexpr (is_one_of(Shape, 'o') &&
+                         details::standard_unsigned_integer<T>) {
+      action(details::parse_number<T, details::radix_8>());
+    } else if constexpr (is_one_of(Shape, 'x', 'X') &&
+                         details::standard_unsigned_integer<T>) {
+      action(details::parse_number<T, details::radix_16>());
+    } else if constexpr (is_one_of(Shape, 'a', 'A') &&
+                         std::is_floating_point_v<T>) {
+      action(details::parse_number<T, details::chars_format::hex>());
+    } else if constexpr (is_one_of(Shape, 'e', 'E') &&
+                         std::is_floating_point_v<T>) {
+      action(details::parse_number<T, details::chars_format::scientific>());
+    } else if constexpr (is_one_of(Shape, 'f', 'F') &&
+                         std::is_floating_point_v<T>) {
+      action(details::parse_number<T, details::chars_format::fixed>());
+    } else if constexpr (is_one_of(Shape, 'g', 'G') &&
+                         std::is_floating_point_v<T>) {
+      action(details::parse_number<T, details::chars_format::general>());
+    } else {
+      static_assert(alignof(T) == 0, "No scan specification for T");
+    }
+
+    return *this;
+  }
+
+  Argument &nargs(std::size_t num_args) {
+    m_num_args_range = NArgsRange{num_args, num_args};
+    return *this;
+  }
+
+  Argument &nargs(std::size_t num_args_min, std::size_t num_args_max) {
+    m_num_args_range = NArgsRange{num_args_min, num_args_max};
+    return *this;
+  }
+
+  Argument &nargs(nargs_pattern pattern) {
+    switch (pattern) {
+    case nargs_pattern::optional:
+      m_num_args_range = NArgsRange{0, 1};
+      break;
+    case nargs_pattern::any:
+      m_num_args_range =
+          NArgsRange{0, (std::numeric_limits<std::size_t>::max)()};
+      break;
+    case nargs_pattern::at_least_one:
+      m_num_args_range =
+          NArgsRange{1, (std::numeric_limits<std::size_t>::max)()};
+      break;
+    }
+    return *this;
+  }
+
+  Argument &remaining() {
+    m_accepts_optional_like_value = true;
+    return nargs(nargs_pattern::any);
+  }
+
+  template <typename T> void add_choice(T &&choice) {
+    static_assert(details::IsChoiceTypeSupported<T>::value,
+                  "Only string or integer type supported for choice");
+    static_assert(std::is_convertible_v<T, std::string_view> ||
+                      details::can_invoke_to_string<T>::value,
+                  "Choice is not convertible to string_type");
+    if (!m_choices.has_value()) {
+      m_choices = std::vector<std::string>{};
+    }
+
+    if constexpr (std::is_convertible_v<T, std::string_view>) {
+      m_choices.value().push_back(
+          std::string{std::string_view{std::forward<T>(choice)}});
+    } else if constexpr (details::can_invoke_to_string<T>::value) {
+      m_choices.value().push_back(std::to_string(std::forward<T>(choice)));
+    }
+  }
+
+  Argument &choices() {
+    if (!m_choices.has_value()) {
+      throw std::runtime_error("Zero choices provided");
+    }
+    return *this;
+  }
+
+  template <typename T, typename... U>
+  Argument &choices(T &&first, U &&... rest) {
+    add_choice(std::forward<T>(first));
+    choices(std::forward<U>(rest)...);
+    return *this;
+  }
+
+  void find_default_value_in_choices_or_throw() const {
+
+    const auto &choices = m_choices.value();
+
+    if (m_default_value.has_value()) {
+      if (std::find(choices.begin(), choices.end(), m_default_value_str) ==
+          choices.end()) {
+        // provided arg not in list of allowed choices
+        // report error
+
+        std::string choices_as_csv =
+            std::accumulate(choices.begin(), choices.end(), std::string(),
+                            [](const std::string &a, const std::string &b) {
+                              return a + (a.empty() ? "" : ", ") + b;
+                            });
+
+        throw std::runtime_error(
+            std::string{"Invalid default value "} + m_default_value_repr +
+            " - allowed options: {" + choices_as_csv + "}");
+      }
+    }
+  }
+
+  template <typename Iterator>
+  bool is_value_in_choices(Iterator option_it) const {
+
+    const auto &choices = m_choices.value();
+
+    return (std::find(choices.begin(), choices.end(), *option_it) !=
+            choices.end());
+  }
+
+  template <typename Iterator>
+  void throw_invalid_arguments_error(Iterator option_it) const {
+    const auto &choices = m_choices.value();
+    const std::string choices_as_csv = std::accumulate(
+        choices.begin(), choices.end(), std::string(),
+        [](const std::string &option_a, const std::string &option_b) {
+          return option_a + (option_a.empty() ? "" : ", ") + option_b;
+        });
+
+    throw std::runtime_error(std::string{"Invalid argument "} +
+                             details::repr(*option_it) +
+                             " - allowed options: {" + choices_as_csv + "}");
+  }
+
+  /* The dry_run parameter can be set to true to avoid running the actions,
+   * and setting m_is_used. This may be used by a pre-processing step to do
+   * a first iteration over arguments.
+   */
+  template <typename Iterator>
+  Iterator consume(Iterator start, Iterator end,
+                   std::string_view used_name = {}, bool dry_run = false) {
+    if (!m_is_repeatable && m_is_used) {
+      throw std::runtime_error(
+          std::string("Duplicate argument ").append(used_name));
+    }
+    m_used_name = used_name;
+
+    std::size_t passed_options = 0;
+
+    if (m_choices.has_value()) {
+      // Check each value in (start, end) and make sure
+      // it is in the list of allowed choices/options
+      const auto max_number_of_args = m_num_args_range.get_max();
+      const auto min_number_of_args = m_num_args_range.get_min();
+      for (auto it = start; it != end; ++it) {
+        if (is_value_in_choices(it)) {
+          passed_options += 1;
+          continue;
+        }
+
+        if ((passed_options >= min_number_of_args) &&
+            (passed_options <= max_number_of_args)) {
+          break;
+        }
+
+        throw_invalid_arguments_error(it);
+      }
+    }
+
+    const auto num_args_max =
+        (m_choices.has_value()) ? passed_options : m_num_args_range.get_max();
+    const auto num_args_min = m_num_args_range.get_min();
+    std::size_t dist = 0;
+    if (num_args_max == 0) {
+      if (!dry_run) {
+        m_values.emplace_back(m_implicit_value);
+        std::visit([](const auto &f) { f({}); }, m_action);
+        m_is_used = true;
+      }
+      return start;
+    }
+    if ((dist = static_cast<std::size_t>(std::distance(start, end))) >=
+        num_args_min) {
+      if (num_args_max < dist) {
+        end = std::next(start, static_cast<typename Iterator::difference_type>(
+                                   num_args_max));
+      }
+      if (!m_accepts_optional_like_value) {
+        end = std::find_if(
+            start, end,
+            std::bind(is_optional, std::placeholders::_1, m_prefix_chars));
+        dist = static_cast<std::size_t>(std::distance(start, end));
+        if (dist < num_args_min) {
+          throw std::runtime_error("Too few arguments for '" +
+                                   std::string(m_used_name) + "'.");
+        }
+      }
+      struct ActionApply {
+        void operator()(valued_action &f) {
+          std::transform(first, last, std::back_inserter(self.m_values), f);
+        }
+
+        void operator()(void_action &f) {
+          std::for_each(first, last, f);
+          if (!self.m_default_value.has_value()) {
+            if (!self.m_accepts_optional_like_value) {
+              self.m_values.resize(
+                  static_cast<std::size_t>(std::distance(first, last)));
+            }
+          }
+        }
+
+        Iterator first, last;
+        Argument &self;
+      };
+      if (!dry_run) {
+        std::visit(ActionApply{start, end, *this}, m_action);
+        m_is_used = true;
+      }
+      return end;
+    }
+    if (m_default_value.has_value()) {
+      if (!dry_run) {
+        m_is_used = true;
+      }
+      return start;
+    }
+    throw std::runtime_error("Too few arguments for '" +
+                             std::string(m_used_name) + "'.");
+  }
+
+  /*
+   * @throws std::runtime_error if argument values are not valid
+   */
+  void validate() const {
+    if (m_is_optional) {
+      // TODO: check if an implicit value was programmed for this argument
+      if (!m_is_used && !m_default_value.has_value() && m_is_required) {
+        throw_required_arg_not_used_error();
+      }
+      if (m_is_used && m_is_required && m_values.empty()) {
+        throw_required_arg_no_value_provided_error();
+      }
+    } else {
+      if (!m_num_args_range.contains(m_values.size()) &&
+          !m_default_value.has_value()) {
+        throw_nargs_range_validation_error();
+      }
+    }
+
+    if (m_choices.has_value()) {
+      // Make sure the default value (if provided)
+      // is in the list of choices
+      find_default_value_in_choices_or_throw();
+    }
+  }
+
+  std::string get_names_csv(char separator = ',') const {
+    return std::accumulate(
+        m_names.begin(), m_names.end(), std::string{""},
+        [&](const std::string &result, const std::string &name) {
+          return result.empty() ? name : result + separator + name;
+        });
+  }
+
+  std::string get_usage_full() const {
+    std::stringstream usage;
+
+    usage << get_names_csv('/');
+    const std::string metavar = !m_metavar.empty() ? m_metavar : "VAR";
+    if (m_num_args_range.get_max() > 0) {
+      usage << " " << metavar;
+      if (m_num_args_range.get_max() > 1) {
+        usage << "...";
+      }
+    }
+    return usage.str();
+  }
+
+  std::string get_inline_usage() const {
+    std::stringstream usage;
+    // Find the longest variant to show in the usage string
+    std::string longest_name = m_names.front();
+    for (const auto &s : m_names) {
+      if (s.size() > longest_name.size()) {
+        longest_name = s;
+      }
+    }
+    if (!m_is_required) {
+      usage << "[";
+    }
+    usage << longest_name;
+    const std::string metavar = !m_metavar.empty() ? m_metavar : "VAR";
+    if (m_num_args_range.get_max() > 0) {
+      usage << " " << metavar;
+      if (m_num_args_range.get_max() > 1 &&
+          m_metavar.find("> <") == std::string::npos) {
+        usage << "...";
+      }
+    }
+    if (!m_is_required) {
+      usage << "]";
+    }
+    if (m_is_repeatable) {
+      usage << "...";
+    }
+    return usage.str();
+  }
+
+  std::size_t get_arguments_length() const {
+
+    std::size_t names_size = std::accumulate(
+        std::begin(m_names), std::end(m_names), std::size_t(0),
+        [](const auto &sum, const auto &s) { return sum + s.size(); });
+
+    if (is_positional(m_names.front(), m_prefix_chars)) {
+      // A set metavar means this replaces the names
+      if (!m_metavar.empty()) {
+        // Indent and metavar
+        return 2 + m_metavar.size();
+      }
+
+      // Indent and space-separated
+      return 2 + names_size + (m_names.size() - 1);
+    }
+    // Is an option - include both names _and_ metavar
+    // size = text + (", " between names)
+    std::size_t size = names_size + 2 * (m_names.size() - 1);
+    if (!m_metavar.empty() && m_num_args_range == NArgsRange{1, 1}) {
+      size += m_metavar.size() + 1;
+    }
+    return size + 2; // indent
+  }
+
+  friend std::ostream &operator<<(std::ostream &stream,
+                                  const Argument &argument) {
+    std::stringstream name_stream;
+    name_stream << "  "; // indent
+    if (argument.is_positional(argument.m_names.front(),
+                               argument.m_prefix_chars)) {
+      if (!argument.m_metavar.empty()) {
+        name_stream << argument.m_metavar;
+      } else {
+        name_stream << details::join(argument.m_names.begin(),
+                                     argument.m_names.end(), " ");
+      }
+    } else {
+      name_stream << details::join(argument.m_names.begin(),
+                                   argument.m_names.end(), ", ");
+      // If we have a metavar, and one narg - print the metavar
+      if (!argument.m_metavar.empty() &&
+          argument.m_num_args_range == NArgsRange{1, 1}) {
+        name_stream << " " << argument.m_metavar;
+      }
+      else if (!argument.m_metavar.empty() &&
+               argument.m_num_args_range.get_min() == argument.m_num_args_range.get_max() &&
+               argument.m_metavar.find("> <") != std::string::npos) {
+        name_stream << " " << argument.m_metavar;
+      }
+    }
+
+    // align multiline help message
+    auto stream_width = stream.width();
+    auto name_padding = std::string(name_stream.str().size(), ' ');
+    auto pos = std::string::size_type{};
+    auto prev = std::string::size_type{};
+    auto first_line = true;
+    auto hspace = "  "; // minimal space between name and help message
+    stream << name_stream.str();
+    std::string_view help_view(argument.m_help);
+    while ((pos = argument.m_help.find('\n', prev)) != std::string::npos) {
+      auto line = help_view.substr(prev, pos - prev + 1);
+      if (first_line) {
+        stream << hspace << line;
+        first_line = false;
+      } else {
+        stream.width(stream_width);
+        stream << name_padding << hspace << line;
+      }
+      prev += pos - prev + 1;
+    }
+    if (first_line) {
+      stream << hspace << argument.m_help;
+    } else {
+      auto leftover = help_view.substr(prev, argument.m_help.size() - prev);
+      if (!leftover.empty()) {
+        stream.width(stream_width);
+        stream << name_padding << hspace << leftover;
+      }
+    }
+
+    // print nargs spec
+    if (!argument.m_help.empty()) {
+      stream << " ";
+    }
+    stream << argument.m_num_args_range;
+
+    bool add_space = false;
+    if (argument.m_default_value.has_value() &&
+        argument.m_num_args_range != NArgsRange{0, 0}) {
+      stream << "[default: " << argument.m_default_value_repr << "]";
+      add_space = true;
+    } else if (argument.m_is_required) {
+      stream << "[required]";
+      add_space = true;
+    }
+    if (argument.m_is_repeatable) {
+      if (add_space) {
+        stream << " ";
+      }
+      stream << "[may be repeated]";
+    }
+    stream << "\n";
+    return stream;
+  }
+
+  template <typename T> bool operator!=(const T &rhs) const {
+    return !(*this == rhs);
+  }
+
+  /*
+   * Compare to an argument value of known type
+   * @throws std::logic_error in case of incompatible types
+   */
+  template <typename T> bool operator==(const T &rhs) const {
+    if constexpr (!details::IsContainer<T>) {
+      return get<T>() == rhs;
+    } else {
+      using ValueType = typename T::value_type;
+      auto lhs = get<T>();
+      return std::equal(std::begin(lhs), std::end(lhs), std::begin(rhs),
+                        std::end(rhs), [](const auto &a, const auto &b) {
+                          return std::any_cast<const ValueType &>(a) == b;
+                        });
+    }
+  }
+
+  /*
+   * positional:
+   *    _empty_
+   *    '-'
+   *    '-' decimal-literal
+   *    !'-' anything
+   */
+  static bool is_positional(std::string_view name,
+                            std::string_view prefix_chars) {
+    auto first = lookahead(name);
+
+    if (first == eof) {
+      return true;
+    }
+    if (prefix_chars.find(static_cast<char>(first)) !=
+                          std::string_view::npos) {
+      name.remove_prefix(1);
+      if (name.empty()) {
+        return true;
+      }
+      return is_decimal_literal(name);
+    }
+    return true;
+  }
+
+private:
+  class NArgsRange {
+    std::size_t m_min;
+    std::size_t m_max;
+
+  public:
+    NArgsRange(std::size_t minimum, std::size_t maximum)
+        : m_min(minimum), m_max(maximum) {
+      if (minimum > maximum) {
+        throw std::logic_error("Range of number of arguments is invalid");
+      }
+    }
+
+    bool contains(std::size_t value) const {
+      return value >= m_min && value <= m_max;
+    }
+
+    bool is_exact() const { return m_min == m_max; }
+
+    bool is_right_bounded() const {
+      return m_max < (std::numeric_limits<std::size_t>::max)();
+    }
+
+    std::size_t get_min() const { return m_min; }
+
+    std::size_t get_max() const { return m_max; }
+
+    // Print help message
+    friend auto operator<<(std::ostream &stream, const NArgsRange &range)
+        -> std::ostream & {
+      if (range.m_min == range.m_max) {
+        if (range.m_min != 0 && range.m_min != 1) {
+          stream << "[nargs: " << range.m_min << "] ";
+        }
+      } else {
+        if (range.m_max == (std::numeric_limits<std::size_t>::max)()) {
+          stream << "[nargs: " << range.m_min << " or more] ";
+        } else {
+          stream << "[nargs=" << range.m_min << ".." << range.m_max << "] ";
+        }
+      }
+      return stream;
+    }
+
+    bool operator==(const NArgsRange &rhs) const {
+      return rhs.m_min == m_min && rhs.m_max == m_max;
+    }
+
+    bool operator!=(const NArgsRange &rhs) const { return !(*this == rhs); }
+  };
+
+  void throw_nargs_range_validation_error() const {
+    std::stringstream stream;
+    if (!m_used_name.empty()) {
+      stream << m_used_name << ": ";
+    } else {
+      stream << m_names.front() << ": ";
+    }
+    if (m_num_args_range.is_exact()) {
+      stream << m_num_args_range.get_min();
+    } else if (m_num_args_range.is_right_bounded()) {
+      stream << m_num_args_range.get_min() << " to "
+             << m_num_args_range.get_max();
+    } else {
+      stream << m_num_args_range.get_min() << " or more";
+    }
+    stream << " argument(s) expected. " << m_values.size() << " provided.";
+    throw std::runtime_error(stream.str());
+  }
+
+  void throw_required_arg_not_used_error() const {
+    std::stringstream stream;
+    stream << m_names.front() << ": required.";
+    throw std::runtime_error(stream.str());
+  }
+
+  void throw_required_arg_no_value_provided_error() const {
+    std::stringstream stream;
+    stream << m_used_name << ": no value provided.";
+    throw std::runtime_error(stream.str());
+  }
+
+  static constexpr int eof = std::char_traits<char>::eof();
+
+  static auto lookahead(std::string_view s) -> int {
+    if (s.empty()) {
+      return eof;
+    }
+    return static_cast<int>(static_cast<unsigned char>(s[0]));
+  }
+
+  /*
+   * decimal-literal:
+   *    '0'
+   *    nonzero-digit digit-sequence_opt
+   *    integer-part fractional-part
+   *    fractional-part
+   *    integer-part '.' exponent-part_opt
+   *    integer-part exponent-part
+   *
+   * integer-part:
+   *    digit-sequence
+   *
+   * fractional-part:
+   *    '.' post-decimal-point
+   *
+   * post-decimal-point:
+   *    digit-sequence exponent-part_opt
+   *
+   * exponent-part:
+   *    'e' post-e
+   *    'E' post-e
+   *
+   * post-e:
+   *    sign_opt digit-sequence
+   *
+   * sign: one of
+   *    '+' '-'
+   */
+  static bool is_decimal_literal(std::string_view s) {
+    auto is_digit = [](auto c) constexpr {
+      switch (c) {
+      case '0':
+      case '1':
+      case '2':
+      case '3':
+      case '4':
+      case '5':
+      case '6':
+      case '7':
+      case '8':
+      case '9':
+        return true;
+      default:
+        return false;
+      }
+    };
+
+    // precondition: we have consumed or will consume at least one digit
+    auto consume_digits = [=](std::string_view sd) {
+      // NOLINTNEXTLINE(readability-qualified-auto)
+      auto it = std::find_if_not(std::begin(sd), std::end(sd), is_digit);
+      return sd.substr(static_cast<std::size_t>(it - std::begin(sd)));
+    };
+
+    switch (lookahead(s)) {
+    case '0': {
+      s.remove_prefix(1);
+      if (s.empty()) {
+        return true;
+      }
+      goto integer_part;
+    }
+    case '1':
+    case '2':
+    case '3':
+    case '4':
+    case '5':
+    case '6':
+    case '7':
+    case '8':
+    case '9': {
+      s = consume_digits(s);
+      if (s.empty()) {
+        return true;
+      }
+      goto integer_part_consumed;
+    }
+    case '.': {
+      s.remove_prefix(1);
+      goto post_decimal_point;
+    }
+    default:
+      return false;
+    }
+
+  integer_part:
+    s = consume_digits(s);
+  integer_part_consumed:
+    switch (lookahead(s)) {
+    case '.': {
+      s.remove_prefix(1);
+      if (is_digit(lookahead(s))) {
+        goto post_decimal_point;
+      } else {
+        goto exponent_part_opt;
+      }
+    }
+    case 'e':
+    case 'E': {
+      s.remove_prefix(1);
+      goto post_e;
+    }
+    default:
+      return false;
+    }
+
+  post_decimal_point:
+    if (is_digit(lookahead(s))) {
+      s = consume_digits(s);
+      goto exponent_part_opt;
+    }
+    return false;
+
+  exponent_part_opt:
+    switch (lookahead(s)) {
+    case eof:
+      return true;
+    case 'e':
+    case 'E': {
+      s.remove_prefix(1);
+      goto post_e;
+    }
+    default:
+      return false;
+    }
+
+  post_e:
+    switch (lookahead(s)) {
+    case '-':
+    case '+':
+      s.remove_prefix(1);
+    }
+    if (is_digit(lookahead(s))) {
+      s = consume_digits(s);
+      return s.empty();
+    }
+    return false;
+  }
+
+  static bool is_optional(std::string_view name,
+                          std::string_view prefix_chars) {
+    return !is_positional(name, prefix_chars);
+  }
+
+  /*
+   * Get argument value given a type
+   * @throws std::logic_error in case of incompatible types
+   */
+  template <typename T> T get() const {
+    if (!m_values.empty()) {
+      if constexpr (details::IsContainer<T>) {
+        return any_cast_container<T>(m_values);
+      } else {
+        return std::any_cast<T>(m_values.front());
+      }
+    }
+    if (m_default_value.has_value()) {
+      return std::any_cast<T>(m_default_value);
+    }
+    if constexpr (details::IsContainer<T>) {
+      if (!m_accepts_optional_like_value) {
+        return any_cast_container<T>(m_values);
+      }
+    }
+
+    throw std::logic_error("No value provided for '" + m_names.back() + "'.");
+  }
+
+  /*
+   * Get argument value given a type.
+   * @pre The object has no default value.
+   * @returns The stored value if any, std::nullopt otherwise.
+   */
+  template <typename T> auto present() const -> std::optional<T> {
+    if (m_default_value.has_value()) {
+      throw std::logic_error("Argument with default value always presents");
+    }
+    if (m_values.empty()) {
+      return std::nullopt;
+    }
+    if constexpr (details::IsContainer<T>) {
+      return any_cast_container<T>(m_values);
+    }
+    return std::any_cast<T>(m_values.front());
+  }
+
+  template <typename T>
+  static auto any_cast_container(const std::vector<std::any> &operand) -> T {
+    using ValueType = typename T::value_type;
+
+    T result;
+    std::transform(
+        std::begin(operand), std::end(operand), std::back_inserter(result),
+        [](const auto &value) { return std::any_cast<ValueType>(value); });
+    return result;
+  }
+
+  void set_usage_newline_counter(int i) { m_usage_newline_counter = i; }
+
+  void set_group_idx(std::size_t i) { m_group_idx = i; }
+
+  std::vector<std::string> m_names;
+  std::string_view m_used_name;
+  std::string m_help;
+  std::string m_metavar;
+  std::any m_default_value;
+  std::string m_default_value_repr;
+  std::optional<std::string>
+      m_default_value_str; // used for checking default_value against choices
+  std::any m_implicit_value;
+  std::optional<std::vector<std::string>> m_choices{std::nullopt};
+  using valued_action = std::function<std::any(const std::string &)>;
+  using void_action = std::function<void(const std::string &)>;
+  std::variant<valued_action, void_action> m_action{
+      std::in_place_type<valued_action>,
+      [](const std::string &value) { return value; }};
+  std::vector<std::any> m_values;
+  NArgsRange m_num_args_range{1, 1};
+  // Bit field of bool values. Set default value in ctor.
+  bool m_accepts_optional_like_value : 1;
+  bool m_is_optional : 1;
+  bool m_is_required : 1;
+  bool m_is_repeatable : 1;
+  bool m_is_used : 1;
+  bool m_is_hidden : 1;            // if set, does not appear in usage or help
+  std::string_view m_prefix_chars; // ArgumentParser has the prefix_chars
+  int m_usage_newline_counter = 0;
+  std::size_t m_group_idx = 0;
+};
+
+class ArgumentParser {
+public:
+  explicit ArgumentParser(std::string program_name = {},
+                          std::string version = "1.0",
+                          default_arguments add_args = default_arguments::all,
+                          bool exit_on_default_arguments = true,
+                          std::ostream &os = std::cout)
+      : m_program_name(std::move(program_name)), m_version(std::move(version)),
+        m_exit_on_default_arguments(exit_on_default_arguments),
+        m_parser_path(m_program_name) {
+    if ((add_args & default_arguments::help) == default_arguments::help) {
+      add_argument("-h", "--help")
+          .action([&](const auto & /*unused*/) {
+            os << help().str();
+            if (m_exit_on_default_arguments) {
+              std::exit(0);
+            }
+          })
+          .default_value(false)
+          .help("shows help message and exits")
+          .implicit_value(true)
+          .nargs(0);
+    }
+    if ((add_args & default_arguments::version) == default_arguments::version) {
+      add_argument("-v", "--version")
+          .action([&](const auto & /*unused*/) {
+            os << m_version << std::endl;
+            if (m_exit_on_default_arguments) {
+              std::exit(0);
+            }
+          })
+          .default_value(false)
+          .help("prints version information and exits")
+          .implicit_value(true)
+          .nargs(0);
+    }
+  }
+
+  ~ArgumentParser() = default;
+
+  // ArgumentParser is meant to be used in a single function.
+  // Setup everything and parse arguments in one place.
+  //
+  // ArgumentParser internally uses std::string_views,
+  // references, iterators, etc.
+  // Many of these elements become invalidated after a copy or move.
+  ArgumentParser(const ArgumentParser &other) = delete;
+  ArgumentParser &operator=(const ArgumentParser &other) = delete;
+  ArgumentParser(ArgumentParser &&) noexcept = delete;
+  ArgumentParser &operator=(ArgumentParser &&) = delete;
+
+  explicit operator bool() const {
+    auto arg_used = std::any_of(m_argument_map.cbegin(), m_argument_map.cend(),
+                                [](auto &it) { return it.second->m_is_used; });
+    auto subparser_used =
+        std::any_of(m_subparser_used.cbegin(), m_subparser_used.cend(),
+                    [](auto &it) { return it.second; });
+
+    return m_is_parsed && (arg_used || subparser_used);
+  }
+
+  // Parameter packing
+  // Call add_argument with variadic number of string arguments
+  template <typename... Targs> Argument &add_argument(Targs... f_args) {
+    using array_of_sv = std::array<std::string_view, sizeof...(Targs)>;
+    auto argument =
+        m_optional_arguments.emplace(std::cend(m_optional_arguments),
+                                     m_prefix_chars, array_of_sv{f_args...});
+
+    if (!argument->m_is_optional) {
+      m_positional_arguments.splice(std::cend(m_positional_arguments),
+                                    m_optional_arguments, argument);
+    }
+    argument->set_usage_newline_counter(m_usage_newline_counter);
+    argument->set_group_idx(m_group_names.size());
+
+    index_argument(argument);
+    return *argument;
+  }
+
+  class MutuallyExclusiveGroup {
+    friend class ArgumentParser;
+
+  public:
+    MutuallyExclusiveGroup() = delete;
+
+    explicit MutuallyExclusiveGroup(ArgumentParser &parent,
+                                    bool required = false)
+        : m_parent(parent), m_required(required), m_elements({}) {}
+
+    MutuallyExclusiveGroup(const MutuallyExclusiveGroup &other) = delete;
+    MutuallyExclusiveGroup &
+    operator=(const MutuallyExclusiveGroup &other) = delete;
+
+    MutuallyExclusiveGroup(MutuallyExclusiveGroup &&other) noexcept
+        : m_parent(other.m_parent), m_required(other.m_required),
+          m_elements(std::move(other.m_elements)) {
+      other.m_elements.clear();
+    }
+
+    template <typename... Targs> Argument &add_argument(Targs... f_args) {
+      auto &argument = m_parent.add_argument(std::forward<Targs>(f_args)...);
+      m_elements.push_back(&argument);
+      argument.set_usage_newline_counter(m_parent.m_usage_newline_counter);
+      argument.set_group_idx(m_parent.m_group_names.size());
+      return argument;
+    }
+
+  private:
+    ArgumentParser &m_parent;
+    bool m_required{false};
+    std::vector<Argument *> m_elements{};
+  };
+
+  MutuallyExclusiveGroup &add_mutually_exclusive_group(bool required = false) {
+    m_mutually_exclusive_groups.emplace_back(*this, required);
+    return m_mutually_exclusive_groups.back();
+  }
+
+  // Parameter packed add_parents method
+  // Accepts a variadic number of ArgumentParser objects
+  template <typename... Targs>
+  ArgumentParser &add_parents(const Targs &... f_args) {
+    for (const ArgumentParser &parent_parser : {std::ref(f_args)...}) {
+      for (const auto &argument : parent_parser.m_positional_arguments) {
+        auto it = m_positional_arguments.insert(
+            std::cend(m_positional_arguments), argument);
+        index_argument(it);
+      }
+      for (const auto &argument : parent_parser.m_optional_arguments) {
+        auto it = m_optional_arguments.insert(std::cend(m_optional_arguments),
+                                              argument);
+        index_argument(it);
+      }
+    }
+    return *this;
+  }
+
+  // Ask for the next optional arguments to be displayed on a separate
+  // line in usage() output. Only effective if set_usage_max_line_width() is
+  // also used.
+  ArgumentParser &add_usage_newline() {
+    ++m_usage_newline_counter;
+    return *this;
+  }
+
+  // Ask for the next optional arguments to be displayed in a separate section
+  // in usage() and help (<< *this) output.
+  // For usage(), this is only effective if set_usage_max_line_width() is
+  // also used.
+  ArgumentParser &add_group(std::string group_name) {
+    m_group_names.emplace_back(std::move(group_name));
+    return *this;
+  }
+
+  ArgumentParser &add_description(std::string description) {
+    m_description = std::move(description);
+    return *this;
+  }
+
+  ArgumentParser &add_epilog(std::string epilog) {
+    m_epilog = std::move(epilog);
+    return *this;
+  }
+
+  // Add a un-documented/hidden alias for an argument.
+  // Ideally we'd want this to be a method of Argument, but Argument
+  // does not own its owing ArgumentParser.
+  ArgumentParser &add_hidden_alias_for(Argument &arg, std::string_view alias) {
+    for (auto it = m_optional_arguments.begin();
+         it != m_optional_arguments.end(); ++it) {
+      if (&(*it) == &arg) {
+        m_argument_map.insert_or_assign(std::string(alias), it);
+        return *this;
+      }
+    }
+    throw std::logic_error(
+        "Argument is not an optional argument of this parser");
+  }
+
+  /* Getter for arguments and subparsers.
+   * @throws std::logic_error in case of an invalid argument or subparser name
+   */
+  template <typename T = Argument> T &at(std::string_view name) {
+    if constexpr (std::is_same_v<T, Argument>) {
+      return (*this)[name];
+    } else {
+      std::string str_name(name);
+      auto subparser_it = m_subparser_map.find(str_name);
+      if (subparser_it != m_subparser_map.end()) {
+        return subparser_it->second->get();
+      }
+      throw std::logic_error("No such subparser: " + str_name);
+    }
+  }
+
+  ArgumentParser &set_prefix_chars(std::string prefix_chars) {
+    m_prefix_chars = std::move(prefix_chars);
+    return *this;
+  }
+
+  ArgumentParser &set_assign_chars(std::string assign_chars) {
+    m_assign_chars = std::move(assign_chars);
+    return *this;
+  }
+
+  /* Call parse_args_internal - which does all the work
+   * Then, validate the parsed arguments
+   * This variant is used mainly for testing
+   * @throws std::runtime_error in case of any invalid argument
+   */
+  void parse_args(const std::vector<std::string> &arguments) {
+    parse_args_internal(arguments);
+    // Check if all arguments are parsed
+    for ([[maybe_unused]] const auto &[unused, argument] : m_argument_map) {
+      argument->validate();
+    }
+
+    // Check each mutually exclusive group and make sure
+    // there are no constraint violations
+    for (const auto &group : m_mutually_exclusive_groups) {
+      auto mutex_argument_used{false};
+      Argument *mutex_argument_it{nullptr};
+      for (Argument *arg : group.m_elements) {
+        if (!mutex_argument_used && arg->m_is_used) {
+          mutex_argument_used = true;
+          mutex_argument_it = arg;
+        } else if (mutex_argument_used && arg->m_is_used) {
+          // Violation
+          throw std::runtime_error("Argument '" + arg->get_usage_full() +
+                                   "' not allowed with '" +
+                                   mutex_argument_it->get_usage_full() + "'");
+        }
+      }
+
+      if (!mutex_argument_used && group.m_required) {
+        // at least one argument from the group is
+        // required
+        std::string argument_names{};
+        std::size_t i = 0;
+        std::size_t size = group.m_elements.size();
+        for (Argument *arg : group.m_elements) {
+          if (i + 1 == size) {
+            // last
+            argument_names += std::string("'") + arg->get_usage_full() + std::string("' ");
+          } else {
+            argument_names += std::string("'") + arg->get_usage_full() + std::string("' or ");
+          }
+          i += 1;
+        }
+        throw std::runtime_error("One of the arguments " + argument_names +
+                                 "is required");
+      }
+    }
+  }
+
+  /* Call parse_known_args_internal - which does all the work
+   * Then, validate the parsed arguments
+   * This variant is used mainly for testing
+   * @throws std::runtime_error in case of any invalid argument
+   */
+  std::vector<std::string>
+  parse_known_args(const std::vector<std::string> &arguments) {
+    auto unknown_arguments = parse_known_args_internal(arguments);
+    // Check if all arguments are parsed
+    for ([[maybe_unused]] const auto &[unused, argument] : m_argument_map) {
+      argument->validate();
+    }
+    return unknown_arguments;
+  }
+
+  /* Main entry point for parsing command-line arguments using this
+   * ArgumentParser
+   * @throws std::runtime_error in case of any invalid argument
+   */
+  // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays)
+  void parse_args(int argc, const char *const argv[]) {
+    parse_args({argv, argv + argc});
+  }
+
+  /* Main entry point for parsing command-line arguments using this
+   * ArgumentParser
+   * @throws std::runtime_error in case of any invalid argument
+   */
+  // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays)
+  auto parse_known_args(int argc, const char *const argv[]) {
+    return parse_known_args({argv, argv + argc});
+  }
+
+  /* Getter for options with default values.
+   * @throws std::logic_error if parse_args() has not been previously called
+   * @throws std::logic_error if there is no such option
+   * @throws std::logic_error if the option has no value
+   * @throws std::bad_any_cast if the option is not of type T
+   */
+  template <typename T = std::string> T get(std::string_view arg_name) const {
+    if (!m_is_parsed) {
+      throw std::logic_error("Nothing parsed, no arguments are available.");
+    }
+    return (*this)[arg_name].get<T>();
+  }
+
+  /* Getter for options without default values.
+   * @pre The option has no default value.
+   * @throws std::logic_error if there is no such option
+   * @throws std::bad_any_cast if the option is not of type T
+   */
+  template <typename T = std::string>
+  auto present(std::string_view arg_name) const -> std::optional<T> {
+    return (*this)[arg_name].present<T>();
+  }
+
+  /* Getter that returns true for user-supplied options. Returns false if not
+   * user-supplied, even with a default value.
+   */
+  auto is_used(std::string_view arg_name) const {
+    return (*this)[arg_name].m_is_used;
+  }
+
+  /* Getter that returns true if a subcommand is used.
+   */
+  auto is_subcommand_used(std::string_view subcommand_name) const {
+    return m_subparser_used.at(std::string(subcommand_name));
+  }
+
+  /* Getter that returns true if a subcommand is used.
+   */
+  auto is_subcommand_used(const ArgumentParser &subparser) const {
+    return is_subcommand_used(subparser.m_program_name);
+  }
+
+  /* Indexing operator. Return a reference to an Argument object
+   * Used in conjunction with Argument.operator== e.g., parser["foo"] == true
+   * @throws std::logic_error in case of an invalid argument name
+   */
+  Argument &operator[](std::string_view arg_name) const {
+    std::string name(arg_name);
+    auto it = m_argument_map.find(name);
+    if (it != m_argument_map.end()) {
+      return *(it->second);
+    }
+    if (!is_valid_prefix_char(arg_name.front())) {
+      const auto legal_prefix_char = get_any_valid_prefix_char();
+      const auto prefix = std::string(1, legal_prefix_char);
+
+      // "-" + arg_name
+      name = prefix + name;
+      it = m_argument_map.find(name);
+      if (it != m_argument_map.end()) {
+        return *(it->second);
+      }
+      // "--" + arg_name
+      name = prefix + name;
+      it = m_argument_map.find(name);
+      if (it != m_argument_map.end()) {
+        return *(it->second);
+      }
+    }
+    throw std::logic_error("No such argument: " + std::string(arg_name));
+  }
+
+  // Print help message
+  friend auto operator<<(std::ostream &stream, const ArgumentParser &parser)
+      -> std::ostream & {
+    stream.setf(std::ios_base::left);
+
+    auto longest_arg_length = parser.get_length_of_longest_argument();
+
+    stream << parser.usage() << "\n\n";
+
+    if (!parser.m_description.empty()) {
+      stream << parser.m_description << "\n\n";
+    }
+
+    const bool has_visible_positional_args = std::find_if(
+      parser.m_positional_arguments.begin(),
+      parser.m_positional_arguments.end(),
+      [](const auto &argument) {
+      return !argument.m_is_hidden; }) !=
+      parser.m_positional_arguments.end();
+    if (has_visible_positional_args) {
+      stream << "Positional arguments:\n";
+    }
+
+    for (const auto &argument : parser.m_positional_arguments) {
+      if (!argument.m_is_hidden) {
+        stream.width(static_cast<std::streamsize>(longest_arg_length));
+        stream << argument;
+      }
+    }
+
+    if (!parser.m_optional_arguments.empty()) {
+      stream << (!has_visible_positional_args ? "" : "\n")
+             << "Optional arguments:\n";
+    }
+
+    for (const auto &argument : parser.m_optional_arguments) {
+      if (argument.m_group_idx == 0 && !argument.m_is_hidden) {
+        stream.width(static_cast<std::streamsize>(longest_arg_length));
+        stream << argument;
+      }
+    }
+
+    for (size_t i_group = 0; i_group < parser.m_group_names.size(); ++i_group) {
+      stream << "\n" << parser.m_group_names[i_group] << " (detailed usage):\n";
+      for (const auto &argument : parser.m_optional_arguments) {
+        if (argument.m_group_idx == i_group + 1 && !argument.m_is_hidden) {
+          stream.width(static_cast<std::streamsize>(longest_arg_length));
+          stream << argument;
+        }
+      }
+    }
+
+    bool has_visible_subcommands = std::any_of(
+        parser.m_subparser_map.begin(), parser.m_subparser_map.end(),
+        [](auto &p) { return !p.second->get().m_suppress; });
+
+    if (has_visible_subcommands) {
+      stream << (parser.m_positional_arguments.empty()
+                     ? (parser.m_optional_arguments.empty() ? "" : "\n")
+                     : "\n")
+             << "Subcommands:\n";
+      for (const auto &[command, subparser] : parser.m_subparser_map) {
+        if (subparser->get().m_suppress) {
+          continue;
+        }
+
+        stream << std::setw(2) << " ";
+        stream << std::setw(static_cast<int>(longest_arg_length - 2))
+               << command;
+        stream << " " << subparser->get().m_description << "\n";
+      }
+    }
+
+    if (!parser.m_epilog.empty()) {
+      stream << '\n';
+      stream << parser.m_epilog << "\n\n";
+    }
+
+    return stream;
+  }
+
+  // Format help message
+  auto help() const -> std::stringstream {
+    std::stringstream out;
+    out << *this;
+    return out;
+  }
+
+  // Sets the maximum width for a line of the Usage message
+  ArgumentParser &set_usage_max_line_width(size_t w) {
+    this->m_usage_max_line_width = w;
+    return *this;
+  }
+
+  // Asks to display arguments of mutually exclusive group on separate lines in
+  // the Usage message
+  ArgumentParser &set_usage_break_on_mutex() {
+    this->m_usage_break_on_mutex = true;
+    return *this;
+  }
+
+  // Format usage part of help only
+  auto usage() const -> std::string {
+    std::stringstream stream;
+
+    std::string curline("Usage: ");
+    curline += this->m_parser_path;
+    const bool multiline_usage =
+        this->m_usage_max_line_width < (std::numeric_limits<std::size_t>::max)();
+    const size_t indent_size = curline.size();
+
+    const auto deal_with_options_of_group = [&](std::size_t group_idx) {
+      bool found_options = false;
+      // Add any options inline here
+      const MutuallyExclusiveGroup *cur_mutex = nullptr;
+      int usage_newline_counter = -1;
+      for (const auto &argument : this->m_optional_arguments) {
+        if (argument.m_is_hidden) {
+          continue;
+        }
+        if (multiline_usage) {
+          if (argument.m_group_idx != group_idx) {
+            continue;
+          }
+          if (usage_newline_counter != argument.m_usage_newline_counter) {
+            if (usage_newline_counter >= 0) {
+              if (curline.size() > indent_size) {
+                stream << curline << std::endl;
+                curline = std::string(indent_size, ' ');
+              }
+            }
+            usage_newline_counter = argument.m_usage_newline_counter;
+          }
+        }
+        found_options = true;
+        const std::string arg_inline_usage = argument.get_inline_usage();
+        const MutuallyExclusiveGroup *arg_mutex =
+            get_belonging_mutex(&argument);
+        if ((cur_mutex != nullptr) && (arg_mutex == nullptr)) {
+          curline += ']';
+          if (this->m_usage_break_on_mutex) {
+            stream << curline << std::endl;
+            curline = std::string(indent_size, ' ');
+          }
+        } else if ((cur_mutex == nullptr) && (arg_mutex != nullptr)) {
+          if ((this->m_usage_break_on_mutex && curline.size() > indent_size) ||
+              curline.size() + 3 + arg_inline_usage.size() >
+                  this->m_usage_max_line_width) {
+            stream << curline << std::endl;
+            curline = std::string(indent_size, ' ');
+          }
+          curline += " [";
+        } else if ((cur_mutex != nullptr) && (arg_mutex != nullptr)) {
+          if (cur_mutex != arg_mutex) {
+            curline += ']';
+            if (this->m_usage_break_on_mutex ||
+                curline.size() + 3 + arg_inline_usage.size() >
+                    this->m_usage_max_line_width) {
+              stream << curline << std::endl;
+              curline = std::string(indent_size, ' ');
+            }
+            curline += " [";
+          } else {
+            curline += '|';
+          }
+        }
+        cur_mutex = arg_mutex;
+        if (curline.size() + 1 + arg_inline_usage.size() >
+            this->m_usage_max_line_width) {
+          stream << curline << std::endl;
+          curline = std::string(indent_size, ' ');
+          curline += " ";
+        } else if (cur_mutex == nullptr) {
+          curline += " ";
+        }
+        curline += arg_inline_usage;
+      }
+      if (cur_mutex != nullptr) {
+        curline += ']';
+      }
+      return found_options;
+    };
+
+    const bool found_options = deal_with_options_of_group(0);
+
+    if (found_options && multiline_usage &&
+        !this->m_positional_arguments.empty()) {
+      stream << curline << std::endl;
+      curline = std::string(indent_size, ' ');
+    }
+    // Put positional arguments after the optionals
+    for (const auto &argument : this->m_positional_arguments) {
+      if (argument.m_is_hidden) {
+        continue;
+      }
+      const std::string pos_arg = !argument.m_metavar.empty()
+                                      ? argument.m_metavar
+                                      : argument.m_names.front();
+      if (curline.size() + 1 + pos_arg.size() > this->m_usage_max_line_width) {
+        stream << curline << std::endl;
+        curline = std::string(indent_size, ' ');
+      }
+      curline += " ";
+      if (argument.m_num_args_range.get_min() == 0 &&
+          !argument.m_num_args_range.is_right_bounded()) {
+        curline += "[";
+        curline += pos_arg;
+        curline += "]...";
+      } else if (argument.m_num_args_range.get_min() == 1 &&
+                 !argument.m_num_args_range.is_right_bounded()) {
+        curline += pos_arg;
+        curline += "...";
+      } else {
+        curline += pos_arg;
+      }
+    }
+
+    if (multiline_usage) {
+      // Display options of other groups
+      for (std::size_t i = 0; i < m_group_names.size(); ++i) {
+        stream << curline << std::endl << std::endl;
+        stream << m_group_names[i] << ":" << std::endl;
+        curline = std::string(indent_size, ' ');
+        deal_with_options_of_group(i + 1);
+      }
+    }
+
+    stream << curline;
+
+    // Put subcommands after positional arguments
+    if (!m_subparser_map.empty()) {
+      stream << " {";
+      std::size_t i{0};
+      for (const auto &[command, subparser] : m_subparser_map) {
+        if (subparser->get().m_suppress) {
+          continue;
+        }
+
+        if (i == 0) {
+          stream << command;
+        } else {
+          stream << "," << command;
+        }
+        ++i;
+      }
+      stream << "}";
+    }
+
+    return stream.str();
+  }
+
+  // Printing the one and only help message
+  // I've stuck with a simple message format, nothing fancy.
+  [[deprecated("Use cout << program; instead.  See also help().")]] std::string
+  print_help() const {
+    auto out = help();
+    std::cout << out.rdbuf();
+    return out.str();
+  }
+
+  void add_subparser(ArgumentParser &parser) {
+    parser.m_parser_path = m_program_name + " " + parser.m_program_name;
+    auto it = m_subparsers.emplace(std::cend(m_subparsers), parser);
+    m_subparser_map.insert_or_assign(parser.m_program_name, it);
+    m_subparser_used.insert_or_assign(parser.m_program_name, false);
+  }
+
+  void set_suppress(bool suppress) { m_suppress = suppress; }
+
+protected:
+  const MutuallyExclusiveGroup *get_belonging_mutex(const Argument *arg) const {
+    for (const auto &mutex : m_mutually_exclusive_groups) {
+      if (std::find(mutex.m_elements.begin(), mutex.m_elements.end(), arg) !=
+          mutex.m_elements.end()) {
+        return &mutex;
+      }
+    }
+    return nullptr;
+  }
+
+  bool is_valid_prefix_char(char c) const {
+    return m_prefix_chars.find(c) != std::string::npos;
+  }
+
+  char get_any_valid_prefix_char() const { return m_prefix_chars[0]; }
+
+  /*
+   * Pre-process this argument list. Anything starting with "--", that
+   * contains an =, where the prefix before the = has an entry in the
+   * options table, should be split.
+   */
+  std::vector<std::string>
+  preprocess_arguments(const std::vector<std::string> &raw_arguments) const {
+    std::vector<std::string> arguments{};
+    for (const auto &arg : raw_arguments) {
+      
+      const auto argument_starts_with_prefix_chars =
+          [this](const std::string &a) -> bool {
+        if (!a.empty()) {
+
+          const auto legal_prefix = [this](char c) -> bool {
+            return m_prefix_chars.find(c) != std::string::npos;
+          };
+
+          // Windows-style
+          // if '/' is a legal prefix char
+          // then allow single '/' followed by argument name, followed by an
+          // assign char, e.g., ':' e.g., 'test.exe /A:Foo'
+          const auto windows_style = legal_prefix('/');
+
+          if (windows_style) {
+            if (legal_prefix(a[0])) {
+              return true;
+            }
+          } else {
+            // Slash '/' is not a legal prefix char
+            // For all other characters, only support long arguments
+            // i.e., the argument must start with 2 prefix chars, e.g,
+            // '--foo' e,g, './test --foo=Bar -DARG=yes'
+            if (a.size() > 1) {
+              return (legal_prefix(a[0]) && legal_prefix(a[1]));
+            }
+          }
+        }
+        return false;
+      };
+
+      // Check that:
+      // - We don't have an argument named exactly this
+      // - The argument starts with a prefix char, e.g., "--"
+      // - The argument contains an assign char, e.g., "="
+      auto assign_char_pos = arg.find_first_of(m_assign_chars);
+
+      if (m_argument_map.find(arg) == m_argument_map.end() &&
+          argument_starts_with_prefix_chars(arg) &&
+          assign_char_pos != std::string::npos) {
+        // Get the name of the potential option, and check it exists
+        std::string opt_name = arg.substr(0, assign_char_pos);
+        if (m_argument_map.find(opt_name) != m_argument_map.end()) {
+          // This is the name of an option! Split it into two parts
+          arguments.push_back(std::move(opt_name));
+          arguments.push_back(arg.substr(assign_char_pos + 1));
+          continue;
+        }
+      }
+      // If we've fallen through to here, then it's a standard argument
+      arguments.push_back(arg);
+    }
+    return arguments;
+  }
+
+  /*
+   * @throws std::runtime_error in case of any invalid argument
+   */
+  void parse_args_internal(const std::vector<std::string> &raw_arguments) {
+    auto arguments = preprocess_arguments(raw_arguments);
+    if (m_program_name.empty() && !arguments.empty()) {
+      m_program_name = arguments.front();
+    }
+    auto end = std::end(arguments);
+    auto positional_argument_it = std::begin(m_positional_arguments);
+    for (auto it = std::next(std::begin(arguments)); it != end;) {
+      const auto &current_argument = *it;
+      if (Argument::is_positional(current_argument, m_prefix_chars)) {
+        if (positional_argument_it == std::end(m_positional_arguments)) {
+
+          // Check sub-parsers
+          auto subparser_it = m_subparser_map.find(current_argument);
+          if (subparser_it != m_subparser_map.end()) {
+
+            // build list of remaining args
+            const auto unprocessed_arguments =
+                std::vector<std::string>(it, end);
+
+            // invoke subparser
+            m_is_parsed = true;
+            m_subparser_used[current_argument] = true;
+            return subparser_it->second->get().parse_args(
+                unprocessed_arguments);
+          }
+
+          if (m_positional_arguments.empty()) {
+
+            // Ask the user if they argument they provided was a typo
+            // for some sub-parser,
+            // e.g., user provided `git totes` instead of `git notes`
+            if (!m_subparser_map.empty()) {
+              throw std::runtime_error(
+                  "Failed to parse '" + current_argument + "', did you mean '" +
+                  std::string{details::get_most_similar_string(
+                      m_subparser_map, current_argument)} +
+                  "'");
+            }
+
+            // Ask the user if they meant to use a specific optional argument
+            if (!m_optional_arguments.empty()) {
+              for (const auto &opt : m_optional_arguments) {
+                if (!opt.m_implicit_value.has_value()) {
+                  // not a flag, requires a value
+                  if (!opt.m_is_used) {
+                    throw std::runtime_error(
+                        "Zero positional arguments expected, did you mean " +
+                        opt.get_usage_full());
+                  }
+                }
+              }
+
+              throw std::runtime_error("Zero positional arguments expected");
+            } else {
+              throw std::runtime_error("Zero positional arguments expected");
+            }
+          } else {
+            throw std::runtime_error("Maximum number of positional arguments "
+                                     "exceeded, failed to parse '" +
+                                     current_argument + "'");
+          }
+        }
+        auto argument = positional_argument_it++;
+
+        // Deal with the situation of <positional_arg1>... <positional_arg2>
+        if (argument->m_num_args_range.get_min() == 1 &&
+            argument->m_num_args_range.get_max() == (std::numeric_limits<std::size_t>::max)() &&
+            positional_argument_it != std::end(m_positional_arguments) &&
+            std::next(positional_argument_it) == std::end(m_positional_arguments) &&
+            positional_argument_it->m_num_args_range.get_min() == 1 &&
+            positional_argument_it->m_num_args_range.get_max() == 1 ) {
+          if (std::next(it) != end) {
+            positional_argument_it->consume(std::prev(end), end);
+            end = std::prev(end);
+          } else {
+            throw std::runtime_error("Missing " + positional_argument_it->m_names.front());
+          }
+        }
+
+        it = argument->consume(it, end);
+        continue;
+      }
+
+      auto arg_map_it = m_argument_map.find(current_argument);
+      if (arg_map_it != m_argument_map.end()) {
+        auto argument = arg_map_it->second;
+        it = argument->consume(std::next(it), end, arg_map_it->first);
+      } else if (const auto &compound_arg = current_argument;
+                 compound_arg.size() > 1 &&
+                 is_valid_prefix_char(compound_arg[0]) &&
+                 !is_valid_prefix_char(compound_arg[1])) {
+        ++it;
+        for (std::size_t j = 1; j < compound_arg.size(); j++) {
+          auto hypothetical_arg = std::string{'-', compound_arg[j]};
+          auto arg_map_it2 = m_argument_map.find(hypothetical_arg);
+          if (arg_map_it2 != m_argument_map.end()) {
+            auto argument = arg_map_it2->second;
+            it = argument->consume(it, end, arg_map_it2->first);
+          } else {
+            throw std::runtime_error("Unknown argument: " + current_argument);
+          }
+        }
+      } else {
+        throw std::runtime_error("Unknown argument: " + current_argument);
+      }
+    }
+    m_is_parsed = true;
+  }
+
+  /*
+   * Like parse_args_internal but collects unused args into a vector<string>
+   */
+  std::vector<std::string>
+  parse_known_args_internal(const std::vector<std::string> &raw_arguments) {
+    auto arguments = preprocess_arguments(raw_arguments);
+
+    std::vector<std::string> unknown_arguments{};
+
+    if (m_program_name.empty() && !arguments.empty()) {
+      m_program_name = arguments.front();
+    }
+    auto end = std::end(arguments);
+    auto positional_argument_it = std::begin(m_positional_arguments);
+    for (auto it = std::next(std::begin(arguments)); it != end;) {
+      const auto &current_argument = *it;
+      if (Argument::is_positional(current_argument, m_prefix_chars)) {
+        if (positional_argument_it == std::end(m_positional_arguments)) {
+
+          // Check sub-parsers
+          auto subparser_it = m_subparser_map.find(current_argument);
+          if (subparser_it != m_subparser_map.end()) {
+
+            // build list of remaining args
+            const auto unprocessed_arguments =
+                std::vector<std::string>(it, end);
+
+            // invoke subparser
+            m_is_parsed = true;
+            m_subparser_used[current_argument] = true;
+            return subparser_it->second->get().parse_known_args_internal(
+                unprocessed_arguments);
+          }
+
+          // save current argument as unknown and go to next argument
+          unknown_arguments.push_back(current_argument);
+          ++it;
+        } else {
+          // current argument is the value of a positional argument
+          // consume it
+          auto argument = positional_argument_it++;
+          it = argument->consume(it, end);
+        }
+        continue;
+      }
+
+      auto arg_map_it = m_argument_map.find(current_argument);
+      if (arg_map_it != m_argument_map.end()) {
+        auto argument = arg_map_it->second;
+        it = argument->consume(std::next(it), end, arg_map_it->first);
+      } else if (const auto &compound_arg = current_argument;
+                 compound_arg.size() > 1 &&
+                 is_valid_prefix_char(compound_arg[0]) &&
+                 !is_valid_prefix_char(compound_arg[1])) {
+        ++it;
+        for (std::size_t j = 1; j < compound_arg.size(); j++) {
+          auto hypothetical_arg = std::string{'-', compound_arg[j]};
+          auto arg_map_it2 = m_argument_map.find(hypothetical_arg);
+          if (arg_map_it2 != m_argument_map.end()) {
+            auto argument = arg_map_it2->second;
+            it = argument->consume(it, end, arg_map_it2->first);
+          } else {
+            unknown_arguments.push_back(current_argument);
+            break;
+          }
+        }
+      } else {
+        // current argument is an optional-like argument that is unknown
+        // save it and move to next argument
+        unknown_arguments.push_back(current_argument);
+        ++it;
+      }
+    }
+    m_is_parsed = true;
+    return unknown_arguments;
+  }
+
+  // Used by print_help.
+  std::size_t get_length_of_longest_argument() const {
+    if (m_argument_map.empty()) {
+      return 0;
+    }
+    std::size_t max_size = 0;
+    for ([[maybe_unused]] const auto &[unused, argument] : m_argument_map) {
+      max_size =
+          std::max<std::size_t>(max_size, argument->get_arguments_length());
+    }
+    for ([[maybe_unused]] const auto &[command, unused] : m_subparser_map) {
+      max_size = std::max<std::size_t>(max_size, command.size());
+    }
+    return max_size;
+  }
+
+  using argument_it = std::list<Argument>::iterator;
+  using mutex_group_it = std::vector<MutuallyExclusiveGroup>::iterator;
+  using argument_parser_it =
+      std::list<std::reference_wrapper<ArgumentParser>>::iterator;
+
+  void index_argument(argument_it it) {
+    for (const auto &name : std::as_const(it->m_names)) {
+      m_argument_map.insert_or_assign(name, it);
+    }
+  }
+
+  std::string m_program_name;
+  std::string m_version;
+  std::string m_description;
+  std::string m_epilog;
+  bool m_exit_on_default_arguments = true;
+  std::string m_prefix_chars{"-"};
+  std::string m_assign_chars{"="};
+  bool m_is_parsed = false;
+  std::list<Argument> m_positional_arguments;
+  std::list<Argument> m_optional_arguments;
+  std::map<std::string, argument_it> m_argument_map;
+  std::string m_parser_path;
+  std::list<std::reference_wrapper<ArgumentParser>> m_subparsers;
+  std::map<std::string, argument_parser_it> m_subparser_map;
+  std::map<std::string, bool> m_subparser_used;
+  std::vector<MutuallyExclusiveGroup> m_mutually_exclusive_groups;
+  bool m_suppress = false;
+  std::size_t m_usage_max_line_width = (std::numeric_limits<std::size_t>::max)();
+  bool m_usage_break_on_mutex = false;
+  int m_usage_newline_counter = 0;
+  std::vector<std::string> m_group_names;
+};
+
+} // namespace argparse
\ No newline at end of file
diff --git a/ext/htslib/.gitignore b/ext/htslib/.gitignore
new file mode 100644
index 0000000..817b123
--- /dev/null
+++ b/ext/htslib/.gitignore
@@ -0,0 +1,85 @@
+*.o
+*.pico
+*.obj
+*.dSYM
+*.exe
+*.dll
+*.dll.a
+*.pc.tmp
+*-uninstalled.pc
+config_vars.h
+/version.h
+
+autom4te.cache
+config.cache
+config.guess
+config.h
+config.h.in
+config.log
+config.mk
+config.status
+config.sub
+configure
+install-sh
+
+hfile_*.bundle
+hfile_*.cygdll
+hfile_*.dll
+hfile_*.so
+
+hts-object-files
+htslib_static.mk
+htscodecs.mk
+
+cyg*.dll
+lib*.a
+lib*.dll
+lib*.dylib
+lib*.so
+lib*.so.*
+
+header-exports.txt
+shlib-exports-*.txt
+
+/annot-tsv
+/bgzip
+/htsfile
+/tabix
+/test/*/FAIL*
+/test/bgzf_boundaries/*.tmp.*
+/test/faidx/*.tmp*
+/test/fieldarith
+/test/hfile
+/test/hts_endian
+/test/longrefs/*.tmp.*
+/test/pileup
+/test/pileup_mod
+/test/plugins-dlhts
+/test/sam
+/test/tabix/*.tmp.*
+/test/test-bcf-sr
+/test/test-bcf-translate
+/test/test-bcf_set_variant_type
+/test/test_bgzf
+/test/test_expr
+/test/test_faidx
+/test/test_index
+/test/test_introspection
+/test/test_kfunc
+/test/test_khash
+/test/test_kstring
+/test/test_mod
+/test/test_nibbles
+/test/test-parse-reg
+/test/test_realn
+/test/test-regidx
+/test/test_str2int
+/test/test_time_funcs
+/test/test-vcf-api
+/test/test-vcf-sweep
+/test/test_view
+/test/thrash_threads[1-7]
+/test/*.tmp
+/test/*.tmp.*
+
+/TAGS
diff --git a/ext/htslib/INSTALL b/ext/htslib/INSTALL
new file mode 100644
index 0000000..eb5048a
--- /dev/null
+++ b/ext/htslib/INSTALL
@@ -0,0 +1,316 @@
+                    Building and Installing HTSlib
+                    ==============================
+
+Requirements
+============
+
+Building HTSlib requires a few programs and libraries to be present.
+See the "System Specific Details" below for guidance on how to install
+these.
+
+At least the following are required:
+
+    GNU make
+    C compiler (e.g. gcc or clang)
+
+In addition, building the configure script requires:
+
+    autoheader
+    autoconf
+    autoreconf
+
+Running the configure script uses awk, along with a number of
+standard UNIX tools (cat, cp, grep, mv, rm, sed, among others).  Almost
+all installations will have these already.
+
+Running the test harness (make test) uses:
+
+    bash
+    perl
+
+HTSlib uses the following external libraries.  Building requires both the
+library itself, and include files needed to compile code that uses functions
+from the library.  Note that some Linux distributions put include files in
+a development ('-dev' or '-devel') package separate from the main library.
+
+    zlib       (required)
+    libbz2     (required, unless configured with --disable-bz2)
+    liblzma    (required, unless configured with --disable-lzma)
+    libcurl    (optional, but strongly recommended)
+    libcrypto  (optional for Amazon S3 support; not needed on MacOS)
+    libdeflate (optional, but strongly recommended for faster gzip)
+
+Disabling libbzip2 and liblzma will make some CRAM files unreadable, so
+is not recommended.
+
+Using libcurl provides HTSlib with network protocol support, for
+example it enables the use of ftp://, http://, and https:// URLs.
+It is also required if direct access to Amazon S3 or Google Cloud
+Storage is enabled.
+
+Amazon S3 support requires an HMAC function to calculate a message
+authentication code.  On MacOS, the CCHmac function from the standard
+library is used.  Systems that do not have CCHmac will get this from
+libcrypto.  libcrypto is part of OpenSSL or one of its derivatives (LibreSSL
+or BoringSSL).
+
+On Microsoft Windows we recommend use of Mingw64/Msys2.  Whilst the
+code may work on Windows with other environments, these have not been
+verified.  Use of the configure script is a requirement too.
+
+Update htscodecs submodule
+==========================
+
+Note that this section only applies to git checkouts.  If you're building
+from a release tar file, you can skip this section.
+
+Some parts of HTSlib are provided by the external "htscodecs" project.  This
+is included as a submodule.  When building from the git repository,
+either clone the project using "git clone --recurse-submodules", or run:
+
+    git submodule update --init --recursive
+
+to ensure the correct version of the submodule is present.
+
+It is also possible to link against an external libhtscodecs library
+by using the '--with-external-htscodecs' configure option.  When
+this is used, the submodule files will be ignored.
+
+Building Configure
+==================
+
+This step is only needed if configure.ac has been changed, or if configure
+does not exist (for example, when building from a git clone).  The
+configure script and config.h.in can be built by running:
+
+    autoreconf -i
+
+Basic Installation
+==================
+
+To build and install HTSlib, 'cd' to the htslib-1.x directory containing
+the package's source and type the following commands:
+
+    ./configure
+    make
+    make install
+
+The './configure' command checks your build environment and allows various
+optional functionality to be enabled (see Configuration below).  If you
+don't want to select any optional functionality, you may wish to omit
+configure and just type 'make; make install' as for previous versions
+of HTSlib.  However if the build fails you should run './configure' as
+it can diagnose the common reasons for build failures.
+
+The 'make' command builds the HTSlib library and various useful
+utilities: bgzip, htsfile, and tabix.  If compilation fails you should
+run './configure' as it can diagnose problems with your build environment
+that cause build failures.
+
+The 'make install' command installs the libraries, library header files,
+utilities, several manual pages, and a pkgconfig file to /usr/local.
+The installation location can be changed by configuring with --prefix=DIR
+or via 'make prefix=DIR install' (see Installation Locations below).
+Shared library permissions can be set via e.g. 'make install LIB_PERM=755'.
+
+
+Configuration
+=============
+
+By default, './configure' examines your build environment, checking for
+requirements such as the zlib development files, and arranges for a plain
+HTSlib build.  The following configure options can be used to enable
+various features and specify further optional external requirements:
+
+--enable-plugins
+    Use plugins to implement exotic file access protocols and other
+    specialised facilities.  This enables such facilities to be developed
+    and packaged outwith HTSlib, and somewhat isolates HTSlib-using programs
+    from their library dependencies.  By default (or with --disable-plugins),
+    any enabled pluggable facilities (such as libcurl file access) are built
+    directly within HTSlib.
+
+    Programs that are statically linked to a libhts.a with plugins enabled
+    need to be linked using -rdynamic or a similar linker option.
+
+    The <https://github.com/samtools/htslib-plugins> repository contains
+    several additional plugins, including the iRODS (<http://irods.org/>)
+    file access plugin previously distributed with HTSlib.
+
+--with-plugin-dir=DIR
+    Specifies the directory into which plugins built while building HTSlib
+    should be installed; by default, LIBEXECDIR/htslib.
+
+--with-plugin-path=DIR:DIR:DIR...
+    Specifies the list of directories that HTSlib will search for plugins.
+    By default, only the directory specified via --with-plugin-dir will be
+    searched; you can use --with-plugin-path='DIR:$(plugindir):DIR' and so
+    on to cause additional directories to be searched.
+
+--with-external-htscodecs
+    Build and link against an external copy of the htscodecs library
+    instead of using the source files in the htscodecs directory.
+
+--enable-libcurl
+    Use libcurl (<http://curl.se/>) to implement network access to
+    remote files via FTP, HTTP, HTTPS, etc.  By default or with
+    --enable-libcurl=check, configure will probe for libcurl and include
+    this functionality if libcurl is available.  Use --disable-libcurl
+    to prevent this.
+
+--enable-gcs
+    Implement network access to Google Cloud Storage.  By default or with
+    --enable-gcs=check, this is enabled when libcurl is enabled.
+
+--enable-s3
+    Implement network access to Amazon AWS S3.  By default or with
+    --enable-s3=check, this is enabled when libcurl is enabled.
+
+--disable-bz2
+    Bzip2 is an optional compression codec format for CRAM, included
+    in HTSlib by default.  It can be disabled with --disable-bz2, but
+    be aware that not all CRAM files may be possible to decode.
+
+--disable-lzma
+    LZMA is an optional compression codec for CRAM, included in HTSlib
+    by default.  It can be disabled with --disable-lzma, but be aware
+    that not all CRAM files may be possible to decode.
+
+--with-libdeflate
+    Libdeflate is a heavily optimized library for DEFLATE-based compression
+    and decompression.  It also includes a fast crc32 implementation.
+    By default, ./configure will probe for libdeflate and use it if
+    available.  To prevent this, use --without-libdeflate.
+
+Each --enable-FEATURE/--disable-FEATURE/--with-PACKAGE/--without-PACKAGE
+option listed also has an opposite, e.g., --without-external-htscodecs
+or --disable-plugins.  However, apart from those options for which the
+default is to probe for related facilities, using these opposite options
+is mostly unnecessary as they just select the default configure behaviour.
+
+The configure script also accepts the usual options and environment variables
+for tuning installation locations and compilers: type './configure --help'
+for details.  For example,
+
+    ./configure CC=icc --prefix=/opt/icc-compiled
+
+would specify that HTSlib is to be built with icc and installed into bin,
+lib, etc subdirectories under /opt/icc-compiled.
+
+If dependencies have been installed in non-standard locations (i.e. not on
+the normal include and library search paths) then the CPPFLAGS and LDFLAGS
+environment variables can be used to set the options needed to find them.
+For example, NetBSD users may use:
+
+    ./configure CPPFLAGS=-I/usr/pkg/include \
+                LDFLAGS='-L/usr/pkg/lib -Wl,-R/usr/pkg/lib'
+
+to allow compiling and linking against dependencies installed via the ports
+collection.
+
+Installation Locations
+======================
+
+By default, 'make install' installs HTSlib libraries under /usr/local/lib,
+HTSlib header files under /usr/local/include, utility programs under
+/usr/local/bin, etc.  (To be precise, the header files are installed within
+a fixed 'htslib' subdirectory under the specified .../include location.)
+
+You can specify a different location to install HTSlib by configuring
+with --prefix=DIR or specify locations for particular parts of HTSlib by
+configuring with --libdir=DIR and so on.  Type './configure --help' for
+the full list of such install directory options.
+
+Alternatively you can specify different locations at install time by
+typing 'make prefix=DIR install' or 'make libdir=DIR install' and so on.
+Consult the list of prefix/exec_prefix/etc variables near the top of the
+Makefile for the full list of such variables that can be overridden.
+
+You can also specify a staging area by typing 'make DESTDIR=DIR install',
+possibly in conjunction with other --prefix or prefix=DIR settings.
+For example,
+
+    make DESTDIR=/tmp/staging prefix=/opt
+
+would install into bin, lib, etc subdirectories under /tmp/staging/opt.
+
+
+System Specific Details
+=======================
+
+Installing the prerequisites is system dependent and there is more
+than one correct way of satisfying these, including downloading them
+from source, compiling and installing them yourself.
+
+For people with super-user access, we provide an example set of commands
+below for installing the dependencies on a variety of operating system
+distributions.  Note these are not specific recommendations on distribution,
+compiler or SSL implementation.  It is assumed you already have the core set
+of packages for the given distribution - the lists may be incomplete if
+this is not the case.
+
+Debian / Ubuntu
+---------------
+
+sudo apt-get update  # Ensure the package list is up to date
+sudo apt-get install autoconf automake make gcc perl zlib1g-dev libbz2-dev liblzma-dev libcurl4-gnutls-dev libssl-dev libdeflate-dev
+
+Note: libcurl4-openssl-dev can be used as an alternative to libcurl4-gnutls-dev.
+
+RedHat / CentOS
+---------------
+
+sudo yum install autoconf automake make gcc perl-Data-Dumper zlib-devel bzip2 bzip2-devel xz-devel curl-devel openssl-devel libdeflate-devel
+
+Note: On some versions perl FindBin will need to be installed to make the tests work.
+
+sudo yum install perl-FindBin
+
+Alpine Linux
+------------
+
+doas apk update  # Ensure the package list is up to date
+doas apk add autoconf automake make gcc musl-dev perl bash zlib-dev bzip2-dev xz-dev curl-dev openssl-dev
+
+Ideally also install a copy of libdeflate-dev for faster (de)compression.
+This can be found in the Alpine community repository.
+
+Note: some older Alpine versions use libressl-dev rather than openssl-dev.
+
+OpenSUSE
+--------
+
+sudo zypper install autoconf automake make gcc perl zlib-devel libbz2-devel xz-devel libcurl-devel libopenssl-devel
+
+Also install libdeflate-devel, available on OpenSUSE Leap 15.4 onwards
+or directly via git releases above.
+
+Windows MSYS2/MINGW64
+---------------------
+
+The configure script must be used as without it the compilation will
+likely fail.
+
+Follow MSYS2 installation instructions at
+https://www.msys2.org/wiki/MSYS2-installation/
+
+Then relaunch to MSYS2 shell using the "MSYS2 MinGW x64" executable.
+Once in that environment (check $MSYSTEM equals "MINGW64") install the
+compilers using pacman -S and the following package list:
+
+base-devel mingw-w64-x86_64-toolchain
+mingw-w64-x86_64-libdeflate mingw-w64-x86_64-zlib mingw-w64-x86_64-bzip2
+mingw-w64-x86_64-xz mingw-w64-x86_64-curl mingw-w64-x86_64-autotools
+mingw-w64-x86_64-tools-git
+
+(The last is only needed for building libraries compatible with MSVC.)
+
+HP-UX
+-----
+
+HP-UX requires that shared libraries have execute permission.  The
+default for HTSlib is to install with permission 644 (read-write for
+owner and read-only for group / other). This can be overridden by
+setting the LIB_PERM variable at install time with:
+
+    make install LIB_PERM=755
diff --git a/ext/htslib/LICENSE b/ext/htslib/LICENSE
new file mode 100644
index 0000000..87931fa
--- /dev/null
+++ b/ext/htslib/LICENSE
@@ -0,0 +1,69 @@
+[Files in this distribution outwith the cram/ subdirectory are distributed
+according to the terms of the following MIT/Expat license.]
+
+The MIT/Expat License
+
+Copyright (C) 2012-2024 Genome Research Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+
+[Files within the cram/ subdirectory in this distribution are distributed
+according to the terms of the following Modified 3-Clause BSD license.]
+
+The Modified-BSD License
+
+Copyright (C) 2012-2024 Genome Research Ltd.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+   this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the names Genome Research Ltd and Wellcome Trust Sanger Institute
+   nor the names of its contributors may be used to endorse or promote products
+   derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR ITS CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+[The use of a range of years within a copyright notice in this distribution
+should be interpreted as being equivalent to a list of years including the
+first and last year specified and all consecutive years between them.
+
+For example, a copyright notice that reads "Copyright (C) 2005, 2007-2009,
+2011-2012" should be interpreted as being identical to a notice that reads
+"Copyright (C) 2005, 2007, 2008, 2009, 2011, 2012" and a copyright notice
+that reads "Copyright (C) 2005-2012" should be interpreted as being identical
+to a notice that reads "Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010,
+2011, 2012".]
diff --git a/ext/htslib/Makefile b/ext/htslib/Makefile
new file mode 100644
index 0000000..c190e22
--- /dev/null
+++ b/ext/htslib/Makefile
@@ -0,0 +1,1006 @@
+# Makefile for htslib, a C library for high-throughput sequencing data formats.
+#
+#    Copyright (C) 2013-2024 Genome Research Ltd.
+#
+#    Author: John Marshall <jm18@sanger.ac.uk>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+CC     = gcc
+AR     = ar
+RANLIB = ranlib
+
+# Default libraries to link if configure is not used
+htslib_default_libs = -lz -lm -lbz2 -llzma -lcurl
+
+CPPFLAGS =
+# TODO: make the 64-bit support for VCF optional via configure, for now add -DVCF_ALLOW_INT64
+#       to CFLAGS manually, here or in config.mk if the latter exists.
+# TODO: probably update cram code to make it compile cleanly with -Wc++-compat
+# For testing strict C99 support add -std=c99 -D_XOPEN_SOURCE=600
+#CFLAGS   = -g -Wall -O2 -pedantic -std=c99 -D_XOPEN_SOURCE=600
+CFLAGS   = -g -Wall -O2 -fvisibility=hidden
+EXTRA_CFLAGS_PIC = -fpic
+TARGET_CFLAGS =
+LDFLAGS  = -fvisibility=hidden
+VERSION_SCRIPT_LDFLAGS = -Wl,-version-script,$(srcprefix)htslib.map
+LIBS     = $(htslib_default_libs)
+
+prefix      = /usr/local
+exec_prefix = $(prefix)
+bindir      = $(exec_prefix)/bin
+includedir  = $(prefix)/include
+libdir      = $(exec_prefix)/lib
+libexecdir  = $(exec_prefix)/libexec
+datarootdir = $(prefix)/share
+mandir      = $(datarootdir)/man
+man1dir     = $(mandir)/man1
+man5dir     = $(mandir)/man5
+man7dir     = $(mandir)/man7
+pkgconfigdir= $(libdir)/pkgconfig
+
+MKDIR_P = mkdir -p
+INSTALL = install -p
+INSTALL_DATA    = $(INSTALL) -m 644
+INSTALL_DIR     = $(MKDIR_P) -m 755
+LIB_PERM        = 644
+INSTALL_LIB     = $(INSTALL) -m $(LIB_PERM)
+INSTALL_MAN     = $(INSTALL_DATA)
+INSTALL_PROGRAM = $(INSTALL)
+
+# Set by config.mk if plugins are enabled
+plugindir =
+
+BUILT_PROGRAMS = \
+	annot-tsv \
+	bgzip \
+	htsfile \
+	tabix
+
+BUILT_TEST_PROGRAMS = 
+#BUILT_TEST_PROGRAMS = \
+#	test/hts_endian \
+#	test/fieldarith \
+#	test/hfile \
+#	test/pileup \
+#	test/pileup_mod \
+#	test/plugins-dlhts \
+#	test/sam \
+#	test/test_bgzf \
+#	test/test_expr \
+#	test/test_faidx \
+#	test/test_kfunc \
+#	test/test_khash \
+#	test/test_kstring \
+#	test/test_mod \
+#	test/test_nibbles \
+#	test/test_realn \
+#	test/test-regidx \
+#	test/test_str2int \
+#	test/test_time_funcs \
+#	test/test_view \
+#	test/test_index \
+#	test/test-vcf-api \
+#	test/test-vcf-sweep \
+#	test/test-bcf-sr \
+#	test/fuzz/hts_open_fuzzer.o \
+#	test/test-bcf-translate \
+#	test/test-parse-reg \
+#	test/test_introspection \
+#	test/test-bcf_set_variant_type
+
+BUILT_THRASH_PROGRAMS = \
+	test/thrash_threads1 \
+	test/thrash_threads2 \
+	test/thrash_threads3 \
+	test/thrash_threads4 \
+	test/thrash_threads5 \
+	test/thrash_threads6 \
+	test/thrash_threads7
+
+all: lib-static lib-shared $(BUILT_PROGRAMS) plugins \
+	$(BUILT_TEST_PROGRAMS) htslib_static.mk htslib-uninstalled.pc
+
+# Report compiler and version
+cc-version:
+	-@$(CC) --version  2>/dev/null || true
+	-@$(CC) --qversion 2>/dev/null || true
+	-@$(CC) -V         2>/dev/null || true
+
+ALL_CPPFLAGS = -I. $(CPPFLAGS)
+
+# Usually htscodecs.mk is generated by running configure or config.status,
+# but if those aren't used create a default here.
+htscodecs.mk:
+	echo '# Default htscodecs.mk generated by Makefile' > $@
+	echo 'include $$(HTSPREFIX)htscodecs_bundled.mk' >> $@
+	$(srcdir)/hts_probe_cc.sh '$(CC)' '$(CFLAGS) $(CPPFLAGS)' '$(LDFLAGS)' >> $@
+
+srcdir = .
+srcprefix =
+HTSPREFIX =
+
+# Flags for SIMD code
+HTS_CFLAGS_AVX2 =
+HTS_CFLAGS_AVX512 =
+HTS_CFLAGS_SSE4 =
+
+# Control building of SIMD code.  Not used if configure has been run.
+HTS_BUILD_AVX2 =
+HTS_BUILD_AVX512 =
+HTS_BUILD_SSE4 =
+
+include htslib_vars.mk
+include htscodecs.mk
+
+# If not using GNU make, you need to copy the version number from version.sh
+# into here.
+PACKAGE_VERSION := $(shell $(srcdir)/version.sh)
+
+LIBHTS_SOVERSION = 3
+
+# Version numbers for the Mac dynamic library.  Note that the leading 3
+# is not strictly necessary and should be removed the next time
+# LIBHTS_SOVERSION is bumped (see #1144 and
+# https://developer.apple.com/library/archive/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html#//apple_ref/doc/uid/TP40002013-SW23)
+MACH_O_COMPATIBILITY_VERSION = 3.1.21
+MACH_O_CURRENT_VERSION = 3.1.21
+
+# Force version.h to be remade if $(PACKAGE_VERSION) has changed.
+version.h: $(if $(wildcard version.h),$(if $(findstring "$(PACKAGE_VERSION)",$(shell cat version.h)),,force))
+
+version.h:
+	echo '#define HTS_VERSION_TEXT "$(PACKAGE_VERSION)"' > $@
+
+print-version:
+	@echo $(PACKAGE_VERSION)
+
+show-version:
+	@echo PACKAGE_VERSION = $(PACKAGE_VERSION)
+
+config_vars.h: override escape=$(subst ',\x27,$(subst ",\",$(subst \,\\,$(1))))
+config_vars.h: override hts_cc_escaped=$(call escape,$(CC))
+config_vars.h: override hts_cppflags_escaped=$(call escape,$(CPPFLAGS))
+config_vars.h: override hts_cflags_escaped=$(call escape,$(CFLAGS))
+config_vars.h: override hts_ldflags_escaped=$(call escape,$(LDFLAGS))
+config_vars.h: override hts_libs_escaped=$(call escape,$(LIBS))
+
+config_vars.h:
+	printf '#define HTS_CC "%s"\n#define HTS_CPPFLAGS "%s"\n#define HTS_CFLAGS "%s"\n#define HTS_LDFLAGS "%s"\n#define HTS_LIBS "%s"\n' \
+	'$(hts_cc_escaped)' \
+	'$(hts_cppflags_escaped)' \
+	'$(hts_cflags_escaped)' \
+	'$(hts_ldflags_escaped)' \
+	'$(hts_libs_escaped)' > $@
+
+.SUFFIXES: .bundle .c .cygdll .dll .o .pico .so
+
+.c.o:
+	$(CC) $(CFLAGS) $(TARGET_CFLAGS) $(ALL_CPPFLAGS) -c -o $@ $<
+
+.c.pico:
+	$(CC) $(CFLAGS) $(TARGET_CFLAGS) $(ALL_CPPFLAGS) $(EXTRA_CFLAGS_PIC) -c -o $@ $<
+
+
+LIBHTS_OBJS = \
+	kfunc.o \
+	kstring.o \
+	bcf_sr_sort.o \
+	bgzf.o \
+	errmod.o \
+	faidx.o \
+	header.o \
+	hfile.o \
+	hts.o \
+	hts_expr.o \
+	hts_os.o\
+	md5.o \
+	multipart.o \
+	probaln.o \
+	realn.o \
+	regidx.o \
+	region.o \
+	sam.o \
+	sam_mods.o \
+	simd.o \
+	synced_bcf_reader.o \
+	vcf_sweep.o \
+	tbx.o \
+	textutils.o \
+	thread_pool.o \
+	vcf.o \
+	vcfutils.o \
+	cram/cram_codecs.o \
+	cram/cram_decode.o \
+	cram/cram_encode.o \
+	cram/cram_external.o \
+	cram/cram_index.o \
+	cram/cram_io.o \
+	cram/cram_stats.o \
+	cram/mFILE.o \
+	cram/open_trace_file.o \
+	cram/pooled_alloc.o \
+	cram/string_alloc.o \
+	$(HTSCODECS_OBJS) \
+	$(NONCONFIGURE_OBJS)
+
+# Without configure we wish to have a rich set of default figures,
+# but we still need conditional inclusion as we wish to still
+# support ./configure --disable-blah.
+NONCONFIGURE_OBJS = hfile_libcurl.o
+
+PLUGIN_EXT  =
+PLUGIN_OBJS =
+
+cram_h = cram/cram.h $(cram_samtools_h) $(header_h) $(cram_structs_h) $(cram_io_h) cram/cram_encode.h cram/cram_decode.h cram/cram_stats.h cram/cram_codecs.h cram/cram_index.h $(htslib_cram_h)
+cram_io_h = cram/cram_io.h $(cram_misc_h)
+cram_misc_h = cram/misc.h
+cram_os_h = cram/os.h $(htslib_hts_endian_h)
+cram_samtools_h = cram/cram_samtools.h $(htslib_sam_h)
+cram_structs_h = cram/cram_structs.h $(htslib_thread_pool_h) $(htslib_cram_h) cram/string_alloc.h cram/mFILE.h $(htslib_khash_h)
+cram_open_trace_file_h = cram/open_trace_file.h cram/mFILE.h
+bcf_sr_sort_h = bcf_sr_sort.h $(htslib_synced_bcf_reader_h) $(htslib_kbitset_h)
+fuzz_settings_h = fuzz_settings.h
+header_h = header.h cram/string_alloc.h cram/pooled_alloc.h $(htslib_khash_h) $(htslib_kstring_h) $(htslib_sam_h)
+hfile_internal_h = hfile_internal.h $(htslib_hts_defs_h) $(htslib_hfile_h) $(textutils_internal_h)
+hts_internal_h = hts_internal.h $(htslib_hts_h) $(textutils_internal_h)
+hts_time_funcs_h = hts_time_funcs.h
+sam_internal_h = sam_internal.h $(htslib_sam_h)
+textutils_internal_h = textutils_internal.h $(htslib_kstring_h)
+thread_pool_internal_h = thread_pool_internal.h $(htslib_thread_pool_h)
+
+# To be effective, config.mk needs to appear after most Makefile variables are
+# set but before most rules appear, so that it can both use previously-set
+# variables in its own rules' prerequisites and also update variables for use
+# in later rules' prerequisites.
+
+# If your make doesn't accept -include, change this to 'include' if you are
+# using the configure script or just comment the line out if you are not.
+-include config.mk
+
+# Usually config.h is generated by running configure or config.status,
+# but if those aren't used create a default config.h here.
+config.h:
+	echo '/* Default config.h generated by Makefile */' > $@
+	echo '#ifndef _XOPEN_SOURCE' >> $@
+	echo '#define _XOPEN_SOURCE 600' >> $@
+	echo '#endif' >> $@
+	echo '#define HAVE_LIBBZ2 1' >> $@
+	echo '#define HAVE_LIBLZMA 1' >> $@
+	echo '#ifndef __APPLE__' >> $@
+	echo '#define HAVE_LZMA_H 1' >> $@
+	echo '#endif' >> $@
+	echo '#define HAVE_DRAND48 1' >> $@
+	echo '#define HAVE_LIBCURL 1' >> $@
+	if [ "x$(HTS_HAVE_CPUID)" != "x" ]; then \
+	    echo '#define HAVE_DECL___CPUID_COUNT 1' >> $@ ; \
+	    echo '#define HAVE_DECL___GET_CPUID_MAX 1' >> $@ ; \
+	fi
+	if [ "x$(HTS_BUILD_SSE4)" != "x" ]; then \
+	    echo '#define HAVE_POPCNT 1' >> $@ ; \
+	    echo '#define HAVE_SSE4_1 1' >> $@ ; \
+	    echo '#define HAVE_SSSE3 1' >> $@ ; \
+	    echo '#if defined(HTS_ALLOW_UNALIGNED) && HTS_ALLOW_UNALIGNED == 0' >> $@ ; \
+	    echo '#define UBSAN 1' >> $@ ; \
+	    echo '#endif' >> $@ ; \
+	fi
+	if [ "x$(HTS_BUILD_AVX2)" != "x" ] ; then \
+	    echo '#define HAVE_AVX2 1' >> $@ ; \
+	fi
+	if [ "x$(HTS_BUILD_AVX512)" != "x" ] ; then \
+	    echo '#define HAVE_AVX512 1' >> $@ ; \
+	fi
+	echo '#if defined __x86_64__ || defined __arm__ || defined __aarch64__' >> $@
+	echo '#define HAVE_ATTRIBUTE_CONSTRUCTOR 1' >> $@
+	echo '#endif' >> $@
+	echo '#if (defined(__x86_64__) || defined(_M_X64))' >> $@
+	echo '#define HAVE_ATTRIBUTE_TARGET 1' >> $@
+	echo '#define HAVE_BUILTIN_CPU_SUPPORT_SSSE3 1' >> $@
+	echo '#endif' >> $@
+
+# And similarly for htslib.pc.tmp ("pkg-config template").  No dependency
+# on htslib.pc.in listed, as if that file is newer the usual way to regenerate
+# this target is via configure or config.status rather than this rule.
+htslib.pc.tmp:
+	sed -e '/^static_libs=/s/@static_LIBS@/$(htslib_default_libs)/;s#@[^-][^@]*@##g' $(srcprefix)htslib.pc.in > $@
+
+# Create a makefile fragment listing the libraries and LDFLAGS needed for
+# static linking.  This can be included by projects that want to build
+# and link against the htslib source tree instead of an installed library.
+htslib_static.mk: htslib.pc.tmp
+	sed -n '/^static_libs=/s/[^=]*=/HTSLIB_static_LIBS = /p;/^static_ldflags=/s/[^=]*=/HTSLIB_static_LDFLAGS = /p' $< > $@
+
+
+lib-static: libhts.a
+
+# $(shell), :=, and ifeq/.../endif are GNU Make-specific.  If you don't have
+# GNU Make, comment out the parts of these conditionals that don't apply.
+ifneq "$(origin PLATFORM)" "file"
+PLATFORM := $(shell uname -s)
+endif
+ifeq "$(PLATFORM)" "Darwin"
+SHLIB_FLAVOUR = dylib
+lib-shared: libhts.dylib
+else ifeq "$(findstring CYGWIN,$(PLATFORM))" "CYGWIN"
+SHLIB_FLAVOUR = cygdll
+lib-shared: cyghts-$(LIBHTS_SOVERSION).dll
+else ifeq "$(findstring MSYS,$(PLATFORM))" "MSYS"
+SHLIB_FLAVOUR = dll
+lib-shared: hts-$(LIBHTS_SOVERSION).dll hts-$(LIBHTS_SOVERSION).def hts-$(LIBHTS_SOVERSION).lib
+else ifeq "$(findstring MINGW,$(PLATFORM))" "MINGW"
+SHLIB_FLAVOUR = dll
+lib-shared: hts-$(LIBHTS_SOVERSION).dll hts-$(LIBHTS_SOVERSION).def hts-$(LIBHTS_SOVERSION).lib
+else
+SHLIB_FLAVOUR = so
+lib-shared: libhts.so
+endif
+
+BUILT_PLUGINS = $(PLUGIN_OBJS:.o=$(PLUGIN_EXT))
+
+ifneq "$(BUILT_PLUGINS)" ""
+plugins: lib-shared
+endif
+plugins: $(BUILT_PLUGINS)
+
+
+libhts.a: $(LIBHTS_OBJS)
+	@-rm -f $@
+	$(AR) -rc $@ $(LIBHTS_OBJS)
+	-$(RANLIB) $@
+
+print-config:
+	@echo HTS_CFLAGS_AVX2 = $(HTS_CFLAGS_AVX2)
+	@echo HTS_CFLAGS_AVX512 = $(HTS_CFLAGS_AVX512)
+	@echo HTS_CFLAGS_SSE4 = $(HTS_CFLAGS_SSE4)
+	@echo LDFLAGS = $(LDFLAGS)
+	@echo LIBHTS_OBJS = $(LIBHTS_OBJS)
+	@echo LIBS = $(LIBS)
+	@echo PLATFORM = $(PLATFORM)
+
+# The target here is libhts.so, as that is the built file that other rules
+# depend upon and that is used when -lhts appears in other program's recipes.
+# As a byproduct invisible to make, libhts.so.NN is also created, as it is the
+# file used at runtime (when $LD_LIBRARY_PATH includes the build directory).
+
+libhts.so: $(LIBHTS_OBJS:.o=.pico)
+	$(CC) -shared -Wl,-soname,libhts.so.$(LIBHTS_SOVERSION) $(VERSION_SCRIPT_LDFLAGS) $(LDFLAGS) -o $@ $(LIBHTS_OBJS:.o=.pico) $(LIBS) -lpthread
+	ln -sf $@ libhts.so.$(LIBHTS_SOVERSION)
+
+# Similarly this also creates libhts.NN.dylib as a byproduct, so that programs
+# when run can find this uninstalled shared library (when $DYLD_LIBRARY_PATH
+# includes this project's build directory).
+
+libhts.dylib: $(LIBHTS_OBJS)
+	$(CC) -dynamiclib -install_name $(libdir)/libhts.$(LIBHTS_SOVERSION).dylib -current_version $(MACH_O_CURRENT_VERSION) -compatibility_version $(MACH_O_COMPATIBILITY_VERSION) $(LDFLAGS) -o $@ $(LIBHTS_OBJS) $(LIBS)
+	ln -sf $@ libhts.$(LIBHTS_SOVERSION).dylib
+
+cyghts-$(LIBHTS_SOVERSION).dll libhts.dll.a: $(LIBHTS_OBJS)
+	$(CC) -shared -Wl,--out-implib=libhts.dll.a -Wl,--enable-auto-import $(LDFLAGS) -o $@ -Wl,--whole-archive $(LIBHTS_OBJS) -Wl,--no-whole-archive $(LIBS) -lpthread
+
+hts-$(LIBHTS_SOVERSION).dll hts.dll.a: $(LIBHTS_OBJS)
+	$(CC) -shared -Wl,--out-implib=hts.dll.a -Wl,--enable-auto-import -Wl,--exclude-all-symbols $(LDFLAGS) -o $@ -Wl,--whole-archive $(LIBHTS_OBJS) -Wl,--no-whole-archive $(LIBS) -lpthread
+
+hts-$(LIBHTS_SOVERSION).def: hts-$(LIBHTS_SOVERSION).dll
+	gendef hts-$(LIBHTS_SOVERSION).dll
+
+hts-$(LIBHTS_SOVERSION).lib: hts-$(LIBHTS_SOVERSION).def
+	dlltool -m i386:x86-64 -d hts-$(LIBHTS_SOVERSION).def -l hts-$(LIBHTS_SOVERSION).lib
+
+# Bundling libraries, binaries, dll dependencies, and licenses into a
+# single directory.  NB: This is not needed for end-users, but a test bed
+# for maintainers building binary distributions.
+#
+# NOTE: only tested on the supported MSYS2/MINGW64 environment.
+dist-windows: DESTDIR=
+dist-windows: prefix=dist-windows
+dist-windows: install
+	cp hts-$(LIBHTS_SOVERSION).def hts-$(LIBHTS_SOVERSION).lib dist-windows/lib
+	cp `ldd hts-$(LIBHTS_SOVERSION).dll| awk '/mingw64/ {print $$3}'` dist-windows/bin
+	mkdir -p dist-windows/share/licenses/htslib
+	-cp -r /mingw64/share/licenses/mingw-w64-libraries \
+	      /mingw64/share/licenses/brotli \
+	      /mingw64/share/licenses/bzip2 \
+	      /mingw64/share/licenses/gcc-libs \
+	      /mingw64/share/licenses/libdeflate \
+	      /mingw64/share/licenses/libpsl \
+	      /mingw64/share/licenses/libtre \
+	      /mingw64/share/licenses/libwinpthread \
+	      /mingw64/share/licenses/openssl \
+	      /mingw64/share/licenses/xz \
+	      /mingw64/share/licenses/zlib \
+	      /mingw64/share/licenses/zstd \
+	      dist-windows/share/licenses/
+	-cp -r /usr/share/licenses/curl \
+	      dist-windows/share/licenses/
+	cp LICENSE dist-windows/share/licenses/htslib/
+
+
+# Target to allow htslib.mk to build all the object files before it
+# links the shared and static libraries.
+hts-object-files: $(LIBHTS_OBJS)
+	touch $@
+
+# On Unix dlopen("libhts.so.NN", RTLD_LAZY) may default to RTLD_LOCAL.
+# Hence plugins need to link to (shared) libhts.so.NN themselves, as they
+# may not be able to access libhts symbols via the main program's libhts
+# if that was dynamically loaded without an explicit RTLD_GLOBAL.
+%.so: %.pico libhts.so
+	$(CC) -shared -Wl,-E $(LDFLAGS) -o $@ $< libhts.so $(LIBS) -lpthread
+
+# For programs *statically* linked to libhts.a, on macOS loading a plugin
+# linked to a shared libhts.NN.dylib would lead to conflicting duplicate
+# symbols.  Fortunately macOS dlopen() defaults to RTLD_GLOBAL so there
+# is less need for plugins to link back to libhts themselves.
+%.bundle: %.o
+	$(CC) -bundle -Wl,-undefined,dynamic_lookup $(LDFLAGS) -o $@ $< $(LIBS)
+
+%.cygdll: %.o libhts.dll.a
+	$(CC) -shared $(LDFLAGS) -o $@ $< libhts.dll.a $(LIBS)
+
+%.dll: %.o hts.dll.a
+	$(CC) -shared $(LDFLAGS) -o $@ $< hts.dll.a $(LIBS)
+
+
+bgzf.o bgzf.pico: bgzf.c config.h $(htslib_hts_h) $(htslib_bgzf_h) $(htslib_hfile_h) $(htslib_thread_pool_h) $(htslib_hts_endian_h) cram/pooled_alloc.h $(hts_internal_h) $(htslib_khash_h)
+errmod.o errmod.pico: errmod.c config.h $(htslib_hts_h) $(htslib_ksort_h) $(htslib_hts_os_h)
+kstring.o kstring.pico: kstring.c config.h $(htslib_kstring_h)
+header.o header.pico: header.c config.h $(textutils_internal_h) $(header_h)
+hfile.o hfile.pico: hfile.c config.h $(htslib_hfile_h) $(hfile_internal_h) $(htslib_kstring_h) $(hts_internal_h) $(htslib_khash_h)
+hfile_gcs.o hfile_gcs.pico: hfile_gcs.c config.h $(htslib_hts_h) $(htslib_kstring_h) $(hfile_internal_h)
+hfile_libcurl.o hfile_libcurl.pico: hfile_libcurl.c config.h $(hfile_internal_h) $(htslib_hts_h) $(htslib_kstring_h) $(htslib_khash_h)
+hfile_s3_write.o hfile_s3_write.pico: hfile_s3_write.c config.h $(hfile_internal_h) $(htslib_hts_h) $(htslib_kstring_h) $(htslib_khash_h)
+hfile_s3.o hfile_s3.pico: hfile_s3.c config.h $(hfile_internal_h) $(htslib_hts_h) $(htslib_kstring_h) $(hts_time_funcs_h)
+hts.o hts.pico: hts.c config.h os/lzma_stub.h $(htslib_hts_h) $(htslib_bgzf_h) $(cram_h) $(htslib_hfile_h) $(htslib_hts_endian_h) version.h config_vars.h $(hts_internal_h) $(hfile_internal_h) $(sam_internal_h) $(htslib_hts_expr_h) $(htslib_hts_os_h) $(htslib_khash_h) $(htslib_kseq_h) $(htslib_ksort_h) $(htslib_tbx_h) $(htscodecs_htscodecs_h)
+hts_expr.o hts_expr.pico: hts_expr.c config.h $(htslib_hts_expr_h) $(htslib_hts_log_h) $(textutils_internal_h)
+hts_os.o hts_os.pico: hts_os.c config.h $(htslib_hts_defs_h) os/rand.c
+vcf.o vcf.pico: vcf.c config.h $(fuzz_settings_h) $(htslib_vcf_h) $(htslib_bgzf_h) $(htslib_tbx_h) $(htslib_hfile_h) $(hts_internal_h) $(htslib_khash_str2int_h) $(htslib_kstring_h) $(htslib_sam_h) $(htslib_khash_h) $(htslib_kseq_h) $(htslib_hts_endian_h)
+sam.o sam.pico: sam.c config.h $(fuzz_settings_h) $(htslib_hts_defs_h) $(htslib_sam_h) $(htslib_bgzf_h) $(cram_h) $(hts_internal_h) $(sam_internal_h) $(htslib_hfile_h) $(htslib_hts_endian_h) $(htslib_hts_expr_h) $(header_h) $(htslib_khash_h) $(htslib_kseq_h) $(htslib_kstring_h)
+sam_mods.o sam_mods.pico: sam_mods.c config.h $(htslib_sam_h) $(textutils_internal_h)
+simd.o simd.pico: simd.c config.h $(htslib_sam_h) $(sam_internal_h)
+tbx.o tbx.pico: tbx.c config.h $(htslib_tbx_h) $(htslib_bgzf_h) $(htslib_hts_endian_h) $(hts_internal_h) $(htslib_khash_h)
+faidx.o faidx.pico: faidx.c config.h $(htslib_bgzf_h) $(htslib_faidx_h) $(htslib_hfile_h) $(htslib_khash_h) $(htslib_kstring_h) $(hts_internal_h)
+bcf_sr_sort.o bcf_sr_sort.pico: bcf_sr_sort.c config.h $(bcf_sr_sort_h) $(htslib_khash_str2int_h) $(htslib_kbitset_h)
+synced_bcf_reader.o synced_bcf_reader.pico: synced_bcf_reader.c config.h $(htslib_synced_bcf_reader_h) $(htslib_kseq_h) $(htslib_khash_str2int_h) $(htslib_bgzf_h) $(htslib_thread_pool_h) $(bcf_sr_sort_h)
+vcf_sweep.o vcf_sweep.pico: vcf_sweep.c config.h $(htslib_vcf_sweep_h) $(htslib_bgzf_h)
+vcfutils.o vcfutils.pico: vcfutils.c config.h $(htslib_vcfutils_h) $(htslib_kbitset_h)
+kfunc.o kfunc.pico: kfunc.c config.h $(htslib_kfunc_h)
+regidx.o regidx.pico: regidx.c config.h $(htslib_hts_h) $(htslib_kstring_h) $(htslib_kseq_h) $(htslib_khash_str2int_h) $(htslib_regidx_h) $(hts_internal_h)
+region.o region.pico: region.c config.h $(htslib_hts_h) $(htslib_khash_h)
+md5.o md5.pico: md5.c config.h $(htslib_hts_h) $(htslib_hts_endian_h)
+multipart.o multipart.pico: multipart.c config.h $(htslib_kstring_h) $(hts_internal_h) $(hfile_internal_h)
+plugin.o plugin.pico: plugin.c config.h $(hts_internal_h) $(htslib_kstring_h)
+probaln.o probaln.pico: probaln.c config.h $(htslib_hts_h)
+realn.o realn.pico: realn.c config.h $(htslib_hts_h) $(htslib_sam_h)
+textutils.o textutils.pico: textutils.c config.h $(htslib_hfile_h) $(htslib_kstring_h) $(htslib_sam_h) $(hts_internal_h)
+
+cram/cram_codecs.o cram/cram_codecs.pico: cram/cram_codecs.c config.h $(fuzz_settings_h) $(htslib_hts_endian_h) $(htscodecs_varint_h) $(htscodecs_pack_h) $(htscodecs_rle_h) $(cram_h)
+cram/cram_decode.o cram/cram_decode.pico: cram/cram_decode.c config.h $(cram_h) $(cram_os_h) $(htslib_hts_h)
+cram/cram_encode.o cram/cram_encode.pico: cram/cram_encode.c config.h $(cram_h) $(cram_os_h) $(sam_internal_h) $(htslib_hts_h) $(htslib_hts_endian_h) $(textutils_internal_h)
+cram/cram_external.o cram/cram_external.pico: cram/cram_external.c config.h $(htscodecs_rANS_static4x16_h) $(htslib_hfile_h) $(cram_h)
+cram/cram_index.o cram/cram_index.pico: cram/cram_index.c config.h $(htslib_bgzf_h) $(htslib_hfile_h) $(hts_internal_h) $(cram_h) $(cram_os_h)
+cram/cram_io.o cram/cram_io.pico: cram/cram_io.c config.h os/lzma_stub.h $(fuzz_settings_h) $(cram_h) $(cram_os_h) $(htslib_hts_h) $(cram_open_trace_file_h) $(htscodecs_rANS_static_h) $(htscodecs_rANS_static4x16_h) $(htscodecs_arith_dynamic_h) $(htscodecs_tokenise_name3_h) $(htscodecs_fqzcomp_qual_h) $(htscodecs_varint_h) $(htslib_hfile_h) $(htslib_bgzf_h) $(htslib_faidx_h) $(hts_internal_h)
+cram/cram_stats.o cram/cram_stats.pico: cram/cram_stats.c config.h $(cram_h) $(cram_os_h)
+cram/mFILE.o cram/mFILE.pico: cram/mFILE.c config.h $(htslib_hts_log_h) $(cram_os_h) cram/mFILE.h
+cram/open_trace_file.o cram/open_trace_file.pico: cram/open_trace_file.c config.h $(cram_os_h) $(cram_open_trace_file_h) $(cram_misc_h) $(htslib_hfile_h) $(htslib_hts_log_h) $(htslib_hts_h)
+cram/pooled_alloc.o cram/pooled_alloc.pico: cram/pooled_alloc.c config.h cram/pooled_alloc.h $(cram_misc_h)
+cram/string_alloc.o cram/string_alloc.pico: cram/string_alloc.c config.h cram/string_alloc.h
+thread_pool.o thread_pool.pico: thread_pool.c config.h $(thread_pool_internal_h) $(htslib_hts_log_h)
+
+htscodecs/htscodecs/arith_dynamic.o htscodecs/htscodecs/arith_dynamic.pico: htscodecs/htscodecs/arith_dynamic.c config.h $(htscodecs_arith_dynamic_h) $(htscodecs_varint_h) $(htscodecs_pack_h) $(htscodecs_utils_h) $(htscodecs_c_simple_model_h)
+htscodecs/htscodecs/fqzcomp_qual.o htscodecs/htscodecs/fqzcomp_qual.pico: htscodecs/htscodecs/fqzcomp_qual.c config.h $(htscodecs_fqzcomp_qual_h) $(htscodecs_varint_h) $(htscodecs_utils_h) $(htscodecs_c_simple_model_h)
+htscodecs/htscodecs/htscodecs.o htscodecs/htscodecs/htscodecs.pico: htscodecs/htscodecs/htscodecs.c $(htscodecs_htscodecs_h) $(htscodecs_version_h)
+htscodecs/htscodecs/pack.o htscodecs/htscodecs/pack.pico: htscodecs/htscodecs/pack.c config.h $(htscodecs_pack_h)
+htscodecs/htscodecs/rANS_static32x16pr.o htscodecs/htscodecs/rANS_static32x16pr.pico: htscodecs/htscodecs/rANS_static32x16pr.c config.h $(htscodecs_rANS_word_h) $(htscodecs_rANS_static4x16_h) $(htscodecs_rANS_static16_int_h) $(htscodecs_varint_h) $(htscodecs_utils_h)
+htscodecs/htscodecs/rANS_static32x16pr_avx2.o htscodecs/htscodecs/rANS_static32x16pr_avx2.pico: htscodecs/htscodecs/rANS_static32x16pr_avx2.c config.h $(htscodecs_rANS_word_h) $(htscodecs_rANS_static4x16_h) $(htscodecs_rANS_static16_int_h) $(htscodecs_varint_h) $(htscodecs_utils_h) $(htscodecs_permute_h)
+htscodecs/htscodecs/rANS_static32x16pr_avx512.o htscodecs/htscodecs/rANS_static32x16pr_avx512.pico: htscodecs/htscodecs/rANS_static32x16pr_avx512.c config.h $(htscodecs_rANS_word_h) $(htscodecs_rANS_static4x16_h) $(htscodecs_rANS_static16_int_h) $(htscodecs_varint_h) $(htscodecs_utils_h)
+htscodecs/htscodecs/rANS_static32x16pr_neon.o htscodecs/htscodecs/rANS_static32x16pr_neon.pico: htscodecs/htscodecs/rANS_static32x16pr_neon.c config.h $(htscodecs_rANS_word_h) $(htscodecs_rANS_static4x16_h) $(htscodecs_rANS_static16_int_h) $(htscodecs_varint_h) $(htscodecs_utils_h)
+htscodecs/htscodecs/rANS_static32x16pr_sse4.o htscodecs/htscodecs/rANS_static32x16pr_sse4.pico: htscodecs/htscodecs/rANS_static32x16pr_sse4.c config.h $(htscodecs_rANS_word_h) $(htscodecs_rANS_static4x16_h) $(htscodecs_rANS_static16_int_h) $(htscodecs_varint_h) $(htscodecs_utils_h)
+htscodecs/htscodecs/rANS_static4x16pr.o htscodecs/htscodecs/rANS_static4x16pr.pico: htscodecs/htscodecs/rANS_static4x16pr.c config.h $(htscodecs_rANS_word_h) $(htscodecs_rANS_static4x16_h) $(htscodecs_rANS_static16_int_h) $(htscodecs_pack_h) $(htscodecs_rle_h) $(htscodecs_utils_h) $(htscodecs_rANS_static32x16pr_h)
+htscodecs/htscodecs/rANS_static.o htscodecs/htscodecs/rANS_static.pico: htscodecs/htscodecs/rANS_static.c config.h $(htscodecs_rANS_byte_h) $(htscodecs_utils_h) $(htscodecs_rANS_static_h)
+htscodecs/htscodecs/rle.o htscodecs/htscodecs/rle.pico: htscodecs/htscodecs/rle.c config.h $(htscodecs_varint_h) $(htscodecs_rle_h)
+htscodecs/htscodecs/tokenise_name3.o htscodecs/htscodecs/tokenise_name3.pico: htscodecs/htscodecs/tokenise_name3.c config.h $(htscodecs_pooled_alloc_h) $(htscodecs_arith_dynamic_h) $(htscodecs_rANS_static4x16_h) $(htscodecs_tokenise_name3_h) $(htscodecs_varint_h) $(htscodecs_utils_h)
+htscodecs/htscodecs/utils.o htscodecs/htscodecs/utils.pico: htscodecs/htscodecs/utils.c config.h $(htscodecs_utils_h)
+
+# Extra CFLAGS for specific files
+htscodecs/htscodecs/rANS_static32x16pr_avx2.o htscodecs/htscodecs/rANS_static32x16pr_avx2.pico: TARGET_CFLAGS = $(HTS_CFLAGS_AVX2)
+htscodecs/htscodecs/rANS_static32x16pr_avx512.o htscodecs/htscodecs/rANS_static32x16pr_avx512.pico: TARGET_CFLAGS = $(HTS_CFLAGS_AVX512)
+htscodecs/htscodecs/rANS_static32x16pr_sse4.o htscodecs/htscodecs/rANS_static32x16pr_sse4.pico: TARGET_CFLAGS = $(HTS_CFLAGS_SSE4)
+
+annot-tsv: annot-tsv.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ annot-tsv.o libhts.a $(LIBS) -lpthread
+
+bgzip: bgzip.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ bgzip.o libhts.a $(LIBS) -lpthread
+
+htsfile: htsfile.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ htsfile.o libhts.a $(LIBS) -lpthread
+
+tabix: tabix.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ tabix.o libhts.a $(LIBS) -lpthread
+
+annot-tsv.o: annot-tsv.c config.h $(htslib_hts_h) $(htslib_hts_defs_h) $(htslib_khash_str2int_h) $(htslib_kstring_h) $(htslib_kseq_h) $(htslib_bgzf_h) $(htslib_regidx_h) $(textutils_internal_h)
+bgzip.o: bgzip.c config.h $(htslib_bgzf_h) $(htslib_hts_h) $(htslib_hfile_h)
+htsfile.o: htsfile.c config.h $(htslib_hfile_h) $(htslib_hts_h) $(htslib_sam_h) $(htslib_vcf_h)
+tabix.o: tabix.c config.h $(htslib_tbx_h) $(htslib_sam_h) $(htslib_vcf_h) $(htslib_kseq_h) $(htslib_bgzf_h) $(htslib_hts_h) $(htslib_regidx_h) $(htslib_hts_defs_h) $(htslib_hts_log_h) $(htslib_thread_pool_h)
+
+# Runes to check that the htscodecs submodule is present
+ifdef HTSCODECS_SOURCES
+htscodecs/htscodecs/%.c: | htscodecs/htscodecs
+	@if test -e htscodecs/.git && test ! -e "$@" ; then \
+	  echo "Missing file '$@'" ; \
+	  echo "  - Do you need to update the htscodecs submodule?" ; \
+	  false ; \
+	fi
+
+htscodecs/htscodecs/%.h: | htscodecs/htscodecs
+	@if test -e htscodecs/.git && test ! -e "$@" ; then \
+	  echo "Missing file '$@'" ; \
+	  echo "  - Do you need to update the htscodecs submodule?" ; \
+	  false ; \
+	fi
+
+htscodecs/htscodecs:
+	@if test -e .git ; then \
+	printf "\\n\\nError: htscodecs submodule files not present for htslib.\\n\
+	Try running: \\n\
+	    git submodule update --init --recursive\\n\
+	in the top-level htslib directory and then re-run make.\\n\\n\\n" ; \
+	else \
+	  printf "\\n\\nError: htscodecs submodule files not present and this is not a git checkout.\\n\
+	  You have an incomplete distribution.  Please try downloading one of the\\n\
+	  official releases from https://www.htslib.org/\\n" ; \
+	fi
+	@false
+
+# Build the htscodecs/htscodecs/version.h file if necessary
+htscodecs/htscodecs/version.h: force
+	@if test -e $(srcdir)/htscodecs/.git && test -e $(srcdir)/htscodecs/configure.ac ; then \
+	  vers=`cd $(srcdir)/htscodecs && git describe --always --dirty --match 'v[0-9]\.[0-9]*'` && \
+	  case "$$vers" in \
+	    v*) vers=$${vers#v} ;; \
+	    *) iv=`awk '/^AC_INIT\(htscodecs,/ { match($$0, /[0-9]+(\.[0-9]+)*/); print substr($$0, RSTART, RLENGTH) }' $(srcdir)/htscodecs/configure.ac` ; vers="$$iv$${vers:+-g$$vers}" ;; \
+	  esac ; \
+	  if ! grep -s -q '"'"$$vers"'"' $@ ; then \
+	    echo 'Updating $@ : #define HTSCODECS_VERSION_TEXT "'"$$vers"'"' ; \
+	    echo '#define HTSCODECS_VERSION_TEXT "'"$$vers"'"' > $@ ; \
+	  fi ; \
+	fi
+endif
+
+# Maintainer extra targets built
+# - compile public headers as C++
+# Maintainer source code checks
+# - copyright boilerplate presence
+# - tab and trailing space detection
+maintainer-check: test/usepublic.o
+	test/maintainer/check_copyright.pl .
+	test/maintainer/check_spaces.pl .
+
+# Look for untracked files in the git repository.
+check-untracked:
+	@if test -e .git && git status --porcelain | grep '^\?'; then \
+	    echo 'Untracked files detected (see above). Please either clean up, add to .gitignore, or for test output files consider naming them to match *.tmp or *.tmp.*' ; \
+	    false ; \
+	fi
+
+# Create a shorthand. We use $(SRC) or $(srcprefix) rather than $(srcdir)/
+# for brevity in test and install rules, and so that build logs do not have
+# ./ sprinkled throughout.
+SRC = $(srcprefix)
+
+# For tests that might use it, set $REF_PATH explicitly to use only reference
+# areas within the test suite (or set it to ':' to use no reference areas).
+#
+# If using MSYS, avoid poor shell expansion via:
+#    MSYS2_ARG_CONV_EXCL="*" make check
+check test: all $(HTSCODECS_TEST_TARGETS)
+	test/hts_endian
+	test/test_expr
+	test/test_kfunc
+	test/test_khash
+	test/test_kstring
+	test/test_nibbles -v
+	test/test_str2int
+	test/test_time_funcs
+	test/fieldarith test/fieldarith.sam
+	test/hfile
+	if test "x$(BUILT_PLUGINS)" != "x"; then \
+	    HTS_PATH=. test/with-shlib.sh test/plugins-dlhts -g ./libhts.$(SHLIB_FLAVOUR); \
+	fi
+	if test "x$(BUILT_PLUGINS)" != "x"; then \
+	  HTS_PATH=. test/with-shlib.sh test/plugins-dlhts -l ./libhts.$(SHLIB_FLAVOUR); \
+	fi
+	test/test_bgzf test/bgziptest.txt
+	test/test-parse-reg -t test/colons.bam
+	cd test/faidx && ./test-faidx.sh faidx.tst
+	cd test/sam_filter && ./filter.sh filter.tst
+	cd test/tabix && ./test-tabix.sh tabix.tst
+	cd test/mpileup && ./test-pileup.sh mpileup.tst
+	cd test/fastq && ./test-fastq.sh
+	cd test/base_mods && ./base-mods.sh base-mods.tst
+	REF_PATH=: test/sam test/ce.fa test/faidx/faidx.fa test/faidx/fastqs.fq
+	test/test-regidx
+	cd test && REF_PATH=: ./test.pl $${TEST_OPTS:-}
+
+test/hts_endian: test/hts_endian.o
+	$(CC) $(LDFLAGS) -o $@ test/hts_endian.o $(LIBS)
+
+# To build the fuzzer, try:
+# make  CC="clang16 -fsanitize=address,undefined,fuzzer" \
+#     CFLAGS="-g -O3 -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION" \
+#     test/fuzz/hts_open_fuzzer
+test/fuzz/hts_open_fuzzer: test/fuzz/hts_open_fuzzer.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ test/fuzz/hts_open_fuzzer.o libhts.a $(LIBS) -lpthread
+
+test/fieldarith: test/fieldarith.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ test/fieldarith.o libhts.a $(LIBS) -lpthread
+
+test/hfile: test/hfile.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ test/hfile.o libhts.a $(LIBS) -lpthread
+
+test/pileup: test/pileup.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ test/pileup.o libhts.a $(LIBS) -lpthread
+
+test/pileup_mod: test/pileup_mod.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ test/pileup_mod.o libhts.a $(LIBS) -lpthread
+
+test/plugins-dlhts: test/plugins-dlhts.o
+	$(CC) $(LDFLAGS) -o $@ test/plugins-dlhts.o $(LIBS)
+
+test/sam: test/sam.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ test/sam.o libhts.a $(LIBS) -lpthread
+
+test/test_bgzf: test/test_bgzf.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ test/test_bgzf.o libhts.a $(LIBS) -lpthread
+
+test/test_expr: test/test_expr.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ test/test_expr.o libhts.a $(LIBS) -lpthread
+
+test/test_faidx: test/test_faidx.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ test/test_faidx.o libhts.a $(LIBS) -lpthread
+
+test/test_kfunc: test/test_kfunc.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ test/test_kfunc.o libhts.a $(LIBS) -lpthread
+
+test/test_khash: test/test_khash.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ test/test_khash.o libhts.a $(LIBS) -lpthread
+
+test/test_kstring: test/test_kstring.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ test/test_kstring.o libhts.a $(LIBS) -lpthread
+
+test/test_mod: test/test_mod.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ test/test_mod.o libhts.a $(LIBS) -lpthread
+
+test/test_nibbles: test/test_nibbles.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ test/test_nibbles.o libhts.a $(LIBS) -lpthread
+
+test/test_realn: test/test_realn.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ test/test_realn.o libhts.a $(LIBS) -lpthread
+
+test/test-regidx: test/test-regidx.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ test/test-regidx.o libhts.a $(LIBS) -lpthread
+
+test/test-parse-reg: test/test-parse-reg.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ test/test-parse-reg.o libhts.a $(LIBS) -lpthread
+
+test/test_str2int: test/test_str2int.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ test/test_str2int.o libhts.a $(LIBS) -lpthread
+
+test/test_time_funcs: test/test_time_funcs.o
+	$(CC) $(LDFLAGS) -o $@ test/test_time_funcs.o
+
+test/test_view: test/test_view.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ test/test_view.o libhts.a $(LIBS) -lpthread
+
+test/test_index: test/test_index.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ test/test_index.o libhts.a $(LIBS) -lpthread
+
+test/test-vcf-api: test/test-vcf-api.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ test/test-vcf-api.o libhts.a $(LIBS) -lpthread
+
+test/test-vcf-sweep: test/test-vcf-sweep.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ test/test-vcf-sweep.o libhts.a $(LIBS) -lpthread
+
+test/test-bcf-sr: test/test-bcf-sr.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ test/test-bcf-sr.o libhts.a $(LIBS) -lpthread
+
+test/test-bcf-translate: test/test-bcf-translate.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ test/test-bcf-translate.o libhts.a $(LIBS) -lpthread
+
+test/test_introspection: test/test_introspection.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ test/test_introspection.o libhts.a $(LIBS) -lpthread
+
+test/test-bcf_set_variant_type: test/test-bcf_set_variant_type.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ test/test-bcf_set_variant_type.o libhts.a $(LIBS) -lpthread
+
+# Extra tests for bundled htscodecs
+test_htscodecs_rans4x8: htscodecs/tests/rans4x8
+	cd htscodecs/tests && srcdir=. && export srcdir && ./rans4x8.test
+
+test_htscodecs_rans4x16: htscodecs/tests/rans4x16pr
+	cd htscodecs/tests && srcdir=. && export srcdir && ./rans4x16.test
+
+test_htscodecs_arith: htscodecs/tests/arith_dynamic
+	cd htscodecs/tests && srcdir=. && export srcdir && ./arith.test
+
+test_htscodecs_tok3: htscodecs/tests/tokenise_name3
+	cd htscodecs/tests && srcdir=. && export srcdir && ./tok3.test
+
+test_htscodecs_fqzcomp: htscodecs/tests/fqzcomp_qual
+	cd htscodecs/tests && srcdir=. && export srcdir && ./fqzcomp.test
+
+test_htscodecs_varint: htscodecs/tests/varint
+	cd htscodecs/tests && ./varint
+
+htscodecs/tests/arith_dynamic: htscodecs/tests/arith_dynamic_test.o $(HTSCODECS_OBJS)
+	$(CC) $(LDFLAGS) -o $@ $^ $(LIBS) -lm -lpthread
+
+htscodecs/tests/fqzcomp_qual: htscodecs/tests/fqzcomp_qual_test.o $(HTSCODECS_OBJS)
+	$(CC) $(LDFLAGS) -o $@ $^ $(LIBS) -lm -lpthread
+
+htscodecs/tests/rans4x16pr: htscodecs/tests/rANS_static4x16pr_test.o $(HTSCODECS_OBJS)
+	$(CC) $(LDFLAGS) -o $@ $^ $(LIBS) -lm -lpthread
+
+htscodecs/tests/rans4x8: htscodecs/tests/rANS_static_test.o $(HTSCODECS_OBJS)
+	$(CC) $(LDFLAGS) -o $@ $^ $(LIBS) -lm -lpthread
+
+htscodecs/tests/tokenise_name3: htscodecs/tests/tokenise_name3_test.o $(HTSCODECS_OBJS)
+	$(CC) $(LDFLAGS) -o $@ $^ $(LIBS) -lm -lpthread
+
+htscodecs/tests/varint: htscodecs/tests/varint_test.o $(HTSCODECS_OBJS)
+	$(CC) $(LDFLAGS) -o $@ $^ $(LIBS) -lm -lpthread
+
+htscodecs/tests/arith_dynamic_test.o: CPPFLAGS += -Ihtscodecs
+htscodecs/tests/arith_dynamic_test.o: htscodecs/tests/arith_dynamic_test.c config.h $(htscodecs_arith_dynamic_h)
+htscodecs/tests/fqzcomp_qual_test.o: CPPFLAGS += -Ihtscodecs
+htscodecs/tests/fqzcomp_qual_test.o: htscodecs/tests/fqzcomp_qual_test.c config.h $(htscodecs_fqzcomp_qual_h) $(htscodecs_varint_h)
+htscodecs/tests/rANS_static4x16pr_test.o: CPPFLAGS += -Ihtscodecs
+htscodecs/tests/rANS_static4x16pr_test.o: htscodecs/tests/rANS_static4x16pr_test.c config.h $(htscodecs_rANS_static4x16_h)
+htscodecs/tests/rANS_static_test.o: CPPFLAGS += -Ihtscodecs
+htscodecs/tests/rANS_static_test.o: htscodecs/tests/rANS_static_test.c config.h $(htscodecs_rANS_static_h)
+htscodecs/tests/tokenise_name3_test.o: CPPFLAGS += -Ihtscodecs
+htscodecs/tests/tokenise_name3_test.o: htscodecs/tests/tokenise_name3_test.c config.h $(htscodecs_tokenise_name3_h)
+htscodecs/tests/varint_test.o: CPPFLAGS += -Ihtscodecs
+htscodecs/tests/varint_test.o: htscodecs/tests/varint_test.c config.h $(htscodecs_varint_h)
+
+test/hts_endian.o: test/hts_endian.c config.h $(htslib_hts_endian_h)
+test/fuzz/hts_open_fuzzer.o: test/fuzz/hts_open_fuzzer.c config.h $(htslib_hfile_h) $(htslib_hts_h) $(htslib_sam_h) $(htslib_vcf_h)
+test/fieldarith.o: test/fieldarith.c config.h $(htslib_sam_h)
+test/hfile.o: test/hfile.c config.h $(htslib_hfile_h) $(htslib_hts_defs_h) $(htslib_kstring_h)
+test/pileup.o: test/pileup.c config.h $(htslib_sam_h) $(htslib_kstring_h)
+test/pileup_mod.o: test/pileup_mod.c config.h $(htslib_sam_h)
+test/plugins-dlhts.o: test/plugins-dlhts.c config.h
+test/sam.o: test/sam.c config.h $(htslib_hts_defs_h) $(htslib_sam_h) $(htslib_faidx_h) $(htslib_khash_h) $(htslib_hts_log_h)
+test/test_bgzf.o: test/test_bgzf.c config.h $(htslib_bgzf_h) $(htslib_hfile_h) $(htslib_hts_log_h) $(hfile_internal_h)
+test/test_expr.o: test/test_expr.c config.h $(htslib_hts_expr_h)
+test/test_kfunc.o: test/test_kfunc.c config.h $(htslib_kfunc_h)
+test/test_khash.o: test/test_khash.c config.h $(htslib_khash_h) $(htslib_kroundup_h)
+test/test_kstring.o: test/test_kstring.c config.h $(htslib_kstring_h)
+test/test_mod.o: test/test_mod.c config.h $(htslib_sam_h)
+test/test_nibbles.o: test/test_nibbles.c config.h $(htslib_sam_h) $(sam_internal_h)
+test/test-parse-reg.o: test/test-parse-reg.c config.h $(htslib_hts_h) $(htslib_sam_h)
+test/test_realn.o: test/test_realn.c config.h $(htslib_hts_h) $(htslib_sam_h) $(htslib_faidx_h)
+test/test-regidx.o: test/test-regidx.c config.h $(htslib_kstring_h) $(htslib_regidx_h) $(htslib_hts_defs_h) $(textutils_internal_h)
+test/test_str2int.o: test/test_str2int.c config.h $(textutils_internal_h)
+test/test_time_funcs.o: test/test_time_funcs.c config.h $(hts_time_funcs_h)
+test/test_view.o: test/test_view.c config.h $(cram_h) $(htslib_sam_h) $(htslib_vcf_h) $(htslib_hts_log_h)
+test/test_faidx.o: test/test_faidx.c config.h $(htslib_faidx_h)
+test/test_index.o: test/test_index.c config.h $(htslib_sam_h) $(htslib_vcf_h)
+test/test-vcf-api.o: test/test-vcf-api.c config.h $(htslib_hts_h) $(htslib_vcf_h) $(htslib_kstring_h) $(htslib_kseq_h)
+test/test-vcf-sweep.o: test/test-vcf-sweep.c config.h $(htslib_vcf_sweep_h)
+test/test-bcf-sr.o: test/test-bcf-sr.c config.h $(htslib_synced_bcf_reader_h) $(htslib_hts_h) $(htslib_vcf_h)
+test/test-bcf-translate.o: test/test-bcf-translate.c config.h $(htslib_vcf_h)
+test/test_introspection.o: test/test_introspection.c config.h $(htslib_hts_h) $(htslib_hfile_h)
+test/test-bcf_set_variant_type.o: test/test-bcf_set_variant_type.c config.h $(htslib_hts_h) vcf.c
+
+# Standalone target not added to $(BUILT_TEST_PROGRAMS) as some may not
+# have a compiler that compiles as C++ when given a .cpp source file.
+test/usepublic.o: test/usepublic.cpp config.h $(htslib_bgzf_h) $(htslib_cram_h) $(htslib_faidx_h) $(htslib_hfile_h) $(htslib_hts_h) $(htslib_hts_defs_h) $(htslib_hts_endian_h) $(htslib_hts_expr_h) $(htslib_hts_log_h) $(htslib_hts_os_h) $(htslib_kbitset_h) $(htslib_kfunc_h) $(htslib_khash_h) $(htslib_khash_str2int_h) $(htslib_klist_h) $(HTSPREFIX)htslib/knetfile.h $(htslib_kroundup_h) $(htslib_kseq_h) $(htslib_ksort_h) $(htslib_kstring_h) $(htslib_regidx_h) $(htslib_sam_h) $(htslib_synced_bcf_reader_h) $(htslib_tbx_h) $(htslib_thread_pool_h) $(htslib_vcf_h) $(htslib_vcf_sweep_h) $(htslib_vcfutils_h)
+	$(CC) $(CFLAGS) $(TARGET_CFLAGS) $(ALL_CPPFLAGS) -c -o $@ test/usepublic.cpp
+
+
+test/thrash_threads1: test/thrash_threads1.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ test/thrash_threads1.o libhts.a $(LIBS) -lpthread
+
+test/thrash_threads2: test/thrash_threads2.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ test/thrash_threads2.o libhts.a $(LIBS) -lpthread
+
+test/thrash_threads3: test/thrash_threads3.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ test/thrash_threads3.o libhts.a $(LIBS) -lpthread
+
+test/thrash_threads4: test/thrash_threads4.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ test/thrash_threads4.o libhts.a $(LIBS) -lpthread
+
+test/thrash_threads5: test/thrash_threads5.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ test/thrash_threads5.o libhts.a $(LIBS) -lpthread
+
+test/thrash_threads6: test/thrash_threads6.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ test/thrash_threads6.o libhts.a $(LIBS) -lpthread
+
+test/thrash_threads7: test/thrash_threads7.o libhts.a
+	$(CC) $(LDFLAGS) -o $@ test/thrash_threads7.o libhts.a $(LIBS) -lpthread
+
+test_thrash: $(BUILT_THRASH_PROGRAMS)
+
+# Test to ensure the functions in the header files are exported by the shared
+# library.  This currently works by comparing the output from ctags on
+# the headers with the list of functions exported by the shared library.
+# Note that functions marked as exported in the .c files and not the public
+# headers will be missed by this test.
+test-shlib-exports: header-exports.txt shlib-exports-$(SHLIB_FLAVOUR).txt
+	@echo "Checking shared library exports"
+	@if test ! -s header-exports.txt ; then echo "Error: header-exports.txt empty" ; false ; fi
+	@if test ! -s shlib-exports-$(SHLIB_FLAVOUR).txt ; then echo "Error: shlib-exports-$(SHLIB_FLAVOUR).txt empty" ; false ; fi
+	@! comm -23 header-exports.txt shlib-exports-$(SHLIB_FLAVOUR).txt | grep . || \
+	( echo "Error: Found unexported symbols (listed above)" ; false )
+
+# Extract symbols that should be exported from public headers using ctags
+# Filter out macros in htslib/hts_defs.h.
+header-exports.txt: test/header_syms.pl htslib/*.h
+	test/header_syms.pl htslib/*.h | sort -u -o $@
+
+shlib-exports-so.txt: libhts.so
+	nm -D -g libhts.so | awk '$$2 == "T" { sub("@.*", "", $$3); print $$3 }' | sort -u -o $@
+
+shlib-exports-dylib.txt: libhts.dylib
+	nm -Ug libhts.dylib | awk '$$2 == "T" { sub("^_", "", $$3); print $$3 }' | sort -u -o $@
+
+shlib-exports-dll.txt: hts.dll.a
+	nm -g hts.dll.a | awk '$$2 == "T" { print $$3 }' | sort -u -o $@
+
+$(srcprefix)htslib.map: libhts.so
+	LC_ALL=C ; export LC_ALL; \
+	curr_vers=`expr 'X$(PACKAGE_VERSION)' : 'X\([0-9]*\.[0-9.]*\)'` ; \
+	last_vers=`awk '/^HTSLIB_[0-9](\.[0-9]+)+/ { lv = $$1 } END { print lv }' htslib.map` ; \
+	if test "x$$curr_vers" = 'x' || test "x$$last_vers" = 'x' ; then \
+	    echo "Version check failed : $$curr_vers / $$las_vers" 1>&2 ; \
+	    exit 1 ; \
+	fi && \
+	if test "HTSLIB_$$curr_vers" = "$$last_vers" ; then \
+	    echo "Refusing to update $@ - HTSlib version not changed" 1>&2 ; \
+	    exit 1 ; \
+	fi && \
+	nm --with-symbol-versions -D -g libhts.so | awk '$$2 ~ /^[DGRT]$$/ && $$3 ~ /@@Base$$/ && $$3 !~ /^(_init|_fini|_edata)@@/ { sub(/@@Base$$/, ";", $$3); print "    " $$3 }' > $@.tmp && \
+	if [ -s $@.tmp ] ; then \
+	    cat $@ > $@.new.tmp && \
+	    printf '\n%s {\n' "HTSLIB_$$curr_vers" >> $@.new.tmp && \
+	    cat $@.tmp >> $@.new.tmp && \
+	    printf '} %s;\n' "$$last_vers" >> $@.new.tmp && \
+	        rm -f $@.tmp && \
+	        mv $@.new.tmp $@ ; \
+	    fi ; \
+	else \
+	    rm -f $@.tmp ; \
+	fi
+
+install: libhts.a $(BUILT_PROGRAMS) $(BUILT_PLUGINS) installdirs install-$(SHLIB_FLAVOUR) install-pkgconfig
+	$(INSTALL_PROGRAM) $(BUILT_PROGRAMS) $(DESTDIR)$(bindir)
+	if test -n "$(BUILT_PLUGINS)"; then $(INSTALL_PROGRAM) $(BUILT_PLUGINS) $(DESTDIR)$(plugindir); fi
+	$(INSTALL_DATA) $(SRC)htslib/*.h $(DESTDIR)$(includedir)/htslib
+	$(INSTALL_DATA) libhts.a $(DESTDIR)$(libdir)/libhts.a
+	$(INSTALL_MAN) $(SRC)annot-tsv.1 $(SRC)bgzip.1 $(SRC)htsfile.1 $(SRC)tabix.1 $(DESTDIR)$(man1dir)
+	$(INSTALL_MAN) $(SRC)faidx.5 $(SRC)sam.5 $(SRC)vcf.5 $(DESTDIR)$(man5dir)
+	$(INSTALL_MAN) $(SRC)htslib-s3-plugin.7 $(DESTDIR)$(man7dir)
+
+installdirs:
+	$(INSTALL_DIR) $(DESTDIR)$(bindir) $(DESTDIR)$(includedir) $(DESTDIR)$(includedir)/htslib $(DESTDIR)$(libdir) $(DESTDIR)$(man1dir) $(DESTDIR)$(man5dir) $(DESTDIR)$(man7dir) $(DESTDIR)$(pkgconfigdir)
+	if test -n "$(plugindir)"; then $(INSTALL_DIR) $(DESTDIR)$(plugindir); fi
+
+# After installation, the real file in $(libdir) will be libhts.so.X.Y.Z,
+# with symlinks libhts.so (used via -lhts during linking of client programs)
+# and libhts.so.NN (used by client executables at runtime).
+
+install-so: libhts.so installdirs
+	$(INSTALL_LIB) libhts.so $(DESTDIR)$(libdir)/libhts.so.$(PACKAGE_VERSION)
+	ln -sf libhts.so.$(PACKAGE_VERSION) $(DESTDIR)$(libdir)/libhts.so
+	ln -sf libhts.so.$(PACKAGE_VERSION) $(DESTDIR)$(libdir)/libhts.so.$(LIBHTS_SOVERSION)
+
+install-cygdll: cyghts-$(LIBHTS_SOVERSION).dll installdirs
+	$(INSTALL_PROGRAM) cyghts-$(LIBHTS_SOVERSION).dll $(DESTDIR)$(bindir)/cyghts-$(LIBHTS_SOVERSION).dll
+	$(INSTALL_PROGRAM) libhts.dll.a $(DESTDIR)$(libdir)/libhts.dll.a
+
+install-dll: hts-$(LIBHTS_SOVERSION).dll installdirs
+	$(INSTALL_PROGRAM) hts-$(LIBHTS_SOVERSION).dll $(DESTDIR)$(bindir)/hts-$(LIBHTS_SOVERSION).dll
+	$(INSTALL_PROGRAM) hts.dll.a $(DESTDIR)$(libdir)/hts.dll.a
+
+install-dylib: libhts.dylib installdirs
+	$(INSTALL_PROGRAM) libhts.dylib $(DESTDIR)$(libdir)/libhts.$(PACKAGE_VERSION).dylib
+	ln -sf libhts.$(PACKAGE_VERSION).dylib $(DESTDIR)$(libdir)/libhts.dylib
+	ln -sf libhts.$(PACKAGE_VERSION).dylib $(DESTDIR)$(libdir)/libhts.$(LIBHTS_SOVERSION).dylib
+
+# Substitute these pseudo-autoconf variables only at install time
+# so that "make install prefix=/prefix/path" etc continue to work.
+install-pkgconfig: htslib.pc.tmp installdirs
+	sed -e 's#@-includedir@#$(includedir)#g;s#@-libdir@#$(libdir)#g;s#@-PACKAGE_VERSION@#$(PACKAGE_VERSION)#g' htslib.pc.tmp > $(DESTDIR)$(pkgconfigdir)/htslib.pc
+	chmod 644 $(DESTDIR)$(pkgconfigdir)/htslib.pc
+
+# A pkg-config file (suitable for copying to $PKG_CONFIG_PATH) that provides
+# flags for building against the uninstalled library in this build directory.
+htslib-uninstalled.pc: htslib.pc.tmp
+	sed -e 's#@-includedir@#'`pwd`'#g;s#@-libdir@#'`pwd`'#g' htslib.pc.tmp > $@
+
+
+testclean:
+	-rm -f test/*.tmp test/*.tmp.* test/faidx/*.tmp* \
+               test/longrefs/*.tmp.* test/tabix/*.tmp.* \
+               test/bgzf_boundaries/*.tmp.* test/*/FAIL* \
+               header-exports.txt shlib-exports-$(SHLIB_FLAVOUR).txt
+	-rm -rf htscodecs/tests/test.out
+
+# Only remove this in git checkouts
+DEL_HTSCODECS_VERSION := $(if $(wildcard htscodecs/.git),htscodecs/htscodecs/version.h)
+
+mostlyclean: testclean
+	-rm -f *.o *.pico cram/*.o cram/*.pico test/*.o test/*.dSYM config_vars.h version.h
+	-rm -f htscodecs/htscodecs/*.o htscodecs/htscodecs/*.pico $(DEL_HTSCODECS_VERSION)
+	-rm -f hts-object-files
+	-rm -f htscodecs/tests/*.o
+
+clean: mostlyclean clean-$(SHLIB_FLAVOUR)
+	-rm -f libhts.a $(BUILT_PROGRAMS) $(BUILT_PLUGINS) $(BUILT_TEST_PROGRAMS) $(BUILT_THRASH_PROGRAMS)
+	-rm -f htscodecs/tests/rans4x8 htscodecs/tests/rans4x16pr htscodecs/tests/arith_dynamic htscodecs/tests/tokenise_name3 htscodecs/tests/fqzcomp_qual htscodecs/tests/varint
+
+distclean maintainer-clean: clean
+	-rm -f config.cache config.h config.log config.mk config.status
+	-rm -f TAGS *.pc.tmp *-uninstalled.pc htslib_static.mk htscodecs.mk
+	-rm -rf autom4te.cache
+
+clean-so:
+	-rm -f libhts.so libhts.so.*
+
+clean-cygdll:
+	-rm -f cyghts-*.dll libhts.dll.a
+
+clean-dll:
+	-rm -f hts-*.dll hts.dll.a
+
+clean-dylib:
+	-rm -f libhts.dylib libhts.*.dylib
+
+
+tags TAGS:
+	ctags -f TAGS *.[ch] cram/*.[ch] htslib/*.h
+
+# We recommend libhts-using programs be built against a separate htslib
+# installation.  However if you feel that you must bundle htslib source
+# code with your program, this hook enables Automake-style "make dist"
+# for this subdirectory.  If you do bundle an htslib snapshot, please
+# add identifying information to $(PACKAGE_VERSION) as appropriate.
+# (The wildcards attempt to omit non-exported files (.git*, README.md,
+# etc) and other detritus that might be in the top-level directory.)
+distdir:
+	@if [ -z "$(distdir)" ]; then echo "Please supply a distdir=DIR argument."; false; fi
+	tar -c *.[ch15] [ILMNRchtv]*[ELSbcekmnth] | (cd $(distdir) && tar -x)
+	+cd $(distdir) && $(MAKE) distclean
+
+force:
+
+
+.PHONY: all check check-untracked clean distclean distdir force
+.PHONY: install install-pkgconfig installdirs lib-shared lib-static
+.PHONY: maintainer-check maintainer-clean mostlyclean plugins
+.PHONY: print-config print-version show-version tags
+.PHONY: test test-shlib-exports test_thrash testclean
+.PHONY: clean-so install-so
+.PHONY: clean-cygdll install-cygdll
+.PHONY: clean-dll install-dll
+.PHONY: clean-dylib install-dylib
+.PHONY: test_htscodecs_rans4x8 test_htscodecs_rans4x16 test_htscodecs_arith
+.PHONY: test_htscodecs_tok3 test_htscodecs_fqzcomp test_htscodecs_varint
+.PHONY: cc-version
diff --git a/ext/htslib/NEWS b/ext/htslib/NEWS
new file mode 100644
index 0000000..8825c30
--- /dev/null
+++ b/ext/htslib/NEWS
@@ -0,0 +1,2520 @@
+Noteworthy changes in release 1.21 (12th September 2024)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The primary user-visible changes in this release are updates to the
+annot-tsv tool and some speed improvements.  Full details of other
+changes and bugs fixed are below.
+
+Notice: this is the last SAMtools / HTSlib release where CRAM 3.0 will be
+the default CRAM version.  From the next we will change to CRAM 3.1
+unless the version is explicitly specified, for example using
+"samtools view -O cram,version=3.0".
+
+
+Updates
+-------
+
+* Extend annot-tsv with several new command line options.
+    --delim permits use of other delimiters.
+    --headers for selection of other header formats.
+    --no-header-idx to suppress column index numbers in header.
+  Also removed -h as it is now short for --headers.  Note --help
+  still works. (PR #1779)
+
+* Allow annot-tsv -a to rename annotations. (PR #1709)
+
+* Extend annot-tsv --overlap to be able to specify the overlap
+  fraction separately for source and target. (PR #1811)
+
+* Added new APIs to facilitate low-level CRAM container manipulations,
+  used by the new "samtools cat" region filtering code. Functions are:
+    cram_container_get_coords()
+    cram_filter_container()
+    cram_index_extents()
+    cram_container_num2offset()
+    cram_container_offset2num()
+    cram_num_containers()
+    cram_num_containers_between()
+  Also improved cram_index_query() to cope with HTS_IDX_NOCOOR regions.
+  (PR #1771)
+
+* Bgzip now retains file modification and access times when
+  compressing and decompressing. (PR #1727, fixes #1718.  Requested by
+  Gert Hulselmans.)
+
+* Use FNV1a for string hashing in khash.  The old algorithm was
+  particularly weak with base-64 style strings and lead to a large
+  number of collisions.  (PR #1806.  Fixes   samtools/samtools#2066,
+  reported by Hans-Joachim Ruscheweyh)
+
+* Improve the speed of the nibble2base() function on Intel (PR
+  #1667, PR #1764, PR #1786, PR #1802, thanks to Ruben Vorderman) and
+  ARM (PR #1795, thanks to John Marshall).
+
+* bgzf_getline() will now warn if it encounters UTF-16 data.
+  (PR #1487, thanks to John Marshall)
+
+* Speed up bgzf_read().  While this does not reduce CPU significantly,
+  it does increase the maximum parallelism available permitting 10-15%
+  faster decoding. (PR #1772, PR #1800, Issue #1798)
+
+* Speed up faidx by use of better isgraph methods (PR #1797) and
+  whole-line reading (PR #1799, thanks to John Marshall).
+
+* Speed up kputll() function, speeding up BAM -> SAM conversion by
+  about 5% and also samtools depth.  (PR #1805)
+
+* Added more example code, covering fasta/fastq indexing, tabix
+  indexing and use of the thread pool. (PR #1666)
+
+Build Changes
+-------------
+
+* Code warning fixes for pedantic compilers (PR #1777) and avoid
+  some undefined behaviour (PR #1810, PR #1816, PR #1828).
+
+* Windows based CI has been migrated from AppVeyor to GitHub Actions.
+  (PR #1796, PR #1803, PR #1808)
+
+* Miscellaneous minor build infrastructure and code fixes.
+  (PR #1807, PR #1829, both thanks to John Marshall)
+
+* Updated htscodecs submodule to version 1.6.1 (PR #1828)
+
+* Fixed an awk script in the Makefile that only worked with gawk. (PR #1831)
+
+Bug fixes
+---------
+
+* Fix small OSS-Fuzz reported issues with CRAM encoding and long
+  CIGARS and/or illegal positions. (PR #1775, PR #1801, PR #1817)
+
+* Fix issues with on-the-fly indexing of VCF/BCF (bcftools --write-index)
+  when not using multiple threads. (PR #1837. Fixes samtools/bcftools#2267,
+  reported by Giulio Genovese)
+
+* Stricter limits on POS / MPOS / TLEN in sam_parse1().  This fixes
+  a signed overflow reported by OSS-Fuzz and should help prevent other
+  as-yet undetected bugs. (PR #1812)
+
+* Check that the underlying file open worked for preload: URLs.  Fixes
+  a NULL pointer dereference reported by OSS-Fuzz. (PR #1821)
+
+* Fix an infinite loop in hts_itr_query() when given extremely large
+  positions which cause integer overflow.  Also adds hts_bin_maxpos()
+  and hts_idx_maxpos() functions.
+  (PR #1774, thanks to John Marshall and reported by Jesus Alberto
+  Munoz Mesa)
+
+* Fix an out of bounds read in hts_itr_multi_next() when switching
+  chromosomes.  This bug is present in releases 1.11 to 1.20.
+  (PR #1788. Fixes samtools/samtools#2063, reported by acorvelo)
+
+* Work around parsing problems with colons in CHROM names.
+  Fixes samtools/bcftools#2139.  (PR #1781, John Marshall / James Bonfield)
+
+* Correct the CPU detection for Mac OS X 10.7.  cpuid is used by
+  htscodecs (see samtools/htscodecs#116), and the corresponding
+  changes in htslib are PR #1785.  Reported by Ryan Carsten Schmidt.
+
+* Make BAM zero-length intervals work the same as CRAM; permitted and
+  returning overlapping records. (PR #1787.  Fixes
+  samtools/samtools#2060, reported by acorvelo)
+
+* Replace assert() with abort() in BCF synced reader.  This is not an
+  ideal solution, but it gives consistent behaviour when compiling
+  with or without NDEBUG.  (PR #1791, thanks to Martin Pollard)
+
+* Fixed failure to change the write block size on compressed SAM or VCF
+  files due to an internal type confusion.  (PR #1826)
+
+* Fixed an out-of-bounds read in cram_codec_iter_next() (PR #1832)
+
+Noteworthy changes in release 1.20 (15th April 2024)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Updates
+-------
+
+* When working on named files, bgzip now sets the modified and access times
+  of the output files it makes to match those of the corresponding input.
+  (PR #1727, feature request #1718.  Requested by Gert Hulselmans)
+
+* It's now possible to use a -o option to specify the output file name in
+  bgzip.
+  (PR #1747, feature request #1726.  Requested by Gert Hulselmans)
+
+* Improved error faidx error messages.
+  (PR #1743, thanks to Nick Moore)
+
+* Faster reading of SAM array (type "B") tags.  These often turn up
+  in ONT and PacBio data.
+  (PR #1741)
+
+* Improved validity checking of base modification tags.
+  (PR #1749)
+
+* mpileup overlap removal now works where one read has a deletion.
+  (PR #1751, fixes samtools/samtools#1992.  Reported by Long Tian)
+
+* The S3 plugin can now find buckets via S3 access point aliases.
+  (PR #1756, thanks to Matt Pawelczyk;
+   fixes samtools/samtools#1984.  Reported by Albert Li)
+
+* Added a --threads option (and -@ short option) to tabix.
+  (PR #1755, feature request #1735.  Requested by Dan Bolser)
+
+* tabix can now index Graph Alignment Format (GAF) files.
+  (See https://github.com/lh3/gfatools/blob/master/doc/rGFA.md)
+  (PR #1763, thanks to Adam Novak)
+
+Bug fixes
+---------
+
+* Security fix: Prevent possible heap overflow in cram_encode_aux() on
+  bad RG:Z tags.
+  (PR #1737)
+
+* Security fix: Prevent attempts to call a NULL pointer if certain URL
+  schemes are used in CRAM @SQ UR: tags.
+  (PR #1757)
+
+* Security fix: Fixed a bug where following certain AWS S3 redirects could
+  downgrade the connection from TLS (i.e. https://) to unencrypted http://.
+  This could happen when using path-based URLs and AWS_DEFAULT_REGION
+  was set to a region other that the one where the data was stored.
+  (PR #1762, fixes #1760. Reported by andaca)
+
+* Fixed arithmetic overflow when loading very long references for CRAM.
+  (PR #1738, fixes #1738.  Reported by Shane McCarthy)
+
+* Fixed faidx and CRAM reference look-ups on compressed	fasta where the .fai
+  index file was present, but the .gzi index of compressed offsets was not.
+  (PR #1745, fixes #1744.  Reported by Theodore Li)
+
+* Fixed BCF indexing on-the-fly bug which produced invalid indexes when
+  using multiple compression threads.
+  (PR #1742, fixes #1740.  Reported by graphenn)
+
+* Ensure that pileup destructors are called by bam_plp_destroy(), to
+  prevent memory leaks.
+  (PR #1749, PR #1754)
+
+* Ensure on-the-fly index timestamps are always older than the data file.
+  Previously the files could be closed out of order, leading to warnings
+  being printed when using the index.
+  (PR #1753, fixes #1732.  Reported by Gert Hulselmans)
+
+* To prevent data corruption when reading (strictly invalid) VCF files
+  with duplicated FORMAT tags, all but the first copy of the data
+  associated with the tag are now dropped with a warning.
+  (PR #1752, PR #1761, fixes #1733.  Reported by anthakki)
+
+* Fixed a bug introduced in release 1.19 (PR #1689) which broke variant
+  record data if it tried to remove an over-long tag.
+  (PR #1752, PR #1761)
+
+* Changed error to warning when complaining about use of the CG tag
+  in SAM or CRAM files.
+  (PR #1758, fixes samtools/samtools#2002)
+
+Noteworthy changes in release 1.19.1 (22nd January 2024)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* Fixed a regression in release 1.19 that caused all aux records to
+  be stored uncompressed in CRAM files.  The resulting files were
+  correctly formatted, but bigger than they needed to be.
+  (PR#1729, fixes samtools#1968.  Reported by Clockris)
+
+* Fixed possible out-of-bounds reads due to an incorrect check on
+  B tag lengths in cram_encode_aux().  (PR#1725)
+
+* Fixed an incorrect check on tag length which could fail to catch a
+  two byte out-of-bounds read in bam_get_aux(). (PR#1728)
+
+* Made errors reported by hts_open_format() less confusing when it can't
+  open the reference file.  (PR#1724, fixes #1723.  Reported by
+  Alex Leonard)
+
+* Made hts_close() fail more gracefully if it's passed a NULL pointer
+  (PR#1724)
+
+Noteworthy changes in release 1.19 (12th December 2023)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Updates
+-------
+
+* A temporary work-around has been put in the VCF parser so that it is
+  less likely to fail on rows with a large number of ALT alleles,
+  where Number=G tags like PL can expand beyond the 2Gb limit enforced
+  by HTSlib.  For now, where this happens the offending tag will be dropped
+  so the data can be processed, albeit without the likelihood data.
+
+  In future work, the library will instead convert such tags into their
+  local alternatives (see https://github.com/samtools/hts-specs/pull/434).
+  (PR #1689)
+
+* New program. Adds annot-tsv which annotates regions in a destination file with
+  texts from overlapping regions in a source file.
+  (PR#1619)
+
+* Change bam_parse_cigar() so that it can modify existing BAM records.  This
+  makes more useful as public API.  Previously it could only handle partially
+  formed BAM records.
+  (PR#1651, fixes #1650. Reported by Oleksii Nikolaienko)
+
+* Add "uncompressed" to hts_format_description() where appropriate.  This adds
+  an "uncompressed" description to uncompressed files that would normally be
+  compressed, such as BAM and BCF.
+  (PR#1656, in relation to samtools#1884.  Thanks to John Marshall)
+
+* Speed up to the VCF parser and writer.
+  (PR#1644 and PR#1663)
+
+* Add an hclen (hard clip length) SAM filter function.
+  (PR#1660, with reference to samtools#813)
+
+* Avoid really closing stdin/stdout in hclose()/hts_close()/et al.
+  See discussion in PR for details.
+  (PR#1665.  Thanks to John Marshall)
+
+* Add support to handle multiple files in bgzip.
+  (PR#1658, fixes #1642.  Requested by bw2)
+
+* Enable auto-vectorisation in CRAM 3.1 codecs.  Speeds decoding on some
+  sequencing platform data.
+  (PR#1669)
+
+* Speed up removal of lines in large headers.
+  (PR#1662, fixes #1460.  Reported by Anže Starič)
+
+* Apply seqtk PR to improve kseq.h parsing performance.  Port of
+  Fabian Klötzl's (kloetzl) lh3/seqtk#123 and attractivechaos/klib#173 to
+  HTSlib.
+  (PR#1674.  Thanks to John Marshall)
+
+Build changes
+-------------
+
+* Updated htscodecs submodule to 1.6.0.
+  (PR#1685, PR#1717, PR#1719)
+
+* Apply the packed attribute to uint*_u types for Clang to prevent
+  -fsanitize=alignment failures.
+  (PR#1667.  Thanks to Fangrui Song)
+
+* Fuzz testing improvements.
+  (PR#1664)
+
+* Add C++ casts for external headers in klist.h and kseq.h.
+  (PR#1683.  See also PR#1674 and PR#1682)
+
+* Add test case compiling the public headers as C++.
+  (PR#1682.  Thanks to John Marshall)
+
+* Enable optimisation level -O3 for SAM QUAL+33 formatting.
+  (PR#1679)
+
+* Make compiler flag detection work with zig cc.
+  (PR#1687)
+
+* Fix unused value warnings when built with NDEBUG.
+  (PR#1688)
+
+* Remove some disused Makefile variables, fix typos and a warning.  Improve
+  bam_parse_basemod() documentation.
+  (PR#1705, Thanks to John Marshall)
+
+Bug fixes
+---------
+
+* Fail bgzf_useek() when offset is above block limits.
+  (PR#1668)
+
+* Fix multi-threaded on-the-fly indexing problems.
+  (PR#1672, fixes samtools#1861 and bcftools#1985.  Reported by Mark Ebbert and
+  lacek)
+
+* Fix hfile_libcurl small seek bug.
+  (PR#1676, fixes samtools#1918.  Also may fix #1037, #1625 and samtools#1622.
+  Reported by Alex Reynolds, Mark Walker, Arthur Gilly and skatragadda-nygc.
+  Thanks to John Marshall)
+
+* Fix a minor memory leak in malformed CRAM EXTERNAL blocks. [fuzz]
+  (PR#1671)
+
+* Fix a cram decode hang from block_resize().
+  (PR#1680. Reported by Sebastian Deorowicz)
+
+* Cram fuzzing improvements.  Fixes a number of cram errors.
+  (PR#1701, fixes #1691, #1692, #1693, #1696, #1697, #1698, #1699 and #1700.
+  Thanks to Octavio Galland for finding and reporting all these)
+
+* Fix crypt4gh redirection.
+  (PR#1675, fixes  grbot/crypt4gh-tutorial#2.  Reported by hth4)
+
+* Fix PG header linking when records make a loop.
+  (PR#1702, fixes #1694.  Reported by Octavio Galland)
+
+* Prevent issues with no-stored-sequence records in CRAM files, by ensuring
+  they are accounted for properly in block size calculations, and by limiting
+  the maximum query length in the CIGAR data.  Originally seen as an overflow
+  by OSS-Fuzz / UBSAN, it turned out this could lead to excessive time and
+  memory use by HTSlib, and could result in it writing out unreadable CRAM
+  files.
+  (PR#1710)
+
+* Fix some illegal shifts and integer overflows found by OSS-Fuzz / UBSAN.
+  (PR#1707, PR#1712, PR#1713)
+
+Noteworthy changes in release 1.18 (25th July 2023)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Updates
+-------
+
+* Using CRAM 3.1 no longer gives a warning about the specification
+  being draft.  Note CRAM 3.0 is still the default output format.
+  (PR#1583)
+
+* Replaced use of sprintf with snprintf, to silence potential warnings
+  from Apple's compilers and those who implement similar checks.
+  (PR#1594, fixes #1586. Reported by Oleksii Nikolaienko)
+
+* Fastq output will now generate empty records for reads with no
+  sequence data (i.e. sequence is "*" in SAM format). (PR#1576,
+  fixes samtools/samtools#1576.  Reported by Nils Homer)
+
+* CRAM decoding speed-ups. (PR#1580)
+
+* A new MN aux tag can now be used to verify that MM/ML base modification
+  data has not been broken by hard clipping. (PR#1590, PR#1612. See also
+  PR samtools/hts-specs#714 and issue samtools/hts-specs#646.
+  Reported by Jared Simpson)
+
+* The base modification API has been improved to make it easier for callers
+  to tell unchecked bases from unmodified ones. (PR#1636, fixes #1550.
+  Requested by Chris Wright)
+
+* A new bam_mods_queryi() API has been added to return additional
+  data about the i-th base modification returned by bam_mods_recorded().
+  (PR#1636, fixes #1550 and #1635.  Requested by Jared Simpson)
+
+* Speed up index look-ups for whole-chromosome queries. (PR#1596)
+
+* Mpileup now merges adjacent (mis)match CIGAR operations, so CIGARs
+  using the X/= operators give the same results as if the M operator
+  was used.  (PR#1607, fixes #1597.  Reported by Marcel Martin)
+
+* It's now possible to call bcf_sr_set_regions() after adding readers
+  using bcf_sr_add_reader() (previously this returned an error).  Doing so
+  will discard any unread data, and reset the readers so they iterate over
+  the new regions.  (PR#1624, fixes samtools/bcftools#1918.  Reported by
+  Gregg Thomas)
+
+* The synced BCF reader can now accept regions with reference names including
+  colons and hyphens, by enclosing them in curly braces.  For example,
+  {chr_part:1-1001}:10-20 will return bases 10 to 20 from reference
+  "chr_part:1-1001".  (PR#1630, fixes #1620.  Reported by Bren)
+
+* Add a "samples" directory with code demonstrating usage of HTSlib plus
+  a tutorial document. (PR#1589)
+
+Build changes
+-------------
+
+* Htscodecs has been updated to 1.5.1 (PR#1654)
+
+* Htscodecs SIMD code now works with Apple multiarch binaries.
+  (PR#1587, HTSlib fix for samtools/htscodecs#76. Reported by John Marshall)
+
+* Improve portability of "expr" usage in version.sh.
+  (PR#1593, fixes #1592.  Reported by John Marshall)
+
+* Improve portability to *BSD targets by ensuring _XOPEN_SOURCE is defined
+  correctly and that source files properly include "config.h".  Perl
+  scripts also now all use #!/usr/bin/env instead of assuming that
+  it's in /usr/bin/perl. (PR#1628, fixes #1606.
+  Reported by Robert Clausecker)
+
+* Fixed NAME entry in htslib-s3-plugin man page so the whatis and apropos
+  commands find it.  (PR#1634, thanks to Étienne Mollier)
+
+* Assorted dependency tracking fixes.  (PR#1653, thanks to John Marshall)
+
+Documentation updates
+---------------------
+
+* Changed Alpine build instructions as they've switched back to using openssl.
+  (PR#1609)
+
+* Recommend using -rdynamic when statically linking a libhts.a with
+  plugins enabled.  (PR#1611, thanks to John Marshall.  Fixes #1600,
+  reported by Jack Wimberley)
+
+* Fixed example in docs for sam_hdr_add_line().  (PR#1618, thanks to kojix2)
+
+* Improved test harness for base modifications API. (PR#1648)
+
+Bug fixes
+---------
+
+* Fix a major bug when searching against a CRAM index where one container
+  has start and end coordinates entirely contained within the previous
+  container. This would occasionally miss data, and sometimes return much
+  more than required.  The bug affected versions 1.11 to 1.17, although the
+  change in 1.11 was bug-fixing multi-threaded index queries.  This bug did
+  not affect index building.  There is no need to reindex your CRAM files.
+  (PR#1574, PR#1640. Fixes #1569, #1639, samtools/samtools#1808,
+  samtools/samtools#1819.  Reported by xuxif, Jens Reeder and Jared Simpson)
+
+* Prevent CRAM blocks from becoming too big in files with short
+  sequences but very long aux tags.  (PR #1613)
+
+* Fix bug where the CRAM decoder for CONST_INT and CONST_BYTE
+  codecs may incorrectly look for extra data in the CORE block.
+  Note that this bug only affected the experimental CRAM v4.0 decoder.
+  (PR#1614)
+
+* Fix crypt4gh redirection so it works in conjunction with non-file
+  IO, such as using htsget. (PR#1577)
+
+* Improve error checking for the VCF POS column, when facing invalid
+  data. (PR#1575, replaces #1570 originally reported and fixed
+  by Colin Nolan.)
+
+* Improved error checking on VCF indexing to validate the data is BGZF
+  compressed. (PR#1581)
+
+* Fix bug where bin number calculation could overflow when making iterators
+  over regions that go to the end of a chromosome. (PR#1595)
+
+* Backport attractivechaos/klib#78 (by Pall Melsted) to HTSlib.
+  Prevents infinite loops in kseq_read() when reading broken gzip files.
+  (PR#1582, fixes #1579.  Reported by Goran Vinterhalter)
+
+* Backport attractivechaos/klib@384277a (by innoink) to HTSlib.
+  Fixes the kh_int_hash_func2() macro definition.
+  (PR#1599, fixes #1598.  Reported by fanxinping)
+
+* Remove a compilation warning on systems with newer libcurl releases.
+  (PR#1572)
+
+* Windows: Fixed BGZF EOF check for recent MinGW releases. (PR#1601,
+  fixes samtools/bcftools#1901)
+
+* Fixed bug where tabix would not return the correct regions for files
+  where the column ordering is end, ..., begin instead of begin, ..., end.
+  (PR#1626, fixes #1622.  Reported by Hiruna Samarakoon)
+
+* sam_format_aux1() now always NUL-terminates Z/H tags. (PR#1631)
+
+* Ensure base modification iterator is reset when no MM tag is present.
+  (PR#1631, PR#1647)
+
+* Fix segfault when attempting to write an uncompressed BAM file opened using
+  hts_open(name, "wbu").  This was attempting to write BAM data without
+  wrapping it in BGZF blocks, which is invalid according to the BAM
+  specification.  "wbu" is now internally converted to "wb0" to output
+  uncompressed data wrapped in BGZF blocks. (PR#1632, fixes #1617.
+  Reported by Joyjit Daw)
+
+* Fixed over-strict bounds check in probaln_glocal() which caused it to make
+  sub-optimal alignments when the requested band width was greater than the
+  query length.  (PR#1616, fixes #1605.  Reported by Jared Simpson)
+
+* Fixed possible double frees when handling errors in bcf_hdr_add_hrec(),
+  if particular memory allocations fail. (PR#1637)
+
+* Ensure that bcf_hdr_remove() clears up all pointers to the items removed
+  from dictionaries.  Failing to do this could have resulted in a call
+  requesting a deleted item via bcf_hdr_get_hrec() returning a stale pointer.
+  (PR#1637)
+
+* Stop the gzip decompresser from finishing prematurely when an empty
+  gzip block is followed by more data. (PR#1643, PR#1646)
+
+Noteworthy changes in release 1.17 (21st February 2023)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* A new API for iterating through a BAM record's aux field.
+  (PR#1354, addresses #1319.  Thanks to John Marshall)
+
+* Text mode for bgzip. Allows bgzip to compress lines of text with block breaks
+  at newlines.
+  (PR#1493, thanks to Mike Lin for the initial version PR#1369)
+
+* Make tabix support CSI indices with large positions.  Unlike SAM and VCF
+  files, BED files do not set a maximum reference length which hindered CSI
+  support.  This change sets an arbitrary large size of 100G to enable it to
+  work.
+  (PR#1506)
+
+* Add a fai_line_length function.  Exposes the internal line-wrap length.
+  (PR#1516)
+
+* Check for invalid barcode tags in fastq output.
+  (PR#1518, fixes samtools#1728.  Reported by Poshi)
+
+* Warn if reference found in a CRAM file is not contained in the specified
+  reference file.
+  (PR#1517 and PR#1521, adds diagnostics for #1515. Reported by Wei WeiDeng)
+
+* Add a faidx_seq_len64 function that can return sequence lengths longer than
+  INT_MAX.  At the same time limit faidx_seq_len to INT_MAX output.  Also add a
+  fai_adjust_region to ensure given ranges do not go beyond the end of the
+  requested sequence.
+  (PR#1519)
+
+* Add a bcf_strerror function to give text descriptions of BCF errors.
+  (PR#1510)
+
+* Add CRAM SQ/M5 header checking when specifying a fasta file.  This is to
+  prevent creating a CRAM that cannot be decoded again.
+  (PR#1522.  In response to samtools#1748 though not a direct fix)
+
+* Improve support for very long input lines (> 2Gbyte).  This is mostly useful
+  for tabix which does not do much interpretation of its input.
+  (PR#1542, a partial fix for #1539)
+
+* Speed up load_ref_portion.  This function has been sped up by about 7x, which
+  speeds up low-depth CRAM decoding by about 10%.
+  (PR#1551)
+
+* Expand CRAM API to cope with new samtools cram_size command.
+  (PR#1546)
+
+* Merges neighbouring I and D ops into one op within pileup. This means
+  4M1D1D1D3M is reported as 4M3D3M.   Fixing this in sam.c means not only is
+  samtools mpileup now looking better, but any tool using the mpileup API will
+  be getting consistent results.
+  (PR#1552, fixes the last remaining part of samtools#139)
+
+* Update the API documentation for bgzf_mt as it refered to a previous
+  iteration.
+  (PR#1556, fixes #1553.  Reported by Raghavendra Padmanabhan)
+
+
+Build changes
+-------------
+
+* Use POSIX grep in testing as egrep and fgrep are considered obsolete.
+  (PR#1509, thanks to David Seifert)
+
+* Switch to building libdefalte with cmake for Cirris CI.
+  (PR#1511)
+
+* Ensure strings in config_vars.h are escaped correctly.
+  (PR#1530, fixes #1527. Reported by Lucas Czech)
+
+* Easier modification of shared library permissions during install.
+  (PR#1532, fixes #1525. Reported by StephDC)
+
+* Fix build on ancient compilers.  Added -std=gnu90 to build tests so older
+  C compilers will still be happy.
+  (PR#1524, fixes #1523.  Reported by Martin Jakt)
+
+* Switch MacOS CI tests to an ARM-based image.
+  (PR#1536)
+
+* Cut down the number of embed_ref=2 tests that get run.
+  (PR#1537)
+
+* Add symbol versions to libhts.so.  This is to aid package developers.
+  (PR#1560 addresses #1505, thanks to John Marshall. Reported by Stefan Bruens)
+
+* htscodecs now updated to v1.4.0.
+  (PR#1563)
+
+* Cleaned up misleading system error reports in test_bgzf.
+  (PR#1565)
+
+Bug fixes
+---------
+
+* VCF. Fix n-squared complexity in sample line with many adjacent tabs [fuzz].
+  (PR#1503)
+
+* Improved bcftools detection and reporting of bgzf decode errors.
+  (PR#1504, thanks to Lilian Janin. PR#1529 thanks to Bergur Ragnarsson, fixes
+  #1528. PR#1554)
+
+* Prevent crash when the only FASTA entry has no sequence [fuzz].
+  (PR#1507)
+
+* Fixed typo in sam.h documentation.
+  (PR#1512, thanks to kojix2)
+
+* Fix buffer read-overrun in bam_plp_insertion_mod.
+  (PR#1520)
+
+* Fix hash keys being left behind by bcf_hdr_remove.
+  (PR#1535, fixes #1533.  Reported by Giulio Genovese in #842)
+
+* Make bcf_hdr_idinfo_exists more robust by checking id value exists.
+  (PR#1544, fixes #1538.  Reported by Giulio Genovese)
+
+* CRAM improvements. Fixed crash with multi-threaded CRAM.  Fixed a bug in the
+  codec parameter learning for CRAM 3.1 name tokeniser. Fixed Cram compression
+  container substitution matrix generation,
+  (PR#1558, PR#1559 and PR#1562) 
+
+Noteworthy changes in release 1.16 (18th August 2022)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* Make hfile_s3 refresh AWS credentials on expiry in order to make HTSlib work
+  better with AWS IAM credentials, which have a limited lifespan.
+  (PR#1462 and PR#1474, addresses #344)
+
+* Allow BAM headers between 2GB and 4GB in size once more.  This is not
+  permitted in the BAM specification but was allowed in an earlier version of
+  HTSlib.  There is now a warning at 2GB and a hard failure at 4GB.
+  (PR#1421, fixes #1420 and samtools#1613. Reported by John Marshall and
+  R C Mueller)
+
+* Improve error message when failing to load an index.
+  (PR#1468, example of the problem samtools#1637)
+
+* Permit MM (base modification) tags containing "." and "?" suffixes.  These
+  define implicit vs explicit coordinates.  See the SAM tags specification for
+  details.
+  (PR#1423 and PR#1426, fixes #1418.  PR#1469, fixes #1466.  Reported
+  by cjw85)
+
+* Warn if spaces instead of tabs are detected in a VCF file to prevent
+  confusion.
+  (PR#1328, fixes bcftools#1575.  Reported by ketkijoshi278)
+
+* Add an "sclen" filter expression keyword.  This is the length of a soft-clip,
+  both left and right end.  It may be combined with qlen (qlen-sclen) to obtain
+  the number of bases in the query sequence that have been aligned to the genome
+  ie it provides a way to compare local-alignment vs global-alignment length.
+  (PR#1441 and PR/samtools#1661, fixes #1436. Requested by Chang Y)
+
+* Improve error messages for CRAM reference mismatches.  If the user specifies
+  the wrong reference, the CRAM slice header MD5sum checks fail.  We now report
+  the SQ line M5 string too so it is possible to validate against the whole
+  chr in the ref.fa file.  The error message has also been improved to report
+  the reference name instead of #num.  Finally, we now hint at the likely cause,
+  which counters the misleading samtools supplied error of "truncated or
+  corrupt" file.
+  (PR#1427, fixes samtools#1640.  Reported by Jian-Guo Zhou)
+
+* Expose more of the CRAM API and add new functionality to extract the reference
+  from a CRAM file.
+  (PR#1429 and PR#1442)
+
+* Improvements to the implementation of embedded references in CRAM where no
+  external reference is specified.
+  (PR#1449, addresses some of the issues in #1445)
+
+* The CRAM writer now allows alignment records with RG:Z: aux tags that
+  don't have a corresponding @RG ID in the file header.  Previously these
+  tags would have been silently dropped.  HTSlib will complain whenever it
+  has to add one though, as such tags do not conform to recommended practice
+  for the SAM, BAM and CRAM formats.
+  (PR#1480, fixes #1479.  Reported by Alex Leonard)
+
+* Set tab delimiter in man page for tabix GFF3 sort.
+  (PR#1457.  Thanks to Colin Diesh)
+
+* When using libdeflate, the 1...9 scale of BGZF compression levels is
+  now remapped to the 1...12 range used by libdeflate instead of being
+  passed directly.  In particular, HTSlib levels 8 and 9 now map to
+  libdeflate levels 10 and 12, so it is possible to select the highest (but
+  slowest) compression offered by libdeflate.
+  (PR#1488, fixes #1477.  Reported by Gert Hulselmans)
+
+* The VCF variant API has been extended so that it can return separate flags
+  for INS and DEL variants as well as the existing INDEL one.  These flags
+  have not been added to the old bcf_get_variant_types() interface as
+  it could break existing users.  To access them, it is necessary to use new
+  functions bcf_has_variant_type() and bcf_has_variant_types().
+  (PR#1467)
+
+* The missing, but trivial, `le_to_u8()` function has been added to hts_endian.
+  (PR#1494, Thanks to John Marshall)
+
+* bcf_format_gt() now works properly on big-endian platforms.
+  (PR#1495, Thanks to John Marshall)
+
+Build changes
+-------------
+
+These are compiler, configuration and makefile based changes.
+
+* Update htscodecs to version 1.3.0 for new SIMD code + various fixes.
+  Updates the htscodecs submodule and adds changes necessary to make HTSlib
+  build the new SIMD codec implementations.
+  (PR#1438, PR#1489, PR#1500)
+
+* Fix clang builds under mingw.  Under mingw, clang requires dllexport to be
+  applied to both function declarations and function definitions.
+  (PR#1435, PR#1497, PR#1498 fixes #1433.  Reported by teepean)
+
+* Fix curl type warning with gcc 12.1 on Windows.
+  (PR#1443)
+
+* Detect ARM Neon support and only build appropriate SIMD object files.
+  (PR#1451, fixes #1450.  Thanks to John Marshall)
+
+* `make print-config` now reports extra CFLAGS that are needed to build the
+  SIMD parts of htscodecs.  These may be of use to third-party build
+  systems that don't use HTSlib's or htscodecs' build infrastructure. (PR#1485.
+  Thanks to John Marshall)
+
+* Fixed some Makefile dependency issues for the "check"/"test" targets
+  and plugins.  In particular, "make check" will now build the "all" target,
+  if not done already, before running the tests.
+  (PR#1496)
+
+Bug fixes
+---------
+
+* Fix bug when reading position -1 in BCF (0 in VCF), which is used to indicate
+  telomeric  regions.  The BCF reader was incorrectly assuming the value stored
+  in the file was unsigned, so a VCF->BCF->VCF round-trip would change it
+  from 0 to 4294967296.
+  (PR#1476, fixes #1475 and bcftools#1753.  Reported by Rodrigo Martin)
+
+* Various bugs and quirks have been fixed in the filter expression engine,
+  mostly related to the handling of absent tags, and the is_true flag.
+  Note that as a result of these fixes, some filter expressions may give
+  different results:
+  - Fixed and-expressions including aux tag values which could give an invalid
+    true result depending on the order of terms.
+  - The expression `![NM]` is now true if only `NM` does not exist.  In
+    earlier versions it would also report true for tags like `NM:i:0` which
+    exist but have a value of zero.
+  - The expression `[X1] != 0` is now false when `X1` does not exist.  Earlier
+    versions would return true for this comparison when the tag was missing.
+  - NULL values due to missing tags now propagate through string, bitwise
+    and mathematical operations.  Logical operations always treat them as
+    false.
+  (PR#1463, fixes samtools#1670.  Reported by Gert Hulselmans;
+   PR#1478, fixes samtools#1677.  Reported by johnsonzcode)
+
+* Fix buffer overrun in bam_plp_insertion_mod.  Memory now grows to the proper
+  size needed for base modification data.
+  (PR#1430, fixes samtools#1652.  Reported by hd2326)
+
+* Remove limit of returned size from fai_retrieve().
+  (PR#1446, fixes samtools#1660.  Reported by Shane McCarthy)
+
+* Cap hts_getline() return value at INT_MAX.  Prevents hts_getline() from
+  returning a negative number (a fail) for very long string length values.
+  (PR#1448.  Thanks to John Marshall)
+
+* Fix breakend detection and test bcf_set_variant_type().
+  (PR#1456, fixes #1455.  Thanks to Martin Pollard)
+
+* Prevent arrays of BCF_BT_NULL values found in BCF files from causing
+  bcf_fmt_array() to call exit() as the type is unsupported.  These are
+  now tested for and caught by bcf_record_check(), which returns an
+  error code instead.  (PR#1486)
+
+* Improved detection of fasta and fastq files that have very long comments
+  following identifiers.  (PR#1491, thanks to John Marshall.
+  Fixes samtools/samtools#1689, reported by cjw85)
+
+* Fixed a SEGV triggered by giving a SAM file to `samtools import`.
+  (PR#1492)
+
+Noteworthy changes in release 1.15.1 (7th April 2022)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* Security fix: Fixed broken error reporting in the sam_prob_realn()
+  function, due to a missing hts_log() parameter.  Prior to this fix
+  (i.e., in HTSlib versions 1.8 to 1.15) it was possible to abuse
+  the log message format string by passing a specially crafted
+  alignment record to this function. (PR#1406)
+
+* HTSlib now uses libhtscodecs release 1.2.2.  This fixes a number
+  of bugs where invalid compressed data could trigger usage of
+  uninitialised values. (PR#1416)
+
+* Fixed excessive memory used by multi-threaded SAM output on
+  long reads. (Part of PR#1384)
+
+* Fixed a bug where tabix would misinterpret region specifiers
+  starting at position 0.  It will also now warn if the file
+  being indexed is supposed to be 1-based but has positions
+  less than or equal to 0. (PR#1411)
+
+* The VCF header parser will now issue a warning if it finds an
+  INFO header with Type=Flag but Number not equal to 0.  It will
+  also ignore the incorrect Number so the flag can be used. (PR#1415)
+
+Noteworthy changes in release 1.15 (21st February 2022)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Features and Updates
+--------------------
+
+* Bgzip now has a --keep option to not remove the input file after
+  compressing. (PR#1331)
+
+* Improved file format detection so some BED files are no longer
+  detected as FASTQ or FASTA. (PR#1350, thanks to John Marshall)
+
+* Added xz (lzma), zstd and D4 formats to the file type detection
+  functions.  We don't actively support reading these data types, but
+  function calls and htsfile can detect them. (PR#1340, thanks to
+  John Marshall)
+
+* CRAM now also uses libdeflate for read-names if the libdeflate
+  version is new enough (1.9 onwards).  Previously we used zlib for
+  this due to poor performance of libdeflate.  This gives a slight
+  speed up and reduction in file size. (PR#1383)
+
+* The VCF and BCF readers will now issue a warning if contig, INFO
+  or FORMAT IDs do not match the formats described in the VCFv4.3
+  specification. Note that while the invalid names will mostly still
+  be accepted, future updates will convert the warnings to errors
+  causing files including invalid names to be rejected.  (PR#1389)
+
+Build changes
+-------------
+
+These are compiler, configuration and makefile based changes.
+
+* HTSlib now uses libhtscodecs release 1.2.1.
+
+* Improved support for compiling and linking against HTSlib with
+  Microsoft Visual Studio. (PR#1380, #1377, #1375.  Thanks to
+  Aidan Bickford and John Marshall)
+
+* Various internal CI improvements.
+
+Bug fixes
+---------
+
+* Fixed CRAM index queries for HTSJDK output (PR#1388, reported by
+  Chris Norman).  Note this also fixes writing CRAM writing, to match
+  the specification (and HTSJDK), from version 3.1 onwards.
+
+* Fixed CRAM index queries when required-fields settings are selected
+  to ignore CIGARs (PR#1372, reported by Giulio Genovese).
+
+* Unmapped but placed (having chr/pos) are now included in the BAM
+  indices. (PR#1352, thanks to John Marshall)
+
+* CRAM now honours the filename##idx##index nomenclature for
+  specifying non-standard index locations. (PR#1360, reported by
+  Michael Cariaso)
+
+* Minor CRAM v1.0 read-group fix (PR#1349, thanks to John Marshall)
+
+* Permit .fa and .fq file type detection as synonyms for FASTA and
+  FASTQ. (PR#1386).
+
+* Empty VCF format fields are now output ":.:" as instead of "::".
+  (PR#1370)
+
+* Repeated bcf_sr_seek calls now work. (PR#1363, reported by
+  Giulio Genovese)
+
+* Bcf_remove_allele_set now works on unpacked BCF records. (PR#1358,
+  reported by Brent Pedersen).
+
+* The hts_parse_decimal() function used to read numbers in region lists
+  is now better at rejecting non-numeric values.  In particular it
+  now rejects a lone 'G' instead of interpreting it as '0G', i.e. zero.
+  (PR#1396, PR#1400, reported by SSSimon Yang; thanks to John Marshall).
+
+* Improve support for GPU issues listed by -Wdouble-promotion.
+  (PR#1365, reported by David Seisert)
+
+* Fix example code in header file documentation. (PR#1381, Thanks to
+  Aidan Bickford)
+
+Noteworthy changes in release 1.14 (22nd October 2021)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Features and Updates
+--------------------
+
+* Added a keep option to bgzip to leave the original file untouched.  This
+  brings bgzip into line with gzip. (PR #1331, thanks to Alex Petty)
+
+* "endpos" has been added to the filter language, giving the position
+  of the rightmost mapped base as measured by the CIGAR string.  For
+  unmapped reads it is the same as "pos". (PR #1307, thanks to John Marshall)
+
+* Interfaces have been added to interpret the new base modification tags
+  added to the SAMtags document in samtools/hts-specs#418.  (PR #1132)
+
+* New API functions hts_flush()/sam_flush()/bcf_flush() for flushing output
+  htsFile/samFile/vcfFile streams. (PR #1326, thanks to John Marshall)
+
+* The synced_bcf_reader now sorts lines with symbolic alleles by END tag as
+  well as POS.  (PR #1321)
+
+* Added synced_bcf_reader options BCF_SR_REGIONS_OVERLAP and
+  BCF_SR_TARGETS_OVERLAP for better control of records that start outside
+  the desired region but overlap it are handled.  Fixes samtools/bcftools#1420
+  and samtools/bcftools#1421 raised by John Marshall.  (PR #1327)
+
+* HTSlib will now accept long-cigar CG:B: tags made by htsjdk which don't
+  quite follow the specification properly (using signed values instead of
+  unsigned).  Thanks to Colin Diesh for reporting an example file. (PR #1317)
+
+* The warning printed when the BGZF reader finds a file with no EOF block
+  has been changed to be less alarming.  Unfortunately some third-party
+  BGZF encoders don't write EOF blocks at the end of files.  Thanks to
+  Keiran Raine for reporting an example file.  (PR #1323)
+
+* The FASTA and FASTQ readers get an option to skip over the first item on
+  the header line, and use the second as the read name.  It allows the original
+  name to be restored on some of the fastq files served from the European
+  Nucleotide Archive (ENA).  (PR #1325)
+
+* HTSlib is now more strict when parsing the VCF samples line (beginning
+  #CHROM).  It will only accept tabs between the mandatory field names and
+  sample names must be separated with tabs. (PR #1328)
+
+* HTSlib will now warn if it looks like the header has been corrupted
+  by diagnostic messages from the program that made it.  This can happen when
+  using `nohup`, which by default mixes stdout and stderr into the same
+  stream.  (PR#1339, thanks to John Marshall)
+
+* File format detection will now recognise signatures for XZ, Zstd and D4
+  files (note that HTSlib will not read them yet).  (PR #1340, thanks to
+  John Marshall)
+
+Build changes
+-------------
+
+These are compiler, configuration and makefile based changes.
+
+* Some redundant tests have been removed from the test harness, speeding it up.
+  (PR #1308)
+
+* The version.sh script now works better on shallow checkouts.  (PR #1324)
+
+* A check-untracked Makefile target has been added to catch untracked files
+  (mostly) left by the test harness.  (PR #1324)
+
+Bug fixes
+---------
+
+* Fixed a case where flushing the thread pool could very occasionally cause
+  a deadlock.  (PR #1309)
+
+* Fixed a bug where some CRAM files could fail to decode if the required_fields
+  option was in use.  Thanks to Matt Sexton for reporting the issue.
+  (PR #1314, fixes samtools/samtools#1475)
+
+* Fixed a regression where the S3 plugin could not read public files unless
+  you supplied some Amazon credentials.  Thanks to Chris Saunders for reporting.
+  (PR #1332, fixes samtools/samtools#1491)
+
+* Fixed a possible CRAM thread deadlock discovered by @ryancaicse.
+  (PR #1330, fixes #1329)
+
+* Some set-but-unused variables have been removed.  (PR #1334)
+
+* Fixed a bug which prevented "flag.read2" from working in the filter
+  language unless it was at the end of the expression.  Thanks to Vamsi Kodali
+  for reporting the issue.  (PR #1342)
+
+* Fixed a memory leak that could happen if CRAM fails to inflate a LZMA
+  block. (PR #1340, thanks to John Marshall)
+
+Noteworthy changes in release 1.13 (7th July 2021)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Features and Updates
+--------------------
+
+* In case a PG header line has multiple ID tags supplied by other applications,
+  the header API now selects the first one encountered as the identifying tag
+  and issues a warning when detecting subsequent ID tags.
+  (#1256; fixed samtools/samtools#1393)
+
+* VCF header reading function (vcf_hdr_read) no longer tries to download a
+  remote index file by default.
+  (#1266; fixes #380)
+
+* Support reading and writing FASTQ format in the same way as SAM, BAM or CRAM.
+  Records read from a FASTQ file will be treated as unmapped data.
+  (#1156)
+
+* Added GCP requester pays bucket access.  Thanks to @indraniel.
+  (#1255)
+
+* Made mpileup's overlap removal choose which copy to remove at random instead
+  of always removing the second one.  This avoids strand bias in experiments
+  where the +ve and -ve strand reads always appear in the same order.
+  (#1273; fixes samtools/bcftools#1459)
+
+* It is now possible to use platform specific BAQ parameters.  This also
+  selects long-read parameters for read lengths bigger than 1kb, which helps
+  bcftools mpileup call SNPs on PacBio CCS reads.
+  (#1275)
+
+* Improved bcf_remove_allele_set.  This fixes a bug that stopped iteration over
+  alleles prematurely, marks removed alleles as 'missing' and does automatic
+  lazy unpacking.
+  (#1288; fixes #1259)
+
+* Improved compression metrics for unsorted CRAM files.  This improves the
+  choice of codecs when handling unsorted data.
+  (#1291)
+
+* Linear index entries for empty intervals are now initialised with the file
+  offset in the next non-empty interval instead of the previous one.  This
+  may reduce the amount of data iterators have to discard before reaching
+  the desired region, when the starting location is in a sequence gap.
+  Thanks to @carsonh for reporting the issue.
+  (#1286; fixes #486)
+
+* A new hts_bin_level API function has been added, to compute the level of a
+  given bin in the binning index.
+  (#1286)
+
+* Related to the above, a new API method, hts_idx_nseq, now returns the total
+  number of contigs from an index.
+  (#1295 and #1299)
+
+* Added bracket handling to bcf_hdr_parse_line, for use with ##META lines.
+  Thanks to Alberto Casas Ortiz.
+  (#1240)
+
+Build changes
+-------------
+
+These are compiler, configuration and makefile based changes.
+
+* HTSlib now uses libhtscodecs release 1.1.1.
+
+* Added a curl/curl.h check to configure and improved INSTALL documentation on
+  build options.  Thanks to Melanie Kirsche and John Marshall.
+  (#1265; fixes #1261)
+
+* Some fixes to address GCC 11.1 warnings.
+  (#1280, #1284, #1285; fixes #1283)
+
+* Supports building HTSlib in a separate directory. Thanks to John Marshall.
+  (#1277; fixes #231)
+
+* Supports building HTSlib on MinGW 32-bit environments. Thanks to
+  John Marshall.
+  (#1301)
+
+Bug fixes
+---------
+
+* Fixed hts_itr_query() et al region queries: fixed bug introduced in
+  HTSlib 1.12, which led to iterators producing very few reads for some
+  queries (especially for larger target regions) when unmapped reads were
+  present. HTSlib 1.11 had a related problem in which iterators would omit
+  a few unmapped reads that should have been produced; cf #1142.
+  Thanks to Daniel Cooke for reporting the issue.
+  (#1281; fixes #1279)
+
+* Removed compressBound assertions on opening bgzf files.  Thanks to
+  Gurt Hulselmans for reporting the issue.
+  (#1258; fixed #1257)
+
+* Duplicate sample name error message for a VCF file now only displays the
+  duplicated name rather the entire same name list.
+  (#1262; fixes samtools/bcftools#1451)
+
+* Fix to make samtools cat work on CRAMs again.
+  (#1276; fixes samtools/samtools#1420)
+
+* Fix for a double memory free in SAM header creation.  Thanks to @ihsineme.
+  (#1274)
+
+* Prevent assert in bcf_sr_set_regions.  Thanks to Dr K D Murray.
+  (#1270)
+
+* Fixed crash in knet_open() etc stubs.  Thanks to John Marshall.
+  (#1289)
+
+* Fixed filter expression "cigar" on unmapped reads.  Stop treating an empty
+  CIGAR string as an error.  Thanks to Chang Y for reporting the issue.
+  (#1298, fixes samtools/samtools#1445)
+
+* Bug fixes in the bundled copy of htscodecs:
+
+  - Fixed an uninitialized access in the name tokeniser decoder.
+    (samtools/htscodecs#23)
+
+  - Fixed a bug with name tokeniser and variable number of names per slice,
+    causing it to incorrectly report an error on certain valid inputs.
+    (samtools/htscodecs#24)
+
+
+Noteworthy changes in release 1.12 (17th March 2021)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Features and Updates
+--------------------
+
+* Added experimental CRAM 3.1 and 4.0 support. (#929)
+
+  These should not be used for long term data storage as the
+  specification still needs to be ratified by GA4GH and may be subject
+  to changes in format. (This is highly likely for 4.0).  However it
+  may be tested using:
+
+    test/test_view -t ref.fa -C -o version=3.1 in.bam -p out31.cram
+
+  For smaller but slower files, try varying the compression profile
+  with an additional "-o small".  Profile choices are fast, normal,
+  small and archive, and can be applied to all CRAM versions.
+
+* Added a general filtering syntax for alignment records in SAM/BAM/CRAM
+  readers. (#1181, #1203)
+
+  An example to find chromosome spanning read-pairs with high mapping
+  quality:  'mqual >= 30 && mrname != rname'
+
+  To find significant sized deletions:
+  'cigar =~ "[0-9]{2}D"' or 'rlen - qlen > 10'.
+
+  To report duplicates that aren't part of a "proper pair":
+  'flag.dup && !flag.proper_pair'
+
+  More details are in the samtools.1 man page under "FILTER EXPRESSIONS".
+
+* The knet networking code has been removed.  It only supported the http
+  and ftp protocols, and a better and safer alternative using libcurl
+  has been available since release 1.3.  If you need access to ftp:// and
+  http:// URLs, HTSlib should be built with libcurl support. (#1200)
+
+* The old htslib/knetfile.h interfaces have been marked as deprecated.  Any
+  code still using them should be updated to use hFILE instead. (#1200)
+
+* Added an introspection API for checking some of the capabilities provided
+  by HTSlib. (#1170) Thanks also to John Marshall for contributions. (#1222)
+    - `hfile_list_schemes`: returns the number of schemes found
+    - `hfile_list_plugins`: returns the number of plugins found
+    - `hfile_has_plugin`: checks if a specific plugin is available
+    - `hts_features`: returns a bit mask with all available features
+    - `hts_test_feature`: test if a feature is available
+    - `hts_feature_string`: return a string summary of enabled features
+
+* Made performance improvements to `probaln_glocal` method, which
+  speeds up mpileup BAQ calculations. (#1188)
+    - Caching of reused loop variables and removal of loop invariants
+    - Code reordering to remove instruction latency.
+    - Other refactoring and tidyups.
+
+* Added a public method for constructing a BAM record from the
+  component pieces. Thanks to Anders Kaplan. (#1159, #1164)
+
+* Added two public methods, `sam_parse_cigar` and `bam_parse_cigar`, as part of
+  a small CIGAR API (#1169, #1182). Thanks to Daniel Cameron for input. (#1147)
+
+* HTSlib, and the included htsfile program, will now recognise the old
+  RAZF compressed file format.  Note that while the format is detected,
+  HTSlib is unable to read it.  It is recommended that RAZF files are
+  uncompressed with `gunzip` before using them with HTSlib.  Thanks to
+  John Marshall (#1244); and Matthew J. Oldach who reported problems
+  with uncompressing some RAZF files (samtools/samtools#1387).
+
+* The S3 plugin now has options to force the address style.  It will recognise
+  the addressing_style and host_bucket entries in the respective aws
+  .credentials and s3cmd .s3cfg files.  There is also a new HTS_S3_ADDRESS_STYLE
+  environment variable.  Details are in the htslib-s3-plugin.7 man file (#1249).
+
+Build changes
+-------------
+
+These are compiler, configuration and makefile based changes.
+
+* Added new Makefile targets for the applications that embed HTSlib and
+  want to run its test suite or clean its generated artefacts. (#1230, #1238)
+
+* The CRAM codecs are now obtained via the htscodecs submodule, hence
+  when cloning it is now best to use "git clone --recursive".  In an
+  existing clone, you may use "git submodule update --init" to obtain
+  the htscodecs submodule checkout.
+
+* Updated CI test configuration to recurse HTSlib submodules. (#1359)
+
+* Added Cirrus-CI integration as a replacement for Travis, which was
+  phased out.  (#1175; #1212)
+
+* Updated the Windows image used by Appveyor to 'Visual Studio 2019'. (#1172;
+  fixed #1166)
+
+* Fixed a buglet in configure.ac, exposed by the release 2.70 of autoconf.
+  Thanks to John Marshall. (#1198)
+
+* Fixed plugin linking on macOS, to prevent symbol conflict when linking
+  with a static HTSlib. Thanks to John Marshall. (#1184)
+
+* Fixed a clang++9 error in `cram_io.h`. Thanks to Pjotr Prins. (#1190)
+
+* Introduced $(ALL_CPPFLAGS) to allow for more flexibility in setting the
+  compiler flags. Thanks to John Marshall. (#1187)
+
+* Added 'fall through' comments to prevent warnings issued by Clang on
+  intentional fall through case statements, when building with
+  `-Wextra flag`. Thanks to John Marshall. (#1163)
+
+* Non-configure builds now define _XOPEN_SOURCE=600 to allow them to work
+  when the `gcc -std=c99` option is used.  Thanks to John Marshall. (#1246)
+
+Bug fixes
+---------
+
+* Fixed VCF `#CHROM` header parsing to only separate columns at tab characters.
+  Thanks to Sam Morris for reporting the issue.
+  (#1237; fixed samtools/bcftools#1408)
+
+* Fixed a crash reported in `bcf_sr_sort_set`, which expects REF to be present.
+  (#1204; fixed samtools/bcftools#1361)
+
+* Fixed a bcf synced reader bug when filtering with a region list, and
+  the first record for a chromosome had the same position as the last
+  record for the previous chromosome. (#1254; fixed samtools/bcftools#1441)
+
+* Fixed a bug in the overlapping logic of mpileup, dealing with iterating over
+  CIGAR segments. Thanks to `@wulj2` for the analysis. (#1202; fixed #1196)
+
+* Fixed a tabix bug that prevented setting the correct number of lines to be
+  skipped in a region file. Thanks to Jim Robinson for reporting it. (#1189;
+  fixed #1186)
+
+* Made `bam_itr_next` an alias for `sam_itr_next`, to prevent it from crashing
+  when working with htsFile pointers. Thanks to Torbjörn Klatt for
+  reporting it. (#1180; fixed #1179)
+
+* Fixed once per outgoing multi-threaded block `bgzf_idx_flush` assertion, to
+  accommodate situations when a single record could span multiple blocks.
+  Thanks to `@lacek`. (#1168; fixed samtools/samtools#1328)
+
+* Fixed assumption of pthread_t being a non-structure, as permitted by POSIX.
+  Thanks also to John Marshall and Anders Kaplan. (#1167, #1153, #1153)
+
+* Fixed the minimum offset of a BAI index bin, to account for unmapped reads.
+  Thanks to John Marshall for spotting the issue. (#1158; fixed #1142)
+
+* Fixed the CRLF handling in `sam_parse_worker` method. Thanks to
+  Anders Kaplan. (#1149; fixed #1148)
+
+* Included unistd.h and errno.h directly in HTSlib files, as opposed to
+  including them indirectly, via third party code. Thanks to
+  Andrew Patterson (#1143) and John Marshall (#1145).
+
+
+Noteworthy changes in release 1.11 (22nd September 2020)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Features and Updates
+--------------------
+
+* Support added for remote reference files.  fai_path() can take a remote
+  reference file and will return the corresponding index file.  Remote indexes
+  can be handled by refs_load_fai().  UR tags in @SQ lines can now be set to
+  remote URIs. (#1017)
+
+* Added tabix --separate-regions option, which adds header comment lines
+  separating different regions' output records when multiple target regions
+  are supplied on the command line. (#1108)
+
+* Added tabix --cache option to set a BGZF block cache size.  Most beneficial
+  when the -R option is used and the same blocks need to be re-read multiple
+  times.  (#1053)
+
+* Improved error checking in tabix and added a --verbosity option so
+  it is possible to change the amount of logging when it runs. (#1040)
+
+* A note about the maximum chromosome length usable with TBI indexes has been
+  added to the tabix manual page.  Thanks to John Marshall.  (#1070)
+
+* New method vcf_open_mode() changes the opening mode of a variant file
+  based on its file extension. Similar to sam_open_mode(). (#1096)
+
+* The VCF parser has been made faster and easier to maintain. (#1057)
+
+* bcf_record_check() has been made faster, giving a 15% speed increase when
+  reading an uncompressed BCF file. (#1130)
+
+* The VCF parser now recognises the "<NON_REF>" symbolic allele produced
+  by GATK. (#1045)
+
+* Support has been added for simultaneous reading of unindexed VCF/BCF files
+  when using the synced_bcf_reader interface.  Input files must have the
+  chromosomes in the same order as each other and be consistent with the order
+  of sequences in the header. (#1089)
+
+* The VCF and BCF readers will now attempt to fix up invalid INFO/END tags
+  where the stored END value is less than POS, resulting in an apparently
+  negative record length.  Such files have been generated by programs which
+  used END incorrectly, and by broken lift-over processes that failed to
+  update any END tags present. (#1021; fixed samtools/bcftools#1154)
+
+* The htsFile interface can now detect the crypt4gh encrypted format (see
+  https://samtools.github.io/hts-specs/crypt4gh.pdf).  If HTSlib is
+  built with external plug-in support, and the hfile_crypt4gh plug-in is
+  present, the file will be passed to it for decryption.  The plug-in
+  can be obtained from https://github.com/samtools/htslib-crypt4gh. (#1046)
+
+* hts_srand48() now seeds the same POSIX-standard sequences of pseudo-random
+  numbers regardless of platform, including on OpenBSD where plain srand48()
+  produces a different cryptographically-strong non-deterministic sequence.
+  Thanks to John Marshall. (#1002)
+
+* Iterators now work with 64 bit positions. (#1018)
+
+* Improved the speed of range queries when using BAI indexes by
+  making better use of the linear index data included in the file.
+  The best improvement is on low-coverage data. (#1031)
+
+* Alignments which consume no reference bases are now considered to have
+  length 1.  This would make such alignments cover 1 reference position in
+  the same manner as alignments that are unmapped or have no CIGAR strings.
+  These alignments can now be returned by iterator-based queries.  Thanks
+  to John Marshall. (#1063; fixed samtools/samtools#1240, see also
+  samtools/hts-specs#521).
+
+* A bam_set_seqi() function to modify a single base in the BAM structure
+  has been added.  This is a companion function to bam_seqi(). (#1022)
+
+* Writing SAM format is around 30% faster. (#1035)
+
+* Added sam_format_aux1() which converts a BAM aux tag to a SAM format string.
+  (#1134)
+
+* bam_aux_update_str() no longer requires NUL-terminated strings.  It
+  is also now possible to create tags containing part of a longer string.
+  (#1088)
+
+* It is now possible to use external plug-ins in language bindings that
+  dynamically load HTSlib.  Note that a side-effect of this change is that
+  some plug-ins now link against libhts.so, which means that they have to be
+  able to find the shared library when they are started up.  Thanks to
+  John Marshall. (#1072)
+
+* bgzf_close(), and therefore hts_close(), will now return non-zero when
+  closing a BGZF handle on which errors have been detected.  (Part of #1117)
+
+* Added a special case to the kt_fisher_exact() test for when the table
+  probability is too small to be represented in a double.  This fixes a
+  bug where it would, for some inputs, fail to correctly determine which
+  side of the distribution the table was on resulting in swapped p-values
+  being returned for the left- and right-tailed tests.  The two-tailed
+  test value was not affected by this problem. (#1126)
+
+* Improved error diagnostics in the CRAM decoder (#1042), BGZF (#1049),
+  the VCF and BCF readers (#1059), and the SAM parser (#1073).
+
+* ks_resize() now allocates 1.5 times the requested size when it needs
+  to expand a kstring instead of rounding up to the next power of two.
+  This has been done mainly to make the inlined function smaller, but it
+  also reduces the overhead of storing data in kstrings at the expense of
+  possibly needing a few more reallocations. (#1129)
+
+CRAM improvements
+-----------------
+
+* Delay CRAM crc32 checks until the data actually needs to be used.  With
+  other changes this leads to a 20x speed up in indexing and other sub-query
+  based actions. (#988)
+
+* CRAM now handles the transition from mapped to unmapped data in a better
+  way, improving compression of the unmapped data. (#961)
+
+* CRAM can now use libdeflate. (#961)
+
+* Fixed bug in MD tag generation with "b" read feature codes, causing the
+  numbers in the tag to be too large.  Note that HTSlib never uses this
+  feature code so it is unlikely that this bug would be seen on real data.
+  The problem was found when testing against hand-crafted CRAM files. (#1086)
+
+* Fixed a regression where the CRAM multi-region iterator became much less
+  efficient when using threads.  It now works more like the single iterator
+  and does not preemptively decode the next container unless it will be used.
+  (#1061)
+
+* Set CRAM default quality in lossy quality modes. If lossy quality is enabled
+  and 'B', 'q' or 'Q' features are used, CRAM starts off with QUAL being all 255
+  (as per BAM spec and "*" quality) and then modifies individual qualities as
+  dictated by the specific features.
+
+  However that then produces ASCII quality " " (space, q=-1) for the unmodified
+  bases.  Instead ASCII quality "?" (q=30) is used, as per HTSJDK. Quality 255
+  is still used for sequences with no modifications at all. (#1094)
+
+
+Build changes
+-------------
+
+These are compiler, configuration and makefile based changes.
+
+* `make all` now also builds htslib_static.mk and htslib-uninstalled.pc.
+  Thanks to John Marshall. (#1011)
+
+* Various cppcheck-1.90 warnings have been fixed. (#995, #1011)
+
+* HTSlib now prefers its own headers when being compiled, fixing build
+  failures on machines that already had a system-installed HTSlib.  Thanks to
+  John Marshall. (#1078; fixed #347)
+
+* Define HTSLIB_EXPORT without using a helper macro to reduce the length of
+  compiler diagnostics that mention exported functions.  Thanks to
+  John Marshall. (#1029)
+
+* Fix dirty default build by including latest pkg.m4 instead of using
+  aclocal.m4. Thanks to Damien Zammit. (#1091)
+
+* Struct tags have been added to htslib/*.h public typedefs.  This makes it
+  possible to forward declare htsFile without including htslib/hts.h. Thanks
+  to Lucas Czech and John Marshall. (#1115; fixed #1106)
+
+* Fixed compiler warnings emitted by the latest gcc and clang releases
+  when compiling HTSlib, along with some -Wextra warnings in the public
+  include files.  Thanks to John Marshall. (#1066, #1063, #1083)
+
+Bug fixes
+---------
+
+* Fixed hfile_libcurl breakage when using libcurl 7.69.1 or later.  Thanks to
+  John Marshall for tracking down the exact libcurl change that caused the
+  incompatibility. (#1105; fixed samtools/samtools#1254 and
+  samtools/samtools#1284)
+
+* Fixed overflows kroundup32() and kroundup_size_t() which caused them to
+  return zero when rounding up values where the most significant bit was
+  set.  When this happens they now return the highest value that can
+  be stored (#1044).  All of the kroundup macro definitions have also been
+  gathered together into a unified implementation (#1051).
+
+* Fixed missing return parameter value in idx_test_and_fetch(). Thanks to
+  Lilian Janin. (#1014)
+
+* Fixed crashes due to inconsistent selection between BGZF and plain (hFILE)
+  interfaces when reading files. [fuzz] (#1019)
+
+* Added and/or fixed byte swapping code for big-endian platforms.  Thanks
+  to Jun Aruga, John Marshall, Michael R Crusoe and Gianfranco Costamagna
+  for their help. (#1023; fixed #119 and #355)
+
+* Fixed a problem with multi-threaded on-the-fly indexes which would
+  occasionally write virtual offsets pointing at the end of a BGZF block.
+  Attempting to read from such an offset caused EOF to be incorrectly
+  reported.  These offsets are now handled correctly, and the indexer
+  has been updated to avoid generating them.  (#1028; fixed
+  samtools/samtools#1197)
+
+* In sam_hdr_create(), free newly allocated SN strings when encountering an
+  error. [fuzz] (#1034)
+
+* Prevent double free in case of idx_test_and_fetch() failure. Thanks to
+  @fanwayne for the bug report. (#1047; fixed #1033)
+
+* In the header, link a new PG line only to valid chains. Prevents an
+  explosive growth of PG lines on headers where PG lines are already present
+  but not linked together correctly. (#1062; fixed samtools/samtools#1235)
+
+* Also in the header, when calling sam_hdr_update_line(), update target arrays
+  only when the name or length is changed. (#1007)
+
+* Fixed buffer overflows in CRAM MD5 calculation triggered by
+  files with invalid compression headers, or files with embedded
+  references that were one byte too short. [fuzz] (#1024, #1068)
+
+* Fix mpileup regression between 1.9 and 1.10 where overlap detection
+  was incorrectly skipped on reads where RNEXT, PNEXT and TLEN were
+  set to the "unavailable" values ("*", 0, 0 in SAM). (#1097)
+
+* kputs() now checks for null pointer in source string. [fuzz] (#1087)
+
+* Fix potential bcf_update_alleles() crash on 0 alleles. Thanks to
+  John Marshall. (#994)
+
+* Added bcf_unpack() calls to some bcf_update functions to fix a bug
+  where updates made after a call to bcf_dup() could be lost. (#1032;
+  fixed #1030)
+
+* Error message typo "Number=R" instead of "Number=G" fixed in
+  bcf_remove_allele_set(). Thanks to Ilya Vorontsov. (#1100)
+
+* Fixed crashes that could occur in BCF files that use IDX= header annotations
+  to create a sparse set of CHROM, FILTER or FORMAT indexes, and
+  include records that use one of the missing index values. [fuzz] (#1092)
+
+* Fixed potential integer overflows in the VCF parser and ensured that
+  the total length of FORMAT fields cannot go over 2Gbytes. [fuzz] (#1044,
+  #1104; latter is CVE-2020-36403 affecting all HTSlib versions up to 1.10.2)
+
+* Download index files atomically in idx_test_and_fetch().  This prevents
+  corruption when running parallel jobs on S3 files.  Thanks to John Marshall.
+  (#1112; samtools/samtools#1242).
+
+* The pileup constructor callback is now given the copy of the bam1_t struct
+  made by pileup instead of the original one passed to bam_plp_push().  This
+  makes it the same as the one passed to the destructor and ensures that
+  cached data, for example the location of an aux tag, will remain valid.
+  (#1127)
+
+* Fixed possible error in code_sort() on negative CRAM Huffman code
+  length. (#1008)
+
+* Fixed possible undefined shift in cram_byte_array_stop_decode_init(). (#1009)
+
+* Fixed a bug where range queries to the end of a given reference
+  would return incorrect results on CRAM files. (#1016;
+  fixed samtools/samtools#1173)
+
+* Fixed an integer overflow in cram_read_slice(). [fuzz] (#1026)
+
+* Fixed a memory leak on failure in cram_decode_slice(). [fuzz] (#1054)
+
+* Fixed a regression which caused cram_transcode_rg() to fail, resulting
+  in a crash in "samtools cat" on CRAM files. (#1093;
+  fixed samtools/samtools#1276)
+
+* Fixed an undersized string reallocation in the threaded SAM reader which
+  caused it to crash when reading SAM files with very long lines.  Numerous
+  memory allocation checks have also been added. (#1117)
+
+
+Noteworthy changes in release 1.10.2 (19th December 2019)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This is a release fix that corrects minor inconsistencies discovered in
+previous deliverables.
+
+
+Noteworthy changes in release 1.10.1 (17th December 2019)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The support for 64-bit coordinates in VCF brought problems for files
+not conforming to VCF/BCF specification. While previous versions would
+make out-of-range values silently overflow creating nonsense values
+but parseable file, the version 1.10 would silently create an invalid BCF.
+
+
+Noteworthy changes in release 1.10 (6th December 2019)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Brief summary
+-------------
+
+There are many changes in this release, so the executive summary is:
+
+* Addition of support for references longer than 2Gb (NB: SAM and VCF
+  formats only, not their binary counterparts).  This may need changes
+  in code using HTSlib.  See README.large_positions.md for more information.
+
+* Added a SAM header API.
+
+* Major speed up to SAM reading and writing.  This also now supports
+  multi-threading.
+
+* We can now auto-index on-the-fly while writing a file.  This also
+  includes to bgzipped SAM.gz.
+
+* Overhaul of the S3 interface, which now supports version 4
+  signatures.  This also makes writing to S3 work.
+
+These also required some ABI changes.  See below for full details.
+
+
+Features / updates
+------------------
+
+* A new SAM/BAM/CRAM header API has been added to HTSlib, allowing header
+  data to be updated without having to parse or rewrite large parts of the
+  header text.  See htslib/sam.h for function definitions and
+  documentation. (#812)
+
+  The header typedef and several pre-existing functions have been renamed
+  to have a sam_hdr_ prefix: sam_hdr_t, sam_hdr_init(), sam_hdr_destroy(),
+  and sam_hdr_dup().  (The existing bam_hdr_-prefixed names are still
+  provided for compatibility with existing code.) (#887, thanks to
+  John Marshall)
+
+* Changes to hfile_s3, which provides support for the AWS S3 API. (#839)
+
+  - hfile_s3 now uses version 4 signatures by default.  Attempting to write to
+    an S3 bucket will also now work correctly.  It is possible to force
+    version 2 signatures by creating environment variable HTS_S3_V2 (the exact
+    value does not matter, it just has to exist).  Note that writing depends
+    on features that need version 4 signatures, so forcing version 2 will
+    disable writes.
+
+  - hfile_s3 will automatically retry requests where the region endpoint
+    was not specified correctly, either by following the 301 redirect (when
+    using path-style requests) or reading the 400 response (when using
+    virtual-hosted style requests and version 4 signatures).  The first
+    region to try can be set by using the AWS_DEFAULT_REGION environment
+    variable, by setting "region" in ".aws/credentials" or by setting
+    "bucket_location" in ".s3cfg".
+
+  - hfile_s3 now percent-escapes the path component of s3:// URLs.  For
+    backwards-compatibility it will ignore any paths that have already
+    been escaped (detected by looking for '%' followed by two hexadecimal
+    digits.)
+
+  - New environment variables HTS_S3_V2, HTS_S3_HOST, HTS_S3_S3CFG
+    and HTS_S3_PART_SIZE to force version-2 signatures, control the
+    S3 server hostname, the configuration file and upload chunk
+    sizes respectively.
+
+* Numerous SAM format improvements.
+
+  - Bgzipped SAM files can now be indexed and queried.  The library now
+    recognises sam.gz as a format name to ease this usage. (#718, #916)
+
+  - The SAM reader and writer now supports multi-threading via the
+    thread-pool. (#916)
+
+    Note that the multi-threaded SAM reader does not currently support seek
+    operations.  Trying to do this (for example with an iterator range request)
+    will result in the SAM readers dropping back to single-threaded mode.
+
+  - Major speed up of SAM decoding and encoding, by around 2x. (#722)
+
+  - SAM format can now handle 64-bit coordinates and references.  This
+    has implications for the ABI too (see below).  Note BAM and CRAM
+    currently cannot handle references longer than 2Gb, however given
+    the speed and threading improvements SAM.gz is a viable workaround. (#709)
+
+* We can now automatically build indices on-the-fly while writing
+  SAM, BAM, CRAM, VCF and BCF files.  (Note for SAM and VCF this only
+  works when bgzipped.) (#718)
+
+* HTSlib now supports the @SQ-AN header field, which lists alternative names
+  for reference sequences.  This means given "@SQ SN:1 AN:chr1", tools like
+  samtools can accept requests for "1" or "chr1" equivalently.  (#931)
+
+* Zero-length files are no longer considered to be valid SAM files
+  (with no header and no alignments).  This has been changed so that pipelines
+  such as `somecmd | samtools ...` with `somecmd` aborting before outputting
+  anything will now propagate the error to the second command. (#721, thanks
+  to John Marshall; #261 reported by Adrian Tan)
+
+* Added support for use of non-standard index names by pasting the
+  data filename and index filename with ##idx##.  For example
+  "/path1/my_data.bam##idx##/path2/my_index.csi" will open bam file
+  "/path1/my_data.bam" and index file "/path2/my_index.csi". (#884)
+
+  This affects hts_idx_load() and hts_open() functions.
+
+* Improved the region parsing code to handle colons in reference
+  names.  Strings can be disambiguated by the use of braces, so for
+  example when reference sequences called "chr1" and "chr1:100-200"
+  are both present, the regions "{chr1}:100-200" and "{chr1:100-200}"
+  unambiguously indicate which reference is being used. (#708)
+
+  A new function hts_parse_region() has been added along with
+  specialisations for sam_parse_region() and fai_parse_region().
+
+* CRAM encoding now has additional checks for MD/NM validity.  If
+  they are incorrect, it stores the (incorrect copy) verbatim so
+  round-trips "work". (#792)
+
+* Sped up decoding of CRAM by around 10% when the MD tag is being
+  generated. (#874)
+
+* CRAM REF_PATH now supports %Ns (where N is a single digit)
+  expansion in http URLs, similar to how it already supported this
+  for directories. (#791)
+
+* BGZF now permits indexing and seeking using virtual offsets in
+  completely uncompressed streams. (#904, thanks to Adam Novak)
+
+* bgzip now asks for extra confirmation before decompressing files
+  that don't have a known compression extension (e.g. .gz).  This avoids
+  `bgzip -d foo.bam.bai` producing a foo.bam file that is very much not
+  a BAM-formatted file. (#927, thanks to John Marshall)
+
+* The htsfile utility can now copy files (including to/from URLs using
+  HTSlib's remote access facilities) with the --copy option, in
+  addition to its existing uses of identifying file formats and
+  displaying sequence or variant data. (#756, thanks to John Marshall)
+
+* Added tabix --min-shift option. (#752, thanks to Garrett Stevens)
+
+* Tabix now has an -D option to disable storing a local copy of a
+  remote index. (#870)
+
+* Improved support for MSYS Windows compiler environment. (#966)
+
+* External htslib plugins are now supported on Windows. (#966)
+
+
+API additions and improvements
+------------------------------
+
+* New API functions bam_set_mempolicy() and bam_get_mempolicy() have
+  been added.  These allow more control over the ownership of bam1_t
+  alignment record data; see documentation in htslib/sam.h for more
+  information. (#922)
+
+* Added more HTS_RESULT_USED checks, this time for VCF I/O. (#805)
+
+* khash can now hash kstrings.  This makes it easier to hash
+  non-NUL-terminated strings. (#713)
+
+* New haddextension() filename extension API function. (#788, thanks to
+  John Marshall)
+
+* New hts_resize() macro, designed to replace uses of hts_expand()
+  and hts_expand0(). (#805)
+
+* Added way of cleaning up unused jobs in the thread pool via the new
+  hts_tpool_dispatch3() function. (#830)
+
+* New API functions hts_reglist_create() and sam_itr_regarray() are added
+  to create hts_reglist_t region lists from `chr:<from>-<to>` type region
+  specifiers. (#836)
+
+* Ksort has been improved to facilitate library use.  See KSORT_INIT2
+  (adds scope / namespace capabilities) and KSORT_INIT_STATIC interfaces.
+  (#851, thanks to John Marshall)
+
+* New kstring functions (#879):
+     KS_INITIALIZE   -  Initializer for structure assignment
+     ks_initialize() -  Initializer for pointed-to kstrings
+     ks_expand()     -  Increase kstring capacity by a given amount
+     ks_clear()      -  Set kstring length to zero
+     ks_free()       -  Free the underlying buffer
+     ks_c_str()      -  Returns the kstring buffer as a const char *,
+                        or an empty string if the length is zero.
+
+* New API functions hts_idx_load3(), sam_index_load3(), tbx_index_load3()
+  and bcf_index_load3() have been added.  These allow control of whether
+  remote indexes should be cached locally, and allow the error message
+  printed when the index does not exist to be suppressed. (#870)
+
+* Improved hts_detect_format() so it no longer assumes all text is
+  SAM unless positively identified otherwise.  It also makes a stab
+  at detecting bzip2 format and identifying BED, FASTA and FASTQ
+  files. (#721, thanks to John Marshall; #200, #719 both reported by
+  Torsten Seemann)
+
+* File format errors now set errno to EFTYPE (BSD, MacOS) when
+  available instead of ENOEXEC. (#721)
+
+* New API function bam_set_qname (#942)
+
+* In addition to the existing hts_version() function, which reflects the
+  HTSlib version being used at runtime, <htslib/hts.h> now also provides
+  HTS_VERSION, a preprocessor macro reflecting the HTSlib version that
+  a program is being compiled against.  (#951, thanks to John Marshall; #794)
+
+
+ABI changes
+-----------
+
+This release contains a number of things which change the Application
+Binary Interface (ABI).  This means code compiled against an earlier
+library will require recompiling.  The shared library soversion has
+been bumped.
+
+* On systems that support it, the default symbol visibility has been
+  changed to hidden and the only exported symbols are ones that form part
+  of the officially supported ABI.  This is to make clear exactly which
+  symbols are considered parts of the library interface.  It also
+  helps packagers who want to check compatibility between HTSlib versions.
+  (#946; see for example issues #311, #616, and #695)
+
+* HTSlib now supports 64 bit reference positions.  This means several
+  structures, function parameters, and return values have been made bigger
+  to allow larger values to be stored.  While most code that uses
+  HTSlib interfaces should still build after this change, some alterations
+  may be needed - notably to printf() formats where the values of structure
+  members are being printed. (#709)
+
+  Due to file format limitations, large positions are only supported
+  when reading and writing SAM and VCF files.
+
+  See README.large_positions.md for more information.
+
+* An extra field has been added to the kbitset_t struct so bitsets can
+  be made smaller (and later enlarged) without involving memory allocation.
+  (#710, thanks to John Marshall)
+
+* A new field has been added to the bam_pileup1_t structure to keep track
+  of which CIGAR operator is being processed.  This is used by a new
+  bam_plp_insertion() function which can be used to return the sequence of
+  any inserted bases at a given pileup location.  If the alignment includes
+  CIGAR P operators, the returned sequence will include pads. (#699)
+
+* The hts_itr_t and hts_itr_multi_t structures have been merged and can be
+  used interchangeably.  Extra fields have been added to hts_itr_t to support
+  this.  hts_itr_multi_t is now a typedef for hts_itr_t; sam_itr_multi_next()
+  is now an alias for sam_itr_next() and hts_itr_multi_destroy() is an alias
+  for hts_itr_destroy(). (#836)
+
+* An improved regidx interface has been added.  To allow this, struct
+  reg_t has been removed, regitr_t has been modified and various new
+  API functions have been added to htslib/regidx.h.  While parts of
+  the old regidx API have been retained for backwards compatibility,
+  it is recommended that all code using regidx should be changed to use
+  the new interface. (#761)
+
+* Elements in the hts_reglist_t structure have been reordered slightly
+  so that they pack together better. (#761)
+
+* bgzf_utell() and bgzf_useek() now use type off_t instead of long for
+  the offset.  This allows them to work correctly on files longer than
+  2G bytes on Windows and 32-bit Linux. (#868)
+
+* A number of functions that used to return void now return int so that
+  they can report problems like memory allocation failures.  Callers
+  should take care to check the return values from these functions. (#834)
+
+  The affected functions are:
+     ksort.h:             ks_introsort(), ks_mergesort()
+     sam.h:               bam_mplp_init_overlaps()
+     synced_bcf_reader.h: bcf_sr_regions_flush()
+     vcf.h:               bcf_format_gt(), bcf_fmt_array(),
+                            bcf_enc_int1(), bcf_enc_size(),
+                          bcf_enc_vchar(), bcf_enc_vfloat(), bcf_enc_vint(),
+                            bcf_hdr_set_version(), bcf_hrec_format()
+     vcfutils.h:          bcf_remove_alleles()
+
+* bcf_set_variant_type() now outputs VCF_OVERLAP for spanning
+  deletions (ALT=*). (#726)
+
+* A new field (hrecs) has been added to the bam_hdr_t structure for
+  use by the new header API.  The old sdict field is now not used and
+  marked as deprecated.  The l_text field has been changed from uint32_t
+  to size_t, to allow for very large headers in SAM files.  The text
+  and l_text fields have been left for backwards compatibility, but
+  should not be accessed directly in code that uses the new header API.
+  To access the header text, the new functions sam_hdr_length() and
+  sam_hdr_str() should be used instead. (#812)
+
+* The old cigar_tab field is now marked as deprecated; use the new
+  bam_cigar_table[] instead. (#891, thanks to John Marshall)
+
+* The bam1_core_t structure's l_qname and l_extranul fields have been
+  rearranged and enlarged; l_qname still includes the extra NULs.
+  (Almost all code should use bam_get_qname(), bam_get_cigar(), etc,
+  and has no need to use these fields directly.)  HTSlib now supports
+  the SAM specification's full 254 QNAME length again.  (#900, thanks
+  to John Marshall; #520)
+
+* bcf_index_load() no longer tries the '.tbi' suffix when looking for
+  BCF index files (.tbi indexes are for text files, not binary BCF). (#870)
+
+* htsFile has a new 'state' member to support SAM multi-threading. (#916)
+
+* A new field has been added to the bam1_t structure, and others
+  have been rearranged to remove structure holes. (#709; #922)
+
+
+Bug fixes
+---------
+
+* Several BGZF format fixes:
+
+  - Support for multi-member gzip files. (#744, thanks to Adam Novak; #742)
+
+  - Fixed error handling code for native gzip formatted files. (64c4927)
+
+  - CRCs checked when threading too (previously only when non-threaded). (#745)
+
+  - Made bgzf_useek function work with threads. (#818)
+
+  - Fixed rare threading deadlocks. (#831)
+
+  - Reading of very short files (<28 bytes) that do not contain an EOF block.
+    (#910)
+
+* Fixed some thread pool deadlocks caused by race conditions. (#746, #906)
+
+* Many additional memory allocation checks in VCF, BCF, SAM and CRAM
+  code. This also changes the return type of some functions.  See ABI
+  changes above. (#920 amongst others)
+
+* Replace some sam parsing abort() calls with proper errors.
+  (#721, thanks to John Marshall; #576)
+
+* Fixed to permit SAM read names of length 252 to 254 (the maximum
+  specified by the SAM specification). (#900, thanks to John Marshall)
+
+* Fixed mpileup overlap detection heuristic to work with BAMs having
+  long CIGARs (more than 65536 operations). (#802)
+
+* Security fix: CIGAR strings starting with the "N" operation can no
+  longer cause underflow on the bam CIGAR structure.  Similarly CIGAR
+  strings that are entirely "D" ops could leak the contents of
+  uninitialised variables. (#699)
+
+* Fixed bug where alignments starting 0M could cause an invalid
+  memory access in sam_prob_realn(). (#699)
+
+* Fixed out of bounds memory access in mpileup when given a reference
+  with binary characters (top-bit set). (#808, thanks to John Marshall)
+
+* Fixed crash in mpileup overlap_push() function. (#882; #852 reported
+  by Pierre Lindenbaum)
+
+* Fixed various potential CRAM memory leaks when recovering from
+  error cases.
+
+* Fixed CRAM index queries for unmapped reads (#911; samtools/samtools#958
+  reported by @acorvelo)
+
+* Fixed the combination of CRAM embedded references and multiple
+  slices per container.  This was incorrectly setting the header
+  MD5sum.  (No impact on default CRAM behaviour.) (b2552fd)
+
+* Removed unwanted explicit data flushing in CRAM writing, which on
+  some OSes caused major slowdowns. (#883)
+
+* Fixed inefficiencies in CRAM encoding when many small references
+  occur within the middle of large chromosomes.  Previously it
+  switched into multi-ref mode, but not back out of it which caused
+  the read POS field to be stored poorly. (#896)
+
+* Fixed CRAM handling of references when the order of sequences in a
+  supplied fasta file differs to the order of the @SQ headers. (#935)
+
+* Fixed BAM and CRAM multi-threaded decoding when used in conjunction
+  with the multi-region iterator. (#830; #577, #822, #926 all reported by
+  Brent Pedersen)
+
+* Removed some unaligned memory accesses in CRAM encoder and
+  undefined behaviour in BCF reading (#867, thanks to David Seifert)
+
+* Repeated calling of bcf_empty() no longer crashes. (#741)
+
+* Fixed bug where some 8 or 16-bit negative integers were stored using values
+  reserved by the BCF specification.  These numbers are now promoted to the
+  next size up, so -121 to -128 are stored using at least 16 bits, and -32761
+  to -32768 are stored using 32 bits.
+
+  Note that while BCF files affected by this bug are technically incorrect,
+  it is still possible to read them.  When converting to VCF format,
+  HTSlib (and therefore bcftools) will interpret the values as intended
+  and write out the correct negative numbers. (#766, thanks to John Marshall;
+  samtools/bcftools#874)
+
+* Allow repeated invocations of bcf_update_info() and bcf_update_format_*()
+  functions. (#856, thanks to John Marshall; #813 reported by Steffen Möller)
+
+* Memory leak removed in knetfile's kftp_parse_url() function. (#759, thanks
+  to David Alexander)
+
+* Fixed various crashes found by libfuzzer (invalid data leading to
+  errors), mostly but not exclusively in CRAM, VCF and BCF decoding. (#805)
+
+* Improved robustness of BAI and CSI index creation and loading. (#870; #967)
+
+* Prevent (invalid) creation of TBI indices for BCF files.
+  (#837; samtools/bcftools#707)
+
+* Better parsing of handling of remote URLs with ?param=val
+  components and their interaction with remote index URLs. (#790; #784
+  reported by Mark Ebbert)
+
+* hts_idx_load() now checks locally for all possible index names before
+  attempting to download a remote index.  It also checks that the remote
+  file it downloads is actually an index before trying to save and use
+  it.  (#870; samtools/samtools#1045 reported by Albert Vilella)
+
+* hts_open_format() now honours the compression field, no longer also
+  requiring an explicit "z" in the mode string.  Also fixed a 1 byte
+  buffer overrun. (#880)
+
+* Removed duplicate hts_tpool_process_flush prototype. (#816, reported by
+  James S Blachly)
+
+* Deleted defunct cram_tell declaration. (66c41e2; #915 reported by
+  Martin Morgan)
+
+* Fixed overly aggressive filename suffix checking in bgzip. (#927, thanks to
+  John Marshall; #129, reported by @hguturu)
+
+* Tabix and bgzip --help output now goes to standard output. (#754, thanks to
+  John Marshall)
+
+* Fixed bgzip index creation when using multiple threads. (#817)
+
+* Made bgzip -b option honour -I (index filename). (#817)
+
+* Bgzip -d no longer attempts to unlink(NULL) when decompressing stdin. (#718)
+
+
+Miscellaneous other changes
+---------------------------
+
+* Integration with Google OSS fuzzing for automatic detection of
+  more bugs. (Thanks to Google for their assistance and the bugs it
+  has found.) (#796, thanks to Markus Kusano)
+
+* aclocal.m4 now has the pkg-config macros. (6ec3b94d; #733 reported by
+  Thomas Hickman)
+
+* Improved C++ compatibility of some header files. (#772; #771 reported
+  by @cwrussell)
+
+* Improved strict C99 compatibility. (#860, thanks to John Marshall)
+
+* Travis and AppVeyor improvements to aid testing. (#747; #773 thanks to
+  Lennard Berger; #781; #809; #804; #860; #909)
+
+* Various minor compiler warnings fixed. (#708; #765; #846, #860, thanks to
+  John Marshall; #865; #966; #973)
+
+* Various new and improved error messages.
+
+* Documentation updates (mostly in the header files).
+
+* Even more testing with "make check".
+
+* Corrected many copyright dates. (#979)
+
+* The default non-configure Makefile now uses libcurl instead of
+  knet, so it can support https. (#895)
+
+
+
+
+
+
+Noteworthy changes in release 1.9 (18th July 2018)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* If `./configure` fails, `make` will stop working until either configure
+  is re-run successfully, or `make distclean` is used.  This makes
+  configuration failures more obvious.  (#711, thanks to John Marshall)
+
+* The default SAM version has been changed to 1.6.  This is in line with the
+  latest version specification and indicates that HTSlib supports the
+  CG tag used to store long CIGAR data in BAM format.
+
+* bgzip integrity check option '--test' (#682, thanks to @sd4B75bJ, @jrayner)
+
+* Faidx can now index fastq files as well as fasta.  The fastq index adds
+  an extra column to the `.fai` index which gives the offset to the quality
+  values.  New interfaces have been added to `htslib/faidx.h` to read the
+  fastq index and retrieve the quality values.  It is possible to open
+  a fastq index as if fasta (only sequences will be returned), but not
+  the other way round. (#701)
+
+* New API interfaces to add or update integer, float and array aux tags. (#694)
+
+* Add `level=<number>` option to `hts_set_opt()` to allow the compression
+  level to be set.  Setting `level=0` enables uncompressed output. (#715)
+
+* Improved bgzip error reporting.
+
+* Better error reporting when CRAM reference files can't be opened. (#706)
+
+* Fixes to make tests work properly on Windows/MinGW - mainly to handle
+  line ending differences. (#716)
+
+* Efficiency improvements:
+
+  - Small speed-up for CRAM indexing.
+
+  - Reduce the number of unnecessary wake-ups in the thread pool. (#703)
+
+  - Avoid some memory copies when writing data, notably for uncompressed
+    BGZF output. (#703)
+
+* Bug fixes:
+
+  - Fix multi-region iterator bugs on CRAM files. (#684)
+
+  - Fixed multi-region iterator bug that caused some reads to be skipped
+    incorrectly when reading BAM files. (#687)
+
+  - Fixed synced_bcf_reader() bug when reading contigs multiple times. (#691,
+    reported by @freeseek)
+
+  - Fixed bug where bcf_hdr_set_samples() did not update the sample dictionary
+    when removing samples. (#692, reported by @freeseek)
+
+  - Fixed bug where the VCF record ref length was calculated incorrectly
+    if an INFO END tag was present. (71b00a)
+
+  - Fixed warnings found when compiling with gcc 8.1.0. (#700)
+
+  - sam_hdr_read() and sam_hdr_write() will now return an error code
+    if passed a NULL file pointer, instead of crashing.
+
+  - Fixed possible negative array look-up in sam_parse1() that somehow escaped
+    previous fuzz testing. (CVE-2018-13845, #731, reported by @fCorleone)
+
+  - Fixed bug where cram range queries could incorrectly report an error
+    when using multiple threads. (#734, reported by Brent Pedersen)
+
+  - Fixed very rare rANS normalisation bug that could cause an assertion
+    failure when writing CRAM files.  (#739, reported by @carsonhh)
+
+Noteworthy changes in release 1.8 (3rd April 2018)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* The URL to get sequences from the EBI reference server has been changed
+  to https://.  This is because the EBI no longer serve sequences via
+  plain HTTP - requests to the http:// endpoint just get redirected.
+  HTSlib needs to be linked against libcurl to download https:// URLs,
+  so CRAM users who want to get references from the EBI will need to
+  run configure and ensure libcurl support is enabled using the
+  --enable-libcurl option.
+
+* Added libdeflate as a build option for alternative faster compression and
+  decompression.  Results vary by CPU but compression should be twice as fast
+  and decompression faster.
+
+* It is now possible to set the compression level in bgzip.  (#675; thanks
+  to Nathan Weeks).
+
+* bgzip now gets its own manual page.
+
+* CRAM encoding now stored MD and NM tags verbatim where the reference
+  contains 'N' characters, to work around ambiguities in the SAM
+  specification (samtools #717/762).
+  Also added "store_md" and "store_nm" cram-options for forcing these
+  tags to be stored at all locations.  This is best when combined with
+  a subsequent decode_md=0 option while reading CRAM.
+
+* Multiple CRAM bug fixes, including a fix to free and the subsequent reuse of
+  references with `-T ref.fa`. (#654; reported by Chris Saunders)
+
+* CRAM multi-threading bugs fixed: don't try to call flush on reading;
+  processing of multiple range queries; problems with multi-slice containers.
+
+* Fixed crashes caused when decoding some cramtools produced CRAM files.
+
+* Fixed a couple of minor rANS issues with handling invalid data.
+
+* Fixed bug where probaln_glocal() tried to allocate far more memory than
+  needed when the query sequence was much longer than the reference.  This
+  caused crashes in samtools and bcftools mpileup when used on data with very
+  long reads. (#572, problem reported by Felix Bemm via minimap2).
+
+* sam_prop_realn() now returns -1 (the same value as for unmapped reads)
+  on reads that do not include at least one 'M', 'X' or '=' CIGAR operator,
+  and no longer adds BQ or ZQ tags.  BAQ adjustments are only made to bases
+  covered by these operators so there is no point in trying to align
+  reads that do not have them. (#572)
+
+Noteworthy changes in release 1.7 (26th January 2018)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* BAM: HTSlib now supports BAMs which include CIGARs with more than
+  65535 operations as per HTS-Specs 18th November (dab57f4 and 2f915a8).
+
+* BCF/VCF:
+  - Removed the need for long double in pileup calculations.
+  - Sped up the synced reader in some situations.
+  - Bug fixing: removed memory leak in bcf_copy.
+
+* CRAM:
+  - Added support for HTS_IDX_START in cram iterators.
+  - Easier to build when lzma header files are absent.
+  - Bug fixing: a region query with REQUIRED_FIELDS option to
+    disable sequence retrieval now gives correct results.
+  - Bug fixing: stop queries to regions starting after the last
+    read on a chromosome from incorrectly reporting errors
+    (#651, #653; reported by Imran Haque and @egafni via pysam).
+
+* Multi-region iterator: The new structure takes a list of regions and
+  iterates over all, deduplicating reads in the process, and producing a
+  full list of file offset intervals. This is usually much faster than
+  repeatedly using the old single-region iterator on a series of regions.
+
+* Curl improvements:
+  - Add Bearer token support via HTS_AUTH_LOCATION env (#600).
+  - Use CURL_CA_BUNDLE environment variable to override the CA (#622;
+    thanks to Garret Kelly & David Alexander).
+  - Speed up (removal of excessive waiting) for both http(s) and ftp.
+  - Avoid repeatedly reconnecting by removal of unnecessary seeks.
+  - Bug fixing: double free when libcurl_open fails.
+
+* BGZF block caching, if enabled, now performs far better (#629; reported
+  by Ram Yalamanchili).
+
+* Added an hFILE layer for in-memory I/O buffers (#590; thanks to Thomas
+  Hickman).
+
+* Tidied up the drand48 support (intended for systems that do not
+  provide this function).
+
+Noteworthy changes in release 1.6 (28th September 2017)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* Fixed bug where iterators on CRAM files did not propagate error return
+  values to the caller correctly.  Thanks go to Chris Saunders.
+
+* Overhauled Windows builds.  Building with msys2/mingw64 now works
+  correctly and passes all tests.
+
+* More improvements to logging output (thanks again to Anders Kaplan).
+
+* Return codes from sam_read1() when reading cram have been made
+  consistent with those returned when reading sam/bam.  Thanks to
+  Chris Saunders (#575).
+
+* BGZF CRC32 checksums are now always verified.
+
+* It's now possible to set nthreads = 1 for cram files.
+
+* hfile_libcurl has been modified to make it thread-safe.  It's also
+  better at handling web servers that do not honour byte range requests
+  when attempting to seek - it now sets errno to ESPIPE and keeps
+  the existing connection open so callers can revert to streaming mode
+  it they want to.
+
+* hfile_s3 now recalculates access tokens if they have become stale.  This
+  fixes a reported problem where authentication failed after a file
+  had been in use for more than 15 minutes.
+
+* Fixed bug where remote index fetches would fail to notice errors when
+  writing files.
+
+* bam_read1() now checks that the query sequence length derived from the
+  CIGAR alignment matches the sequence length in the BAM record.
+
+Noteworthy changes in release 1.5 (21st June 2017)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* Added a new logging API: hts_log(), along with hts_log_error(),
+  hts_log_warn() etc. convenience macros.  Thanks go to Anders Kaplan
+  for the implementation. (#499, #543, #551)
+
+* Added a new file I/O option "block_size" (HTS_OPT_BLOCK_SIZE) to
+  alter the hFILE buffer size.
+
+* Fixed various bugs, including compilation issues samtools/bcftools#610,
+  samtools/bcftools#611 and robustness to corrupted data #537, #538,
+  #541, #546, #548, #549, #554.
+
+
+Noteworthy changes in release 1.4.1  (8th May 2017)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This is primarily a security bug fix update.
+
+* Fixed SECURITY (CVE-2017-1000206) issue with buffer overruns with
+  malicious data. (#514)
+
+* S3 support for non Amazon AWS endpoints. (#506)
+
+* Support for variant breakpoints in bcftools. (#516)
+
+* Improved handling of BCF NaNs. (#485)
+
+* Compilation / portability improvements. (#255, #423, #498, #488)
+
+* Miscellaneous bug fixes (#482, #521, #522, #523, #524).
+
+* Sanitise headers (#509)
+
+
+Release 1.4 (13 March 2017)
+
+* Incompatible changes: several functions and data types have been changed
+  in this release, and the shared library soversion has been bumped to 2.
+
+  - bam_pileup1_t has an additional field (which holds user data)
+  - bam1_core_t has been modified to allow for >64K CIGAR operations
+    and (along with bam1_t) so that CIGAR entries are aligned in memory
+  - hopen() has vararg arguments for setting URL scheme-dependent options
+  - the various tbx_conf_* presets are now const
+  - auxiliary fields in bam1_t are now always stored in little-endian byte
+    order (previously this depended on if you read a bam, sam or cram file)
+  - index metadata (accessible via hts_idx_get_meta()) is now always
+    stored in little-endian byte order (previously this depended on if
+    the index was in tbi or csi format)
+  - bam_aux2i() now returns an int64_t value
+  - fai_load() will no longer save local copies of remote fasta indexes
+  - hts_idx_get_meta() now takes a uint32_t * for l_meta (was int32_t *)
+
+* HTSlib now links against libbz2 and liblzma by default.  To remove these
+  dependencies, run configure with options --disable-bz2 and --disable-lzma,
+  but note that this may make some CRAM files produced elsewhere unreadable.
+
+* Added a thread pool interface and replaced the bgzf multi-threading
+  code to use this pool.  BAM and CRAM decoding is now multi-threaded
+  too, using the pool to automatically balance the number of threads
+  between decode, encode and any data processing jobs.
+
+* New errmod_cal(), probaln_glocal(), sam_cap_mapq(), and sam_prob_realn()
+  functions, previously internal to SAMtools, have been added to HTSlib.
+
+* Files can now be accessed via Google Cloud Storage using gs: URLs, when
+  HTSlib is configured to use libcurl for network file access rather than
+  the included basic knetfile networking.
+
+* S3 file access now also supports the "host_base" setting in the
+  $HOME/.s3cfg configuration file.
+
+* Data URLs ("data:,text") now follow the standard RFC 2397 format and may
+  be base64-encoded (when written as "data:;base64,text") or may include
+  percent-encoded characters.  HTSlib's previous over-simplified "data:text"
+  format is no longer supported -- you will need to add an initial comma.
+
+* When plugins are enabled, S3 support is now provided by a separate
+  hfile_s3 plugin rather than by hfile_libcurl itself as previously.
+  When --enable-libcurl is used, by default both GCS and S3 support
+  and plugins will also be built; they can be individually disabled
+  via --disable-gcs and --disable-s3.
+
+* The iRODS file access plugin has been moved to a separate repository.
+  Configure no longer has a --with-irods option; instead build the plugin
+  found at <https://github.com/samtools/htslib-plugins>.
+
+* APIs to portably read and write (possibly unaligned) data in little-endian
+  byte order have been added.
+
+* New functions bam_auxB_len(), bam_auxB2i() and bam_auxB2f() have been
+  added to make accessing array-type auxiliary data easier.  bam_aux2i()
+  can now return the full range of values that can be stored in an integer
+  tag (including unsigned 32 bit tags).  bam_aux2f() will return the value
+  of integer tags (as a double) as well as floating-point ones.  All of
+  the bam_aux2 and bam_auxB2 functions will set errno if the requested
+  conversion is not valid.
+
+* New functions fai_load3() and fai_build3() allow fasta indexes to be
+  stored in a different location to the indexed fasta file.
+
+* New functions bgzf_index_dump_hfile() and bgzf_index_load_hfile()
+  allow bgzf index files (.gzi) to be written to / read from an existing
+  hFILE handle.
+
+* hts_idx_push() will report when trying to add a range to an index that
+  is beyond the limits that the given index can handle.  This means trying
+  to index chromosomes longer than 2^29 bases with a .bai or .tbi index
+  will report an error instead of apparently working but creating an invalid
+  index entry.
+
+* VCF formatting is now approximately 4x faster.  (Whether this is
+  noticeable depends on what was creating the VCF.)
+
+* CRAM lossy_names mode now works with TLEN of 0 or TLEN within +/- 1
+  of the computed value.  Note in these situations TLEN will be
+  generated / fixed during CRAM decode.
+
+* CRAM now supports bzip2 and lzma codecs.  Within htslib these are
+  disabled by default, but can be enabled by specifying "use_bzip2" or
+  "use_lzma" in an hts_opt_add() call or via the mode string of the
+  hts_open_format() function.
+
+Noteworthy changes in release 1.3.2  (13 September 2016)
+
+* Corrected bin calculation when converting directly from CRAM to BAM.
+  Previously a small fraction of converted reads would fail Picard's
+  validation with "bin field of BAM record does not equal value computed"
+  (SAMtools issue #574).
+
+* Plugins can now signal to HTSlib which of RTLD_LOCAL and RTLD_GLOBAL
+  they wish to be opened with -- previously they were always RTLD_LOCAL.
+
+
+Noteworthy changes in release 1.3.1  (22 April 2016)
+
+* Improved error checking and reporting, especially of I/O errors when
+  writing output files (#17, #315, PR #271, PR #317).
+
+* Build fixes for 32-bit systems; be sure to run configure to enable
+  large file support and access to 2GiB+ files.
+
+* Numerous VCF parsing fixes (#321, #322, #323, #324, #325; PR #370).
+  Particular thanks to Kostya Kortchinsky of the Google Security Team
+  for testing and numerous input parsing bug reports.
+
+* HTSlib now prints an informational message when initially creating a
+  CRAM reference cache in the default location under your $HOME directory.
+  (No message is printed if you are using $REF_CACHE to specify a location.)
+
+* Avoided rare race condition when caching downloaded CRAM reference sequence
+  files, by using distinctive names for temporary files (in addition to O_EXCL,
+  which has always been used).  Occasional corruption would previously occur
+  when multiple tools were simultaneously caching the same reference sequences
+  on an NFS filesystem that did not support O_EXCL (PR #320).
+
+* Prevented race condition in file access plugin loading (PR #341).
+
+* Fixed mpileup memory leak, so no more "[bam_plp_destroy] memory leak [...]
+  Continue anyway" warning messages (#299).
+
+* Various minor CRAM fixes.
+
+* Fixed documentation problems #348 and #358.
+
+
+Noteworthy changes in release 1.3  (15 December 2015)
+
+* Files can now be accessed via HTTPS and Amazon S3 in addition to HTTP
+  and FTP, when HTSlib is configured to use libcurl for network file access
+  rather than the included basic knetfile networking.
+
+* HTSlib can be built to use remote access hFILE backends (such as iRODS
+  and libcurl) via a plugin mechanism.  This allows other backends to be
+  easily added and facilitates building tools that use HTSlib, as they
+  don't need to be linked with the backends' various required libraries.
+
+* When writing CRAM output, sam_open() etc now default to writing CRAM v3.0
+  rather than v2.1.
+
+* fai_build() and samtools faidx now accept initial whitespace in ">"
+  headers (e.g., "> chr1 description" is taken to refer to "chr1").
+
+* tabix --only-header works again (was broken in 1.2.x; #249).
+
+* HTSlib's configure script and Makefile now fully support the standard
+  convention of allowing CC/CPPFLAGS/CFLAGS/LDFLAGS/LIBS to be overridden
+  as needed.  Previously the Makefile listened to $(LDLIBS) instead; if you
+  were overriding that, you should now override LIBS rather than LDLIBS.
+
+* Fixed bugs #168, #172, #176, #197, #206, #225, #245, #265, #295, and #296.
+
+
+Noteworthy changes in release 1.2.1  (3 February 2015)
+
+* Reinstated hts_file_type() and FT_* macros, which were available until 1.1
+  but briefly removed in 1.2.  This function is deprecated and will be removed
+  in a future release -- you should use hts_detect_format() etc instead
+
+
+Noteworthy changes in release 1.2  (2 February 2015)
+
+* HTSlib now has a configure script which checks your build environment
+  and allows for selection of optional extras.  See INSTALL for details
+
+* By default, reference sequences are fetched from the EBI CRAM Reference
+  Registry and cached in your $HOME cache directory.  This behaviour can
+  be controlled by setting REF_PATH and REF_CACHE environment variables
+  (see the samtools(1) man page for details)
+
+* Numerous CRAM improvements:
+  - Support for CRAM v3.0, an upcoming revision to CRAM supporting
+    better compression and per-container checksums
+  - EOF checking for v2.1 and v3.0 (similar to checking BAM EOF blocks)
+  - Non-standard values for PNEXT and TLEN fields are now preserved
+  - hts_set_fai_filename() now provides a reference file when encoding
+  - Generated read names are now numbered from 1, rather than being
+    labelled 'slice:record-in-slice'
+  - Multi-threading and speed improvements
+
+* New htsfile command for identifying file formats, and corresponding
+  file format detection APIs
+
+* New tabix --regions FILE, --targets FILE options for filtering via BED files
+
+* Optional iRODS file access, disabled by default.  Configure with --with-irods
+  to enable accessing iRODS data objects directly via 'irods:DATAOBJ'
+
+* All occurrences of 2^29 in the source have been eliminated, so indexing
+  and querying against reference sequences larger than 512Mbp works (when
+  using CSI indices)
+
+* Support for plain GZIP compression in various places
+
+* VCF header editing speed improvements
+
+* Added seq_nt16_int[] (equivalent to the samtools API's bam_nt16_nt4_table)
+
+* Reinstated faidx_fetch_nseq(), which was accidentally removed from 1.1.
+  Now faidx_fetch_nseq() and faidx_nseq() are equivalent; eventually
+  faidx_fetch_nseq() will be deprecated and removed [#156]
+
+* Fixed bugs #141, #152, #155, #158, #159, and various memory leaks
diff --git a/ext/htslib/README b/ext/htslib/README
new file mode 100644
index 0000000..db368af
--- /dev/null
+++ b/ext/htslib/README
@@ -0,0 +1,27 @@
+HTSlib is an implementation of a unified C library for accessing common file
+formats, such as SAM, CRAM, VCF, and BCF, used for high-throughput sequencing
+data.  It is the core library used by samtools and bcftools.
+
+See INSTALL for building and installation instructions.
+
+Please cite this paper when using HTSlib for your publications:
+
+HTSlib: C library for reading/writing high-throughput sequencing data
+James K Bonfield, John Marshall, Petr Danecek, Heng Li, Valeriu Ohan, Andrew Whitwham, Thomas Keane, Robert M Davies
+GigaScience, Volume 10, Issue 2, February 2021, giab007, https://doi.org/10.1093/gigascience/giab007
+
+@article{10.1093/gigascience/giab007,
+    author = {Bonfield, James K and Marshall, John and Danecek, Petr and Li, Heng and Ohan, Valeriu and Whitwham, Andrew and Keane, Thomas and Davies, Robert M},
+    title = "{HTSlib: C library for reading/writing high-throughput sequencing data}",
+    journal = {GigaScience},
+    volume = {10},
+    number = {2},
+    year = {2021},
+    month = {02},
+    abstract = "{Since the original publication of the VCF and SAM formats, an explosion of software tools have been created to process these data files. To facilitate this a library was produced out of the original SAMtools implementation, with a focus on performance and robustness. The file formats themselves have become international standards under the jurisdiction of the Global Alliance for Genomics and Health.We present a software library for providing programmatic access to sequencing alignment and variant formats. It was born out of the widely used SAMtools and BCFtools applications. Considerable improvements have been made to the original code plus many new features including newer access protocols, the addition of the CRAM file format, better indexing and iterators, and better use of threading.Since the original Samtools release, performance has been considerably improved, with a BAM read-write loop running 5 times faster and BAM to SAM conversion 13 times faster (both using 16 threads, compared to Samtools 0.1.19). Widespread adoption has seen HTSlib downloaded \\&gt;1 million times from GitHub and conda. The C library has been used directly by an estimated 900 GitHub projects and has been incorporated into Perl, Python, Rust, and R, significantly expanding the number of uses via other languages. HTSlib is open source and is freely available from htslib.org under MIT/BSD license.}",
+    issn = {2047-217X},
+    doi = {10.1093/gigascience/giab007},
+    url = {https://doi.org/10.1093/gigascience/giab007},
+    note = {giab007},
+    eprint = {https://academic.oup.com/gigascience/article-pdf/10/2/giab007/36332285/giab007.pdf},
+}
diff --git a/ext/htslib/README.large_positions.md b/ext/htslib/README.large_positions.md
new file mode 100644
index 0000000..3e2b2c9
--- /dev/null
+++ b/ext/htslib/README.large_positions.md
@@ -0,0 +1,234 @@
+# HTSlib 64 bit reference positions
+
+HTSlib version 1.10 onwards internally use 64 bit reference positions.  This
+is to support analysis of species like axolotl, tulip and marbled lungfish
+which have, or are expected to have,  chromosomes longer than two gigabases.
+
+# File format support
+
+Currently 64 bit positions can only be stored in SAM and VCF format files.
+Binary BAM, CRAM and BCF cannot be used due to limitations in the formats
+themselves.  As SAM and VCF are text formats, they have no limit on the
+size of numeric values. Note that while 64 bit positions are supported by
+default for SAM, for VCF they must be enabled explicitly at compile time
+by editing Makefile and adding -DVCF_ALLOW_INT64=1 to CFLAGS.
+
+# Compatibility issues to check
+
+Various data structure members, function parameters, and return values have
+been expanded from 32 to 64 bits.  As a result, some changes may be needed to
+code that uses the library, even if it does not support long references.
+
+## Variadic functions taking format strings
+
+The type of various structure members (e.g. `bam1_core_t::pos`) and return
+values from some functions (e.g. `bam_cigar2rlen()`) have been changed to
+`hts_pos_t`, which is a 64-bit signed integer.  Using these in 32-bit
+code will generally work (as long as the stored positions are within range),
+however care needs to be taken when these values are passed directly
+to functions like `printf()` which take a variable-length argument list and
+a format string.
+
+Header file `htslib/hts.h` defines macro `PRIhts_pos` which can be
+used in `printf()` format strings to get the correct format specifier for
+an `hts_pos_t` value.  Code that needs to print positions should be
+changed from:
+
+```c
+printf("Position is %d\n", bam->core.pos);
+```
+
+to:
+
+```c
+printf("Position is %"PRIhts_pos"\n", bam->core.pos);
+```
+
+If for some reason compatibility with older versions of HTSlib (which do
+not have `hts_pos_t` or `PRIhts_pos`) is needed, the value can be cast to
+`int64_t` and printed as an explicitly 64-bit value:
+
+```c
+#include <inttypes.h> // For PRId64 and int64_t
+
+printf("Position is %" PRId64 "\n", (int64_t) bam->core.pos);
+```
+
+Passing incorrect types to variadic functions like `printf()` can lead
+to incorrect behaviour and security risks, so it important to track down
+and fix all of the places where this may happen.  Modern C compilers like
+gcc (version 3.0 onwards) and clang can check `printf()` and `scanf()`
+parameter types for compatibility against the format string.  To
+enable this, build code with `-Wall` or `-Wformat` and fix all the
+reported warnings.
+
+Where functions that take `printf`-style format strings are implemented,
+they should use the appropriate gcc attributes to enable format string
+checking.  `htslib/hts_defs.h` includes macros `HTS_FORMAT` and
+`HTS_PRINTF_FMT` which can be used to provide the attribute declaration
+in a portable way.  For example, `test/sam.c` uses them for a function
+that prints error messages:
+
+```
+void HTS_FORMAT(HTS_PRINTF_FMT, 1, 2) fail(const char *fmt, ...) { /* ... */ }
+```
+
+## Implicit type conversions
+
+Conversion of signed `int` or `int32_t` to `hts_pos_t` will always work.
+
+Conversion of `hts_pos_t` to `int` or `int32_t` will work as long as the value
+converted is within the range that can be stored in the destination.
+
+Code that casts unsigned `uint32_t` values to signed with the expectation
+that the result may be negative will no longer work as `hts_pos_t` can store
+values over UINT32_MAX.  Such code should be changed to use signed values.
+
+Functions hts_parse_region() and hts_parse_reg64() return special value
+`HTS_POS_MAX` for regions which extend to the end of the reference.
+This value is slightly smaller than INT64_MAX, but should be larger than
+any reference that is likely to be used.  When cast to `int32_t` the
+result should be `INT32_MAX`.
+
+# Upgrading code to work with 64 bit positions
+
+Variables used to store reference positions should be changed to
+type `hts_pos_t`.  Use `PRIhts_pos` in format strings when printing them.
+
+When converting positions stored in strings, use `strtoll()` in place of
+`atoi()` or `strtol()` (which produces a 32 bit value on 64-bit Windows and
+all 32-bit platforms).
+
+Programs which need to look up a reference sequence length from a `sam_hdr_t`
+structure should use `sam_hdr_tid2len()` instead of the old
+`sam_hdr_t::target_len` array (which is left as 32-bit for reasons of
+compatibility).  `sam_hdr_tid2len()` returns `hts_pos_t`, so works correctly
+for large references.
+
+Various functions which take pointer arguments have new versions which
+support `hts_pos_t *` arguments.  Code supporting 64-bit positions should
+use the new versions.  These are:
+
+Original function  | 64-bit version
+------------------ | --------------------
+fai_fetch()        | fai_fetch64()
+fai_fetchqual()    | fai_fetchqual64()
+faidx_fetch_seq()  | faidx_fetch_seq64()
+faidx_fetch_qual() | faidx_fetch_qual64()
+hts_parse_reg()    | hts_parse_reg64() or hts_parse_region()
+bam_plp_auto()     | bam_plp64_auto()
+bam_plp_next()     | bam_plp64_next()
+bam_mplp_auto()    | bam_mplp64_auto()
+
+Limited support has been added for 64-bit INFO values in VCF files, for large
+values in structural variant END tags.  New functions `bcf_update_info_int64()`
+and `bcf_get_info_int64()` can be used to set and fetch 64-bit INFO values.
+They both take arrays of `int64_t`.  `bcf_int64_missing` and
+`bcf_int64_vector_end` can be used to set missing and vector end values in
+these arrays.  The INFO data is stored in the minimum size needed, so there
+is no harm in using these functions to store smaller integer values.
+
+# Structure members that have changed size
+
+```
+File htslib/hts.h:
+   hts_pair32_t::begin
+   hts_pair32_t::end
+
+   (typedef hts_pair_pos_t is provided as a better-named replacement for hts_pair32_t)
+
+   hts_reglist_t::min_beg
+   hts_reglist_t::max_end
+
+   hts_itr_t::beg
+   hts_itr_t::end
+   hts_itr_t::curr_beg
+   hts_itr_t::curr_end
+
+File htslib/regidx.h:
+   reg_t::start
+   reg_t::end
+
+File htslib/sam.h:
+   bam1_core_t::pos
+   bam1_core_t::mpos
+   bam1_core_t::isize
+
+File htslib/synced_bcf_reader.h:
+   bcf_sr_regions_t::start
+   bcf_sr_regions_t::end
+   bcf_sr_regions_t::prev_start
+
+File htslib/vcf.h:
+   bcf_idinfo_t::info
+
+   bcf_info_t::v1::i
+
+   bcf1_t::pos
+   bcf1_t::rlen
+```
+
+# Functions where parameters or the return value have changed size
+
+Functions are annotated as follows:
+
+* `[new]`  The function has been added since version 1.9
+* `[parameters]` Function parameters have changed size
+* `[return]` Function return value has changed size
+
+```
+File htslib/faidx.h:
+
+   [new]        fai_fetch64()
+   [new]        fai_fetchqual64()
+   [new]        faidx_fetch_seq64()
+   [new]        faidx_fetch_qual64()
+   [new]        fai_parse_region()
+
+File htslib/hts.h:
+
+   [parameters] hts_idx_push()
+   [new]        hts_parse_reg64()
+   [parameters] hts_itr_query()
+   [parameters] hts_reg2bin()
+
+File htslib/kstring.h:
+
+   [new]        kputll()
+
+File htslib/regidx.h:
+
+   [parameters] regidx_overlap()
+
+File htslib/sam.h:
+
+   [new]        sam_hdr_tid2len()
+   [return]     bam_cigar2qlen()
+   [return]     bam_cigar2rlen()
+   [return]     bam_endpos()
+   [parameters] bam_itr_queryi()
+   [parameters] sam_itr_queryi()
+   [new]        bam_plp64_next()
+   [new]        bam_plp64_auto()
+   [new]        bam_mplp64_auto()
+   [parameters] sam_cap_mapq()
+   [parameters] sam_prob_realn()
+
+File htslib/synced_bcf_reader.h:
+
+   [parameters] bcf_sr_seek()
+   [parameters] bcf_sr_regions_overlap()
+
+File htslib/tbx.h:
+
+   [parameters] tbx_readrec()
+
+File htslib/vcf.h:
+
+   [parameters] bcf_readrec()
+   [new]        bcf_update_info_int64()
+   [new]        bcf_get_info_int64()
+   [return]     bcf_dec_int1()
+   [return]     bcf_dec_typed_int1()
+
+```
diff --git a/ext/htslib/annot-tsv.1 b/ext/htslib/annot-tsv.1
new file mode 100644
index 0000000..3a6034b
--- /dev/null
+++ b/ext/htslib/annot-tsv.1
@@ -0,0 +1,259 @@
+'\" t
+.TH annot-tsv 1 "12 September 2024" "htslib-1.21" "Bioinformatics tools"
+.\"
+.\" Copyright (C) 2015, 2017-2018, 2023-2024 Genome Research Ltd.
+.\"
+.\" Author: Petr Danecek
+.\"
+.\" Permission is hereby granted, free of charge, to any person obtaining a
+.\" copy of this software and associated documentation files (the "Software"),
+.\" to deal in the Software without restriction, including without limitation
+.\" the rights to use, copy, modify, merge, publish, distribute, sublicense,
+.\" and/or sell copies of the Software, and to permit persons to whom the
+.\" Software is furnished to do so, subject to the following conditions:
+.\"
+.\" The above copyright notice and this permission notice shall be included in
+.\" all copies or substantial portions of the Software.
+.\"
+.\" THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+.\" IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+.\" FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+.\" THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+.\" LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+.\" FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+.\" DEALINGS IN THE SOFTWARE.
+.\"
+.
+.\" For code blocks and examples (cf groff's Ultrix-specific man macros)
+.de EX
+
+.  in +\\$1
+.  nf
+.  ft CR
+..
+.de EE
+.  ft
+.  fi
+.  in
+
+..
+.SH NAME
+annot\-tsv \- transfer annotations from one TSV (tab\-separated values) file into another
+.SH SYNOPSIS
+.PP
+.B annot-tsv
+.RI [ OPTIONS ]
+.SH DESCRIPTION
+The program finds overlaps in two sets of genomic regions (for example two CNV call sets) and annotates regions of the target file
+.RB ( \-t ", " \-\-target\-file )
+with information from overlapping regions of the source file
+.RB ( \-s ", " \-\-source\-file ).
+
+It can transfer one or multiple columns
+.RB ( \-f ", " \-\-transfer )
+and the transfer can be conditioned on requiring matching values in one or more columns
+.RB ( \-m ", " \-\-match ).
+In addition to column transfer
+.RB ( \-f )
+and special annotations
+.RB ( \-a ", " \-\-annotate ),
+the program can operate in a simple grep-like mode and print matching lines (when neither
+.B \-f
+nor
+.B \-a
+are given) or drop matching lines
+.RB ( \-x ", " \-\-drop-overlaps ).
+
+All indexes and coordinates are 1-based and inclusive.
+.SH OPTIONS
+.SS "Common Options"
+.PP
+.BR \-c ", " \-\-core " SRC:TGT"
+.RS 4
+List of names of the core columns, in the order of chromosome, start and end positions, irrespective of the header name and order in which they appear in source or target files (for example "chr,beg,end:CHROM,START,END").
+If both files use the same header names, the TGT names can be omitted (for example "chr,beg,end").
+If SRC or TGT file has no header, 1-based indexes can be given instead (for example "chr,beg,end:3,1,2").
+Note that regions are not required, the program can work with a list of positions (for example "chr,beg,end:CHROM,POS,POS").
+.RE
+.PP
+.BR \-f ", " \-\-transfer " SRC:TGT"
+.RS 4
+Comma-separated list of columns to transfer. If the SRC column does not exist, interpret it as the default value to fill in when a match is found or a dot (".") when a match is not found. If the TGT column does not exist, a new column is created. If the TGT column already exists, its values will be overwritten when overlap is found and left as is otherwise.
+.RE
+.PP
+.BR \-m ", " \-\-match " SRC:TGT"
+.RS 4
+The columns required to be identical
+.RE
+.PP
+.BR \-o ", " \-\-output " FILE"
+.RS 4
+Output file name, by default the result is printed on standard output
+.RE
+.PP
+.BR \-s ", " \-\-source\-file " FILE"
+.RS 4
+Source file with annotations to transfer
+.RE
+.PP
+.BR \-t ", " \-\-target\-file " FILE"
+.RS 4
+Target file to be extend with annotations from
+.BR \-s ", " \-\-source\-file
+.RE
+.SS "Other options"
+.PP
+.B \-\-allow\-dups
+.RS 4
+Add the same annotations multiple times if multiple overlaps are found
+.RE
+.PP
+.B \-\-help
+.RS 4
+This help message
+.RE
+.PP
+.BR \-\-max\-annots " INT"
+.RS 4
+Add at most INT annotations per column to save time when many overlaps are found with a single region
+.RE
+.PP
+.B \-\-version
+.RS 4
+Print version string and exit
+.RE
+.PP
+.BR \-a ", " \-\-annotate " LIST"
+.RS 4
+Add one or more special annotation and its target name separated by ':'. If no target name is given, the special annotation's name will be used in output header.
+.PP
+.I cnt
+.RS 4
+number of overlapping regions
+.RE
+.PP
+.I frac
+.RS 4
+fraction of the target region with an overlap
+.RE
+.PP
+.I nbp
+.RS 4
+number of source base pairs in the overlap
+.RE
+.RE
+.PP
+.BR \-d ", " \-\-delim " SRC:TGT"
+.RS 4
+Column delimiter in the source and the target file. For example, if both files are comma-delimited, run with
+"--delim ,:," or simply "--delim ,". If the source file is comma-delimited and the target file is tab-delimited,
+run with "-d $',:\\t'".
+.RE
+.PP
+.BR \-h ", " \-\-headers " SRC:TGT"
+.RS 4
+Line number of the header row with column names. By default the first line is interpreted as header if it starts with the comment
+character ("#"), otherwise expects numeric indices. However, if the first line does not start with "#" but still
+contains the column names, use "--headers 1:1". To ignore existing header (skip comment lines) and use numeric indices,
+use "--headers 0:0" which is equivalent to "--ignore-headers". When negative value is given, it is interpreted as the number of
+lines from the end of the comment block. Specifically, "--headers -1" takes the column names from the last line of
+the comment block (e.g., the "#CHROM" line in the VCF format).
+.RE
+.PP
+.BR \-H ", " \-\-ignore\-headers
+.RS 4
+Ignore the headers completely and use numeric indexes even when a header exists
+.RE
+.PP
+.BR \-I ", " \-\-no\-hdr\-idx
+.RS 4
+Suppress index numbers in the printed header. If given twice, drop the entire header.
+.RE
+.PP
+.BR \-O ", " \-\-overlap " FLOAT,[FLOAT]"
+.RS 4
+Minimum overlap as a fraction of region length in SRC and TGT, respectively (with two numbers), or in
+at least one of the overlapping regions (with a single number). If also
+.BR \-r ", " \-\-reciprocal
+is given, require at least
+.I FLOAT
+overlap with respect to both regions. Two identical numbers are equivalent to running with
+.BR \-r ", " \-\-reciprocal
+.RE
+.PP
+.BR \-r ", " \-\-reciprocal
+.RS 4
+Require the
+.BR \-O ", " \-\-overlap
+with respect to both overlapping regions
+.RE
+.PP
+.BR \-x ", " \-\-drop-overlaps
+.RS 4
+Drop overlapping regions (cannot be combined with
+.BR \-f ", " \-\-transfer )
+.RE
+.SH EXAMPLE
+
+Both SRC and TGT input files must be tab-delimited files with or without a header, their columns can be named differently, can appear in arbitrary order. For example consider the source file
+
+.EX
+#chr   beg   end   sample   type   qual
+chr1   100   200   smpl1    DEL    10
+chr1   300   400   smpl2    DUP    30
+.EE
+and the target file
+.EX
+150   200   chr1   smpl1
+150   200   chr1   smpl2
+350   400   chr1   smpl1
+350   400   chr1   smpl2
+.EE
+In the first example we transfer type and quality but only for regions with matching sample. Notice that the header is present in SRC but not in TGT, therefore we use column indexes for the latter
+.EX
+annot-tsv -s src.txt.gz -t tgt.txt.gz -c chr,beg,end:3,1,2 -m sample:4 -f type,qual
+150   200   chr1   smpl1   DEL   10
+150   200   chr1   smpl2   .     .
+350   400   chr1   smpl1   .     .
+350   400   chr1   smpl2   DUP   30
+.EE
+The next example demonstrates the special annotations nbp and cnt,
+with target name as pair,count.
+In this case we use a target file with headers so that column names will
+be copied to the output:
+.EX
+#from	to	chrom	sample
+150	200	chr1	smpl1
+150	200	chr1	smpl2
+350	400	chr1	smpl1
+350	400	chr1	smpl2
+.EE
+
+.EX
+annot-tsv -s src.txt.gz -t tgt_hdr.txt.gz -c chr,beg,end:chrom,from,to -m sample -f type,qual -a nbp,cnt:pair,count
+#[1]from	[2]to	[3]chrom	[4]sample	[5]type	[6]qual	[7]pair	[8]count
+150	200	chr1	smpl1	DEL	10	51	1
+150	200	chr1	smpl2	.	.	0	0
+350	400	chr1	smpl1	.	.	0	0
+350	400	chr1	smpl2	DUP	30	51	1
+.EE
+One of the SRC or TGT file can be streamed from stdin
+.EX
+cat src.txt | annot\-tsv \-t tgt.txt \-c chr,beg,end:3,2,1 \-m sample:4 \-f type,qual \-o output.txt
+cat tgt.txt | annot\-tsv \-s src.txt \-c chr,beg,end:3,2,1 \-m sample:4 \-f type,qual \-o output.txt
+.EE
+
+The program can be used in a grep-like mode to print only matching regions of the target file without modifying the records
+
+.EX
+annot\-tsv \-s src.txt \-t tgt.txt \-c chr,beg,end:3,2,1 \-m sample:4
+150   200   chr1   smpl1
+350   400   chr1   smpl2
+.EE
+
+.SH AUTHORS
+The program was written by Petr Danecek and was originally published on github as annot\-regs
+.SH COPYING
+The MIT/Expat License, see the LICENSE document for details.
+.br
+Copyright (c) Genome Research Ltd.
diff --git a/ext/htslib/annot-tsv.c b/ext/htslib/annot-tsv.c
new file mode 100644
index 0000000..494c437
--- /dev/null
+++ b/ext/htslib/annot-tsv.c
@@ -0,0 +1,1038 @@
+/*
+    Copyright (C) 2018-2024 Genome Research Ltd.
+
+    Author: Petr Danecek <pd3@sanger.ac.uk>
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in
+    all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+    DEALINGS IN THE SOFTWARE.
+*/
+
+/*
+    Note: The original code comes from https://github.com/pd3/utils/tree/master/annot-regs.
+    In annot-tsv we changed the naming from "destination" file to "target" file, however
+    the code still internally uses the original naming convention.
+*/
+
+#include <config.h>
+
+#include <unistd.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <getopt.h>
+#include <assert.h>
+#include <string.h>
+#include <strings.h>
+#include "htslib/hts.h"
+#include "htslib/hts_defs.h"
+#include "htslib/khash_str2int.h"
+#include "htslib/kstring.h"
+#include "htslib/kseq.h"
+#include "htslib/bgzf.h"
+#include "htslib/regidx.h"
+#include "textutils_internal.h"
+
+#define ANN_NBP     1
+#define ANN_FRAC    2
+#define ANN_CNT     4
+
+typedef struct
+{
+    uint32_t n,m;
+    char **off, *rmme;
+}
+cols_t;
+
+typedef struct
+{
+    void *name2idx;
+    cols_t *cols, *annots;
+    int dummy;
+}
+hdr_t;
+
+typedef struct
+{
+    char *fname;
+    hdr_t hdr;
+    cols_t *core, *match, *transfer, *annots;
+    int *core_idx, *match_idx, *transfer_idx, *annots_idx;
+    int *nannots_added; // for --max-annots: the number of annotations added
+    char delim;
+    int grow_n;
+    kstring_t line;     // one buffered line, a byproduct of reading the header
+    htsFile *fp;
+}
+dat_t;
+
+// This is for the special -a annotations, keeps a list of
+// source regions that hit the destination region. The start
+// coordinates are converted to beg<<1 and end coordinates
+// to (end<<1)+1.
+#define NBP_SET_BEG(x) ((x)<<1)
+#define NBP_SET_END(x) (((x)<<1)+1)
+#define NBP_GET(x)     ((x)>>1)
+#define NBP_IS_BEG(x)  (((x)&1)==0)
+#define NBP_IS_END(x)  (((x)&1)==1)
+typedef struct
+{
+    size_t n,m;          // n is a multiple of two: breakpoints are stored in regs, not regions
+    hts_pos_t *regs;
+    hts_pos_t beg,end;   // the current destination interval
+}
+nbp_t;
+
+#define PRINT_MATCHING    1
+#define PRINT_NONMATCHING 2
+typedef struct
+{
+    nbp_t *nbp;
+    dat_t dst, src;
+    char *core_str, *match_str, *transfer_str, *annots_str, *headers_str, *delim_str;
+    char *temp_dir, *out_fname;
+    BGZF *out_fp;
+    int allow_dups, max_annots, mode, no_write_hdr, overlap_either;
+    double overlap_src, overlap_dst;
+    regidx_t *idx;
+    regitr_t *itr;
+    kstring_t tmp_kstr;
+    cols_t *tmp_cols;               // the -t transfer fields to write for each line
+    khash_t(str2int) **tmp_hash;    // lookup tables for tmp_cols
+}
+args_t;
+
+static void HTS_FORMAT(HTS_PRINTF_FMT, 1, 2) HTS_NORETURN
+error(const char *format, ...)
+{
+    va_list ap;
+    fflush(stdout);
+    va_start(ap, format);
+    vfprintf(stderr, format, ap);
+    va_end(ap);
+    fflush(stderr);
+    exit(EXIT_FAILURE);
+}
+
+static nbp_t *nbp_init(void)
+{
+    nbp_t *nbp = calloc(1,sizeof(nbp_t));
+    if ( !nbp ) error("Out of memory, failed to allocate %zu bytes\n",sizeof(nbp_t));
+    return nbp;
+}
+static void nbp_destroy(nbp_t *nbp)
+{
+    free(nbp->regs);
+    free(nbp);
+}
+static inline void nbp_reset(nbp_t *nbp, hts_pos_t beg, hts_pos_t end)
+{
+    nbp->n   = 0;
+    nbp->beg = beg;
+    nbp->end = end;
+}
+static inline void nbp_add(nbp_t *nbp, hts_pos_t beg, hts_pos_t end)
+{
+    nbp->n += 2;
+    if ( nbp->n >= nbp->m )
+    {
+        nbp->m += 2;
+        nbp->regs = realloc(nbp->regs, nbp->m*sizeof(*nbp->regs));
+        if ( !nbp->regs ) error("Out of memory, failed to allocate %zu bytes\n",nbp->m*sizeof(*nbp->regs));
+    }
+    nbp->regs[nbp->n - 2] = NBP_SET_BEG(beg);
+    nbp->regs[nbp->n - 1] = NBP_SET_END(end);
+}
+static int compare_hts_pos(const void *aptr, const void *bptr)
+{
+    hts_pos_t a = *(const hts_pos_t*) aptr;
+    hts_pos_t b = *(const hts_pos_t*) bptr;
+    if (a < b) return -1;
+    if (a > b) return 1;
+    return 0;
+}
+static hts_pos_t nbp_length(nbp_t *nbp)
+{
+    qsort(nbp->regs, nbp->n, sizeof(*nbp->regs), compare_hts_pos);
+    int i, nopen = 0;
+    hts_pos_t beg = 0, length = 0;
+    for (i=0; i<nbp->n; i++)
+    {
+        if ( NBP_IS_BEG(nbp->regs[i]) )
+        {
+            if ( !nopen ) beg = NBP_GET(nbp->regs[i]);
+            nopen++;
+        }
+        else nopen--;
+        assert( nopen>=0 );
+        if ( nopen==0 && beg>0 ) length += NBP_GET(nbp->regs[i]) - beg + 1;
+    }
+    return length;
+}
+
+cols_t *cols_split(const char *line, cols_t *cols, char delim)
+{
+    if ( !cols ) cols = (cols_t*) calloc(1,sizeof(cols_t));
+    if ( !cols ) error("Out of memory, failed to allocate %zu bytes\n",sizeof(cols_t));
+    if ( cols->rmme ) free(cols->rmme);
+    cols->n = 0;
+    cols->rmme = strdup(line);
+    if ( !cols->rmme ) error("Out of memory\n");
+    char *ss = cols->rmme;
+    while (1)
+    {
+        char *se = ss;
+        while ( *se && *se!=delim ) se++;
+        char tmp = *se;
+        *se = 0;
+        cols->n++;
+        if ( cols->n > cols->m )
+        {
+            cols->m += 10;
+            cols->off = realloc(cols->off, sizeof(*cols->off)*cols->m);
+            if ( !cols->off ) error("Out of memory, failed to allocate %zu bytes\n",sizeof(*cols->off)*cols->m);
+        }
+        cols->off[ cols->n - 1 ] = ss;
+        if ( !tmp ) break;
+        ss = se + 1;
+    }
+    return cols;
+}
+// Can be combined with cols_split() but is much slower.
+// The string must exist throughout the life of cols unless initialized with cols_split().
+void cols_append(cols_t *cols, char *str)
+{
+    if ( cols->rmme )
+    {
+        size_t str_len = strlen(str);
+        size_t lst_len = strlen(cols->off[ cols->n - 1 ]);
+        size_t tot_len = 2 + str_len + lst_len + (cols->off[ cols->n - 1 ] - cols->rmme);
+
+        cols_t *tmp_cols = (cols_t*)calloc(1,sizeof(cols_t));
+        if ( !tmp_cols ) error("Out of memory, failed to allocate %zu bytes\n",sizeof(cols_t));
+        tmp_cols->rmme = calloc(tot_len,1);
+        tmp_cols->off  = calloc(cols->n+1,sizeof(*tmp_cols->off));
+        if ( !tmp_cols->rmme || !tmp_cols->off ) error("Out of memory\n");
+
+        char *ptr = tmp_cols->rmme;
+        int i;
+        for (i=0; i<cols->n; i++)
+        {
+            size_t len = strlen(cols->off[i]);
+            memcpy(ptr, cols->off[i], len);
+            tmp_cols->off[i] = ptr;
+            ptr += len + 1;
+        }
+        memcpy(ptr, str, str_len);
+        tmp_cols->off[i] = ptr;
+
+        free(cols->off);
+        free(cols->rmme);
+        cols->rmme = tmp_cols->rmme;
+        cols->off  = tmp_cols->off;
+        cols->n    = cols->n+1;
+        cols->m    = cols->n;
+        free(tmp_cols);
+        return;
+    }
+    cols->n++;
+    if ( cols->n > cols->m )
+    {
+        cols->m++;
+        cols->off = realloc(cols->off,sizeof(*cols->off)*cols->m);
+        if ( !cols->off ) error("Out of memory, failed to allocate %zu bytes\n",sizeof(*cols->off)*cols->m);
+    }
+    cols->off[cols->n-1] = str;
+}
+void cols_clear(cols_t *cols)
+{
+    if ( !cols ) return;
+    free(cols->rmme);
+    free(cols->off);
+    cols->rmme = NULL;
+    cols->off  = NULL;
+}
+void cols_destroy(cols_t *cols)
+{
+    if ( !cols ) return;
+    cols_clear(cols);
+    free(cols);
+}
+
+int parse_tab_with_payload(const char *line, char **chr_beg, char **chr_end, hts_pos_t *beg, hts_pos_t *end, void *payload, void *usr)
+{
+    static int beg_end_warned = 0;
+
+    if ( line[0]=='#' )
+    {
+        *((cols_t**)payload) = NULL;
+        return -1;
+    }
+
+    dat_t *dat = (dat_t*) usr;
+
+    cols_t *cols = cols_split(line, NULL, dat->delim);
+    *((cols_t**)payload) = cols;
+
+    if ( cols->n < dat->core_idx[0] ) error("Expected at least %d columns, found %d: %s\n",dat->core_idx[0]+1,cols->n,line);
+    *chr_beg = cols->off[ dat->core_idx[0] ];
+    *chr_end = *chr_beg + strlen(*chr_beg) - 1;
+
+    if ( cols->n < dat->core_idx[1] ) error("Expected at least %d columns, found %d: %s\n",dat->core_idx[1]+1,cols->n,line);
+    char *tmp, *ptr = cols->off[ dat->core_idx[1] ];
+    *beg = strtod(ptr, &tmp);
+    if ( tmp==ptr ) error("Expected numeric value, found \"%s\": %s\n",ptr,line);
+
+    if ( cols->n < dat->core_idx[2] ) error("Expected at least %d columns, found %d: %s\n",dat->core_idx[2]+1,cols->n,line);
+    ptr = cols->off[ dat->core_idx[2] ];
+    *end = strtod(ptr, &tmp);
+    if ( tmp==ptr ) error("Expected numeric value, found \"%s\": %s\n",ptr,line);
+
+    if ( *end < *beg )
+    {
+        if ( !beg_end_warned )
+            fprintf(stderr,"Warning: the start coordinate is bigger than the end coordinate:\n\t%s\nThis message is printed only once.\n",line);
+        beg_end_warned = 1;
+        hts_pos_t tmp = *beg; *beg = *end; *end = tmp;
+    }
+
+    return 0;
+}
+void free_payload(void *payload)
+{
+    cols_t *cols = *((cols_t**)payload);
+    cols_destroy(cols);
+}
+
+// Parse header if present, the parameter irow indicates the header row line number:
+//      0   .. ignore headers, create numeric fields names, 1-based indices
+//      N>0 .. N-th line, all previous lines are discarded
+//      N<0 .. N-th line from the end of the comment block (comment lines are prefixed with #),
+//             all preceding lines are discarded.
+// When autodetect is set, the argument nth_row is ignored.
+// Note this makes no attempt to preserve comment lines on output
+void parse_header(dat_t *dat, char *fname, int nth_row, int autodetect)
+{
+    dat->fp = hts_open(fname,"r");
+    if ( !dat->fp ) error("Failed to open: %s\n", fname);
+
+    // buffer comment lines when N<0
+    int nbuf = 0;
+    char **buf = NULL;
+    if ( nth_row < 0 )
+    {
+        buf = calloc(-nth_row,sizeof(*buf));
+        if ( !buf ) error("Out of memory, failed to allocate %zu bytes\n",(-nth_row)*sizeof(*buf));
+    }
+
+    int irow = 0;
+    cols_t *cols = NULL;
+    while ( hts_getline(dat->fp, KS_SEP_LINE, &dat->line) > 0 )
+    {
+        if ( autodetect )
+        {
+            // if the first line is comment line, use it as a header. Otherwise go
+            // with numeric indices
+            nth_row = dat->line.s[0]=='#' ? 1 : 0;
+            break;
+        }
+        if ( nth_row==0 )
+        {
+            // N=0 .. comment lines to be ignored, read until we get to the first data line
+            if ( dat->line.s[0]=='#' ) continue;
+            break;
+        }
+        if ( nth_row>0 )
+        {
+            // N>1 .. regardless of this being a comment or data line, read until Nth line
+            if ( ++irow < nth_row ) continue;
+            break;
+        }
+        // N<0 .. keep abs(N) comment lines in a sliding buffer
+        if ( dat->line.s[0]!='#' ) break;   // data line
+        if ( nbuf == -nth_row )
+        {
+            // one more comment line and the buffer is full. We could use round buffer
+            // for efficiency, but the assumption is abs(nth_row) is small
+            free(buf[0]);
+            memmove(buf, &buf[1], (nbuf-1)*sizeof(*buf));
+            nbuf--;
+        }
+        buf[nbuf++] = strdup(dat->line.s);
+    }
+
+    int keep_line = 0;
+    if ( nth_row < 0 )
+    {
+        if ( nbuf!=-nth_row )
+            error("Found %d header lines in %s, cannot fetch N=%d from the end\n",nbuf,fname,-nth_row);
+        cols = cols_split(buf[0], NULL, dat->delim);
+        keep_line = 1;
+    }
+    else
+        cols = cols_split(dat->line.s, NULL, dat->delim);
+
+    if ( !dat->line.l ) error("Failed to read: %s\n", fname);
+    assert(cols && cols->n);
+
+    if ( nth_row == 0 ) // create numeric indices
+    {
+        // create a dummy header with numeric field names
+        kstring_t str = {0,0,0};
+        int i, n = cols->n;
+        for (i=0; i<n; i++)
+        {
+            if ( i>0 ) kputc(dat->delim, &str);
+            kputw(i+1, &str);
+        }
+        cols_destroy(cols);
+        cols = cols_split(str.s, NULL, dat->delim);
+        free(str.s);
+        dat->hdr.dummy = 1;
+        keep_line = 1;
+    }
+
+    dat->hdr.name2idx = khash_str2int_init();
+    int i;
+    for (i=0; i<cols->n; i++)
+    {
+        char *ss = cols->off[i];
+        while ( *ss && (*ss=='#' || isspace_c(*ss)) ) ss++;
+        if ( !*ss ) error("Could not parse the header field \"%s\": %s\n", cols->off[i],dat->line.s);
+        if ( *ss=='[' )
+        {
+            char *se = ss+1;
+            while ( *se && isdigit_c(*se) ) se++;
+            if ( *se==']' ) ss = se + 1;
+        }
+        while ( *ss && (*ss=='#' || isspace_c(*ss)) ) ss++;
+        if ( !*ss ) error("Could not parse the header field \"%s\": %s\n", cols->off[i],dat->line.s);
+        cols->off[i] = ss;
+        khash_str2int_set(dat->hdr.name2idx, cols->off[i], i);
+    }
+    dat->hdr.cols = cols;
+    if ( !keep_line ) dat->line.l = 0;
+
+    for (i=0; i<nbuf; i++) free(buf[i]);
+    free(buf);
+}
+void write_header(args_t *args, dat_t *dat)
+{
+    if ( dat->hdr.dummy ) return;
+    if ( args->no_write_hdr>1 ) return;
+    int i;
+    kstring_t str = {0,0,0};
+    kputc('#', &str);
+    for (i=0; i<dat->hdr.cols->n; i++)
+    {
+        if ( i>0 ) kputc(dat->delim, &str);
+        if ( !args->no_write_hdr ) ksprintf(&str,"[%d]", i+1);
+        kputs(dat->hdr.cols->off[i], &str);
+    }
+    if ( dat->hdr.annots )
+    {
+        for (i=0; i<dat->hdr.annots->n; i++)
+        {
+            if ( str.l > 1 ) kputc(dat->delim, &str);
+            kputs(dat->hdr.annots->off[i], &str);
+        }
+    }
+    kputc('\n',&str);
+    if ( bgzf_write(args->out_fp, str.s, str.l) != str.l ) error("Failed to write %zd bytes\n", str.l);
+    free(str.s);
+}
+void destroy_header(dat_t *dat)
+{
+    if ( dat->hdr.cols ) cols_destroy(dat->hdr.cols);
+    khash_str2int_destroy(dat->hdr.name2idx);
+}
+
+static int read_next_line(dat_t *dat)
+{
+    if ( dat->line.l ) return dat->line.l;
+    int ret = hts_getline(dat->fp, KS_SEP_LINE, &dat->line);
+    if ( ret > 0 ) return dat->line.l;
+    if ( ret < -1 ) error("Error encountered while reading %s\n",dat->fname);
+    return 0;
+}
+
+void sanity_check_columns(char *fname, hdr_t *hdr, cols_t *cols, int **col2idx, int force)
+{
+    *col2idx = (int*)malloc(sizeof(int)*cols->n);
+    if ( !*col2idx ) error("Out of memory, failed to allocate %zu bytes\n",sizeof(int)*cols->n);
+    int i, idx;
+    for (i=0; i<cols->n; i++)
+    {
+        if ( khash_str2int_get(hdr->name2idx, cols->off[i], &idx) < 0 )
+        {
+            if ( !force ) error("The key \"%s\" not found in %s\n", cols->off[i],fname);
+            idx = -1;
+        }
+        (*col2idx)[i] = idx;
+    }
+}
+void init_data(args_t *args)
+{
+    if ( !args->delim_str )
+        args->dst.delim = args->src.delim = '\t';
+    else if ( strlen(args->delim_str)==1 )
+        args->dst.delim = args->src.delim = *args->delim_str;
+    else if ( strlen(args->delim_str)==3 && args->delim_str[1]==':' )
+        args->src.delim = args->delim_str[0], args->dst.delim = args->delim_str[2];
+    else
+        error("Could not parse the option --delim %s\n",args->delim_str);
+
+    // --headers, determine header row index
+    int isrc = 0, idst = 0, autodetect = 1;
+    if ( args->headers_str )
+    {
+        cols_t *tmp = cols_split(args->headers_str, NULL, ':');
+        char *rmme;
+        isrc = strtol(tmp->off[0],&rmme,10);
+        if ( *rmme || tmp->off[0]==rmme ) error("Could not parse the option --headers %s\n",args->headers_str);
+        idst = strtol(tmp->n==2 ? tmp->off[1] : tmp->off[0],&rmme,10);
+        if ( *rmme || (tmp->n==2 ? tmp->off[1] : tmp->off[0])==rmme ) error("Could not parse the option --headers %s\n",args->headers_str);
+        cols_destroy(tmp);
+        autodetect = 0;
+    }
+    parse_header(&args->dst, args->dst.fname, idst, autodetect);
+    parse_header(&args->src, args->src.fname, isrc, autodetect);
+
+    // -c, core columns
+    if ( !args->core_str ) args->core_str = "chr,beg,end:chr,beg,end";
+    cols_t *tmp = cols_split(args->core_str, NULL, ':');
+    args->src.core = cols_split(tmp->off[0],NULL,',');
+    args->dst.core = cols_split(tmp->n==2 ? tmp->off[1] : tmp->off[0],NULL,',');
+    sanity_check_columns(args->src.fname, &args->src.hdr, args->src.core, &args->src.core_idx, 0);
+    sanity_check_columns(args->dst.fname, &args->dst.hdr, args->dst.core, &args->dst.core_idx, 0);
+    if ( args->src.core->n!=3 || args->dst.core->n!=3 ) error("Expected three columns: %s\n", args->core_str);
+    cols_destroy(tmp);
+
+    // -m, match columns
+    if ( args->match_str )
+    {
+        tmp = cols_split(args->match_str, NULL, ':');
+        args->src.match = cols_split(tmp->off[0],NULL,',');
+        args->dst.match = cols_split(tmp->n==2 ? tmp->off[1] : tmp->off[0],NULL,',');
+        sanity_check_columns(args->src.fname, &args->src.hdr, args->src.match, &args->src.match_idx, 0);
+        sanity_check_columns(args->dst.fname, &args->dst.hdr, args->dst.match, &args->dst.match_idx, 0);
+        if ( args->src.match->n != args->dst.match->n ) error("Expected equal number of columns: %s\n", args->match_str);
+        cols_destroy(tmp);
+    }
+
+    // -t, transfer columns
+    int i;
+    if ( args->transfer_str )
+    {
+        tmp = cols_split(args->transfer_str, NULL, ':');
+        args->src.transfer = cols_split(tmp->off[0],NULL,',');
+        args->dst.transfer = cols_split(tmp->n==2 ? tmp->off[1] : tmp->off[0],NULL,',');
+        sanity_check_columns(args->src.fname, &args->src.hdr, args->src.transfer, &args->src.transfer_idx, 1);
+        sanity_check_columns(args->dst.fname, &args->dst.hdr, args->dst.transfer, &args->dst.transfer_idx, 1);
+        if ( args->src.transfer->n != args->dst.transfer->n ) error("Expected equal number of columns: %s\n", args->transfer_str);
+        for (i=0; i<args->src.transfer->n; i++)
+        {
+            if ( args->src.transfer_idx[i]==-1 )
+            {
+                cols_append(args->src.hdr.cols,args->src.transfer->off[i]);
+                args->src.transfer_idx[i] = -args->src.hdr.cols->n;    // negative index indicates different ptr location
+                args->src.grow_n++;
+            }
+        }
+        for (i=0; i<args->dst.transfer->n; i++)
+        {
+            if ( args->dst.transfer_idx[i]==-1 )
+            {
+                cols_append(args->dst.hdr.cols,args->dst.transfer->off[i]);
+                args->dst.transfer_idx[i] = args->dst.hdr.cols->n - 1;
+                args->dst.grow_n++;
+            }
+        }
+        args->tmp_cols = (cols_t*)calloc(args->src.transfer->n,sizeof(cols_t));
+        args->tmp_hash = (khash_t(str2int)**)calloc(args->src.transfer->n,sizeof(khash_t(str2int)*));
+        if ( !args->tmp_cols || !args->tmp_hash ) error("Out of memory\n");
+        for (i=0; i<args->src.transfer->n; i++)
+            args->tmp_hash[i] = khash_str2int_init();
+        cols_destroy(tmp);
+    }
+    else
+    {
+        args->src.transfer = calloc(1,sizeof(*args->src.transfer));
+        if ( !args->src.transfer ) error("Out of memory\n");
+    }
+    args->src.nannots_added = calloc(args->src.transfer->n,sizeof(*args->src.nannots_added));
+    if ( !args->src.nannots_added ) error("Out of memory\n");
+
+    // -a, annotation columns
+    if ( args->annots_str )
+    {
+        tmp = cols_split(args->annots_str, NULL, ':');
+        args->src.annots = cols_split(tmp->off[0],NULL,',');
+        args->dst.annots = cols_split(tmp->n==2 ? tmp->off[1] : tmp->off[0],NULL,',');
+        if ( args->src.annots->n!=args->dst.annots->n ) error("Different number of src and dst columns in %s\n",args->annots_str);
+        args->dst.annots_idx = (int*) malloc(sizeof(int)*args->dst.annots->n);
+        if ( !args->dst.annots_idx ) error("Out of memory\n");
+        for (i=0; i<args->src.annots->n; i++)
+        {
+            if ( !strcasecmp(args->src.annots->off[i],"nbp") )
+            {
+                args->dst.annots_idx[i] = ANN_NBP;
+                cols_append(args->dst.hdr.cols,tmp->n==2?args->dst.annots->off[i]:"nbp");
+            }
+            else if ( !strcasecmp(args->src.annots->off[i],"frac") )
+            {
+                args->dst.annots_idx[i] = ANN_FRAC;
+                cols_append(args->dst.hdr.cols,tmp->n==2?args->dst.annots->off[i]:"frac");
+            }
+            else if ( !strcasecmp(args->src.annots->off[i],"cnt") )
+            {
+                args->dst.annots_idx[i] = ANN_CNT;
+                cols_append(args->dst.hdr.cols,tmp->n==2?args->dst.annots->off[i]:"cnt");
+            }
+            else error("The annotation \"%s\" is not recognised\n", args->src.annots->off[i]);
+        }
+        args->nbp = nbp_init();
+        cols_destroy(tmp);
+    }
+
+    args->idx = regidx_init(NULL, parse_tab_with_payload,free_payload,sizeof(cols_t),&args->src);
+    while ( read_next_line(&args->src) )
+    {
+        if ( regidx_insert(args->idx,args->src.line.s) !=0 ) error("Could not parse the region in %s: %s\n",args->src.fname,args->src.line.s);
+        args->src.line.l = 0;
+    }
+    args->itr = regitr_init(args->idx);
+    if ( hts_close(args->src.fp)!=0 ) error("Failed to close: %s\n", args->src.fname);
+
+    int len = args->out_fname ? strlen(args->out_fname) : 0;
+    if ( len )
+    {
+        int compress_output = 0;
+        if ( !strcasecmp(".gz",args->out_fname+len-3) || !strcasecmp(".bgz",args->out_fname+len-4) ) compress_output = 1;
+        args->out_fp = bgzf_open(args->out_fname, compress_output ? "wg" : "wu");
+    }
+    else
+        args->out_fp = bgzf_open("-","wu");
+    if ( !args->out_fp ) error("Could not open file for writing: %s\n",args->out_fname?args->out_fname:"stdout");
+}
+void destroy_data(args_t *args)
+{
+    if ( bgzf_close(args->out_fp)!=0 ) error("Failed to close: %s\n", args->out_fname?args->out_fname:"stdout");
+    if ( hts_close(args->dst.fp)!=0 ) error("Failed to close: %s\n", args->dst.fname);
+    int i;
+    for (i=0; i<args->src.transfer->n; i++)
+        khash_str2int_destroy(args->tmp_hash[i]);
+    free(args->tmp_hash);
+    for (i=0; i<args->src.transfer->n; i++) cols_clear(&args->tmp_cols[i]);
+    free(args->tmp_cols);
+    cols_destroy(args->src.core);
+    cols_destroy(args->dst.core);
+    cols_destroy(args->src.match);
+    cols_destroy(args->dst.match);
+    cols_destroy(args->src.transfer);
+    cols_destroy(args->dst.transfer);
+    if ( args->src.annots ) cols_destroy(args->src.annots);
+    if ( args->dst.annots ) cols_destroy(args->dst.annots);
+    if ( args->nbp ) nbp_destroy(args->nbp);
+    destroy_header(&args->src);
+    destroy_header(&args->dst);
+    free(args->src.nannots_added);
+    free(args->src.core_idx);
+    free(args->dst.core_idx);
+    free(args->src.match_idx);
+    free(args->dst.match_idx);
+    free(args->src.transfer_idx);
+    free(args->dst.transfer_idx);
+    free(args->src.annots_idx);
+    free(args->dst.annots_idx);
+    free(args->src.line.s);
+    free(args->dst.line.s);
+    if (args->itr) regitr_destroy(args->itr);
+    if (args->idx) regidx_destroy(args->idx);
+    free(args->tmp_kstr.s);
+}
+
+static inline void write_string(args_t *args, char *str, size_t len)
+{
+    if ( len==0 ) len = strlen(str);
+    if ( len==0 ) str = ".", len = 1;
+    if ( bgzf_write(args->out_fp, str, len) != len ) error("Failed to write %zd bytes\n", len);
+}
+static void write_annots(args_t *args)
+{
+    if ( !args->dst.annots ) return;
+
+    args->tmp_kstr.l = 0;
+    int i;
+    hts_pos_t len = nbp_length(args->nbp);
+    for (i=0; i<args->dst.annots->n; i++)
+    {
+        if ( args->dst.annots_idx[i]==ANN_NBP )
+        {
+            kputc(args->dst.delim,&args->tmp_kstr);
+            kputw(len,&args->tmp_kstr);
+        }
+        else if ( args->dst.annots_idx[i]==ANN_FRAC )
+        {
+            kputc(args->dst.delim,&args->tmp_kstr);
+            kputd((double)len/(args->nbp->end - args->nbp->beg + 1),&args->tmp_kstr);
+        }
+        else if ( args->dst.annots_idx[i]==ANN_CNT )
+        {
+            kputc(args->dst.delim,&args->tmp_kstr);
+            kputw(args->nbp->n/2,&args->tmp_kstr);
+        }
+    }
+    write_string(args, args->tmp_kstr.s, args->tmp_kstr.l);
+}
+
+void process_line(args_t *args, char *line, size_t size)
+{
+    char *chr_beg, *chr_end;
+    hts_pos_t beg, end;
+    cols_t *dst_cols = NULL;
+    int i,j;
+    int ret = parse_tab_with_payload(line, &chr_beg, &chr_end, &beg, &end, &dst_cols, &args->dst);
+    if ( ret==-1 )
+    {
+        cols_destroy(dst_cols);
+        return;
+    }
+
+    if ( args->nbp ) nbp_reset(args->nbp,beg,end);
+
+    if ( !regidx_overlap(args->idx, chr_beg,beg,end, args->itr) )
+    {
+        if ( args->mode & PRINT_NONMATCHING )
+        {
+            write_string(args, line, size);
+            write_annots(args);
+            write_string(args, "\n", 1);
+        }
+        cols_destroy(dst_cols);
+        return;
+    }
+
+    for (i=0; i<args->src.transfer->n; i++)
+    {
+        args->src.nannots_added[i] = 0;
+        args->tmp_cols[i].n = 0;
+        kh_clear(str2int, args->tmp_hash[i]);
+    }
+
+    int has_match = 0, annot_len = 0;
+    while ( regitr_overlap(args->itr) )
+    {
+        if ( args->overlap_src || args->overlap_dst )
+        {
+            double len_dst = end - beg + 1;
+            double len_src = args->itr->end - args->itr->beg + 1;
+            double isec = (args->itr->end < end ? args->itr->end : end) - (args->itr->beg > beg ? args->itr->beg : beg) + 1;
+            int pass_dst = isec/len_dst < args->overlap_dst ? 0 : 1;
+            int pass_src = isec/len_src < args->overlap_src ? 0 : 1;
+            if ( args->overlap_either )
+            {
+                if ( !pass_dst && !pass_src ) continue;
+            }
+            else
+            {
+                if ( !pass_dst || !pass_src ) continue;
+            }
+        }
+        cols_t *src_cols = regitr_payload(args->itr,cols_t*);
+        if ( args->dst.match && args->dst.match->n )
+        {
+            for (i=0; i<args->dst.match->n; i++)
+            {
+                if ( args->dst.match_idx[i] > dst_cols->n ) error("Expected at least %d columns, found %d: %s\n",args->dst.match_idx[i],dst_cols->n,line);
+                char *dst = dst_cols->off[ args->dst.match_idx[i] ];
+                char *src = src_cols->off[ args->src.match_idx[i] ];
+                if ( strcmp(dst,src) ) break;
+            }
+            if ( i != args->dst.match->n ) continue;
+        }
+        has_match = 1;
+
+        if ( args->nbp )
+            nbp_add(args->nbp, args->itr->beg >= beg ? args->itr->beg : beg, args->itr->end <= end ? args->itr->end : end);
+
+        int max_annots_reached = 0;
+        for (i=0; i<args->src.transfer->n; i++)
+        {
+            char *str;
+            if ( args->src.transfer_idx[i] >= 0 )
+                str = src_cols->off[ args->src.transfer_idx[i] ];                   // transfer a value from the src file
+            else
+                str = args->src.hdr.cols->off[ -args->src.transfer_idx[i] - 1 ];    // non-existent field in src, use a default value
+
+            if ( !str || !*str ) str = ".";     // provide a nill dot value when the field is empty
+
+            if ( !args->allow_dups )
+            {
+                if ( khash_str2int_has_key(args->tmp_hash[i],str) ) continue;
+                khash_str2int_set(args->tmp_hash[i],str,1);
+            }
+            if ( args->max_annots )
+            {
+                if ( ++args->src.nannots_added[i] >= args->max_annots ) max_annots_reached = 1;
+            }
+            cols_append(&args->tmp_cols[i], str);
+            annot_len += strlen(str);
+        }
+        if ( max_annots_reached ) break;
+    }
+
+    if ( !has_match )
+    {
+        if ( args->mode & PRINT_NONMATCHING )
+        {
+            write_string(args, line, size);
+            write_annots(args);
+            write_string(args, "\n", 1);
+        }
+        cols_destroy(dst_cols);
+        return;
+    }
+    if ( !(args->mode & PRINT_MATCHING) )
+    {
+        cols_destroy(dst_cols);
+        return;
+    }
+
+    size_t len;
+    args->tmp_kstr.l = 0;
+    ks_resize(&args->tmp_kstr, annot_len*3 + args->src.transfer->n*2);
+    for (i=0; i<args->src.transfer->n; i++)
+    {
+        char *off = dst_cols->off[ args->dst.transfer_idx[i] ] = args->tmp_kstr.s + args->tmp_kstr.l;
+        cols_t *ann = &args->tmp_cols[i];
+        if ( !ann->n ) { off[0] = '.'; off[1] = 0; args->tmp_kstr.l += 2; continue; }
+        for (j=0; j<ann->n; j++)
+        {
+            if ( j>0 ) { off[0] = ','; off++; args->tmp_kstr.l++; }
+            len = strlen(ann->off[j]);
+            memcpy(off, ann->off[j], len);
+            off += len;
+            args->tmp_kstr.l += len;
+        }
+        off[0] = 0;
+        args->tmp_kstr.l++;
+    }
+    write_string(args, dst_cols->off[0], 0);
+    for (i=1; i<dst_cols->n; i++)
+    {
+        write_string(args, &args->dst.delim, 1);
+        write_string(args, dst_cols->off[i], 0);
+    }
+    write_annots(args);
+    write_string(args, "\n", 1);
+    cols_destroy(dst_cols);
+}
+
+static const char *usage_text(void)
+{
+    return
+        "About: Annotate regions of the target file (TGT) with information from\n"
+        "       overlapping regions of the source file (SRC). Multiple columns can be\n"
+        "       transferred (-f) and the transfer can be conditioned on requiring\n"
+        "       matching values in one or more columns (-m).\n"
+        "       In addition to column transfer (-f) and special annotations (-a), the\n"
+        "       program can operate in a simple grep-like mode and print matching lines\n"
+        "       (when neither -f nor -a are given) or drop matching lines (-x).\n"
+        "       All indexes and coordinates are 1-based and inclusive.\n"
+        "\n"
+        "Usage: annot-tsv [OPTIONS] -s source.txt -t target.txt > output.txt\n"
+        "\n"
+        "Common options:\n"
+        "   -c, --core SRC:TGT      Core columns in SRC and TGT file\n"
+        "                             [chr,beg,end:chr,beg,end]\n"
+        "   -f, --transfer SRC:TGT  Columns to transfer. If SRC column does not exist,\n"
+        "                           interpret as the default value to use. If the TGT\n"
+        "                           column does not exist, a new column is created. If\n"
+        "                           the TGT column does exist, its values are overwritten\n"
+        "                           when overlap is found or left as is otherwise.\n"
+        "   -m, --match SRC:TGT     Require match in these columns for annotation\n"
+        "                           transfer\n"
+        "   -o, --output FILE       Output file name [STDOUT]\n"
+        "   -s, --source-file FILE  Source file to take annotations from\n"
+        "   -t, --target-file FILE  Target file to be extend with annotations from -s\n"
+        "\n"
+        "Other options:\n"
+        "       --allow-dups        Add annotations multiple times\n"
+        "       --help              This help message\n"
+        "       --max-annots INT    Adding at most INT annotations per column to save\n"
+        "                           time in big regions\n"
+        "       --version           Print version string and exit\n"
+        "   -a, --annotate LIST     Add special annotations, one or more of:\n"
+        "                             cnt  .. number of overlapping regions\n"
+        "                             frac .. fraction of the target region with an\n"
+        "                                       overlap\n"
+        "                             nbp  .. number of source base pairs in the overlap\n"
+        "   -d, --delim SRC:TGT     Column delimiter in SRC and TGT file\n"
+        "   -h, --headers SRC:TGT   Header row line number, 0:0 is equivalent to -H, negative\n"
+        "                             value counts from the end of comment line block [1:1]\n"
+        "   -H, --ignore-headers    Use numeric indices, ignore the headers completely\n"
+        "   -I, --no-header-idx     Suppress index numbers in the printed header. If given\n"
+        "                           twice, drop the entire header\n"
+        "   -O, --overlap FLOAT[,FLOAT]     Minimum required overlap with respect to SRC,TGT.\n"
+        "                           If single value, the bigger overlap is considered.\n"
+        "                           Identical values are equivalent to running with -r.\n"
+        "   -r, --reciprocal        Apply the -O requirement to both overlapping\n"
+        "                           intervals\n"
+        "   -x, --drop-overlaps     Drop overlapping regions (precludes -f)\n"
+        "\n"
+        "Examples:\n"
+        "   # Header is present, match and transfer by column name\n"
+        "   annot-tsv -s src.txt.gz -t tgt.txt.gz -c chr,beg,end:CHR,POS,POS \\\n"
+        "       -m type,sample:TYPE,SMPL -f info:INFO\n"
+        "\n"
+        "   # Header is not present, match and transfer by column index (1-based)\n"
+        "   annot-tsv -s src.txt.gz -t tgt.txt.gz -c 1,2,3:1,2,3 -m 4,5:4,5 -f 6:6\n"
+        "\n"
+        "   # If the TGT part is not given, the program assumes that the SRC:TGT columns\n"
+        "   # are identical\n"
+        "   annot-tsv -s src.txt.gz -t tgt.txt.gz -c chr,beg,end -m type,sample -f info\n"
+        "\n"
+        "   # One of the SRC or TGT file can be streamed from stdin\n"
+        "   gunzip -c src.txt.gz | \\\n"
+        "       annot-tsv -t tgt.txt.gz -c chr,beg,end -m type,sample -f info\n"
+        "   gunzip -c tgt.txt.gz | \\\n"
+        "       annot-tsv -s src.txt.gz -c chr,beg,end -m type,sample -f info\n"
+        "\n"
+        "   # Print matching regions as above but without modifying the records\n"
+        "   gunzip -c src.txt.gz | annot-tsv -t tgt.txt.gz -c chr,beg,end -m type,sample\n"
+        "\n";
+}
+
+int main(int argc, char **argv)
+{
+    args_t *args = (args_t*) calloc(1,sizeof(args_t));
+    static struct option loptions[] =
+    {
+        {"core",required_argument,NULL,'c'},
+        {"transfer",required_argument,NULL,'f'},
+        {"match",required_argument,NULL,'m'},
+        {"output",required_argument,NULL,'o'},
+        {"source-file",required_argument,NULL,'s'},
+        {"target-file",required_argument,NULL,'t'},
+        {"allow-dups",no_argument,NULL,0},
+        {"max-annots",required_argument,NULL,2},
+        {"no-header-idx",required_argument,NULL,'I'},
+        {"version",no_argument,NULL,1},
+        {"annotate",required_argument,NULL,'a'},
+        {"headers",no_argument,NULL,'h'},
+        {"ignore-headers",no_argument,NULL,'H'},
+        {"overlap",required_argument,NULL,'O'},
+        {"reciprocal",no_argument,NULL,'r'},
+        {"drop-overlaps",no_argument,NULL,'x'},
+        {"delim",required_argument,NULL,'d'},
+        {"help",no_argument,NULL,4},
+        {NULL,0,NULL,0}
+    };
+    char *tmp = NULL;
+    int c;
+    int reciprocal = 0;
+    while ((c = getopt_long(argc, argv, "c:f:m:o:s:t:a:HO:rxh:Id:",loptions,NULL)) >= 0)
+    {
+        switch (c)
+        {
+            case  0 : args->allow_dups = 1; break;
+            case  1 :
+                printf(
+"annot-tsv (htslib) %s\n"
+"Copyright (C) 2024 Genome Research Ltd.\n", hts_version());
+                return EXIT_SUCCESS;
+                break;
+            case  2 :
+                args->max_annots = strtod(optarg, &tmp);
+                if ( tmp==optarg || *tmp ) error("Could not parse --max-annots  %s\n", optarg);
+                break;
+            case 'I': args->no_write_hdr++; break;
+            case 'd': args->delim_str = optarg; break;
+            case 'h': args->headers_str = optarg; break;
+            case 'H': args->headers_str = "0:0"; break;
+            case 'r': reciprocal = 1; break;
+            case 'c': args->core_str  = optarg; break;
+            case 't': args->dst.fname = optarg; break;
+            case 'm': args->match_str = optarg; break;
+            case 'a': args->annots_str = optarg; break;
+            case 'o': args->out_fname = optarg; break;
+            case 'O':
+                args->overlap_src = strtod(optarg, &tmp);
+                if ( tmp==optarg || (*tmp && *tmp!=',') ) error("Could not parse --overlap %s\n", optarg);
+                if ( args->overlap_src<0 || args->overlap_src>1 ) error("Expected value(s) from the interval [0,1]: --overlap %s\n", optarg);
+                if ( *tmp )
+                {
+                    args->overlap_dst = strtod(tmp+1, &tmp);
+                    if ( *tmp ) error("Could not parse --overlap %s\n", optarg);
+                    if ( args->overlap_dst<0 || args->overlap_dst>1 ) error("Expected value(s) from the interval [0,1]: --overlap %s\n", optarg);
+                }
+                else
+                    args->overlap_either = 1;
+                break;
+            case 's': args->src.fname = optarg; break;
+            case 'f': args->transfer_str = optarg; break;
+            case 'x': args->mode = PRINT_NONMATCHING; break;
+            case  4 : printf("\nVersion: %s\n%s\n",hts_version(),usage_text()); exit(EXIT_SUCCESS); break;
+            case '?': // fall through
+            default: error("\nVersion: %s\n%s\n",hts_version(),usage_text()); break;
+        }
+    }
+    if ( argc==1 ) error("\nVersion: %s\n%s\n",hts_version(),usage_text());
+    if ( !args->dst.fname && !args->src.fname ) error("Missing the -s and -t options\n");
+    if ( !args->dst.fname || !args->src.fname )
+    {
+        if ( isatty(fileno((FILE *)stdin)) ) error("Missing the %s option\n",args->dst.fname?"-s":"-t");
+        // reading from stdin
+        if ( !args->dst.fname ) args->dst.fname = "-";
+        if ( !args->src.fname ) args->src.fname = "-";
+    }
+    if ( !args->mode )
+    {
+        if ( !args->transfer_str && !args->annots_str ) args->mode = PRINT_MATCHING;
+        else args->mode = PRINT_MATCHING|PRINT_NONMATCHING;
+    }
+    if ( (args->transfer_str || args->annots_str) && !(args->mode & PRINT_MATCHING) ) error("The option -x cannot be combined with -f and -a\n");
+    if ( reciprocal )
+    {
+        if ( args->overlap_dst && args->overlap_src && args->overlap_dst!=args->overlap_src )
+            error("The combination of --reciprocal with --overlap %f,%f makes no sense: expected single value or identical values\n",args->overlap_src,args->overlap_dst);
+        if ( !args->overlap_src )
+            args->overlap_src = args->overlap_dst;
+        else
+            args->overlap_dst = args->overlap_src;
+        args->overlap_either = 0;
+    }
+
+    init_data(args);
+    write_header(args, &args->dst);
+    while ( read_next_line(&args->dst) )
+    {
+        int i;
+        for (i=0; i<args->dst.grow_n; i++)
+        {
+            kputc(args->dst.delim, &args->dst.line);
+            kputc('.', &args->dst.line);
+        }
+        process_line(args, args->dst.line.s, args->dst.line.l);
+        args->dst.line.l = 0;
+    }
+    destroy_data(args);
+    free(args);
+
+    return 0;
+}
+
diff --git a/ext/htslib/bcf_sr_sort.c b/ext/htslib/bcf_sr_sort.c
new file mode 100644
index 0000000..01e98bb
--- /dev/null
+++ b/ext/htslib/bcf_sr_sort.c
@@ -0,0 +1,707 @@
+/*
+    Copyright (C) 2017-2021 Genome Research Ltd.
+
+    Author: Petr Danecek <pd3@sanger.ac.uk>
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in
+    all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+    THE SOFTWARE.
+*/
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+
+#include <assert.h>
+#include <strings.h>
+
+#include "bcf_sr_sort.h"
+#include "htslib/khash_str2int.h"
+#include "htslib/kbitset.h"
+
+#define SR_REF   1
+#define SR_SNP   2
+#define SR_INDEL 4
+#define SR_OTHER 8
+#define SR_SCORE(srt,a,b) (srt)->score[((a)<<4)|(b)]
+
+// Logical AND
+static inline int kbs_logical_and(kbitset_t *bs1, kbitset_t *bs2)
+{
+    // General case, bitsets of unequal size:
+    //  int i, n = bs1->n < bs2->n ? bs1->n : bs2->n;
+    int i, n = bs1->n;
+
+    for (i=0; i<n; i++) if ( bs1->b[i] & bs2->b[i] ) return 1;
+    return 0;
+}
+
+// Bitwise OR, dst will be modified, src will be left unchanged
+static inline void kbs_bitwise_or(kbitset_t *dst, kbitset_t *src)
+{
+    int i;
+    for (i=0; i<dst->n; i++) dst->b[i] |= src->b[i];
+}
+
+
+static void bcf_sr_init_scores(sr_sort_t *srt)
+{
+    int i,jbit,kbit;
+
+    // lower number = lower priority, zero means forbidden
+
+    if ( srt->pair & BCF_SR_PAIR_ANY ) srt->pair |= (BCF_SR_PAIR_SNPS | BCF_SR_PAIR_INDELS | BCF_SR_PAIR_SNP_REF | BCF_SR_PAIR_INDEL_REF);
+    if ( srt->pair & BCF_SR_PAIR_SNPS ) SR_SCORE(srt,SR_SNP,SR_SNP) = 3;
+    if ( srt->pair & BCF_SR_PAIR_INDELS ) SR_SCORE(srt,SR_INDEL,SR_INDEL) = 3;
+    if ( srt->pair & BCF_SR_PAIR_SNP_REF )
+    {
+        SR_SCORE(srt,SR_SNP,SR_REF) = 2;
+        SR_SCORE(srt,SR_REF,SR_SNP) = 2;
+    }
+    if ( srt->pair & BCF_SR_PAIR_INDEL_REF )
+    {
+        SR_SCORE(srt,SR_INDEL,SR_REF) = 2;
+        SR_SCORE(srt,SR_REF,SR_INDEL) = 2;
+    }
+    if ( srt->pair & BCF_SR_PAIR_ANY )
+    {
+        for (i=0; i<256; i++)
+            if ( !srt->score[i] ) srt->score[i] = 1;
+    }
+
+    // set all combinations
+    for (i=0; i<256; i++)
+    {
+        if ( srt->score[i] ) continue;      // already set
+        int max = 0;
+        for (jbit=0; jbit<4; jbit++)        // high bits
+        {
+            int j = 1<<jbit;
+            if ( !(i & (j<<4)) ) continue;
+            for (kbit=0; kbit<4; kbit++)    // low bits
+            {
+                int k = 1<<kbit;
+                if ( !(i & k) ) continue;
+                if ( max < SR_SCORE(srt,j,k) ) max = SR_SCORE(srt,j,k);
+            }
+        }
+        srt->score[i] = max;
+    }
+}
+static int multi_is_exact(var_t *avar, var_t *bvar)
+{
+    if ( avar->nalt != bvar->nalt ) return 0;
+
+    int alen = strlen(avar->str);
+    int blen = strlen(bvar->str);
+    if ( alen != blen ) return 0;
+
+    char *abeg = avar->str;
+    while ( *abeg )
+    {
+        char *aend = abeg;
+        while ( *aend && *aend!=',' ) aend++;
+
+        char *bbeg = bvar->str;
+        while ( *bbeg )
+        {
+            char *bend = bbeg;
+            while ( *bend && *bend!=',' ) bend++;
+            if ( bend - bbeg == aend - abeg && !strncasecmp(abeg,bbeg,bend-bbeg) ) break;
+            bbeg = *bend ? bend+1 : bend;
+        }
+        if ( !*bbeg ) return 0;
+
+        abeg = *aend ? aend+1 : aend;
+    }
+    return 1;
+}
+static int multi_is_subset(var_t *avar, var_t *bvar)
+{
+    char *abeg = avar->str;
+    while ( *abeg )
+    {
+        char *aend = abeg;
+        while ( *aend && *aend!=',' ) aend++;
+
+        char *bbeg = bvar->str;
+        while ( *bbeg )
+        {
+            char *bend = bbeg;
+            while ( *bend && *bend!=',' ) bend++;
+            if ( bend - bbeg == aend - abeg && !strncasecmp(abeg,bbeg,bend-bbeg) ) return 1;
+            bbeg = *bend ? bend+1 : bend;
+        }
+        abeg = *aend ? aend+1 : aend;
+    }
+    return 0;
+}
+static uint32_t pairing_score(sr_sort_t *srt, int ivset, int jvset)
+{
+    varset_t *iv = &srt->vset[ivset];
+    varset_t *jv = &srt->vset[jvset];
+
+    // Restrictive logic: the strictest type from a group is selected,
+    // so that, for example, snp+ref does not lead to the inclusion of an indel
+    int i,j;
+    uint32_t min = UINT32_MAX;
+    for (i=0; i<iv->nvar; i++)
+    {
+        var_t *ivar = &srt->var[iv->var[i]];
+        for (j=0; j<jv->nvar; j++)
+        {
+            var_t *jvar = &srt->var[jv->var[j]];
+            if ( srt->pair & BCF_SR_PAIR_EXACT )
+            {
+                if ( ivar->type != jvar->type ) continue;
+                if ( !strcmp(ivar->str,jvar->str) ) return UINT32_MAX;  // exact match, best possibility
+                if ( multi_is_exact(ivar,jvar) ) return UINT32_MAX; // identical alleles
+                continue;
+            }
+            if ( ivar->type==jvar->type && !strcmp(ivar->str,jvar->str) ) return UINT32_MAX;  // exact match, best possibility
+            if ( ivar->type & jvar->type && multi_is_subset(ivar,jvar) ) return UINT32_MAX; // one of the alleles is identical
+
+            uint32_t score = SR_SCORE(srt,ivar->type,jvar->type);
+            if ( !score ) return 0;     // some of the varsets in the two groups are not compatible, will not pair
+            if ( min>score ) min = score;
+        }
+    }
+    if ( srt->pair & BCF_SR_PAIR_EXACT ) return 0;
+
+    assert( min!=UINT32_MAX );
+
+    uint32_t cnt = 0;
+    for (i=0; i<iv->nvar; i++) cnt += srt->var[iv->var[i]].nvcf;
+    for (j=0; j<jv->nvar; j++) cnt += srt->var[jv->var[j]].nvcf;
+
+    return (1u<<(28+min)) + cnt;
+}
+static void remove_vset(sr_sort_t *srt, int jvset)
+{
+    if ( jvset+1 < srt->nvset )
+    {
+        varset_t tmp = srt->vset[jvset];
+        memmove(&srt->vset[jvset], &srt->vset[jvset+1], sizeof(varset_t)*(srt->nvset - jvset - 1));
+        srt->vset[srt->nvset-1] = tmp;
+
+        int *jmat = srt->pmat + jvset*srt->ngrp;
+        memmove(jmat, &jmat[srt->ngrp],sizeof(int)*(srt->nvset - jvset - 1)*srt->ngrp);
+
+        memmove(&srt->cnt[jvset], &srt->cnt[jvset+1], sizeof(int)*(srt->nvset - jvset - 1));
+    }
+    srt->nvset--;
+}
+static int merge_vsets(sr_sort_t *srt, int ivset, int jvset)
+{
+    int i,j;
+    if ( ivset > jvset ) { i = ivset; ivset = jvset; jvset = i; }
+
+    varset_t *iv = &srt->vset[ivset];
+    varset_t *jv = &srt->vset[jvset];
+
+    kbs_bitwise_or(iv->mask,jv->mask);
+
+    i = iv->nvar;
+    iv->nvar += jv->nvar;
+    hts_expand(int, iv->nvar, iv->mvar, iv->var);
+    for (j=0; j<jv->nvar; j++,i++) iv->var[i] = jv->var[j];
+
+    int *imat = srt->pmat + ivset*srt->ngrp;
+    int *jmat = srt->pmat + jvset*srt->ngrp;
+    for (i=0; i<srt->ngrp; i++) imat[i] += jmat[i];
+    srt->cnt[ivset] += srt->cnt[jvset];
+
+    remove_vset(srt, jvset);
+
+    return ivset;
+}
+
+static int push_vset(sr_sort_t *srt, int ivset)
+{
+    varset_t *iv = &srt->vset[ivset];
+    int i,j;
+    for (i=0; i<srt->sr->nreaders; i++)
+    {
+        vcf_buf_t *buf = &srt->vcf_buf[i];
+        buf->nrec++;
+        hts_expand(bcf1_t*,buf->nrec,buf->mrec,buf->rec);
+        buf->rec[buf->nrec-1] = NULL;
+    }
+    for (i=0; i<iv->nvar; i++)
+    {
+        var_t *var = &srt->var[ iv->var[i] ];
+        for (j=0; j<var->nvcf; j++)
+        {
+            int jvcf = var->vcf[j];
+            vcf_buf_t *buf = &srt->vcf_buf[jvcf];
+            buf->rec[buf->nrec-1] = var->rec[j];
+        }
+    }
+    remove_vset(srt, ivset);
+    return 0; // FIXME: check for errs in this function
+}
+
+static int cmpstringp(const void *p1, const void *p2)
+{
+    return strcmp(* (char * const *) p1, * (char * const *) p2);
+}
+
+#define DEBUG_VSETS 0
+#if DEBUG_VSETS
+void debug_vsets(sr_sort_t *srt)
+{
+    int i,j,k;
+    for (i=0; i<srt->nvset; i++)
+    {
+        fprintf(stderr,"dbg_vset %d:", i);
+        for (j=0; j<srt->vset[i].mask->n; j++) fprintf(stderr,"%c%lu",j==0?' ':':',srt->vset[i].mask->b[j]);
+        fprintf(stderr,"\t");
+        for (j=0; j<srt->vset[i].nvar; j++)
+        {
+            var_t *var = &srt->var[srt->vset[i].var[j]];
+            fprintf(stderr,"\t%s",var->str);
+            for (k=0; k<var->nvcf; k++)
+                fprintf(stderr,"%c%d", k==0?':':',',var->vcf[k]);
+        }
+        fprintf(stderr,"\n");
+    }
+}
+#endif
+
+#define DEBUG_VBUF 0
+#if DEBUG_VBUF
+void debug_vbuf(sr_sort_t *srt)
+{
+    int i, j;
+    for (j=0; j<srt->vcf_buf[0].nrec; j++)
+    {
+        fprintf(stderr,"dbg_vbuf %d:\t", j);
+        for (i=0; i<srt->sr->nreaders; i++)
+        {
+            vcf_buf_t *buf = &srt->vcf_buf[i];
+            fprintf(stderr,"\t%"PRIhts_pos, buf->rec[j] ? buf->rec[j]->pos+1 : 0);
+        }
+        fprintf(stderr,"\n");
+    }
+}
+#endif
+
+static char *grp_create_key(sr_sort_t *srt)
+{
+    if ( !srt->str.l ) return strdup("");
+    int i;
+    hts_expand(char*,srt->noff,srt->mcharp,srt->charp);
+    for (i=0; i<srt->noff; i++)
+    {
+        srt->charp[i] = srt->str.s + srt->off[i];
+        if ( i>0 ) srt->charp[i][-1] = 0;
+    }
+    qsort(srt->charp, srt->noff, sizeof(*srt->charp), cmpstringp);
+    char *ret = (char*) malloc(srt->str.l + 1), *ptr = ret;
+    for (i=0; i<srt->noff; i++)
+    {
+        int len = strlen(srt->charp[i]);
+        memcpy(ptr, srt->charp[i], len);
+        ptr += len + 1;
+        ptr[-1] = i+1==srt->noff ? 0 : ';';
+    }
+    return ret;
+}
+int bcf_sr_sort_set_active(sr_sort_t *srt, int idx)
+{
+    hts_expand(int,idx+1,srt->mactive,srt->active);
+    srt->nactive = 1;
+    srt->active[srt->nactive - 1] = idx;
+    return 0; // FIXME: check for errs in this function
+}
+int bcf_sr_sort_add_active(sr_sort_t *srt, int idx)
+{
+    hts_expand(int,idx+1,srt->mactive,srt->active);
+    srt->nactive++;
+    srt->active[srt->nactive - 1] = idx;
+    return 0; // FIXME: check for errs in this function
+}
+static int bcf_sr_sort_set(bcf_srs_t *readers, sr_sort_t *srt, const char *chr, hts_pos_t min_pos)
+{
+    if ( !srt->grp_str2int )
+    {
+        // first time here, initialize
+        if ( !srt->pair )
+        {
+            if ( readers->collapse==COLLAPSE_NONE ) readers->collapse = BCF_SR_PAIR_EXACT;
+            bcf_sr_set_opt(readers, BCF_SR_PAIR_LOGIC, readers->collapse);
+        }
+        bcf_sr_init_scores(srt);
+        srt->grp_str2int = khash_str2int_init();
+        srt->var_str2int = khash_str2int_init();
+    }
+    int k;
+    khash_t(str2int) *hash;
+    hash = srt->grp_str2int;
+    for (k=0; k < kh_end(hash); k++)
+        if ( kh_exist(hash,k) ) free((char*)kh_key(hash,k));
+    hash = srt->var_str2int;
+    for (k=0; k < kh_end(hash); k++)
+        if ( kh_exist(hash,k) ) free((char*)kh_key(hash,k));
+    kh_clear(str2int, srt->grp_str2int);
+    kh_clear(str2int, srt->var_str2int);
+    srt->ngrp = srt->nvar = srt->nvset = 0;
+
+    grp_t grp;
+    memset(&grp,0,sizeof(grp_t));
+
+    // group VCFs into groups, each with a unique combination of variants in the duplicate lines
+    int ireader,ivar,irec,igrp,ivset,iact;
+    for (ireader=0; ireader<readers->nreaders; ireader++) srt->vcf_buf[ireader].nrec = 0;
+    for (iact=0; iact<srt->nactive; iact++)
+    {
+        ireader = srt->active[iact];
+        bcf_sr_t *reader = &readers->readers[ireader];
+        int rid   = bcf_hdr_name2id(reader->header, chr);
+        grp.nvar  = 0;
+        hts_expand(int,reader->nbuffer,srt->moff,srt->off);
+        srt->noff  = 0;
+        srt->str.l = 0;
+        for (irec=1; irec<=reader->nbuffer; irec++)
+        {
+            bcf1_t *line = reader->buffer[irec];
+            if ( line->rid!=rid || line->pos!=min_pos ) break;
+
+            if ( srt->str.l ) kputc(';',&srt->str);
+            srt->off[srt->noff++] = srt->str.l;
+            size_t beg  = srt->str.l;
+            int end_pos = -1;
+            for (ivar=1; ivar<line->n_allele; ivar++)
+            {
+                if ( ivar>1 ) kputc(',',&srt->str);
+                kputs(line->d.allele[0],&srt->str);
+                kputc('>',&srt->str);
+                kputs(line->d.allele[ivar],&srt->str);
+
+                // If symbolic allele, check also the END tag in case there are multiple events,
+                // such as <DEL>s, starting at the same positions
+                if ( line->d.allele[ivar][0]=='<' )
+                {
+                    if ( end_pos==-1 )
+                    {
+                        bcf_info_t *end_info = bcf_get_info(reader->header,line,"END");
+                        if ( end_info )
+                            end_pos = (int)end_info->v1.i;  // this is only to create a unique id, we don't mind a potential int64 overflow
+                        else
+                            end_pos = 0;
+                    }
+                    if ( end_pos )
+                    {
+                        kputc('/',&srt->str);
+                        kputw(end_pos, &srt->str);
+                    }
+                }
+            }
+            if ( line->n_allele==1 )
+            {
+                kputs(line->d.allele[0],&srt->str);
+                kputsn(">.",2,&srt->str);
+            }
+
+            // Create new variant or attach to existing one. But careful, there can be duplicate
+            // records with the same POS,REF,ALT (e.g. in dbSNP-b142)
+            char *var_str = beg + srt->str.s;
+            int ret, var_idx = 0, var_end = srt->str.l;
+            while ( 1 )
+            {
+                ret = khash_str2int_get(srt->var_str2int, var_str, &ivar);
+                if ( ret==-1 ) break;
+
+                var_t *var = &srt->var[ivar];
+                if ( var->vcf[var->nvcf-1] != ireader ) break;
+
+                srt->str.l = var_end;
+                kputw(var_idx, &srt->str);
+                var_str = beg + srt->str.s;
+                var_idx++;
+            }
+            if ( ret==-1 )
+            {
+                ivar = srt->nvar++;
+                hts_expand0(var_t,srt->nvar,srt->mvar,srt->var);
+                srt->var[ivar].nvcf = 0;
+                khash_str2int_set(srt->var_str2int, strdup(var_str), ivar);
+                free(srt->var[ivar].str);   // possible left-over from the previous position
+            }
+            var_t *var = &srt->var[ivar];
+            var->nalt = line->n_allele - 1;
+            var->type = bcf_get_variant_types(line);
+            srt->str.s[var_end] = 0;
+            if ( ret==-1 )
+                var->str = strdup(var_str);
+
+            int mvcf = var->mvcf;
+            var->nvcf++;
+            hts_expand0(int*, var->nvcf, var->mvcf, var->vcf);
+            if ( mvcf != var->mvcf ) var->rec = (bcf1_t **) realloc(var->rec,sizeof(bcf1_t*)*var->mvcf);
+            var->vcf[var->nvcf-1] = ireader;
+            var->rec[var->nvcf-1] = line;
+
+            grp.nvar++;
+            hts_expand(var_t,grp.nvar,grp.mvar,grp.var);
+            grp.var[grp.nvar-1] = ivar;
+        }
+        char *grp_key = grp_create_key(srt);
+        int ret = khash_str2int_get(srt->grp_str2int, grp_key, &igrp);
+        if ( ret==-1 )
+        {
+            igrp = srt->ngrp++;
+            hts_expand0(grp_t, srt->ngrp, srt->mgrp, srt->grp);
+            free(srt->grp[igrp].var);
+            srt->grp[igrp] = grp;
+            srt->grp[igrp].key = grp_key;
+            khash_str2int_set(srt->grp_str2int, grp_key, igrp);
+            memset(&grp,0,sizeof(grp_t));
+        }
+        else
+            free(grp_key);
+        srt->grp[igrp].nvcf++;
+    }
+    free(grp.var);
+
+    // initialize bitmask - which groups is the variant present in
+    for (ivar=0; ivar<srt->nvar; ivar++)
+    {
+        if ( kbs_resize(&srt->var[ivar].mask, srt->ngrp) < 0 )
+        {
+            fprintf(stderr, "[%s:%d %s] kbs_resize failed\n", __FILE__,__LINE__,__func__);
+            exit(1);
+        }
+        kbs_clear(srt->var[ivar].mask);
+    }
+    for (igrp=0; igrp<srt->ngrp; igrp++)
+    {
+        for (ivar=0; ivar<srt->grp[igrp].nvar; ivar++)
+        {
+            int i = srt->grp[igrp].var[ivar];
+            kbs_insert(srt->var[i].mask, igrp);
+        }
+    }
+
+    // create the initial list of variant sets
+    for (ivar=0; ivar<srt->nvar; ivar++)
+    {
+        ivset = srt->nvset++;
+        hts_expand0(varset_t, srt->nvset, srt->mvset, srt->vset);
+
+        varset_t *vset = &srt->vset[ivset];
+        vset->nvar = 1;
+        hts_expand0(var_t, vset->nvar, vset->mvar, vset->var);
+        vset->var[vset->nvar-1] = ivar;
+        var_t *var  = &srt->var[ivar];
+        vset->cnt   = var->nvcf;
+        if ( kbs_resize(&vset->mask, srt->ngrp) < 0 )
+        {
+            fprintf(stderr, "[%s:%d %s] kbs_resize failed\n", __FILE__,__LINE__,__func__);
+            exit(1);
+        }
+        kbs_clear(vset->mask);
+        kbs_bitwise_or(vset->mask, var->mask);
+
+        int type = 0;
+        if ( var->type==VCF_REF ) type |= SR_REF;
+        else
+        {
+            if ( var->type & VCF_SNP ) type |= SR_SNP;
+            if ( var->type & VCF_MNP ) type |= SR_SNP;
+            if ( var->type & VCF_INDEL ) type |= SR_INDEL;
+            if ( var->type & VCF_OTHER ) type |= SR_OTHER;
+        }
+        var->type = type;
+    }
+#if DEBUG_VSETS
+    debug_vsets(srt);
+#endif
+
+    // initialize the pairing matrix
+    hts_expand(int, srt->ngrp*srt->nvset, srt->mpmat, srt->pmat);
+    hts_expand(int, srt->nvset, srt->mcnt, srt->cnt);
+    memset(srt->pmat, 0, sizeof(*srt->pmat)*srt->ngrp*srt->nvset);
+    for (ivset=0; ivset<srt->nvset; ivset++)
+    {
+        varset_t *vset = &srt->vset[ivset];
+        for (igrp=0; igrp<srt->ngrp; igrp++) srt->pmat[ivset*srt->ngrp+igrp] = 0;
+        srt->cnt[ivset] = vset->cnt;
+    }
+
+    // pair the lines
+    while ( srt->nvset )
+    {
+#if DEBUG_VSETS
+    fprintf(stderr,"\n");
+    debug_vsets(srt);
+#endif
+
+        int imax = 0;
+        for (ivset=1; ivset<srt->nvset; ivset++)
+            if ( srt->cnt[imax] < srt->cnt[ivset] ) imax = ivset;
+
+        int ipair = -1;
+        uint32_t max_score = 0;
+        for (ivset=0; ivset<srt->nvset; ivset++)
+        {
+            if ( kbs_logical_and(srt->vset[imax].mask,srt->vset[ivset].mask) ) continue;   // cannot be merged
+            uint32_t score = pairing_score(srt, imax, ivset);
+            // fprintf(stderr,"score: %d %d, logic=%d \t..\t %u\n", imax,ivset,srt->pair,score);
+            if ( max_score < score ) { max_score = score; ipair = ivset; }
+        }
+
+        // merge rows creating a new variant set this way
+        if ( ipair!=-1 && ipair!=imax )
+        {
+            imax = merge_vsets(srt, imax, ipair);
+            continue;
+        }
+
+        push_vset(srt, imax);
+    }
+
+    srt->chr = chr;
+    srt->pos = min_pos;
+
+    return 0;  // FIXME: check for errs in this function
+}
+
+int bcf_sr_sort_next(bcf_srs_t *readers, sr_sort_t *srt, const char *chr, hts_pos_t min_pos)
+{
+    int i,j;
+    assert( srt->nactive>0 );
+
+    if ( srt->nsr != readers->nreaders )
+    {
+        srt->sr = readers;
+        if ( srt->nsr < readers->nreaders )
+        {
+            srt->vcf_buf = (vcf_buf_t*) realloc(srt->vcf_buf,readers->nreaders*sizeof(vcf_buf_t));
+            memset(srt->vcf_buf + srt->nsr, 0, sizeof(vcf_buf_t)*(readers->nreaders - srt->nsr));
+            if ( srt->msr < srt->nsr ) srt->msr = srt->nsr;
+        }
+        srt->nsr = readers->nreaders;
+        srt->chr = NULL;
+    }
+    if ( srt->nactive == 1 )
+    {
+        if ( readers->nreaders>1 )
+            memset(readers->has_line, 0, readers->nreaders*sizeof(*readers->has_line));
+        bcf_sr_t *reader = &readers->readers[srt->active[0]];
+        assert( reader->buffer[1]->pos==min_pos );
+        bcf1_t *tmp = reader->buffer[0];
+        for (j=1; j<=reader->nbuffer; j++) reader->buffer[j-1] = reader->buffer[j];
+        reader->buffer[ reader->nbuffer ] = tmp;
+        reader->nbuffer--;
+        readers->has_line[srt->active[0]] = 1;
+        return 1;
+    }
+    if ( !srt->chr || srt->pos!=min_pos || strcmp(srt->chr,chr) ) bcf_sr_sort_set(readers, srt, chr, min_pos);
+
+    if ( !srt->vcf_buf[0].nrec ) return 0;
+
+#if DEBUG_VBUF
+    debug_vbuf(srt);
+#endif
+
+    int nret = 0;
+    for (i=0; i<srt->sr->nreaders; i++)
+    {
+        vcf_buf_t *buf = &srt->vcf_buf[i];
+
+        if ( buf->rec[0] )
+        {
+            bcf_sr_t *reader = &srt->sr->readers[i];
+            for (j=1; j<=reader->nbuffer; j++)
+                if ( reader->buffer[j] == buf->rec[0] ) break;
+
+            assert( j<=reader->nbuffer );
+
+            bcf1_t *tmp = reader->buffer[0];
+            reader->buffer[0] = reader->buffer[j++];
+            for (; j<=reader->nbuffer; j++) reader->buffer[j-1] = reader->buffer[j];
+            reader->buffer[ reader->nbuffer ] = tmp;
+            reader->nbuffer--;
+
+            nret++;
+            srt->sr->has_line[i] = 1;
+        }
+        else
+            srt->sr->has_line[i] = 0;
+
+        buf->nrec--;
+        if ( buf->nrec > 0 )
+            memmove(buf->rec, &buf->rec[1], buf->nrec*sizeof(bcf1_t*));
+    }
+    return nret;
+}
+void bcf_sr_sort_remove_reader(bcf_srs_t *readers, sr_sort_t *srt, int i)
+{
+    //vcf_buf is allocated only in bcf_sr_sort_next
+    //So, a call to bcf_sr_add_reader() followed immediately by bcf_sr_remove_reader()
+    //would cause the program to crash in this segment
+    if (srt->vcf_buf)
+    {
+        free(srt->vcf_buf[i].rec);
+        if ( i+1 < srt->nsr )
+            memmove(&srt->vcf_buf[i], &srt->vcf_buf[i+1], (srt->nsr - i - 1)*sizeof(vcf_buf_t));
+        memset(srt->vcf_buf + srt->nsr - 1, 0, sizeof(vcf_buf_t));
+    }
+}
+sr_sort_t *bcf_sr_sort_init(sr_sort_t *srt)
+{
+    if ( !srt ) return calloc(1,sizeof(sr_sort_t));
+    memset(srt,0,sizeof(sr_sort_t));
+    return srt;
+}
+void bcf_sr_sort_reset(sr_sort_t *srt)
+{
+    srt->chr = NULL;
+}
+void bcf_sr_sort_destroy(sr_sort_t *srt)
+{
+    free(srt->active);
+    if ( srt->var_str2int ) khash_str2int_destroy_free(srt->var_str2int);
+    if ( srt->grp_str2int ) khash_str2int_destroy_free(srt->grp_str2int);
+    int i;
+    for (i=0; i<srt->nsr; i++) free(srt->vcf_buf[i].rec);
+    free(srt->vcf_buf);
+    for (i=0; i<srt->mvar; i++)
+    {
+        free(srt->var[i].str);
+        free(srt->var[i].vcf);
+        free(srt->var[i].rec);
+        kbs_destroy(srt->var[i].mask);
+    }
+    free(srt->var);
+    for (i=0; i<srt->mgrp; i++)
+        free(srt->grp[i].var);
+    free(srt->grp);
+    for (i=0; i<srt->mvset; i++)
+    {
+        kbs_destroy(srt->vset[i].mask);
+        free(srt->vset[i].var);
+    }
+    free(srt->vset);
+    free(srt->str.s);
+    free(srt->off);
+    free(srt->charp);
+    free(srt->cnt);
+    free(srt->pmat);
+    memset(srt,0,sizeof(*srt));
+}
+
diff --git a/ext/htslib/bcf_sr_sort.h b/ext/htslib/bcf_sr_sort.h
new file mode 100644
index 0000000..c8bd787
--- /dev/null
+++ b/ext/htslib/bcf_sr_sort.h
@@ -0,0 +1,108 @@
+/*
+    Copyright (C) 2017 Genome Research Ltd.
+
+    Author: Petr Danecek <pd3@sanger.ac.uk>
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in
+    all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+    THE SOFTWARE.
+*/
+
+/*
+    Reorder duplicate lines so that compatible variant types are
+    returned together by bcf_sr_next_line()
+
+    - readers grouped by variants. Even with many readers there will be
+      typically only several groups
+
+*/
+
+#ifndef BCF_SR_SORT_H
+#define BCF_SR_SORT_H
+
+#include "htslib/synced_bcf_reader.h"
+#include "htslib/kbitset.h"
+
+typedef struct
+{
+    int nrec, mrec;
+    bcf1_t **rec;
+}
+vcf_buf_t;
+
+typedef struct
+{
+    char *str;      // "A>C" for biallelic records or "A>C,A>CC" for multiallelic records
+    int type;       // VCF_SNP, VCF_REF, etc.
+    int nalt;       // number of alternate alleles in this record
+    int nvcf, mvcf, *vcf;   // the list of readers with the same variants
+    bcf1_t **rec;           // list of VCF records in the readers
+    kbitset_t *mask;        // which groups contain the variant
+}
+var_t;
+
+typedef struct
+{
+    char *key;              // only for debugging
+    int nvar, mvar, *var;   // the variants and their type
+    int nvcf;               // number of readers with the same variants
+}
+grp_t;
+
+typedef struct
+{
+    int nvar, mvar, *var;   // list of compatible variants that can be output together
+    int cnt;                // number of readers in this group
+    kbitset_t *mask;        // which groups are populated in this set (replace with expandable bitmask)
+}
+varset_t;
+
+typedef struct
+{
+    uint8_t score[256];
+    int nvar, mvar;
+    var_t *var;             // list of all variants from all readers
+    int nvset, mvset;
+    int mpmat, *pmat;       // pairing matrix, i-th vset and j-th group accessible as i*ngrp+j
+    int ngrp, mgrp;
+    int mcnt, *cnt;         // number of VCF covered by a varset
+    grp_t *grp;             // list of VCF representatives, each with a unique combination of duplicate lines
+    varset_t *vset;         // list of variant sets - combinations of compatible variants across multiple groups ready for output
+    vcf_buf_t *vcf_buf;     // records sorted in output order, for each VCF
+    bcf_srs_t *sr;
+    void *grp_str2int;
+    void *var_str2int;
+    kstring_t str;
+    int moff, noff, *off, mcharp;
+    char **charp;
+    const char *chr;
+    hts_pos_t pos;
+    int nsr, msr;
+    int pair;
+    int nactive, mactive, *active;  // list of readers with lines at the current pos
+}
+sr_sort_t;
+
+sr_sort_t *bcf_sr_sort_init(sr_sort_t *srt);
+void bcf_sr_sort_reset(sr_sort_t *srt);
+int bcf_sr_sort_next(bcf_srs_t *readers, sr_sort_t *srt, const char *chr, hts_pos_t pos);
+int bcf_sr_sort_set_active(sr_sort_t *srt, int i);
+int bcf_sr_sort_add_active(sr_sort_t *srt, int i);
+void bcf_sr_sort_destroy(sr_sort_t *srt);
+void bcf_sr_sort_remove_reader(bcf_srs_t *readers, sr_sort_t *srt, int i);
+
+#endif
diff --git a/ext/htslib/bgzf.c b/ext/htslib/bgzf.c
new file mode 100644
index 0000000..8092c7b
--- /dev/null
+++ b/ext/htslib/bgzf.c
@@ -0,0 +1,2602 @@
+/* The MIT License
+
+   Copyright (c) 2008 Broad Institute / Massachusetts Institute of Technology
+                 2011, 2012 Attractive Chaos <attractor@live.co.uk>
+   Copyright (C) 2009, 2013-2023 Genome Research Ltd
+
+   Permission is hereby granted, free of charge, to any person obtaining a copy
+   of this software and associated documentation files (the "Software"), to deal
+   in the Software without restriction, including without limitation the rights
+   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+   copies of the Software, and to permit persons to whom the Software is
+   furnished to do so, subject to the following conditions:
+
+   The above copyright notice and this permission notice shall be included in
+   all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+   THE SOFTWARE.
+*/
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <unistd.h>
+#include <assert.h>
+#include <pthread.h>
+#include <sys/types.h>
+#include <inttypes.h>
+#include <zlib.h>
+
+#ifdef HAVE_LIBDEFLATE
+#include <libdeflate.h>
+#endif
+
+#include "htslib/hts.h"
+#include "htslib/bgzf.h"
+#include "htslib/hfile.h"
+#include "htslib/thread_pool.h"
+#include "htslib/hts_endian.h"
+#include "cram/pooled_alloc.h"
+#include "hts_internal.h"
+
+#ifndef EFTYPE
+#define EFTYPE ENOEXEC
+#endif
+
+#define BGZF_CACHE
+#define BGZF_MT
+
+#define BLOCK_HEADER_LENGTH 18
+#define BLOCK_FOOTER_LENGTH 8
+
+
+/* BGZF/GZIP header (specialized from RFC 1952; little endian):
+ +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+
+ | 31|139|  8|  4|              0|  0|255|      6| 66| 67|      2|BLK_LEN|
+ +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+
+  BGZF extension:
+                ^                              ^   ^   ^
+                |                              |   |   |
+               FLG.EXTRA                     XLEN  B   C
+
+  BGZF format is compatible with GZIP. It limits the size of each compressed
+  block to 2^16 bytes and adds and an extra "BC" field in the gzip header which
+  records the size.
+
+*/
+static const uint8_t g_magic[19] = "\037\213\010\4\0\0\0\0\0\377\6\0\102\103\2\0\0\0";
+
+#ifdef BGZF_CACHE
+typedef struct {
+    int size;
+    uint8_t *block;
+    int64_t end_offset;
+} cache_t;
+
+#include "htslib/khash.h"
+KHASH_MAP_INIT_INT64(cache, cache_t)
+#endif
+
+struct bgzf_cache_t {
+    khash_t(cache) *h;
+    khint_t last_pos;
+};
+
+#ifdef BGZF_MT
+
+typedef struct bgzf_job {
+    BGZF *fp;
+    unsigned char comp_data[BGZF_MAX_BLOCK_SIZE];
+    size_t comp_len;
+    unsigned char uncomp_data[BGZF_MAX_BLOCK_SIZE];
+    size_t uncomp_len;
+    int errcode;
+    int64_t block_address;
+    int hit_eof;
+} bgzf_job;
+
+enum mtaux_cmd {
+    NONE = 0,
+    SEEK,
+    SEEK_DONE,
+    HAS_EOF,
+    HAS_EOF_DONE,
+    CLOSE,
+};
+
+// When multi-threaded bgzf_tell won't work, so we delay the hts_idx_push
+// until we've written the last block.
+typedef struct {
+    hts_pos_t beg, end;
+    int tid, is_mapped;  // args for hts_idx_push
+    uint64_t offset, block_number;
+} hts_idx_cache_entry;
+
+typedef struct {
+    int nentries, mentries; // used and allocated
+    hts_idx_cache_entry *e; // hts_idx elements
+} hts_idx_cache_t;
+
+typedef struct bgzf_mtaux_t {
+    // Memory pool for bgzf_job structs, to avoid many malloc/free
+    pool_alloc_t *job_pool;
+    bgzf_job *curr_job;
+
+    // Thread pool
+    int n_threads;
+    int own_pool;
+    hts_tpool *pool;
+
+    // Output queue holding completed bgzf_jobs
+    hts_tpool_process *out_queue;
+
+    // I/O thread.
+    pthread_t io_task;
+    pthread_mutex_t job_pool_m;
+    int jobs_pending; // number of jobs waiting
+    int flush_pending;
+    void *free_block;
+    int hit_eof;  // r/w entirely within main thread
+
+    // Message passing to the reader thread; eg seek requests
+    int errcode;
+    uint64_t block_address;
+    int eof;
+    pthread_mutex_t command_m; // Set whenever fp is being updated
+    pthread_cond_t command_c;
+    enum mtaux_cmd command;
+
+    // For multi-threaded on-the-fly indexing. See bgzf_idx_push below.
+    pthread_mutex_t idx_m;
+    hts_idx_t *hts_idx;
+    uint64_t block_number, block_written;
+    hts_idx_cache_t idx_cache;
+} mtaux_t;
+#endif
+
+typedef struct
+{
+    uint64_t uaddr;  // offset w.r.t. uncompressed data
+    uint64_t caddr;  // offset w.r.t. compressed data
+}
+bgzidx1_t;
+
+struct bgzidx_t
+{
+    int noffs, moffs;       // the size of the index, n:used, m:allocated
+    bgzidx1_t *offs;        // offsets
+    uint64_t ublock_addr;   // offset of the current block (uncompressed data)
+};
+
+/*
+ * Buffers up arguments to hts_idx_push for later use, once we've written all bar
+ * this block.  This is necessary when multiple blocks are in flight (threading)
+ * and fp->block_address isn't known at the time of call as we have in-flight
+ * blocks that haven't yet been compressed.
+ *
+ * NB: this only matters when we're indexing on the fly (writing).
+ * Normal indexing is threaded reads, but we already know block sizes
+ * so it's a simpler process
+ *
+ * Returns 0 on success,
+ *        -1 on failure
+ */
+int bgzf_idx_push(BGZF *fp, hts_idx_t *hidx, int tid, hts_pos_t beg, hts_pos_t end, uint64_t offset, int is_mapped) {
+    hts_idx_cache_entry *e;
+    mtaux_t *mt = fp->mt;
+
+    if (!mt)
+        return hts_idx_push(hidx, tid, beg, end, offset, is_mapped);
+
+    // Early check for out of range positions which would fail in hts_idx_push()
+    if (hts_idx_check_range(hidx, tid, beg, end) < 0)
+        return -1;
+
+    pthread_mutex_lock(&mt->idx_m);
+
+    mt->hts_idx = hidx;
+    hts_idx_cache_t *ic = &mt->idx_cache;
+
+    if (ic->nentries >= ic->mentries) {
+        int new_sz = ic->mentries ? ic->mentries*2 : 1024;
+        if (!(e = realloc(ic->e, new_sz * sizeof(*ic->e)))) {
+            pthread_mutex_unlock(&mt->idx_m);
+            return -1;
+        }
+        ic->e = e;
+        ic->mentries = new_sz;
+    }
+
+    e = &ic->e[ic->nentries++];
+    e->tid = tid;
+    e->beg = beg;
+    e->end = end;
+    e->is_mapped = is_mapped;
+    e->offset = offset & 0xffff;
+    e->block_number = mt->block_number;
+
+    pthread_mutex_unlock(&mt->idx_m);
+
+    return 0;
+}
+
+static int bgzf_idx_flush(BGZF *fp,
+                          size_t block_uncomp_len, size_t block_comp_len) {
+    mtaux_t *mt = fp->mt;
+
+    if (!mt->idx_cache.e) {
+        mt->block_written++;
+        return 0;
+    }
+
+    pthread_mutex_lock(&mt->idx_m);
+
+    hts_idx_cache_entry *e = mt->idx_cache.e;
+    int i;
+
+    assert(mt->idx_cache.nentries == 0 || mt->block_written <= e[0].block_number);
+
+    for (i = 0; i < mt->idx_cache.nentries && e[i].block_number == mt->block_written; i++) {
+        if (block_uncomp_len > 0 && e[i].offset == block_uncomp_len) {
+            /*
+             * If the virtual offset is at the end of the current block,
+             * adjust it to point to the start of the next one.  This
+             * is needed when on-the-fly indexing has recorded a virtual
+             * offset just before a new block has been started, and makes
+             * on-the-fly and standard indexing give exactly the same results.
+             *
+             * In theory the two virtual offsets are equivalent, but pointing
+             * to the end of a block is inefficient, and caused problems with
+             * versions of HTSlib before 1.11 where bgzf_read() would
+             * incorrectly return EOF.
+             */
+
+            // Assert that this is the last entry for the current block_number
+            assert(i == mt->idx_cache.nentries - 1
+                   || e[i].block_number < e[i + 1].block_number);
+
+            // Work out where the next block starts.  For this entry, the
+            // offset will be zero.
+            uint64_t next_block_addr = mt->block_address + block_comp_len;
+            if (hts_idx_push(mt->hts_idx, e[i].tid, e[i].beg, e[i].end,
+                             next_block_addr << 16, e[i].is_mapped) < 0) {
+                pthread_mutex_unlock(&mt->idx_m);
+                return -1;
+            }
+            // Count this entry and drop out of the loop
+            i++;
+            break;
+        }
+
+        if (hts_idx_push(mt->hts_idx, e[i].tid, e[i].beg, e[i].end,
+                         (mt->block_address << 16) + e[i].offset,
+                         e[i].is_mapped) < 0) {
+            pthread_mutex_unlock(&mt->idx_m);
+            return -1;
+        }
+    }
+
+    memmove(&e[0], &e[i], (mt->idx_cache.nentries - i) * sizeof(*e));
+    mt->idx_cache.nentries -= i;
+    mt->block_written++;
+
+    pthread_mutex_unlock(&mt->idx_m);
+    return 0;
+}
+
+void bgzf_index_destroy(BGZF *fp);
+int bgzf_index_add_block(BGZF *fp);
+static int mt_destroy(mtaux_t *mt);
+
+static inline void packInt16(uint8_t *buffer, uint16_t value)
+{
+    buffer[0] = value;
+    buffer[1] = value >> 8;
+}
+
+static inline int unpackInt16(const uint8_t *buffer)
+{
+    return buffer[0] | buffer[1] << 8;
+}
+
+static inline void packInt32(uint8_t *buffer, uint32_t value)
+{
+    buffer[0] = value;
+    buffer[1] = value >> 8;
+    buffer[2] = value >> 16;
+    buffer[3] = value >> 24;
+}
+
+static void razf_info(hFILE *hfp, const char *filename)
+{
+    uint64_t usize, csize;
+    off_t sizes_pos;
+
+    if (filename == NULL || strcmp(filename, "-") == 0) filename = "FILE";
+
+    // RAZF files end with USIZE,CSIZE stored as big-endian uint64_t
+    if ((sizes_pos = hseek(hfp, -16, SEEK_END)) < 0) goto no_sizes;
+    if (hread(hfp, &usize, 8) != 8 || hread(hfp, &csize, 8) != 8) goto no_sizes;
+    if (!ed_is_big()) ed_swap_8p(&usize), ed_swap_8p(&csize);
+    if (csize >= sizes_pos) goto no_sizes; // Very basic validity check
+
+    hts_log_error(
+"To decompress this file, use the following commands:\n"
+"    truncate -s %" PRIu64 " %s\n"
+"    gunzip %s\n"
+"The resulting uncompressed file should be %" PRIu64 " bytes in length.\n"
+"If you do not have a truncate command, skip that step (though gunzip will\n"
+"likely produce a \"trailing garbage ignored\" message, which can be ignored).",
+                  csize, filename, filename, usize);
+    return;
+
+no_sizes:
+    hts_log_error(
+"To decompress this file, use the following command:\n"
+"    gunzip %s\n"
+"This will likely produce a \"trailing garbage ignored\" message, which can\n"
+"usually be safely ignored.", filename);
+}
+
+static const char *bgzf_zerr(int errnum, z_stream *zs)
+{
+    static char buffer[32];
+
+    /* Return zs->msg if available.
+       zlib doesn't set this very reliably.  Looking at the source suggests
+       that it may get set to a useful message for deflateInit2, inflateInit2
+       and inflate when it returns Z_DATA_ERROR. For inflate with other
+       return codes, deflate, deflateEnd and inflateEnd it doesn't appear
+       to be useful.  For the likely non-useful cases, the caller should
+       pass NULL into zs. */
+
+    if (zs && zs->msg) return zs->msg;
+
+    // gzerror OF((gzFile file, int *errnum)
+    switch (errnum) {
+    case Z_ERRNO:
+        return strerror(errno);
+    case Z_STREAM_ERROR:
+        return "invalid parameter/compression level, or inconsistent stream state";
+    case Z_DATA_ERROR:
+        return "invalid or incomplete IO";
+    case Z_MEM_ERROR:
+        return "out of memory";
+    case Z_BUF_ERROR:
+        return "progress temporarily not possible, or in() / out() returned an error";
+    case Z_VERSION_ERROR:
+        return "zlib version mismatch";
+    case Z_NEED_DICT:
+        return "data was compressed using a dictionary";
+    case Z_OK: // 0: maybe gzgets error Z_NULL
+    default:
+        snprintf(buffer, sizeof(buffer), "[%d] unknown", errnum);
+        return buffer;  // FIXME: Not thread-safe.
+    }
+}
+
+static BGZF *bgzf_read_init(hFILE *hfpr, const char *filename)
+{
+    BGZF *fp;
+    uint8_t magic[18];
+    ssize_t n = hpeek(hfpr, magic, 18);
+    if (n < 0) return NULL;
+
+    fp = (BGZF*)calloc(1, sizeof(BGZF));
+    if (fp == NULL) return NULL;
+
+    fp->is_write = 0;
+    fp->uncompressed_block = malloc(2 * BGZF_MAX_BLOCK_SIZE);
+    if (fp->uncompressed_block == NULL) { free(fp); return NULL; }
+    fp->compressed_block = (char *)fp->uncompressed_block + BGZF_MAX_BLOCK_SIZE;
+    fp->is_compressed = (n==18 && magic[0]==0x1f && magic[1]==0x8b);
+    fp->is_gzip = ( !fp->is_compressed || ((magic[3]&4) && memcmp(&magic[12], "BC\2\0",4)==0) ) ? 0 : 1;
+    if (fp->is_compressed && (magic[3]&4) && memcmp(&magic[12], "RAZF", 4)==0) {
+        hts_log_error("Cannot decompress legacy RAZF format");
+        razf_info(hfpr, filename);
+        free(fp->uncompressed_block);
+        free(fp);
+        errno = EFTYPE;
+        return NULL;
+    }
+#ifdef BGZF_CACHE
+    if (!(fp->cache = malloc(sizeof(*fp->cache)))) {
+        free(fp->uncompressed_block);
+        free(fp);
+        return NULL;
+    }
+    if (!(fp->cache->h = kh_init(cache))) {
+        free(fp->uncompressed_block);
+        free(fp->cache);
+        free(fp);
+        return NULL;
+    }
+    fp->cache->last_pos = 0;
+#endif
+    return fp;
+}
+
+// get the compress level from the mode string: compress_level==-1 for the default level, -2 plain uncompressed
+static int mode2level(const char *mode)
+{
+    int i, compress_level = -1;
+    for (i = 0; mode[i]; ++i)
+        if (mode[i] >= '0' && mode[i] <= '9') break;
+    if (mode[i]) compress_level = (int)mode[i] - '0';
+    if (strchr(mode, 'u')) compress_level = -2;
+    return compress_level;
+}
+static BGZF *bgzf_write_init(const char *mode)
+{
+    BGZF *fp;
+    fp = (BGZF*)calloc(1, sizeof(BGZF));
+    if (fp == NULL) goto mem_fail;
+    fp->is_write = 1;
+    int compress_level = mode2level(mode);
+    if ( compress_level==-2 )
+    {
+        fp->is_compressed = 0;
+        return fp;
+    }
+    fp->is_compressed = 1;
+
+    fp->uncompressed_block = malloc(2 * BGZF_MAX_BLOCK_SIZE);
+    if (fp->uncompressed_block == NULL) goto mem_fail;
+    fp->compressed_block = (char *)fp->uncompressed_block + BGZF_MAX_BLOCK_SIZE;
+
+    fp->compress_level = compress_level < 0? Z_DEFAULT_COMPRESSION : compress_level; // Z_DEFAULT_COMPRESSION==-1
+    if (fp->compress_level > 9) fp->compress_level = Z_DEFAULT_COMPRESSION;
+    if ( strchr(mode,'g') )
+    {
+        // gzip output
+        fp->is_gzip = 1;
+        fp->gz_stream = (z_stream*)calloc(1,sizeof(z_stream));
+        if (fp->gz_stream == NULL) goto mem_fail;
+        fp->gz_stream->zalloc = NULL;
+        fp->gz_stream->zfree  = NULL;
+        fp->gz_stream->msg    = NULL;
+
+        int ret = deflateInit2(fp->gz_stream, fp->compress_level, Z_DEFLATED, 15|16, 8, Z_DEFAULT_STRATEGY);
+        if (ret!=Z_OK) {
+            hts_log_error("Call to deflateInit2 failed: %s", bgzf_zerr(ret, fp->gz_stream));
+            goto fail;
+        }
+    }
+    return fp;
+
+mem_fail:
+    hts_log_error("%s", strerror(errno));
+
+fail:
+    if (fp != NULL) {
+        free(fp->uncompressed_block);
+        free(fp->gz_stream);
+        free(fp);
+    }
+    return NULL;
+}
+
+BGZF *bgzf_open(const char *path, const char *mode)
+{
+    BGZF *fp = 0;
+    if (strchr(mode, 'r')) {
+        hFILE *fpr;
+        if ((fpr = hopen(path, mode)) == 0) return 0;
+        fp = bgzf_read_init(fpr, path);
+        if (fp == 0) { hclose_abruptly(fpr); return NULL; }
+        fp->fp = fpr;
+    } else if (strchr(mode, 'w') || strchr(mode, 'a')) {
+        hFILE *fpw;
+        if ((fpw = hopen(path, mode)) == 0) return 0;
+        fp = bgzf_write_init(mode);
+        if (fp == NULL) return NULL;
+        fp->fp = fpw;
+    }
+    else { errno = EINVAL; return 0; }
+
+    fp->is_be = ed_is_big();
+    return fp;
+}
+
+BGZF *bgzf_dopen(int fd, const char *mode)
+{
+    BGZF *fp = 0;
+    if (strchr(mode, 'r')) {
+        hFILE *fpr;
+        if ((fpr = hdopen(fd, mode)) == 0) return 0;
+        fp = bgzf_read_init(fpr, NULL);
+        if (fp == 0) { hclose_abruptly(fpr); return NULL; } // FIXME this closes fd
+        fp->fp = fpr;
+    } else if (strchr(mode, 'w') || strchr(mode, 'a')) {
+        hFILE *fpw;
+        if ((fpw = hdopen(fd, mode)) == 0) return 0;
+        fp = bgzf_write_init(mode);
+        if (fp == NULL) return NULL;
+        fp->fp = fpw;
+    }
+    else { errno = EINVAL; return 0; }
+
+    fp->is_be = ed_is_big();
+    return fp;
+}
+
+BGZF *bgzf_hopen(hFILE *hfp, const char *mode)
+{
+    BGZF *fp = NULL;
+    if (strchr(mode, 'r')) {
+        fp = bgzf_read_init(hfp, NULL);
+        if (fp == NULL) return NULL;
+    } else if (strchr(mode, 'w') || strchr(mode, 'a')) {
+        fp = bgzf_write_init(mode);
+        if (fp == NULL) return NULL;
+    }
+    else { errno = EINVAL; return 0; }
+
+    fp->fp = hfp;
+    fp->is_be = ed_is_big();
+    return fp;
+}
+
+#ifdef HAVE_LIBDEFLATE
+int bgzf_compress(void *_dst, size_t *dlen, const void *src, size_t slen, int level)
+{
+    if (slen == 0) {
+        // EOF block
+        if (*dlen < 28) return -1;
+        memcpy(_dst, "\037\213\010\4\0\0\0\0\0\377\6\0\102\103\2\0\033\0\3\0\0\0\0\0\0\0\0\0", 28);
+        *dlen = 28;
+        return 0;
+    }
+
+    uint8_t *dst = (uint8_t*)_dst;
+
+    if (level == 0) {
+        // Uncompressed data
+        if (*dlen < slen+5 + BLOCK_HEADER_LENGTH + BLOCK_FOOTER_LENGTH) return -1;
+        dst[BLOCK_HEADER_LENGTH] = 1; // BFINAL=1, BTYPE=00; see RFC1951
+        u16_to_le(slen,  &dst[BLOCK_HEADER_LENGTH+1]); // length
+        u16_to_le(~slen, &dst[BLOCK_HEADER_LENGTH+3]); // ones-complement length
+        memcpy(dst + BLOCK_HEADER_LENGTH+5, src, slen);
+        *dlen = slen+5 + BLOCK_HEADER_LENGTH + BLOCK_FOOTER_LENGTH;
+
+    } else {
+        level = level > 0 ? level : 6; // libdeflate doesn't honour -1 as default
+        // NB levels go up to 12 here.
+        int lvl_map[] = {0,1,2,3,5,6,7,8,10,12};
+        level = lvl_map[level>9 ?9 :level];
+        struct libdeflate_compressor *z = libdeflate_alloc_compressor(level);
+        if (!z) return -1;
+
+        // Raw deflate
+        size_t clen =
+            libdeflate_deflate_compress(z, src, slen,
+                                        dst + BLOCK_HEADER_LENGTH,
+                                        *dlen - BLOCK_HEADER_LENGTH - BLOCK_FOOTER_LENGTH);
+
+        if (clen <= 0) {
+            hts_log_error("Call to libdeflate_deflate_compress failed");
+            libdeflate_free_compressor(z);
+            return -1;
+        }
+
+        *dlen = clen + BLOCK_HEADER_LENGTH + BLOCK_FOOTER_LENGTH;
+
+        libdeflate_free_compressor(z);
+    }
+
+    // write the header
+    memcpy(dst, g_magic, BLOCK_HEADER_LENGTH); // the last two bytes are a place holder for the length of the block
+    packInt16(&dst[16], *dlen - 1); // write the compressed length; -1 to fit 2 bytes
+
+    // write the footer
+    uint32_t crc = libdeflate_crc32(0, src, slen);
+    packInt32((uint8_t*)&dst[*dlen - 8], crc);
+    packInt32((uint8_t*)&dst[*dlen - 4], slen);
+    return 0;
+}
+
+#else
+
+int bgzf_compress(void *_dst, size_t *dlen, const void *src, size_t slen, int level)
+{
+    uint32_t crc;
+    z_stream zs;
+    uint8_t *dst = (uint8_t*)_dst;
+
+    if (level == 0) {
+    uncomp:
+        // Uncompressed data
+        if (*dlen < slen+5 + BLOCK_HEADER_LENGTH + BLOCK_FOOTER_LENGTH) return -1;
+        dst[BLOCK_HEADER_LENGTH] = 1; // BFINAL=1, BTYPE=00; see RFC1951
+        u16_to_le(slen,  &dst[BLOCK_HEADER_LENGTH+1]); // length
+        u16_to_le(~slen, &dst[BLOCK_HEADER_LENGTH+3]); // ones-complement length
+        memcpy(dst + BLOCK_HEADER_LENGTH+5, src, slen);
+        *dlen = slen+5 + BLOCK_HEADER_LENGTH + BLOCK_FOOTER_LENGTH;
+    } else {
+        // compress the body
+        zs.zalloc = NULL; zs.zfree = NULL;
+        zs.msg = NULL;
+        zs.next_in  = (Bytef*)src;
+        zs.avail_in = slen;
+        zs.next_out = dst + BLOCK_HEADER_LENGTH;
+        zs.avail_out = *dlen - BLOCK_HEADER_LENGTH - BLOCK_FOOTER_LENGTH;
+        int ret = deflateInit2(&zs, level, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY); // -15 to disable zlib header/footer
+        if (ret!=Z_OK) {
+            hts_log_error("Call to deflateInit2 failed: %s", bgzf_zerr(ret, &zs));
+            return -1;
+        }
+        if ((ret = deflate(&zs, Z_FINISH)) != Z_STREAM_END) {
+            if (ret == Z_OK && zs.avail_out == 0) {
+                deflateEnd(&zs);
+                goto uncomp;
+            } else {
+                hts_log_error("Deflate operation failed: %s", bgzf_zerr(ret, ret == Z_DATA_ERROR ? &zs : NULL));
+            }
+            return -1;
+        }
+        // If we used up the entire output buffer, then we either ran out of
+        // room or we *just* fitted, but either way we may as well store
+        // uncompressed for faster decode.
+        if (zs.avail_out == 0) {
+            deflateEnd(&zs);
+            goto uncomp;
+        }
+        if ((ret = deflateEnd(&zs)) != Z_OK) {
+            hts_log_error("Call to deflateEnd failed: %s", bgzf_zerr(ret, NULL));
+            return -1;
+        }
+        *dlen = zs.total_out + BLOCK_HEADER_LENGTH + BLOCK_FOOTER_LENGTH;
+    }
+
+    // write the header
+    memcpy(dst, g_magic, BLOCK_HEADER_LENGTH); // the last two bytes are a place holder for the length of the block
+    packInt16(&dst[16], *dlen - 1); // write the compressed length; -1 to fit 2 bytes
+    // write the footer
+    crc = crc32(crc32(0L, NULL, 0L), (Bytef*)src, slen);
+    packInt32((uint8_t*)&dst[*dlen - 8], crc);
+    packInt32((uint8_t*)&dst[*dlen - 4], slen);
+    return 0;
+}
+#endif // HAVE_LIBDEFLATE
+
+static int bgzf_gzip_compress(BGZF *fp, void *_dst, size_t *dlen, const void *src, size_t slen, int level)
+{
+    uint8_t *dst = (uint8_t*)_dst;
+    z_stream *zs = fp->gz_stream;
+    int flush = slen ? Z_PARTIAL_FLUSH : Z_FINISH;
+    zs->next_in   = (Bytef*)src;
+    zs->avail_in  = slen;
+    zs->next_out  = dst;
+    zs->avail_out = *dlen;
+    int ret = deflate(zs, flush);
+    if (ret == Z_STREAM_ERROR) {
+        hts_log_error("Deflate operation failed: %s", bgzf_zerr(ret, NULL));
+        return -1;
+    }
+    if (zs->avail_in != 0) {
+        hts_log_error("Deflate block too large for output buffer");
+        return -1;
+    }
+    *dlen = *dlen - zs->avail_out;
+    return 0;
+}
+
+// Deflate the block in fp->uncompressed_block into fp->compressed_block. Also adds an extra field that stores the compressed block length.
+static int deflate_block(BGZF *fp, int block_length)
+{
+    size_t comp_size = BGZF_MAX_BLOCK_SIZE;
+    int ret;
+    if ( !fp->is_gzip )
+        ret = bgzf_compress(fp->compressed_block, &comp_size, fp->uncompressed_block, block_length, fp->compress_level);
+    else
+        ret = bgzf_gzip_compress(fp, fp->compressed_block, &comp_size, fp->uncompressed_block, block_length, fp->compress_level);
+
+    if ( ret != 0 )
+    {
+        hts_log_debug("Compression error %d", ret);
+        fp->errcode |= BGZF_ERR_ZLIB;
+        return -1;
+    }
+    fp->block_offset = 0;
+    return comp_size;
+}
+
+#ifdef HAVE_LIBDEFLATE
+
+static int bgzf_uncompress(uint8_t *dst, size_t *dlen,
+                           const uint8_t *src, size_t slen,
+                           uint32_t expected_crc) {
+    struct libdeflate_decompressor *z = libdeflate_alloc_decompressor();
+    if (!z) {
+        hts_log_error("Call to libdeflate_alloc_decompressor failed");
+        return -1;
+    }
+
+    int ret = libdeflate_deflate_decompress(z, src, slen, dst, *dlen, dlen);
+    libdeflate_free_decompressor(z);
+
+    if (ret != LIBDEFLATE_SUCCESS) {
+        hts_log_error("Inflate operation failed: %d", ret);
+        return -1;
+    }
+
+    uint32_t crc = libdeflate_crc32(0, (unsigned char *)dst, *dlen);
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    // Pretend the CRC was OK so the fuzzer doesn't have to get it right
+    crc = expected_crc;
+#endif
+    if (crc != expected_crc) {
+        hts_log_error("CRC32 checksum mismatch");
+        return -2;
+    }
+
+    return 0;
+}
+
+#else
+
+static int bgzf_uncompress(uint8_t *dst, size_t *dlen,
+                           const uint8_t *src, size_t slen,
+                           uint32_t expected_crc) {
+    z_stream zs = {
+        .zalloc = NULL,
+        .zfree = NULL,
+        .msg = NULL,
+        .next_in = (Bytef*)src,
+        .avail_in = slen,
+        .next_out = (Bytef*)dst,
+        .avail_out = *dlen
+    };
+
+    int ret = inflateInit2(&zs, -15);
+    if (ret != Z_OK) {
+        hts_log_error("Call to inflateInit2 failed: %s", bgzf_zerr(ret, &zs));
+        return -1;
+    }
+    if ((ret = inflate(&zs, Z_FINISH)) != Z_STREAM_END) {
+        hts_log_error("Inflate operation failed: %s", bgzf_zerr(ret, ret == Z_DATA_ERROR ? &zs : NULL));
+        if ((ret = inflateEnd(&zs)) != Z_OK) {
+            hts_log_warning("Call to inflateEnd failed: %s", bgzf_zerr(ret, NULL));
+        }
+        return -1;
+    }
+    if ((ret = inflateEnd(&zs)) != Z_OK) {
+        hts_log_error("Call to inflateEnd failed: %s", bgzf_zerr(ret, NULL));
+        return -1;
+    }
+    *dlen = *dlen - zs.avail_out;
+
+    uint32_t crc = crc32(crc32(0L, NULL, 0L), (unsigned char *)dst, *dlen);
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    // Pretend the CRC was OK so the fuzzer doesn't have to get it right
+    crc = expected_crc;
+#endif
+    if (crc != expected_crc) {
+        hts_log_error("CRC32 checksum mismatch");
+        return -2;
+    }
+
+    return 0;
+}
+#endif // HAVE_LIBDEFLATE
+
+// Inflate the block in fp->compressed_block into fp->uncompressed_block
+static int inflate_block(BGZF* fp, int block_length)
+{
+    size_t dlen = BGZF_MAX_BLOCK_SIZE;
+    uint32_t crc = le_to_u32((uint8_t *)fp->compressed_block + block_length-8);
+    int ret = bgzf_uncompress(fp->uncompressed_block, &dlen,
+                              (Bytef*)fp->compressed_block + 18,
+                              block_length - 18, crc);
+    if (ret < 0) {
+        if (ret == -2)
+            fp->errcode |= BGZF_ERR_CRC;
+        else
+            fp->errcode |= BGZF_ERR_ZLIB;
+        return -1;
+    }
+
+    return dlen;
+}
+
+// Decompress the next part of a non-blocked GZIP file.
+// Return the number of uncompressed bytes read, 0 on EOF, or a negative number on error.
+// Will fill the output buffer unless the end of the GZIP file is reached.
+static int inflate_gzip_block(BGZF *fp)
+{
+    // we will set this to true when we detect EOF, so we don't bang against the EOF more than once per call
+    int input_eof = 0;
+
+    // write to the part of the output buffer after block_offset
+    fp->gz_stream->next_out = (Bytef*)fp->uncompressed_block + fp->block_offset;
+    fp->gz_stream->avail_out = BGZF_MAX_BLOCK_SIZE - fp->block_offset;
+
+    while ( fp->gz_stream->avail_out != 0 ) {
+        // until we fill the output buffer (or hit EOF)
+
+        if ( !input_eof && fp->gz_stream->avail_in == 0 ) {
+            // we are out of input data in the buffer. Get more.
+            fp->gz_stream->next_in = fp->compressed_block;
+            int ret = hread(fp->fp, fp->compressed_block, BGZF_BLOCK_SIZE);
+            if ( ret < 0 ) {
+                // hread had an error. Pass it on.
+                return ret;
+            }
+            fp->gz_stream->avail_in = ret;
+            if ( fp->gz_stream->avail_in < BGZF_BLOCK_SIZE ) {
+                // we have reached EOF but the decompressor hasn't necessarily
+                input_eof = 1;
+            }
+        }
+
+        fp->gz_stream->msg = NULL;
+        // decompress as much data as we can
+        int ret = inflate(fp->gz_stream, Z_SYNC_FLUSH);
+
+        if ( (ret < 0 && ret != Z_BUF_ERROR) || ret == Z_NEED_DICT ) {
+            // an error occurred, other than running out of space
+            hts_log_error("Inflate operation failed: %s", bgzf_zerr(ret, ret == Z_DATA_ERROR ? fp->gz_stream : NULL));
+            fp->errcode |= BGZF_ERR_ZLIB;
+            return -1;
+        } else if ( ret == Z_STREAM_END ) {
+            // we finished a GZIP member
+
+            // scratch for peeking to see if the file is over
+            char c;
+            if (fp->gz_stream->avail_in > 0 || hpeek(fp->fp, &c, 1) == 1) {
+                // there is more data; try and read another GZIP member in the remaining data
+                int reset_ret = inflateReset(fp->gz_stream);
+                if (reset_ret != Z_OK) {
+                    hts_log_error("Call to inflateReset failed: %s", bgzf_zerr(reset_ret, NULL));
+                    fp->errcode |= BGZF_ERR_ZLIB;
+                    return -1;
+                }
+            } else {
+                // we consumed all the input data and hit Z_STREAM_END
+                // so stop looping, even if we never fill the output buffer
+                break;
+            }
+        } else if ( ret == Z_BUF_ERROR && input_eof && fp->gz_stream->avail_out > 0 ) {
+            // the gzip file has ended prematurely
+            hts_log_error("Gzip file truncated");
+            fp->errcode |= BGZF_ERR_IO;
+            return -1;
+        }
+    }
+
+    // when we get here, the buffer is full or there is an EOF after a complete gzip member
+    return BGZF_MAX_BLOCK_SIZE - fp->gz_stream->avail_out;
+}
+
+// Returns: 0 on success (BGZF header); -1 on non-BGZF GZIP header; -2 on error
+static int check_header(const uint8_t *header)
+{
+    if ( header[0] != 31 || header[1] != 139 || header[2] != 8 ) return -2;
+    return ((header[3] & 4) != 0
+            && unpackInt16((uint8_t*)&header[10]) == 6
+            && header[12] == 'B' && header[13] == 'C'
+            && unpackInt16((uint8_t*)&header[14]) == 2) ? 0 : -1;
+}
+
+#ifdef BGZF_CACHE
+static void free_cache(BGZF *fp)
+{
+    khint_t k;
+    if (fp->is_write) return;
+    khash_t(cache) *h = fp->cache->h;
+    for (k = kh_begin(h); k < kh_end(h); ++k)
+        if (kh_exist(h, k)) free(kh_val(h, k).block);
+    kh_destroy(cache, h);
+    free(fp->cache);
+}
+
+static int load_block_from_cache(BGZF *fp, int64_t block_address)
+{
+    khint_t k;
+    cache_t *p;
+
+    khash_t(cache) *h = fp->cache->h;
+    k = kh_get(cache, h, block_address);
+    if (k == kh_end(h)) return 0;
+    p = &kh_val(h, k);
+    if (fp->block_length != 0) fp->block_offset = 0;
+    fp->block_address = block_address;
+    fp->block_length = p->size;
+    memcpy(fp->uncompressed_block, p->block, p->size);
+    if ( hseek(fp->fp, p->end_offset, SEEK_SET) < 0 )
+    {
+        // todo: move the error up
+        hts_log_error("Could not hseek to %" PRId64, p->end_offset);
+        exit(1);
+    }
+    return p->size;
+}
+
+static void cache_block(BGZF *fp, int size)
+{
+    int ret;
+    khint_t k, k_orig;
+    uint8_t *block = NULL;
+    cache_t *p;
+    //fprintf(stderr, "Cache block at %llx\n", (int)fp->block_address);
+    khash_t(cache) *h = fp->cache->h;
+    if (BGZF_MAX_BLOCK_SIZE >= fp->cache_size) return;
+    if (fp->block_length < 0 || fp->block_length > BGZF_MAX_BLOCK_SIZE) return;
+    if ((kh_size(h) + 1) * BGZF_MAX_BLOCK_SIZE > (uint32_t)fp->cache_size) {
+        /* Remove uniformly from any position in the hash by a simple
+         * round-robin approach.  An alternative strategy would be to
+         * remove the least recently accessed block, but the round-robin
+         * removal is simpler and is not expected to have a big impact
+         * on performance */
+        if (fp->cache->last_pos >= kh_end(h)) fp->cache->last_pos = kh_begin(h);
+        k_orig = k = fp->cache->last_pos;
+        if (++k >= kh_end(h)) k = kh_begin(h);
+        while (k != k_orig) {
+            if (kh_exist(h, k))
+                break;
+            if (++k == kh_end(h))
+                k = kh_begin(h);
+        }
+        fp->cache->last_pos = k;
+
+        if (k != k_orig) {
+            block = kh_val(h, k).block;
+            kh_del(cache, h, k);
+        }
+    } else {
+        block = (uint8_t*)malloc(BGZF_MAX_BLOCK_SIZE);
+    }
+    if (!block) return;
+    k = kh_put(cache, h, fp->block_address, &ret);
+    if (ret <= 0) { // kh_put failed, or in there already (shouldn't happen)
+        free(block);
+        return;
+    }
+    p = &kh_val(h, k);
+    p->size = fp->block_length;
+    p->end_offset = fp->block_address + size;
+    p->block = block;
+    memcpy(p->block, fp->uncompressed_block, p->size);
+}
+#else
+static void free_cache(BGZF *fp) {}
+static int load_block_from_cache(BGZF *fp, int64_t block_address) {return 0;}
+static void cache_block(BGZF *fp, int size) {}
+#endif
+
+/*
+ * Absolute htell in this compressed file.
+ *
+ * Do not confuse with the external bgzf_tell macro which returns the virtual
+ * offset.
+ */
+static off_t bgzf_htell(BGZF *fp) {
+    if (fp->mt) {
+        pthread_mutex_lock(&fp->mt->job_pool_m);
+        off_t pos = fp->block_address + fp->block_clength;
+        pthread_mutex_unlock(&fp->mt->job_pool_m);
+        return pos;
+    } else {
+        return htell(fp->fp);
+    }
+}
+
+int bgzf_read_block(BGZF *fp)
+{
+    hts_tpool_result *r;
+
+    if (fp->errcode) return -1;
+
+    if (fp->mt) {
+    again:
+        if (fp->mt->hit_eof) {
+            // Further reading at EOF will always return 0
+            fp->block_length = 0;
+            return 0;
+        }
+        r = hts_tpool_next_result_wait(fp->mt->out_queue);
+        bgzf_job *j = r ? (bgzf_job *)hts_tpool_result_data(r) : NULL;
+
+        if (!j || j->errcode == BGZF_ERR_MT) {
+            if (!fp->mt->free_block) {
+                fp->uncompressed_block = malloc(2 * BGZF_MAX_BLOCK_SIZE);
+                if (fp->uncompressed_block == NULL) return -1;
+                fp->compressed_block = (char *)fp->uncompressed_block + BGZF_MAX_BLOCK_SIZE;
+            } // else it's already allocated with malloc, maybe even in-use.
+            if (mt_destroy(fp->mt) < 0) {
+                fp->errcode = BGZF_ERR_IO;
+            }
+            fp->mt = NULL;
+            hts_tpool_delete_result(r, 0);
+            if (fp->errcode) {
+                return -1;
+            }
+            goto single_threaded;
+        }
+
+        if (j->errcode) {
+            fp->errcode = j->errcode;
+            hts_log_error("BGZF decode jobs returned error %d "
+                          "for block offset %"PRId64,
+                          j->errcode, j->block_address);
+            hts_tpool_delete_result(r, 0);
+            return -1;
+        }
+
+        if (j->hit_eof) {
+            if (!fp->last_block_eof && !fp->no_eof_block) {
+                fp->no_eof_block = 1;
+                hts_log_warning("EOF marker is absent. The input may be truncated");
+            }
+            fp->mt->hit_eof = 1;
+        }
+
+        // Zero length blocks in the middle of a file are (wrongly)
+        // considered as EOF by many callers.  We work around this by
+        // trying again to see if we hit a genuine EOF.
+        if (!j->hit_eof && j->uncomp_len == 0) {
+            fp->last_block_eof = 1;
+            hts_tpool_delete_result(r, 0);
+            goto again;
+        }
+
+        // block_length=0 and block_offset set by bgzf_seek.
+        if (fp->block_length != 0) fp->block_offset = 0;
+        if (!j->hit_eof) fp->block_address = j->block_address;
+        fp->block_clength = j->comp_len;
+        fp->block_length = j->uncomp_len;
+        // bgzf_read() can change fp->block_length
+        fp->last_block_eof = (fp->block_length == 0);
+
+        if ( j->uncomp_len && j->fp->idx_build_otf )
+        {
+            bgzf_index_add_block(j->fp);
+            j->fp->idx->ublock_addr += j->uncomp_len;
+        }
+
+        // Steal the data block as it's quicker than a memcpy.
+        // We just need to make sure we delay the pool free.
+        if (fp->mt->curr_job) {
+            pthread_mutex_lock(&fp->mt->job_pool_m);
+            pool_free(fp->mt->job_pool, fp->mt->curr_job);
+            pthread_mutex_unlock(&fp->mt->job_pool_m);
+        }
+        fp->uncompressed_block = j->uncomp_data;
+        fp->mt->curr_job = j;
+        if (fp->mt->free_block) {
+            free(fp->mt->free_block); // clear up last non-mt block
+            fp->mt->free_block = NULL;
+        }
+
+        hts_tpool_delete_result(r, 0);
+        return 0;
+    }
+
+    uint8_t header[BLOCK_HEADER_LENGTH], *compressed_block;
+    int count, size, block_length, remaining;
+
+ single_threaded:
+    size = 0;
+
+    int64_t block_address;
+    block_address = bgzf_htell(fp);
+
+    // Reading an uncompressed file
+    if ( !fp->is_compressed )
+    {
+        count = hread(fp->fp, fp->uncompressed_block, BGZF_MAX_BLOCK_SIZE);
+        if (count < 0)  // Error
+        {
+            hts_log_error("Failed to read uncompressed data "
+                          "at offset %"PRId64"%s%s",
+                          block_address, errno ? ": " : "", strerror(errno));
+            fp->errcode |= BGZF_ERR_IO;
+            return -1;
+        }
+        else if (count == 0)  // EOF
+        {
+            fp->block_length = 0;
+            return 0;
+        }
+        if (fp->block_length != 0) fp->block_offset = 0;
+        fp->block_address = block_address;
+        fp->block_length = count;
+        return 0;
+    }
+
+    // Reading compressed file
+    if ( fp->is_gzip && fp->gz_stream ) // is this is an initialized gzip stream?
+    {
+        count = inflate_gzip_block(fp);
+        if ( count<0 )
+        {
+            hts_log_error("Reading GZIP stream failed at offset %"PRId64,
+                          block_address);
+            fp->errcode |= BGZF_ERR_ZLIB;
+            return -1;
+        }
+        fp->block_length = count;
+        fp->block_address = block_address;
+        return 0;
+    }
+    if (fp->cache_size && load_block_from_cache(fp, block_address)) return 0;
+
+    // loop to skip empty bgzf blocks
+    while (1)
+    {
+        count = hread(fp->fp, header, sizeof(header));
+        if (count == 0) { // no data read
+            if (!fp->last_block_eof && !fp->no_eof_block && !fp->is_gzip) {
+                fp->no_eof_block = 1;
+                hts_log_warning("EOF marker is absent. The input may be truncated");
+            }
+            fp->block_length = 0;
+            return 0;
+        }
+        int ret = 0;
+        if ( count != sizeof(header) || (ret=check_header(header))==-2 )
+        {
+            fp->errcode |= BGZF_ERR_HEADER;
+            hts_log_error("%s BGZF header at offset %"PRId64,
+                          ret ? "Invalid" : "Failed to read",
+                          block_address);
+            return -1;
+        }
+        if ( ret==-1 )
+        {
+            // GZIP, not BGZF
+            uint8_t *cblock = (uint8_t*)fp->compressed_block;
+            memcpy(cblock, header, sizeof(header));
+            count = hread(fp->fp, cblock+sizeof(header), BGZF_BLOCK_SIZE - sizeof(header)) + sizeof(header);
+
+            fp->is_gzip = 1;
+            fp->gz_stream = (z_stream*) calloc(1,sizeof(z_stream));
+            // Set up zlib, using a window size of 15, and its built-in GZIP header processing (+16).
+            int ret = inflateInit2(fp->gz_stream, 15 + 16);
+            if (ret != Z_OK)
+            {
+                hts_log_error("Call to inflateInit2 failed: %s", bgzf_zerr(ret, fp->gz_stream));
+                fp->errcode |= BGZF_ERR_ZLIB;
+                return -1;
+            }
+            fp->gz_stream->avail_in = count;
+            fp->gz_stream->next_in  = cblock;
+            count = inflate_gzip_block(fp);
+            if ( count<0 )
+            {
+                hts_log_error("Reading GZIP stream failed at offset %"PRId64,
+                              block_address);
+                fp->errcode |= BGZF_ERR_ZLIB;
+                return -1;
+            }
+            fp->block_length = count;
+            fp->block_address = block_address;
+            if ( fp->idx_build_otf ) return -1; // cannot build index for gzip
+            return 0;
+        }
+        size = count;
+        block_length = unpackInt16((uint8_t*)&header[16]) + 1; // +1 because when writing this number, we used "-1"
+        if (block_length < BLOCK_HEADER_LENGTH)
+        {
+            hts_log_error("Invalid BGZF block length at offset %"PRId64,
+                          block_address);
+            fp->errcode |= BGZF_ERR_HEADER;
+            return -1;
+        }
+        compressed_block = (uint8_t*)fp->compressed_block;
+        memcpy(compressed_block, header, BLOCK_HEADER_LENGTH);
+        remaining = block_length - BLOCK_HEADER_LENGTH;
+        count = hread(fp->fp, &compressed_block[BLOCK_HEADER_LENGTH], remaining);
+        if (count != remaining) {
+            hts_log_error("Failed to read BGZF block data at offset %"PRId64
+                          " expected %d bytes; hread returned %d",
+                          block_address, remaining, count);
+            fp->errcode |= BGZF_ERR_IO;
+            return -1;
+        }
+        size += count;
+        if ((count = inflate_block(fp, block_length)) < 0) {
+            hts_log_debug("Inflate block operation failed for "
+                          "block at offset %"PRId64": %s",
+                          block_address, bgzf_zerr(count, NULL));
+            fp->errcode |= BGZF_ERR_ZLIB;
+            return -1;
+        }
+        fp->last_block_eof = (count == 0);
+        if ( count ) break;     // otherwise an empty bgzf block
+        block_address = bgzf_htell(fp); // update for new block start
+    }
+    if (fp->block_length != 0) fp->block_offset = 0; // Do not reset offset if this read follows a seek.
+    fp->block_address = block_address;
+    fp->block_length = count;
+    if ( fp->idx_build_otf )
+    {
+        bgzf_index_add_block(fp);
+        fp->idx->ublock_addr += count;
+    }
+    cache_block(fp, size);
+    return 0;
+}
+
+ssize_t bgzf_read(BGZF *fp, void *data, size_t length)
+{
+    ssize_t bytes_read = 0;
+    uint8_t *output = (uint8_t*)data;
+    if (length <= 0) return 0;
+    assert(fp->is_write == 0);
+    while (bytes_read < length) {
+        int copy_length, available = fp->block_length - fp->block_offset;
+        uint8_t *buffer;
+        if (available <= 0) {
+            int ret = bgzf_read_block(fp);
+            if (ret != 0) {
+                hts_log_error("Read block operation failed with error %d after %zd of %zu bytes", fp->errcode, bytes_read, length);
+                fp->errcode |= BGZF_ERR_ZLIB;
+                return -1;
+            }
+            available = fp->block_length - fp->block_offset;
+            if (available == 0) {
+                if (fp->block_length == 0)
+                    break; // EOF
+
+                // Offset was at end of block (see commit e9863a0)
+                fp->block_address = bgzf_htell(fp);
+                fp->block_offset = fp->block_length = 0;
+                continue;
+            } else if (available < 0) {
+                // Block offset was set to an invalid coordinate
+                hts_log_error("BGZF block offset %d set beyond block size %d",
+                              fp->block_offset, fp->block_length);
+                fp->errcode |= BGZF_ERR_MISUSE;
+                return -1;
+            }
+        }
+        copy_length = length - bytes_read < available? length - bytes_read : available;
+        buffer = (uint8_t*)fp->uncompressed_block;
+        memcpy(output, buffer + fp->block_offset, copy_length);
+        fp->block_offset += copy_length;
+        output += copy_length;
+        bytes_read += copy_length;
+
+        // For raw gzip streams this avoids short reads.
+        if (fp->block_offset == fp->block_length) {
+            fp->block_address = bgzf_htell(fp);
+            fp->block_offset = fp->block_length = 0;
+        }
+    }
+
+    fp->uncompressed_address += bytes_read;
+
+    return bytes_read;
+}
+
+// -1 for EOF, -2 for error, 0-255 for byte.
+int bgzf_peek(BGZF *fp) {
+    int available = fp->block_length - fp->block_offset;
+    if (available <= 0) {
+        if (bgzf_read_block(fp) < 0) {
+            hts_log_error("Read block operation failed with error %d", fp->errcode);
+            fp->errcode = BGZF_ERR_ZLIB;
+            return -2;
+        }
+    }
+    available = fp->block_length - fp->block_offset;
+    if (available)
+        return ((unsigned char *)fp->uncompressed_block)[fp->block_offset];
+
+    return -1;
+}
+
+ssize_t bgzf_raw_read(BGZF *fp, void *data, size_t length)
+{
+    ssize_t ret = hread(fp->fp, data, length);
+    if (ret < 0) fp->errcode |= BGZF_ERR_IO;
+    return ret;
+}
+
+#ifdef BGZF_MT
+
+/* Function to clean up when jobs are discarded (e.g. during seek)
+ * This works for results too, as results are the same struct with
+ * decompressed data stored in it. */
+static void job_cleanup(void *arg) {
+    bgzf_job *j = (bgzf_job *)arg;
+    mtaux_t *mt = j->fp->mt;
+    pthread_mutex_lock(&mt->job_pool_m);
+    pool_free(mt->job_pool, j);
+    pthread_mutex_unlock(&mt->job_pool_m);
+}
+
+static void *bgzf_encode_func(void *arg) {
+    bgzf_job *j = (bgzf_job *)arg;
+
+    j->comp_len = BGZF_MAX_BLOCK_SIZE;
+    int ret = bgzf_compress(j->comp_data, &j->comp_len,
+                            j->uncomp_data, j->uncomp_len,
+                            j->fp->compress_level);
+    if (ret != 0)
+        j->errcode |= BGZF_ERR_ZLIB;
+
+    return arg;
+}
+
+// Optimisation for compression level 0 (uncompressed deflate blocks)
+// Avoids memcpy of the data from uncompressed to compressed buffer.
+static void *bgzf_encode_level0_func(void *arg) {
+    bgzf_job *j = (bgzf_job *)arg;
+    uint32_t crc;
+    j->comp_len = j->uncomp_len + BLOCK_HEADER_LENGTH + BLOCK_FOOTER_LENGTH + 5;
+
+    // Data will have already been copied in to
+    // j->comp_data + BLOCK_HEADER_LENGTH + 5
+
+    // Add preamble
+    memcpy(j->comp_data, g_magic, BLOCK_HEADER_LENGTH);
+    u16_to_le(j->comp_len-1, j->comp_data + 16);
+
+    // Deflate uncompressed data header
+    j->comp_data[BLOCK_HEADER_LENGTH] = 1; // BFINAL=1, BTYPE=00; see RFC1951
+    u16_to_le(j->uncomp_len, j->comp_data + BLOCK_HEADER_LENGTH + 1);
+    u16_to_le(~j->uncomp_len, j->comp_data + BLOCK_HEADER_LENGTH + 3);
+
+    // Trailer (CRC, uncompressed length)
+#ifdef HAVE_LIBDEFLATE
+    crc = libdeflate_crc32(0, j->comp_data + BLOCK_HEADER_LENGTH + 5,
+                           j->uncomp_len);
+#else
+    crc = crc32(crc32(0L, NULL, 0L),
+                (Bytef*)j->comp_data + BLOCK_HEADER_LENGTH + 5, j->uncomp_len);
+#endif
+    u32_to_le(crc, j->comp_data +  j->comp_len - 8);
+    u32_to_le(j->uncomp_len, j->comp_data + j->comp_len - 4);
+
+    return arg;
+}
+
+// Our input block has already been decoded by bgzf_mt_read_block().
+// We need to split that into a fetch block (compressed) and make this
+// do the actual decompression step.
+static void *bgzf_decode_func(void *arg) {
+    bgzf_job *j = (bgzf_job *)arg;
+
+    j->uncomp_len = BGZF_MAX_BLOCK_SIZE;
+    uint32_t crc = le_to_u32((uint8_t *)j->comp_data + j->comp_len-8);
+    int ret = bgzf_uncompress(j->uncomp_data, &j->uncomp_len,
+                              j->comp_data+18, j->comp_len-18, crc);
+    if (ret != 0)
+        j->errcode |= BGZF_ERR_ZLIB;
+
+    return arg;
+}
+
+/*
+ * Nul function so we can dispatch a job with the correct serial
+ * to mark failure or to indicate an empty read (EOF).
+ */
+static void *bgzf_nul_func(void *arg) { return arg; }
+
+/*
+ * Takes compressed blocks off the results queue and calls hwrite to
+ * punt them to the output stream.
+ *
+ * Returns NULL when no more are left, or -1 on error
+ */
+static void *bgzf_mt_writer(void *vp) {
+    BGZF *fp = (BGZF *)vp;
+    mtaux_t *mt = fp->mt;
+    hts_tpool_result *r;
+
+    if (fp->idx_build_otf) {
+        fp->idx->moffs = fp->idx->noffs = 1;
+        fp->idx->offs = (bgzidx1_t*) calloc(fp->idx->moffs, sizeof(bgzidx1_t));
+        if (!fp->idx->offs) goto err;
+    }
+
+    // Iterates until result queue is shutdown, where it returns NULL.
+    while ((r = hts_tpool_next_result_wait(mt->out_queue))) {
+        bgzf_job *j = (bgzf_job *)hts_tpool_result_data(r);
+        assert(j);
+
+        if (fp->idx_build_otf) {
+            fp->idx->noffs++;
+            if ( fp->idx->noffs > fp->idx->moffs )
+            {
+                fp->idx->moffs = fp->idx->noffs;
+                kroundup32(fp->idx->moffs);
+                fp->idx->offs = (bgzidx1_t*) realloc(fp->idx->offs, fp->idx->moffs*sizeof(bgzidx1_t));
+                if ( !fp->idx->offs ) goto err;
+            }
+            fp->idx->offs[ fp->idx->noffs-1 ].uaddr = fp->idx->offs[ fp->idx->noffs-2 ].uaddr + j->uncomp_len;
+            fp->idx->offs[ fp->idx->noffs-1 ].caddr = fp->idx->offs[ fp->idx->noffs-2 ].caddr + j->comp_len;
+        }
+
+        // Flush any cached hts_idx_push calls
+        if (bgzf_idx_flush(fp, j->uncomp_len, j->comp_len) < 0)
+            goto err;
+
+        if (hwrite(fp->fp, j->comp_data, j->comp_len) != j->comp_len)
+            goto err;
+
+        // Update our local block_address.  Cannot be fp->block_address due to no
+        // locking in bgzf_tell.
+        pthread_mutex_lock(&mt->idx_m);
+        mt->block_address += j->comp_len;
+        pthread_mutex_unlock(&mt->idx_m);
+
+        /*
+         * Periodically call hflush (which calls fsync when on a file).
+         * This avoids the fsync being done at the bgzf_close stage,
+         * which can sometimes cause significant delays.  As this is in
+         * a separate thread, spreading the sync delays throughout the
+         * program execution seems better.
+         * Frequency of 1/512 has been chosen by experimentation
+         * across local XFS, NFS and Lustre tests.
+         */
+        if (++mt->flush_pending % 512 == 0)
+            if (hflush(fp->fp) != 0)
+                goto err;
+
+
+        hts_tpool_delete_result(r, 0);
+
+        // Also updated by main thread
+        pthread_mutex_lock(&mt->job_pool_m);
+        pool_free(mt->job_pool, j);
+        mt->jobs_pending--;
+        pthread_mutex_unlock(&mt->job_pool_m);
+    }
+
+    if (hflush(fp->fp) != 0)
+        goto err;
+
+    hts_tpool_process_destroy(mt->out_queue);
+
+    return NULL;
+
+ err:
+    hts_tpool_process_destroy(mt->out_queue);
+    return (void *)-1;
+}
+
+
+/*
+ * Reads a compressed block of data using hread and dispatches it to
+ * the thread pool for decompression.  This is the analogue of the old
+ * non-threaded bgzf_read_block() function, but without modifying fp
+ * in any way (except for the read offset).  All output goes via the
+ * supplied bgzf_job struct.
+ *
+ * Returns NULL when no more are left, or -1 on error
+ */
+int bgzf_mt_read_block(BGZF *fp, bgzf_job *j)
+{
+    uint8_t header[BLOCK_HEADER_LENGTH], *compressed_block;
+    int count, block_length, remaining;
+
+    // NOTE: Guaranteed to be compressed as we block multi-threading in
+    // uncompressed mode.  However it may be gzip compression instead
+    // of bgzf.
+
+    // Reading compressed file
+    int64_t block_address;
+    block_address = htell(fp->fp);
+
+    j->block_address = block_address;  // in case we exit with j->errcode
+
+    if (fp->cache_size && load_block_from_cache(fp, block_address)) return 0;
+    count = hpeek(fp->fp, header, sizeof(header));
+    if (count == 0) // no data read
+        return -1;
+    int ret;
+    if ( count != sizeof(header) || (ret=check_header(header))==-2 )
+    {
+        j->errcode |= BGZF_ERR_HEADER;
+        return -1;
+    }
+    if (ret == -1) {
+        j->errcode |= BGZF_ERR_MT;
+        return -1;
+    }
+
+    count = hread(fp->fp, header, sizeof(header));
+    if (count != sizeof(header)) // no data read
+        return -1;
+
+    block_length = unpackInt16((uint8_t*)&header[16]) + 1; // +1 because when writing this number, we used "-1"
+    if (block_length < BLOCK_HEADER_LENGTH) {
+        j->errcode |= BGZF_ERR_HEADER;
+        return -1;
+    }
+    compressed_block = (uint8_t*)j->comp_data;
+    memcpy(compressed_block, header, BLOCK_HEADER_LENGTH);
+    remaining = block_length - BLOCK_HEADER_LENGTH;
+    count = hread(fp->fp, &compressed_block[BLOCK_HEADER_LENGTH], remaining);
+    if (count != remaining) {
+        j->errcode |= BGZF_ERR_IO;
+        return -1;
+    }
+    j->comp_len = block_length;
+    j->uncomp_len = BGZF_MAX_BLOCK_SIZE;
+    j->block_address = block_address;
+    j->fp = fp;
+    j->errcode = 0;
+
+    return 0;
+}
+
+
+static int bgzf_check_EOF_common(BGZF *fp)
+{
+    uint8_t buf[28];
+    off_t offset = htell(fp->fp);
+    if (hseek(fp->fp, -28, SEEK_END) < 0) {
+        if (errno == ESPIPE) { hclearerr(fp->fp); return 2; }
+#ifdef _WIN32
+        if (errno == EINVAL) { hclearerr(fp->fp); return 2; }
+#else
+        // Assume that EINVAL was due to the file being less than 28 bytes
+        // long, rather than being a random error return from an hfile backend.
+        // This should be reported as "no EOF block" rather than an error.
+        if (errno == EINVAL) { hclearerr(fp->fp); return 0; }
+#endif
+        return -1;
+    }
+    if ( hread(fp->fp, buf, 28) != 28 ) return -1;
+    if ( hseek(fp->fp, offset, SEEK_SET) < 0 ) return -1;
+    return (memcmp("\037\213\010\4\0\0\0\0\0\377\6\0\102\103\2\0\033\0\3\0\0\0\0\0\0\0\0\0", buf, 28) == 0)? 1 : 0;
+}
+
+/*
+ * Checks EOF from the reader thread.
+ */
+static void bgzf_mt_eof(BGZF *fp) {
+    mtaux_t *mt = fp->mt;
+
+    pthread_mutex_lock(&mt->job_pool_m);
+    mt->eof = bgzf_check_EOF_common(fp);
+    pthread_mutex_unlock(&mt->job_pool_m);
+    mt->command = HAS_EOF_DONE;
+    pthread_cond_signal(&mt->command_c);
+}
+
+
+/*
+ * Performs the seek (called by reader thread).
+ *
+ * This simply drains the entire queue, throwing away blocks, seeks,
+ * and starts it up again.  Brute force, but maybe sufficient.
+ */
+static void bgzf_mt_seek(BGZF *fp) {
+    mtaux_t *mt = fp->mt;
+
+    hts_tpool_process_reset(mt->out_queue, 0);
+    pthread_mutex_lock(&mt->job_pool_m);
+    mt->errcode = 0;
+
+    if (hseek(fp->fp, mt->block_address, SEEK_SET) < 0)
+        mt->errcode = BGZF_ERR_IO;
+
+    pthread_mutex_unlock(&mt->job_pool_m);
+    mt->command = SEEK_DONE;
+    pthread_cond_signal(&mt->command_c);
+}
+
+static void *bgzf_mt_reader(void *vp) {
+    BGZF *fp = (BGZF *)vp;
+    mtaux_t *mt = fp->mt;
+
+restart:
+    pthread_mutex_lock(&mt->job_pool_m);
+    bgzf_job *j = pool_alloc(mt->job_pool);
+    pthread_mutex_unlock(&mt->job_pool_m);
+    if (!j) goto err;
+    j->errcode = 0;
+    j->comp_len = 0;
+    j->uncomp_len = 0;
+    j->hit_eof = 0;
+    j->fp = fp;
+
+    while (bgzf_mt_read_block(fp, j) == 0) {
+        // Dispatch
+        if (hts_tpool_dispatch3(mt->pool, mt->out_queue, bgzf_decode_func, j,
+                                job_cleanup, job_cleanup, 0) < 0) {
+            job_cleanup(j);
+            goto err;
+        }
+
+        // Check for command
+        pthread_mutex_lock(&mt->command_m);
+        switch (mt->command) {
+        case SEEK:
+            bgzf_mt_seek(fp);  // Sets mt->command to SEEK_DONE
+            pthread_mutex_unlock(&mt->command_m);
+            goto restart;
+
+        case HAS_EOF:
+            bgzf_mt_eof(fp);   // Sets mt->command to HAS_EOF_DONE
+            break;
+
+        case SEEK_DONE:
+        case HAS_EOF_DONE:
+            pthread_cond_signal(&mt->command_c);
+            break;
+
+        case CLOSE:
+            pthread_cond_signal(&mt->command_c);
+            pthread_mutex_unlock(&mt->command_m);
+            hts_tpool_process_destroy(mt->out_queue);
+            return NULL;
+
+        default:
+            break;
+        }
+        pthread_mutex_unlock(&mt->command_m);
+
+        // Allocate buffer for next block
+        pthread_mutex_lock(&mt->job_pool_m);
+        j = pool_alloc(mt->job_pool);
+        pthread_mutex_unlock(&mt->job_pool_m);
+        if (!j) {
+            hts_tpool_process_destroy(mt->out_queue);
+            return NULL;
+        }
+        j->errcode = 0;
+        j->comp_len = 0;
+        j->uncomp_len = 0;
+        j->hit_eof = 0;
+        j->fp = fp;
+    }
+
+    if (j->errcode == BGZF_ERR_MT) {
+        // Attempt to multi-thread decode a raw gzip stream cannot be done.
+        // We tear down the multi-threaded decoder and revert to the old code.
+        if (hts_tpool_dispatch3(mt->pool, mt->out_queue, bgzf_nul_func, j,
+                                job_cleanup, job_cleanup, 0) < 0) {
+            job_cleanup(j);
+            hts_tpool_process_destroy(mt->out_queue);
+            return NULL;
+        }
+        hts_tpool_process_ref_decr(mt->out_queue);
+        return &j->errcode;
+    }
+
+    // Dispatch an empty block so EOF is spotted.
+    // We also use this mechanism for returning errors, in which case
+    // j->errcode is set already.
+
+    j->hit_eof = 1;
+    if (hts_tpool_dispatch3(mt->pool, mt->out_queue, bgzf_nul_func, j,
+                            job_cleanup, job_cleanup, 0) < 0) {
+        job_cleanup(j);
+        hts_tpool_process_destroy(mt->out_queue);
+        return NULL;
+    }
+    if (j->errcode != 0) {
+        hts_tpool_process_destroy(mt->out_queue);
+        return &j->errcode;
+    }
+
+    // We hit EOF so can stop reading, but we may get a subsequent
+    // seek request.  In this case we need to restart the reader.
+    //
+    // To handle this we wait on a condition variable and then
+    // monitor the command. (This could be either seek or close.)
+    for (;;) {
+        pthread_mutex_lock(&mt->command_m);
+        if (mt->command == NONE)
+            pthread_cond_wait(&mt->command_c, &mt->command_m);
+        switch(mt->command) {
+        default:
+            pthread_mutex_unlock(&mt->command_m);
+            break;
+
+        case SEEK:
+            bgzf_mt_seek(fp);
+            pthread_mutex_unlock(&mt->command_m);
+            goto restart;
+
+        case HAS_EOF:
+            bgzf_mt_eof(fp);   // Sets mt->command to HAS_EOF_DONE
+            pthread_mutex_unlock(&mt->command_m);
+            break;
+
+        case SEEK_DONE:
+        case HAS_EOF_DONE:
+            pthread_cond_signal(&mt->command_c);
+            pthread_mutex_unlock(&mt->command_m);
+            break;
+
+        case CLOSE:
+            pthread_cond_signal(&mt->command_c);
+            pthread_mutex_unlock(&mt->command_m);
+            hts_tpool_process_destroy(mt->out_queue);
+            return NULL;
+        }
+    }
+
+ err:
+    pthread_mutex_lock(&mt->command_m);
+    mt->command = CLOSE;
+    pthread_cond_signal(&mt->command_c);
+    pthread_mutex_unlock(&mt->command_m);
+    hts_tpool_process_destroy(mt->out_queue);
+    return NULL;
+}
+
+int bgzf_thread_pool(BGZF *fp, hts_tpool *pool, int qsize) {
+    // No gain from multi-threading when not compressed
+    if (!fp->is_compressed)
+        return 0;
+
+    mtaux_t *mt;
+    mt = (mtaux_t*)calloc(1, sizeof(mtaux_t));
+    if (!mt) return -1;
+    fp->mt = mt;
+
+    mt->pool = pool;
+    mt->n_threads = hts_tpool_size(pool);
+    if (!qsize)
+        qsize = mt->n_threads*2;
+    if (!(mt->out_queue = hts_tpool_process_init(mt->pool, qsize, 0)))
+        goto err;
+    hts_tpool_process_ref_incr(mt->out_queue);
+
+    mt->job_pool = pool_create(sizeof(bgzf_job));
+    if (!mt->job_pool)
+        goto err;
+
+    pthread_mutex_init(&mt->job_pool_m, NULL);
+    pthread_mutex_init(&mt->command_m, NULL);
+    pthread_mutex_init(&mt->idx_m, NULL);
+    pthread_cond_init(&mt->command_c, NULL);
+    mt->flush_pending = 0;
+    mt->jobs_pending = 0;
+    mt->free_block = fp->uncompressed_block; // currently in-use block
+    mt->block_address = fp->block_address;
+    pthread_create(&mt->io_task, NULL,
+                   fp->is_write ? bgzf_mt_writer : bgzf_mt_reader, fp);
+
+    return 0;
+
+ err:
+    free(mt);
+    fp->mt = NULL;
+    return -1;
+}
+
+int bgzf_mt(BGZF *fp, int n_threads, int n_sub_blks)
+{
+    // No gain from multi-threading when not compressed
+    if (!fp->is_compressed || fp->is_gzip)
+        return 0;
+
+    if (n_threads < 1) return -1;
+    hts_tpool *p = hts_tpool_init(n_threads);
+    if (!p)
+        return -1;
+
+    if (bgzf_thread_pool(fp, p, 0) != 0) {
+        hts_tpool_destroy(p);
+        return -1;
+    }
+
+    fp->mt->own_pool = 1;
+
+    return 0;
+}
+
+static int mt_destroy(mtaux_t *mt)
+{
+    int ret = 0;
+
+    // Tell the reader to shut down
+    pthread_mutex_lock(&mt->command_m);
+    mt->command = CLOSE;
+    pthread_cond_signal(&mt->command_c);
+    hts_tpool_wake_dispatch(mt->out_queue); // unstick the reader
+    pthread_mutex_unlock(&mt->command_m);
+
+    // Check for thread worker failure, indicated by is_shutdown returning 2
+    // It's possible really late errors might be missed, but we can live with
+    // that.
+    ret = -(hts_tpool_process_is_shutdown(mt->out_queue) > 1);
+    // Destroying the queue first forces the writer to exit.
+    // mt->out_queue is reference counted, so destroy gets called in both
+    // this and the IO threads.  The last to do it will clean up.
+    hts_tpool_process_destroy(mt->out_queue);
+
+    // IO thread will now exit.  Wait for it and perform final clean-up.
+    // If it returned non-NULL, it was not happy.
+    void *retval = NULL;
+    pthread_join(mt->io_task, &retval);
+    ret = retval != NULL ? -1 : ret;
+
+    pthread_mutex_destroy(&mt->job_pool_m);
+    pthread_mutex_destroy(&mt->command_m);
+    pthread_mutex_destroy(&mt->idx_m);
+    pthread_cond_destroy(&mt->command_c);
+    if (mt->curr_job)
+        pool_free(mt->job_pool, mt->curr_job);
+
+    if (mt->own_pool)
+        hts_tpool_destroy(mt->pool);
+
+    pool_destroy(mt->job_pool);
+
+    if (mt->idx_cache.e)
+        free(mt->idx_cache.e);
+
+    free(mt);
+    fflush(stderr);
+
+    return ret;
+}
+
+static int mt_queue(BGZF *fp)
+{
+    mtaux_t *mt = fp->mt;
+
+    mt->block_number++;
+
+    // Also updated by writer thread
+    pthread_mutex_lock(&mt->job_pool_m);
+    bgzf_job *j = pool_alloc(mt->job_pool);
+    if (j) mt->jobs_pending++;
+    pthread_mutex_unlock(&mt->job_pool_m);
+    if (!j) return -1;
+
+    j->fp = fp;
+    j->errcode = 0;
+    j->uncomp_len  = fp->block_offset;
+    if (fp->compress_level == 0) {
+        memcpy(j->comp_data + BLOCK_HEADER_LENGTH + 5, fp->uncompressed_block,
+               j->uncomp_len);
+        if (hts_tpool_dispatch3(mt->pool, mt->out_queue,
+                                bgzf_encode_level0_func, j,
+                                job_cleanup, job_cleanup, 0) < 0) {
+            goto fail;
+        }
+    } else {
+        memcpy(j->uncomp_data, fp->uncompressed_block, j->uncomp_len);
+
+        // Need non-block vers & job_pending?
+        if (hts_tpool_dispatch3(mt->pool, mt->out_queue, bgzf_encode_func, j,
+                                job_cleanup, job_cleanup, 0) < 0) {
+            goto fail;
+        }
+    }
+
+    fp->block_offset = 0;
+    return 0;
+
+ fail:
+    job_cleanup(j);
+    pthread_mutex_lock(&mt->job_pool_m);
+    mt->jobs_pending--;
+    pthread_mutex_unlock(&mt->job_pool_m);
+    return -1;
+}
+
+static int mt_flush_queue(BGZF *fp)
+{
+    mtaux_t *mt = fp->mt;
+
+    // Drain the encoder jobs.
+    // We cannot use hts_tpool_flush here as it can cause deadlock if
+    // the queue is full up of decoder tasks.  The best solution would
+    // be to have one input queue per type of job, but we don't right now.
+    //hts_tpool_flush(mt->pool);
+    pthread_mutex_lock(&mt->job_pool_m);
+    int shutdown = 0;
+    while (mt->jobs_pending != 0) {
+        if ((shutdown = hts_tpool_process_is_shutdown(mt->out_queue)))
+            break;
+        pthread_mutex_unlock(&mt->job_pool_m);
+        usleep(10000); // FIXME: replace by condition variable
+        pthread_mutex_lock(&mt->job_pool_m);
+    }
+    pthread_mutex_unlock(&mt->job_pool_m);
+
+    if (shutdown)
+        return -1;
+
+    // Wait on bgzf_mt_writer to drain the queue
+    if (hts_tpool_process_flush(mt->out_queue) != 0)
+        return -1;
+
+    return (fp->errcode == 0)? 0 : -1;
+}
+
+static int lazy_flush(BGZF *fp)
+{
+    if (fp->mt)
+        return fp->block_offset ? mt_queue(fp) : 0;
+    else
+        return bgzf_flush(fp);
+}
+
+#else  // ~ #ifdef BGZF_MT
+
+int bgzf_mt(BGZF *fp, int n_threads, int n_sub_blks)
+{
+    return 0;
+}
+
+static inline int lazy_flush(BGZF *fp)
+{
+    return bgzf_flush(fp);
+}
+
+#endif // ~ #ifdef BGZF_MT
+
+int bgzf_flush(BGZF *fp)
+{
+    if (!fp->is_write) return 0;
+#ifdef BGZF_MT
+    if (fp->mt) {
+        int ret = 0;
+        if (fp->block_offset) ret = mt_queue(fp);
+        if (!ret) ret = mt_flush_queue(fp);
+
+        // We maintain mt->block_address when threading as the
+        // main code can call bgzf_tell without any locks.
+        // (The result from tell are wrong, but we only care about the last
+        // 16-bits worth except for the final flush process.
+        pthread_mutex_lock(&fp->mt->idx_m);
+        fp->block_address = fp->mt->block_address;
+        pthread_mutex_unlock(&fp->mt->idx_m);
+
+        return ret;
+    }
+#endif
+
+    if (!fp->is_compressed) {
+        return hflush(fp->fp);
+    }
+
+    while (fp->block_offset > 0) {
+        int block_length;
+        if ( fp->idx_build_otf )
+        {
+            bgzf_index_add_block(fp);
+            fp->idx->ublock_addr += fp->block_offset;
+        }
+        block_length = deflate_block(fp, fp->block_offset);
+        if (block_length < 0) {
+            hts_log_debug("Deflate block operation failed: %s", bgzf_zerr(block_length, NULL));
+            return -1;
+        }
+        if (hwrite(fp->fp, fp->compressed_block, block_length) != block_length) {
+            hts_log_error("File write failed (wrong size)");
+            fp->errcode |= BGZF_ERR_IO; // possibly truncated file
+            return -1;
+        }
+        fp->block_address += block_length;
+    }
+    return 0;
+}
+
+int bgzf_flush_try(BGZF *fp, ssize_t size)
+{
+    if (fp->block_offset + size > BGZF_BLOCK_SIZE) return lazy_flush(fp);
+    return 0;
+}
+
+ssize_t bgzf_write(BGZF *fp, const void *data, size_t length)
+{
+    if ( !fp->is_compressed ) {
+        size_t push = length + (size_t) fp->block_offset;
+        fp->block_offset = push % BGZF_MAX_BLOCK_SIZE;
+        fp->block_address += (push - fp->block_offset);
+        return hwrite(fp->fp, data, length);
+    }
+
+    const uint8_t *input = (const uint8_t*)data;
+    ssize_t remaining = length;
+    assert(fp->is_write);
+    while (remaining > 0) {
+        uint8_t* buffer = (uint8_t*)fp->uncompressed_block;
+        int copy_length = BGZF_BLOCK_SIZE - fp->block_offset;
+        if (copy_length > remaining) copy_length = remaining;
+        memcpy(buffer + fp->block_offset, input, copy_length);
+        fp->block_offset += copy_length;
+        input += copy_length;
+        remaining -= copy_length;
+        if (fp->block_offset == BGZF_BLOCK_SIZE) {
+            if (lazy_flush(fp) != 0) return -1;
+        }
+    }
+    return length - remaining;
+}
+
+ssize_t bgzf_block_write(BGZF *fp, const void *data, size_t length)
+{
+    if ( !fp->is_compressed ) {
+        size_t push = length + (size_t) fp->block_offset;
+        fp->block_offset = push % BGZF_MAX_BLOCK_SIZE;
+        fp->block_address += (push - fp->block_offset);
+        return hwrite(fp->fp, data, length);
+    }
+
+    const uint8_t *input = (const uint8_t*)data;
+    ssize_t remaining = length;
+    assert(fp->is_write);
+    uint64_t current_block; //keep track of current block
+    uint64_t ublock_size; // amount of uncompressed data to be fed into next block
+    while (remaining > 0) {
+        current_block = fp->idx->moffs - fp->idx->noffs;
+        ublock_size = current_block + 1 < fp->idx->moffs ? fp->idx->offs[current_block+1].uaddr-fp->idx->offs[current_block].uaddr : BGZF_MAX_BLOCK_SIZE;
+        uint8_t* buffer = (uint8_t*)fp->uncompressed_block;
+        int copy_length = ublock_size - fp->block_offset;
+        if (copy_length > remaining) copy_length = remaining;
+        memcpy(buffer + fp->block_offset, input, copy_length);
+        fp->block_offset += copy_length;
+        input += copy_length;
+        remaining -= copy_length;
+        if (fp->block_offset == ublock_size) {
+            if (lazy_flush(fp) != 0) return -1;
+            if (fp->idx->noffs > 0)
+                fp->idx->noffs--;  // decrement noffs to track the blocks
+        }
+    }
+    return length - remaining;
+}
+
+
+ssize_t bgzf_raw_write(BGZF *fp, const void *data, size_t length)
+{
+    ssize_t ret = hwrite(fp->fp, data, length);
+    if (ret < 0) fp->errcode |= BGZF_ERR_IO;
+    return ret;
+}
+
+// Helper function for tidying up fp->mt and setting errcode
+static void bgzf_close_mt(BGZF *fp) {
+    if (fp->mt) {
+        if (!fp->mt->free_block)
+            fp->uncompressed_block = NULL;
+        if (mt_destroy(fp->mt) < 0)
+            fp->errcode = BGZF_ERR_IO;
+    }
+}
+
+int bgzf_close(BGZF* fp)
+{
+    int ret, block_length;
+    if (fp == 0) return -1;
+    if (fp->is_write && fp->is_compressed) {
+        if (bgzf_flush(fp) != 0) {
+            bgzf_close_mt(fp);
+            return -1;
+        }
+        fp->compress_level = -1;
+        block_length = deflate_block(fp, 0); // write an empty block
+        if (block_length < 0) {
+            hts_log_debug("Deflate block operation failed: %s", bgzf_zerr(block_length, NULL));
+            bgzf_close_mt(fp);
+            return -1;
+        }
+        if (hwrite(fp->fp, fp->compressed_block, block_length) < 0
+            || hflush(fp->fp) != 0) {
+            hts_log_error("File write failed");
+            fp->errcode |= BGZF_ERR_IO;
+            return -1;
+        }
+    }
+
+    bgzf_close_mt(fp);
+
+    if ( fp->is_gzip )
+    {
+        if (fp->gz_stream == NULL) ret = Z_OK;
+        else if (!fp->is_write) ret = inflateEnd(fp->gz_stream);
+        else ret = deflateEnd(fp->gz_stream);
+        if (ret != Z_OK) {
+            hts_log_error("Call to inflateEnd/deflateEnd failed: %s", bgzf_zerr(ret, NULL));
+        }
+        free(fp->gz_stream);
+    }
+    ret = hclose(fp->fp);
+    if (ret != 0) return -1;
+    bgzf_index_destroy(fp);
+    free(fp->uncompressed_block);
+    free_cache(fp);
+    ret = fp->errcode ? -1 : 0;
+    free(fp);
+    return ret;
+}
+
+void bgzf_set_cache_size(BGZF *fp, int cache_size)
+{
+    if (fp && fp->mt) return; // Not appropriate when multi-threading
+    if (fp && fp->cache) fp->cache_size = cache_size;
+}
+
+int bgzf_check_EOF(BGZF *fp) {
+    int has_eof;
+
+    if (fp->mt) {
+        pthread_mutex_lock(&fp->mt->command_m);
+        // fp->mt->command state transitions should be:
+        // NONE -> HAS_EOF -> HAS_EOF_DONE -> NONE
+        // (HAS_EOF -> HAS_EOF_DONE happens in bgzf_mt_reader thread)
+        if (fp->mt->command != CLOSE)
+            fp->mt->command = HAS_EOF;
+        pthread_cond_signal(&fp->mt->command_c);
+        hts_tpool_wake_dispatch(fp->mt->out_queue);
+        do {
+            if (fp->mt->command == CLOSE) {
+                // possible error in bgzf_mt_reader
+                pthread_mutex_unlock(&fp->mt->command_m);
+                return 0;
+            }
+            pthread_cond_wait(&fp->mt->command_c, &fp->mt->command_m);
+            switch (fp->mt->command) {
+            case HAS_EOF_DONE: break;
+            case HAS_EOF:
+                // Resend signal intended for bgzf_mt_reader()
+                pthread_cond_signal(&fp->mt->command_c);
+                break;
+            case CLOSE:
+                continue;
+            default:
+                abort();  // Should not get to any other state
+            }
+        } while (fp->mt->command != HAS_EOF_DONE);
+        fp->mt->command = NONE;
+        has_eof = fp->mt->eof;
+        pthread_mutex_unlock(&fp->mt->command_m);
+    } else {
+        has_eof = bgzf_check_EOF_common(fp);
+    }
+
+    fp->no_eof_block = (has_eof == 0);
+
+    return has_eof;
+}
+
+static inline int64_t bgzf_seek_common(BGZF* fp,
+                                       int64_t block_address, int block_offset)
+{
+    if (fp->mt) {
+        // The reader runs asynchronous and does loops of:
+        //    Read block
+        //    Check & process command
+        //    Dispatch decode job
+        //
+        // Once at EOF it then switches to loops of
+        //    Wait for command
+        //    Process command (possibly switching back to above loop).
+        //
+        // To seek we therefore send the reader thread a SEEK command,
+        // waking it up if blocked in dispatch and signalling if
+        // waiting for a command.  We then wait for the response so we
+        // know the seek succeeded.
+        pthread_mutex_lock(&fp->mt->command_m);
+        fp->mt->hit_eof = 0;
+        // fp->mt->command state transitions should be:
+        // NONE -> SEEK -> SEEK_DONE -> NONE
+        // (SEEK -> SEEK_DONE happens in bgzf_mt_reader thread)
+        fp->mt->command = SEEK;
+        fp->mt->block_address = block_address;
+        pthread_cond_signal(&fp->mt->command_c);
+        hts_tpool_wake_dispatch(fp->mt->out_queue);
+        do {
+            pthread_cond_wait(&fp->mt->command_c, &fp->mt->command_m);
+            switch (fp->mt->command) {
+            case SEEK_DONE: break;
+            case SEEK:
+                // Resend signal intended for bgzf_mt_reader()
+                pthread_cond_signal(&fp->mt->command_c);
+                break;
+            default:
+                abort();  // Should not get to any other state
+            }
+        } while (fp->mt->command != SEEK_DONE);
+        fp->mt->command = NONE;
+
+        fp->block_length = 0;  // indicates current block has not been loaded
+        fp->block_address = block_address;
+        fp->block_offset = block_offset;
+
+        pthread_mutex_unlock(&fp->mt->command_m);
+    } else {
+        if (hseek(fp->fp, block_address, SEEK_SET) < 0) {
+            fp->errcode |= BGZF_ERR_IO;
+            return -1;
+        }
+        fp->block_length = 0;  // indicates current block has not been loaded
+        fp->block_address = block_address;
+        fp->block_offset = block_offset;
+    }
+
+    return 0;
+}
+
+int64_t bgzf_seek(BGZF* fp, int64_t pos, int where)
+{
+    if (fp->is_write || where != SEEK_SET || fp->is_gzip) {
+        fp->errcode |= BGZF_ERR_MISUSE;
+        return -1;
+    }
+
+    // This is a flag to indicate we've jumped elsewhere in the stream, to act
+    // as a hint to any other code which is wrapping up bgzf for its own
+    // purposes.  We may not be able to tell when seek happens as it can be
+    // done on our behalf, eg by the iterator.
+    //
+    // This is never cleared here.  Any tool that needs to handle it is also
+    // responsible for clearing it.
+    fp->seeked = pos;
+
+    return bgzf_seek_common(fp, pos >> 16, pos & 0xFFFF);
+}
+
+int bgzf_is_bgzf(const char *fn)
+{
+    uint8_t buf[16];
+    int n;
+    hFILE *fp;
+    if ((fp = hopen(fn, "r")) == 0) return 0;
+    n = hread(fp, buf, 16);
+    if (hclose(fp) < 0) return 0;
+    if (n != 16) return 0;
+    return check_header(buf) == 0? 1 : 0;
+}
+
+int bgzf_compression(BGZF *fp)
+{
+    return (!fp->is_compressed)? no_compression : (fp->is_gzip)? gzip : bgzf;
+}
+
+int bgzf_getc(BGZF *fp)
+{
+    if (fp->block_offset+1 < fp->block_length) {
+        fp->uncompressed_address++;
+        return ((unsigned char*)fp->uncompressed_block)[fp->block_offset++];
+    }
+
+    int c;
+    if (fp->block_offset >= fp->block_length) {
+        if (bgzf_read_block(fp) != 0) return -2; /* error */
+        if (fp->block_length == 0) return -1; /* end-of-file */
+    }
+    c = ((unsigned char*)fp->uncompressed_block)[fp->block_offset++];
+    if (fp->block_offset == fp->block_length) {
+        fp->block_address = bgzf_htell(fp);
+        fp->block_offset = 0;
+        fp->block_length = 0;
+    }
+    fp->uncompressed_address++;
+    return c;
+}
+
+int bgzf_getline(BGZF *fp, int delim, kstring_t *str)
+{
+    int l, state = 0;
+    str->l = 0;
+    do {
+        if (fp->block_offset >= fp->block_length) {
+            if (bgzf_read_block(fp) != 0) { state = -2; break; }
+            if (fp->block_length == 0) { state = -1; break; }
+        }
+        unsigned char *buf = fp->uncompressed_block;
+
+        // Equivalent to a naive byte by byte search from
+        // buf + block_offset to buf + block_length.
+        void *e = memchr(&buf[fp->block_offset], delim,
+                         fp->block_length - fp->block_offset);
+        l = e ? (unsigned char *)e - buf : fp->block_length;
+
+        if (l < fp->block_length) state = 1;
+        l -= fp->block_offset;
+        if (ks_expand(str, l + 2) < 0) { state = -3; break; }
+        memcpy(str->s + str->l, buf + fp->block_offset, l);
+        str->l += l;
+        fp->block_offset += l + 1;
+        if (fp->block_offset >= fp->block_length) {
+            fp->block_address = bgzf_htell(fp);
+            fp->block_offset = 0;
+            fp->block_length = 0;
+        }
+    } while (state == 0);
+    if (state < -1) return state;
+    if (str->l == 0 && state < 0) return state;
+    fp->uncompressed_address += str->l + 1;
+    if ( delim=='\n' && str->l>0 && str->s[str->l-1]=='\r' ) str->l--;
+    str->s[str->l] = 0;
+    return str->l <= INT_MAX ? (int) str->l : INT_MAX;
+}
+
+void bgzf_index_destroy(BGZF *fp)
+{
+    if ( !fp->idx ) return;
+    free(fp->idx->offs);
+    free(fp->idx);
+    fp->idx = NULL;
+    fp->idx_build_otf = 0;
+}
+
+int bgzf_index_build_init(BGZF *fp)
+{
+    bgzf_index_destroy(fp);
+    fp->idx = (bgzidx_t*) calloc(1,sizeof(bgzidx_t));
+    if ( !fp->idx ) return -1;
+    fp->idx_build_otf = 1;  // build index on the fly
+    return 0;
+}
+
+int bgzf_index_add_block(BGZF *fp)
+{
+    fp->idx->noffs++;
+    if ( fp->idx->noffs > fp->idx->moffs )
+    {
+        fp->idx->moffs = fp->idx->noffs;
+        kroundup32(fp->idx->moffs);
+        fp->idx->offs = (bgzidx1_t*) realloc(fp->idx->offs, fp->idx->moffs*sizeof(bgzidx1_t));
+        if ( !fp->idx->offs ) return -1;
+    }
+    fp->idx->offs[ fp->idx->noffs-1 ].uaddr = fp->idx->ublock_addr;
+    fp->idx->offs[ fp->idx->noffs-1 ].caddr = fp->block_address;
+    return 0;
+}
+
+static inline int hwrite_uint64(uint64_t x, hFILE *f)
+{
+    if (ed_is_big()) x = ed_swap_8(x);
+    if (hwrite(f, &x, sizeof(x)) != sizeof(x)) return -1;
+    return 0;
+}
+
+static char * get_name_suffix(const char *bname, const char *suffix)
+{
+    size_t len = strlen(bname) + strlen(suffix) + 1;
+    char *buff = malloc(len);
+    if (!buff) return NULL;
+    snprintf(buff, len, "%s%s", bname, suffix);
+    return buff;
+}
+
+int bgzf_index_dump_hfile(BGZF *fp, struct hFILE *idx, const char *name)
+{
+    // Note that the index contains one extra record when indexing files opened
+    // for reading. The terminating record is not present when opened for writing.
+    // This is not a bug.
+
+    int i;
+
+    if (!fp->idx) {
+        hts_log_error("Called for BGZF handle with no index");
+        errno = EINVAL;
+        return -1;
+    }
+
+    if (bgzf_flush(fp) != 0) return -1;
+
+    // discard the entry marking the end of the file
+    if (fp->mt && fp->idx)
+        fp->idx->noffs--;
+
+    if (hwrite_uint64(fp->idx->noffs - 1, idx) < 0) goto fail;
+    for (i=1; i<fp->idx->noffs; i++)
+    {
+        if (hwrite_uint64(fp->idx->offs[i].caddr, idx) < 0) goto fail;
+        if (hwrite_uint64(fp->idx->offs[i].uaddr, idx) < 0) goto fail;
+    }
+    return 0;
+
+ fail:
+    hts_log_error("Error writing to %s : %s", name ? name : "index", strerror(errno));
+    return -1;
+}
+
+int bgzf_index_dump(BGZF *fp, const char *bname, const char *suffix)
+{
+    const char *name = bname, *msg = NULL;
+    char *tmp = NULL;
+    hFILE *idx = NULL;
+
+    if (!fp->idx) {
+        hts_log_error("Called for BGZF handle with no index");
+        errno = EINVAL;
+        return -1;
+    }
+
+    if ( suffix )
+    {
+        tmp = get_name_suffix(bname, suffix);
+        if ( !tmp ) return -1;
+        name = tmp;
+    }
+
+    idx = hopen(name, "wb");
+    if ( !idx ) {
+        msg = "Error opening";
+        goto fail;
+    }
+
+    if (bgzf_index_dump_hfile(fp, idx, name) != 0) goto fail;
+
+    if (hclose(idx) < 0)
+    {
+        idx = NULL;
+        msg = "Error on closing";
+        goto fail;
+    }
+
+    free(tmp);
+    return 0;
+
+ fail:
+    if (msg != NULL) {
+        hts_log_error("%s %s : %s", msg, name, strerror(errno));
+    }
+    if (idx) hclose_abruptly(idx);
+    free(tmp);
+    return -1;
+}
+
+static inline int hread_uint64(uint64_t *xptr, hFILE *f)
+{
+    if (hread(f, xptr, sizeof(*xptr)) != sizeof(*xptr)) return -1;
+    if (ed_is_big()) ed_swap_8p(xptr);
+    return 0;
+}
+
+int bgzf_index_load_hfile(BGZF *fp, struct hFILE *idx, const char *name)
+{
+    fp->idx = (bgzidx_t*) calloc(1,sizeof(bgzidx_t));
+    if (fp->idx == NULL) goto fail;
+    uint64_t x;
+    if (hread_uint64(&x, idx) < 0) goto fail;
+
+    fp->idx->noffs = fp->idx->moffs = x + 1;
+    fp->idx->offs  = (bgzidx1_t*) malloc(fp->idx->moffs*sizeof(bgzidx1_t));
+    if (fp->idx->offs == NULL) goto fail;
+    fp->idx->offs[0].caddr = fp->idx->offs[0].uaddr = 0;
+
+    int i;
+    for (i=1; i<fp->idx->noffs; i++)
+    {
+        if (hread_uint64(&fp->idx->offs[i].caddr, idx) < 0) goto fail;
+        if (hread_uint64(&fp->idx->offs[i].uaddr, idx) < 0) goto fail;
+    }
+
+    return 0;
+
+ fail:
+    hts_log_error("Error reading %s : %s", name ? name : "index", strerror(errno));
+    if (fp->idx) {
+        free(fp->idx->offs);
+        free(fp->idx);
+        fp->idx = NULL;
+    }
+    return -1;
+}
+
+int bgzf_index_load(BGZF *fp, const char *bname, const char *suffix)
+{
+    const char *name = bname, *msg = NULL;
+    char *tmp = NULL;
+    hFILE *idx = NULL;
+    if ( suffix )
+    {
+        tmp = get_name_suffix(bname, suffix);
+        if ( !tmp ) return -1;
+        name = tmp;
+    }
+
+    idx = hopen(name, "rb");
+    if ( !idx ) {
+        msg = "Error opening";
+        goto fail;
+    }
+
+    if (bgzf_index_load_hfile(fp, idx, name) != 0) goto fail;
+
+    if (hclose(idx) != 0) {
+        idx = NULL;
+        msg = "Error closing";
+        goto fail;
+    }
+
+    free(tmp);
+    return 0;
+
+ fail:
+    if (msg != NULL) {
+        hts_log_error("%s %s : %s", msg, name, strerror(errno));
+    }
+    if (idx) hclose_abruptly(idx);
+    free(tmp);
+    return -1;
+}
+
+int bgzf_useek(BGZF *fp, off_t uoffset, int where)
+{
+    if (fp->is_write || where != SEEK_SET || fp->is_gzip) {
+        fp->errcode |= BGZF_ERR_MISUSE;
+        return -1;
+    }
+    if (uoffset >= fp->uncompressed_address - fp->block_offset &&
+        uoffset < fp->uncompressed_address + fp->block_length - fp->block_offset) {
+        // Can seek into existing data
+        fp->block_offset += uoffset - fp->uncompressed_address;
+        fp->uncompressed_address = uoffset;
+        return 0;
+    }
+    if ( !fp->is_compressed )
+    {
+        if (hseek(fp->fp, uoffset, SEEK_SET) < 0)
+        {
+            fp->errcode |= BGZF_ERR_IO;
+            return -1;
+        }
+        fp->block_length = 0;  // indicates current block has not been loaded
+        fp->block_address = uoffset;
+        fp->block_offset = 0;
+        if (bgzf_read_block(fp) < 0) {
+            fp->errcode |= BGZF_ERR_IO;
+            return -1;
+        }
+        fp->uncompressed_address = uoffset;
+        return 0;
+    }
+
+    if ( !fp->idx )
+    {
+        fp->errcode |= BGZF_ERR_IO;
+        return -1;
+    }
+
+    // binary search
+    int ilo = 0, ihi = fp->idx->noffs - 1;
+    while ( ilo<=ihi )
+    {
+        int i = (ilo+ihi)*0.5;
+        if ( uoffset < fp->idx->offs[i].uaddr ) ihi = i - 1;
+        else if ( uoffset >= fp->idx->offs[i].uaddr ) ilo = i + 1;
+        else break;
+    }
+    int i = ilo-1;
+    off_t offset = 0;
+    if (bgzf_seek_common(fp, fp->idx->offs[i].caddr, 0) < 0)
+        return -1;
+
+    if ( bgzf_read_block(fp) < 0 ) {
+        fp->errcode |= BGZF_ERR_IO;
+        return -1;
+    }
+    offset = uoffset - fp->idx->offs[i].uaddr;
+    if ( offset > 0 )
+    {
+        if (offset > fp->block_length) {
+            fp->errcode |= BGZF_ERR_IO;
+            return -1;                                      //offset outside the available data
+        }
+        fp->block_offset = offset;
+        assert( fp->block_offset <= fp->block_length );     // todo: skipped, unindexed, blocks
+    }
+    fp->uncompressed_address = uoffset;
+    return 0;
+}
+
+off_t bgzf_utell(BGZF *fp)
+{
+    return fp->uncompressed_address;    // currently maintained only when reading
+}
+
+/* prototype is in hfile_internal.h */
+struct hFILE *bgzf_hfile(struct BGZF *fp) {
+    return fp->fp;
+}
diff --git a/ext/htslib/bgzip.1 b/ext/htslib/bgzip.1
new file mode 100644
index 0000000..1e115d0
--- /dev/null
+++ b/ext/htslib/bgzip.1
@@ -0,0 +1,206 @@
+.TH bgzip 1 "12 September 2024" "htslib-1.21" "Bioinformatics tools"
+.SH NAME
+.PP
+bgzip \- Block compression/decompression utility
+.\"
+.\" Copyright (C) 2009-2011 Broad Institute.
+.\" Copyright (C) 2018, 2021-2024 Genome Research Limited.
+.\"
+.\" Author: Heng Li <lh3@sanger.ac.uk>
+.\"
+.\" Permission is hereby granted, free of charge, to any person obtaining a
+.\" copy of this software and associated documentation files (the "Software"),
+.\" to deal in the Software without restriction, including without limitation
+.\" the rights to use, copy, modify, merge, publish, distribute, sublicense,
+.\" and/or sell copies of the Software, and to permit persons to whom the
+.\" Software is furnished to do so, subject to the following conditions:
+.\"
+.\" The above copyright notice and this permission notice shall be included in
+.\" all copies or substantial portions of the Software.
+.\"
+.\" THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+.\" IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+.\" FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+.\" THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+.\" LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+.\" FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+.\" DEALINGS IN THE SOFTWARE.
+.\"
+.
+.\" For code blocks and examples (cf groff's Ultrix-specific man macros)
+.de EX
+
+.  in +\\$1
+.  nf
+.  ft CR
+..
+.de EE
+.  ft
+.  fi
+.  in
+
+..
+.SH SYNOPSIS
+.PP
+.B bgzip
+.RB [ -cdfhikrt ]
+.RB [ -b
+.IR virtualOffset ]
+.RB [ -I
+.IR index_name ]
+.RB [ -l
+.IR compression_level ]
+.RB [ -o
+.IR outfile ]
+.RB [ -s
+.IR size ]
+.RB [ -@
+.IR threads ]
+.RI [ file " ...]"
+.PP
+.SH DESCRIPTION
+.PP
+Bgzip compresses files in a similar manner to, and compatible with, gzip(1).
+The file is compressed into a series of small (less than 64K) 'BGZF' blocks.
+This allows indexes to be built against the compressed file and used to
+retrieve portions of the data without having to decompress the entire file. 
+
+If no files are specified on the command line, bgzip will compress (or
+decompress if the -d option is used) standard input to standard output.
+If a file is specified, it will be compressed (or decompressed with -d).
+If the -c option is used, the result will be written to standard output,
+otherwise when compressing bgzip will write to a new file with a .gz
+suffix and remove the original.  When decompressing the input file must
+have a .gz suffix, which will be removed to make the output name.  Again
+after decompression completes the input file will be removed. When multiple
+files are given as input, the operation is performed on all of them. Access
+and modification time of input file from filesystem is set to output file. 
+Note, access time may get updated by system when it deems appropriate.
+
+.SH OPTIONS
+.TP 10
+.B "--binary"
+Bgzip will attempt to ensure BGZF blocks end on a newline when the
+input is a text file.  The exception to this is where a single line is
+larger than a BGZF block (64Kb).  This can aid tools that use the
+index to perform random access on the compressed stream, as the start
+of a block is likely to also be the start of a text record.
+
+This option processes text files as if they were binary content,
+ignoring the location of newlines.  This also restores the behaviour
+for text files to bgzip version 1.15 and earlier.
+.TP
+.BI "-b, --offset " INT
+Decompress to standard output from virtual file position (0-based uncompressed
+offset).
+Implies -c and -d.
+.TP
+.B "-c, --stdout"
+Write to standard output, keep original files unchanged.
+.TP
+.B "-d, --decompress"
+Decompress.
+.TP
+.B "-f, --force"
+Overwrite files without asking, or decompress files that don't have a known
+compression filename extension (e.g., \fI.gz\fR) without asking.
+Use \fB--force\fR twice to do both without asking.
+.TP
+.B "-g, --rebgzip"
+Try to use an existing index to create a compressed file with matching
+block offsets.  The index must be specified using the \fB-I
+\fIfile.gzi\fR option.
+Note that this assumes that the same compression library and level are in use
+as when making the original file.
+Don't use it unless you know what you're doing.
+.TP
+.B "-h, --help"
+Displays a help message.
+.TP
+.B "-i, --index"
+Create a BGZF index while compressing.
+Unless the -I option is used, this will have the name of the compressed
+file with .gzi appended to it.
+.TP
+.BI "-I, --index-name " FILE
+Index file name.
+.TP
+.B "-k, --keep"
+Do not delete input file during operation.
+.TP
+.BI "-l, --compress-level " INT
+Compression level to use when compressing.
+From 0 to 9, or -1 for the default level set by the compression library. [-1]
+.TP
+.BI "-o, --output " FILE
+Write to a file, keep original files unchanged, will overwrite an existing
+file.
+.TP
+.B "-r, --reindex"
+Rebuild the index on an existing compressed file.
+.TP
+.BI "-s, --size " INT
+Decompress INT bytes (uncompressed size) to standard output.
+Implies -c.
+.TP
+.B "-t, --test"
+Test the integrity of the compressed file.
+.TP
+.BI "-@, --threads " INT
+Number of threads to use [1].
+.PP
+
+.SH BGZF FORMAT
+The BGZF format written by bgzip is described in the SAM format specification
+available from http://samtools.github.io/hts-specs/SAMv1.pdf.
+
+It makes use of a gzip feature which allows compressed files to be
+concatenated.
+The input data is divided into blocks which are no larger than 64 kilobytes
+both before and after compression (including compression headers).
+Each block is compressed into a gzip file.
+The gzip header includes an extra sub-field with identifier 'BC' and the length
+of the compressed block, including all headers.
+
+.SH GZI FORMAT
+The index format is a binary file listing pairs of compressed and
+uncompressed offsets in a BGZF file.
+Each compressed offset points to the start of a BGZF block.
+The uncompressed offset is the corresponding location in the uncompressed
+data stream.
+
+All values are stored as little-endian 64-bit unsigned integers.
+
+The file contents are:
+.EX 4
+uint64_t number_entries
+.EE
+followed by number_entries pairs of:
+.EX 4
+uint64_t compressed_offset
+uint64_t uncompressed_offset
+.EE
+
+.SH EXAMPLES
+.EX 4
+# Compress stdin to stdout
+bgzip < /usr/share/dict/words > /tmp/words.gz
+
+# Make a .gzi index
+bgzip -r /tmp/words.gz
+
+# Extract part of the data using the index
+bgzip -b 367635 -s 4 /tmp/words.gz 
+
+# Uncompress the whole file, removing the compressed copy
+bgzip -d /tmp/words.gz
+.EE
+
+.SH AUTHOR
+.PP
+The BGZF library was originally implemented by Bob Handsaker and modified
+by Heng Li for remote file access and in-memory caching.
+
+.SH SEE ALSO
+.IR gzip (1),
+.IR tabix (1)
diff --git a/ext/htslib/bgzip.c b/ext/htslib/bgzip.c
new file mode 100644
index 0000000..687b29d
--- /dev/null
+++ b/ext/htslib/bgzip.c
@@ -0,0 +1,771 @@
+/* bgzip.c -- Block compression/decompression utility.
+
+   Copyright (C) 2008, 2009 Broad Institute / Massachusetts Institute of Technology
+   Copyright (C) 2010, 2013-2019, 2021-2024 Genome Research Ltd.
+
+   Permission is hereby granted, free of charge, to any person obtaining a copy
+   of this software and associated documentation files (the "Software"), to deal
+   in the Software without restriction, including without limitation the rights
+   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+   copies of the Software, and to permit persons to whom the Software is
+   furnished to do so, subject to the following conditions:
+
+   The above copyright notices and this permission notice shall be included in
+   all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+   THE SOFTWARE.
+*/
+
+#include <config.h>
+
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+#include <stdio.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <errno.h>
+#include <stdarg.h>
+#include <getopt.h>
+#include <inttypes.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include "htslib/bgzf.h"
+#include "htslib/hts.h"
+#include "htslib/hfile.h"
+
+#ifdef _WIN32
+#  define WIN32_LEAN_AND_MEAN
+#  include <windows.h>
+#  include <sys/utime.h>
+#endif
+
+static const int WINDOW_SIZE = BGZF_BLOCK_SIZE;
+
+static void HTS_FORMAT(HTS_PRINTF_FMT, 1, 2) error(const char *format, ...)
+{
+    va_list ap;
+    va_start(ap, format);
+    vfprintf(stderr, format, ap);
+    va_end(ap);
+    exit(EXIT_FAILURE);
+}
+
+static int ask_yn(void)
+{
+    char line[1024];
+    if (fgets(line, sizeof line, stdin) == NULL)
+        return 0;
+    return line[0] == 'Y' || line[0] == 'y';
+}
+
+static int confirm_overwrite(const char *fn)
+{
+    int save_errno = errno;
+    int ret = 0;
+
+    if (isatty(STDIN_FILENO)) {
+        fprintf(stderr, "[bgzip] %s already exists; do you wish to overwrite (y or n)? ", fn);
+        if (ask_yn()) ret = 1;
+    }
+
+    errno = save_errno;
+    return ret;
+}
+
+static int known_extension(const char *ext)
+{
+    static const char *known[] = {
+        "gz", "bgz", "bgzf",
+        NULL
+    };
+
+    const char **p;
+    for (p = known; *p; p++)
+        if (strcasecmp(ext, *p) == 0) return 1;
+    return 0;
+}
+
+static int confirm_filename(int *is_forced, const char *name, const char *ext)
+{
+    if (*is_forced) {
+        (*is_forced)--;
+        return 1;
+    }
+
+    if (!isatty(STDIN_FILENO))
+        return 0;
+
+    fprintf(stderr, "[bgzip] .%s is not a known extension; do you wish to decompress to %s (y or n)? ", ext, name);
+    return ask_yn();
+}
+
+/* getfilespec - get file status data
+   path        - file path for which status to be retrieved
+   status      - pointer to status structure in which the data to be stored
+   returns 0 on success and -1 on failure
+*/
+static int getfilespec(const char *path, struct stat *status)
+{
+    if (!path || !status) {     //invalid
+        return -1;
+    }
+    if (!strcmp(path, "-")) {   //cant get / set for stdin/out, return success
+        return 0;
+    }
+    if (stat(path, status) < 0) {
+        return -1;
+    }
+    return 0;
+}
+
+/* setfilespec - set file status data
+   path        - file path for which status to be set
+   status      - pointer to status structure in which the data is present
+   returns 0 on success and -1 on failure
+   sets only the time as of now.
+*/
+static int setfilespec(const char *path, const struct stat *status)
+{
+    if (!path || !status) {     //invalid
+        return -1;
+    }
+    if (!strcmp(path, "-")) {   //cant get / set for stdin/out, return success
+        return 0;
+    }
+
+#ifdef _WIN32
+    struct _utimbuf tval;
+    //time upto sec - access & modification time
+    tval.actime = status->st_atime;
+    tval.modtime = status->st_mtime;
+    if (_utime(path, &tval) < 0) {
+        fprintf(stderr, "[bgzip] Failed to set file specifications.\n");
+        return -1;
+    }
+#else
+    struct timeval tval[2];
+    memset(&tval[0], 0, sizeof(tval));
+    //time upto sec - access time
+    tval[0].tv_sec = status->st_atime;
+    //time upto sec - modification time
+    tval[1].tv_sec = status->st_mtime;
+    if (utimes(path, &tval[0]) < 0) {
+        fprintf(stderr, "[bgzip] Failed to set file specifications.\n");
+        return -1;
+    }
+#endif //_WIN32
+    return 0;
+}
+
+
+static int check_name_and_extension(char *name, int *forced) {
+    size_t pos;
+    char *ext;
+
+    for (pos = strlen(name); pos > 0; --pos)
+        if (name[pos] == '.' || name[pos] == '/') break;
+
+    if (pos == 0 || name[pos] != '.') {
+        fprintf(stderr, "[bgzip] can't find an extension in %s -- please rename\n", name);
+        return 1;
+    }
+
+    name[pos] = '\0';
+    ext = &name[pos+1];
+
+    if (!(known_extension(ext) || confirm_filename(forced, name, ext))) {
+        fprintf(stderr, "[bgzip] unknown extension .%s -- declining to decompress to %s\n", ext, name);
+        return 2;                            //explicit N, continue and return 2
+    }
+
+    return 0;
+}
+
+
+static int bgzip_main_usage(FILE *fp, int status)
+{
+    fprintf(fp, "\n");
+    fprintf(fp, "Version: %s\n", hts_version());
+    fprintf(fp, "Usage:   bgzip [OPTIONS] [FILE] ...\n");
+    fprintf(fp, "Options:\n");
+    fprintf(fp, "   -b, --offset INT           decompress at virtual file pointer (0-based uncompressed offset)\n");
+    fprintf(fp, "   -c, --stdout               write on standard output, keep original files unchanged\n");
+    fprintf(fp, "   -d, --decompress           decompress\n");
+    fprintf(fp, "   -f, --force                overwrite files without asking\n");
+    fprintf(fp, "   -g, --rebgzip              use an index file to bgzip a file\n");
+    fprintf(fp, "   -h, --help                 give this help\n");
+    fprintf(fp, "   -i, --index                compress and create BGZF index\n");
+    fprintf(fp, "   -I, --index-name FILE      name of BGZF index file [file.gz.gzi]\n");
+    fprintf(fp, "   -k, --keep                 don't delete input files during operation\n");
+    fprintf(fp, "   -l, --compress-level INT   Compression level to use when compressing; 0 to 9, or -1 for default [-1]\n");
+    fprintf(fp, "   -o, --output FILE          write to file, keep original files unchanged\n");
+    fprintf(fp, "   -r, --reindex              (re)index compressed file\n");
+    fprintf(fp, "   -s, --size INT             decompress INT bytes (uncompressed size)\n");
+    fprintf(fp, "   -t, --test                 test integrity of compressed file\n");
+    fprintf(fp, "       --binary               Don't align blocks with text lines\n");
+    fprintf(fp, "   -@, --threads INT          number of compression threads to use [1]\n");
+    return status;
+}
+
+int main(int argc, char **argv)
+{
+    int c, compress, compress_level = -1, pstdout, is_forced, test, index = 0, rebgzip = 0, reindex = 0, keep, binary;
+    BGZF *fp;
+    char *buffer;
+    long start, end, size;
+    struct stat filestat;
+    char *statfilename = NULL;
+    char *index_fname = NULL, *write_fname = NULL;
+    int threads = 1, isstdin = 0, usedstdout = 0, ret = 0, exp_out_open = 0, f_dst = -1;
+
+    static const struct option loptions[] =
+    {
+        {"help", no_argument, NULL, 'h'},
+        {"offset", required_argument, NULL, 'b'},
+        {"stdout", no_argument, NULL, 'c'},
+        {"decompress", no_argument, NULL, 'd'},
+        {"force", no_argument, NULL, 'f'},
+        {"index", no_argument, NULL, 'i'},
+        {"index-name", required_argument, NULL, 'I'},
+        {"compress-level", required_argument, NULL, 'l'},
+        {"reindex", no_argument, NULL, 'r'},
+        {"rebgzip",no_argument,NULL,'g'},
+        {"size", required_argument, NULL, 's'},
+        {"threads", required_argument, NULL, '@'},
+        {"test", no_argument, NULL, 't'},
+        {"version", no_argument, NULL, 1},
+        {"keep", no_argument, NULL, 'k'},
+        {"binary", no_argument, NULL, 2},
+        {"output", required_argument, NULL, 'o'},
+        {NULL, 0, NULL, 0}
+    };
+
+    compress = 1; pstdout = 0; start = 0; size = -1; end = -1; is_forced = 0; test = 0; keep = 0; binary = 0;
+    while((c  = getopt_long(argc, argv, "cdh?fb:@:s:iI:l:grtko:",loptions,NULL)) >= 0){
+        switch(c){
+        case 'd': compress = 0; break;
+        case 'c': pstdout = 1; break;
+        case 'b': start = atol(optarg); compress = 0; pstdout = 1; break;
+        case 's': size = atol(optarg); pstdout = 1; break;
+        case 'f': is_forced++; break;
+        case 'i': index = 1; break;
+        case 'I': index_fname = optarg; break;
+        case 'l': compress_level = atol(optarg); break;
+        case 'g': rebgzip = 1; break;
+        case 'r': reindex = 1; compress = 0; break;
+        case '@': threads = atoi(optarg); break;
+        case 't': test = 1; compress = 0; reindex = 0; break;
+        case 'k': keep = 1; break;
+        case 'o': write_fname = optarg; break;
+        case 1:
+            printf(
+"bgzip (htslib) %s\n"
+"Copyright (C) 2024 Genome Research Ltd.\n", hts_version());
+            return EXIT_SUCCESS;
+        case  2:  binary = 1; break;
+        case 'h': return bgzip_main_usage(stdout, EXIT_SUCCESS);
+        case '?': return bgzip_main_usage(stderr, EXIT_FAILURE);
+        }
+    }
+    if (size >= 0) end = start + size;
+    if (end >= 0 && end < start) {
+        fprintf(stderr, "[bgzip] Illegal region: [%ld, %ld]\n", start, end);
+        return 1;
+    }
+
+    if ( (index || reindex) && rebgzip )
+    {
+        fprintf(stderr, "[bgzip] Can't produce a index and rebgzip simultaneously\n");
+        return 1;
+    }
+    if ( rebgzip && !index_fname )
+    {
+        fprintf(stderr, "[bgzip] Index file name expected with rebgzip.  See -I option.\n");
+        return 1;
+    }
+    /* avoid -I / indexfile with multiple inputs while index/reindex. these wont be set during
+    read/decompress and are not considered even if set */
+    if ( (index || reindex) && !write_fname && index_fname && argc - optind > 1) {
+        fprintf(stderr, "[bgzip] Cannot specify index filename with multiple data file on index, reindex.\n");
+        return 1;
+    }
+
+    if (write_fname) {
+        if (pstdout) {
+            fprintf(stderr, "[bgzip] Cannot write to %s and stdout at the same time.\n", write_fname);
+            return 1;
+        } else if (strncmp(write_fname, "-", strlen(write_fname)) == 0) {
+            // stdout has special handling so treat as -c
+            pstdout = 1;
+            write_fname = NULL;
+        }
+    }
+
+    do {
+        isstdin = optind >= argc ? 1 : !strcmp("-", argv[optind]);          //using stdin or not?
+        /* when a named output file is not used, stdout is in use when explicitly
+        selected or when stdin in is in use, it needs to be closed
+        explicitly to get all io errors*/
+
+        if (!write_fname)
+            usedstdout |= isstdin || pstdout || test;
+
+        statfilename = NULL;
+
+        if (compress == 1) {
+            hFILE* f_src = NULL;
+            char out_mode[3] = "w\0";
+            char out_mode_exclusive[4] = "wx\0";
+
+            if (compress_level < -1 || compress_level > 9) {
+                fprintf(stderr, "[bgzip] Invalid compress-level: %d\n", compress_level);
+                return 1;
+            }
+            if (compress_level >= 0) {
+                out_mode[1] = compress_level + '0';
+                out_mode_exclusive[2] = compress_level + '0';
+            }
+            if (!(f_src = hopen(!isstdin ? argv[optind] : "-", "r"))) {
+                fprintf(stderr, "[bgzip] %s: %s\n", strerror(errno), isstdin ? "stdin" : argv[optind]);
+                return 1;
+            }
+
+            if (write_fname) {
+                if (!exp_out_open) {  // only open this file once for writing, close at the end
+                    if ((fp = bgzf_open(write_fname, out_mode)) == NULL) {
+                        fprintf(stderr, "[bgzip] can't create %s: %s\n", write_fname, strerror(errno));
+                        return 1;
+                    } else {
+                        exp_out_open = 1;
+                    }
+                }
+            } else if ( argc>optind && !isstdin )            //named input file that isn't an explicit "-"
+            {
+                if (pstdout)
+                    fp = bgzf_open("-", out_mode);
+                else
+                {
+                    char *name = malloc(strlen(argv[optind]) + 5);
+                    strcpy(name, argv[optind]);
+                    strcat(name, ".gz");
+                    fp = bgzf_open(name, is_forced? out_mode : out_mode_exclusive);
+                    if (fp == NULL && errno == EEXIST) {
+                        if (confirm_overwrite(name)) {
+                            fp = bgzf_open(name, out_mode);
+                        }
+                        else {
+                            ret = 2;                        //explicit N - no overwrite, continue and return 2
+                            hclose_abruptly(f_src);
+                            free(name);
+                            continue;
+                        }
+                    }
+                    if (fp == NULL) {
+                        fprintf(stderr, "[bgzip] can't create %s: %s\n", name, strerror(errno));
+                        free(name);
+                        return 1;
+                    }
+                    statfilename = name;
+                }
+            }
+            else if (!pstdout && isatty(fileno((FILE *)stdout)) )
+                return bgzip_main_usage(stderr, EXIT_FAILURE);
+            else if ( index && !index_fname )
+            {
+                fprintf(stderr, "[bgzip] Index file name expected when writing to stdout\n");
+                return 1;
+            }
+            else
+                fp = bgzf_open("-", out_mode);
+
+            if ( index ) bgzf_index_build_init(fp);
+            if (threads > 1)
+                bgzf_mt(fp, threads, 256);
+
+            buffer = malloc(WINDOW_SIZE);
+            if (!buffer) {
+                if (statfilename) {
+                    free(statfilename);
+                }
+                return 1;
+            }
+            if (rebgzip){
+                if ( bgzf_index_load(fp, index_fname, NULL) < 0 ) error("Could not load index: %s.%s\n", !isstdin ? argv[optind] : index_fname, !isstdin ? "gzi" : "");
+
+                while ((c = hread(f_src, buffer, WINDOW_SIZE)) > 0)
+                    if (bgzf_block_write(fp, buffer, c) < 0) error("Could not write %d bytes: Error %d\n", c, fp->errcode);
+            }
+            else {
+                htsFormat fmt;
+                int textual = 0;
+                if (!binary
+                    && hts_detect_format(f_src, &fmt) == 0
+                    && fmt.compression == no_compression) {
+                    switch(fmt.format) {
+                    case text_format:
+                    case sam:
+                    case vcf:
+                    case bed:
+                    case fasta_format:
+                    case fastq_format:
+                    case fai_format:
+                    case fqi_format:
+                        textual = 1;
+                        break;
+                    default: break; // silence clang warnings
+                    }
+                }
+
+                if (binary || !textual) {
+                    // Binary data, either detected or explicit
+                    while ((c = hread(f_src, buffer, WINDOW_SIZE)) > 0)
+                        if (bgzf_write(fp, buffer, c) < 0)
+                            error("Could not write %d bytes: Error %d\n",
+                                c, fp->errcode);
+                } else {
+                    /* Text mode, try a flush after a newline */
+                    int in_header = 1, n = 0, long_line = 0;
+                    while ((c = hread(f_src, buffer+n, WINDOW_SIZE-n)) > 0) {
+                        int c2 = c+n;
+                        int flush = 0;
+                        if (in_header &&
+                            (long_line || buffer[0] == '@' || buffer[0] == '#')) {
+                            // Scan forward to find the last header line.
+                            int last_start = 0;
+                            n = 0;
+                            while (n < c2) {
+                                if (buffer[n++] != '\n')
+                                    continue;
+
+                                last_start = n;
+                                if (n < c2 &&
+                                    !(buffer[n] == '@' || buffer[n] == '#')) {
+                                    in_header = 0;
+                                    break;
+                                }
+                            }
+                            if (!last_start) {
+                                n = c2;
+                                long_line = 1;
+                            } else {
+                                n = last_start;
+                                flush = 1;
+                                long_line = 0;
+                            }
+                        } else {
+                            // Scan backwards to find the last newline.
+                            n += c; // c read plus previous n overflow
+                            while (--n >= 0 && ((char *)buffer)[n] != '\n')
+                                ;
+
+                            if (n >= 0) {
+                                flush = 1;
+                                n++;
+                            } else {
+                                n = c2;
+                            }
+                        }
+
+                        // Pos n is either at the end of the buffer with flush==0,
+                        // or the first byte after a newline and a flush point.
+                        if (bgzf_write(fp, buffer, n) < 0)
+                            error("Could not write %d bytes: Error %d\n",
+                                n, fp->errcode);
+                        if (flush)
+                            if (bgzf_flush_try(fp, 65536) < 0) {// force
+                                if (statfilename) {
+                                    free(statfilename);
+                                }
+                                return -1;
+                            }
+
+                        memmove(buffer, buffer+n, c2-n);
+                        n = c2-n;
+                    }
+
+                    // Trailing data.
+                    if (bgzf_write(fp, buffer, n) < 0)
+                        error("Could not write %d bytes: Error %d\n",
+                            n, fp->errcode);
+                }
+            }
+            if ( index && !write_fname )
+            {
+                if (index_fname) {
+                    if (bgzf_index_dump(fp, index_fname, NULL) < 0)
+                        error("Could not write index to '%s'\n", index_fname);
+                } else if (!isstdin) {
+                    if (bgzf_index_dump(fp, argv[optind], ".gz.gzi") < 0)
+                        error("Could not write index to '%s.gz.gzi'\n", argv[optind]);
+                }
+                else {
+                    //stdin, cant create index file as name is not present "-.gz.gzi" not a valid one!
+                    error("Can not write index for stdin data without index filename, use -I option to set index file.\n");
+                }
+            }
+
+            if (!write_fname) {
+                if (bgzf_close(fp) < 0)
+                    error("Output close failed: Error %d\n", fp->errcode);
+            }
+
+            if (hclose(f_src) < 0)
+                error("Input close failed\n");
+
+            if (statfilename) {
+                //get input file timestamp
+                if (!getfilespec(argv[optind], &filestat)) {
+                    //set output file timestamp
+                    if (setfilespec(statfilename, &filestat) < 0) {
+                        fprintf(stderr, "[bgzip] Failed to set file specification.\n");
+                    }
+                }
+                else {
+                    fprintf(stderr, "[bgzip] Failed to get file specification.\n");
+                }
+                free(statfilename);
+            }
+
+            if (argc > optind && !pstdout && !keep && !isstdin && !write_fname) unlink(argv[optind]);
+
+            free(buffer);
+        }
+        else if ( reindex )
+        {
+            if ( argc>optind && !isstdin )
+            {
+                fp = bgzf_open(argv[optind], "r");
+                if ( !fp ) error("[bgzip] Could not open file: %s\n", argv[optind]);
+            }
+            else
+            {
+                if ( !index_fname ) error("[bgzip] Index file name expected when reading from stdin\n");
+                fp = bgzf_open("-", "r");
+                if ( !fp ) error("[bgzip] Could not read from stdin: %s\n", strerror(errno));
+            }
+
+            buffer = malloc(BGZF_BLOCK_SIZE);
+            bgzf_index_build_init(fp);
+            int ret;
+            while ( (ret=bgzf_read(fp, buffer, BGZF_BLOCK_SIZE))>0 ) ;
+            free(buffer);
+            if ( ret<0 ) error("Is the file gzipped or bgzipped? The latter is required for indexing.\n");
+
+            if ( index_fname ) {
+                if (bgzf_index_dump(fp, index_fname, NULL) < 0)
+                    error("Could not write index to '%s'\n", index_fname);
+            } else if (!isstdin) {
+                if (bgzf_index_dump(fp, argv[optind], ".gzi") < 0)
+                    error("Could not write index to '%s.gzi'\n", argv[optind]);
+            }
+            else {
+                //stdin, cant create index file as name is not present "-.gzi" not a valid one!
+                error("Can not write index for stdin data without index filename, use -I option to set index file.\n");
+            }
+
+            if ( bgzf_close(fp)<0 ) error("Close failed: Error %d\n",fp->errcode);
+        }
+        else
+        {
+            int is_forced_tmp = is_forced;
+
+            if ( argc>optind && !isstdin )
+            {
+                fp = bgzf_open(argv[optind], "r");
+                if (fp == NULL) {
+                    fprintf(stderr, "[bgzip] Could not open %s: %s\n", argv[optind], strerror(errno));
+                    return 1;
+                }
+                if (bgzf_compression(fp) == no_compression) {
+                    fprintf(stderr, "[bgzip] %s: not a compressed file -- ignored\n", argv[optind]);
+                    bgzf_close(fp);
+                    return 1;
+                }
+
+                if (pstdout || test) {
+                    f_dst = fileno(stdout);
+                } else {
+                    const int wrflags = O_WRONLY | O_CREAT | O_TRUNC;
+                    char *name;
+                    int check;
+
+                    if (!(name = strdup(argv[optind]))) {
+                        fprintf(stderr, "[bgzip] unable to allocate memory for output file name.\n");
+                        bgzf_close(fp);
+                        return 1;
+                    }
+
+                    if ((check = check_name_and_extension(name, &is_forced_tmp))) {
+                        bgzf_close(fp);
+
+                        if (check == 1) {
+                            return 1;
+                        } else {
+                            ret = 2;
+                            continue;
+                        }
+                    }
+
+                    if (!exp_out_open) {
+                        if (write_fname) { // only open file once and don't care about overwriting
+                            is_forced_tmp = 1;
+                            exp_out_open = 1;
+                        }
+
+                        f_dst = open(write_fname ? write_fname : name, is_forced_tmp? wrflags : wrflags|O_EXCL, 0666);
+
+                        if (f_dst < 0 && errno == EEXIST) {
+                            if (confirm_overwrite(name)) {
+                                f_dst = open(name, wrflags, 0666);
+                            }
+                            else {
+                                ret = 2;                        //explicit N - no overwrite, continue and return 2
+                                bgzf_close(fp);
+                                free(name);
+                                continue;
+                            }
+                        }
+                        if (f_dst < 0) {
+                            fprintf(stderr, "[bgzip] can't create %s: %s\n", name, strerror(errno));
+                            free(name);
+                            return 1;
+                        }
+                    }
+
+                    statfilename = name;
+                }
+            }
+            else if (!pstdout && isatty(fileno((FILE *)stdin)) )
+                return bgzip_main_usage(stderr, EXIT_FAILURE);
+            else
+            {
+                f_dst = fileno(stdout);
+                fp = bgzf_open("-", "r");
+                if (fp == NULL) {
+                    fprintf(stderr, "[bgzip] Could not read from stdin: %s\n", strerror(errno));
+                    return 1;
+                }
+                if (bgzf_compression(fp) == no_compression) {
+                    fprintf(stderr, "[bgzip] stdin is not compressed -- ignored\n");
+                    bgzf_close(fp);
+                    return 1;
+                }
+
+                if (!write_fname) {
+                    f_dst = fileno(stdout);
+                } else {
+                    if (!exp_out_open) {
+                        exp_out_open = 1;
+
+                        f_dst = open(write_fname, O_WRONLY | O_CREAT | O_TRUNC, 0666);
+
+                        if (f_dst < 0) {
+                            fprintf(stderr, "[bgzip] can't create %s: %s\n", write_fname, strerror(errno));
+                            return 1;
+                        }
+                    }
+                }
+            }
+
+            buffer = malloc(WINDOW_SIZE);
+            if ( start>0 )
+            {
+                if (index_fname) {
+                    if ( bgzf_index_load(fp, index_fname, NULL) < 0 )
+                        error("Could not load index: %s\n", index_fname);
+                } else {
+                    if (optind >= argc || isstdin) {
+                        error("The -b option requires -I when reading from stdin "
+                            "(and stdin must be seekable)\n");
+                    }
+                    if ( bgzf_index_load(fp, argv[optind], ".gzi") < 0 )
+                        error("Could not load index: %s.gzi\n", argv[optind]);
+                }
+                if ( bgzf_useek(fp, start, SEEK_SET) < 0 ) error("Could not seek to %ld-th (uncompressd) byte\n", start);
+            }
+
+            if (threads > 1)
+                bgzf_mt(fp, threads, 256);
+
+    #ifdef _WIN32
+            _setmode(f_dst, O_BINARY);
+    #endif
+            long start_reg = start, end_reg = end;
+            while (1) {
+                if (end < 0) c = bgzf_read(fp, buffer, WINDOW_SIZE);
+                else c = bgzf_read(fp, buffer, (end - start > WINDOW_SIZE)? WINDOW_SIZE:(end - start));
+                if (c == 0) break;
+                if (c < 0) error("Error %d in block starting at offset %" PRId64 "(%" PRIX64 ")\n", fp->errcode, fp->block_address, fp->block_address);
+                start += c;
+                if ( !test && write(f_dst, buffer, c) != c ) {
+    #ifdef _WIN32
+                    if (GetLastError() != ERROR_NO_DATA)
+    #endif
+                    error("Could not write %d bytes\n", c);
+                }
+                if (end >= 0 && start >= end) break;
+            }
+            start = start_reg;
+            end = end_reg;
+            free(buffer);
+            if (bgzf_close(fp) < 0) error("Close failed: Error %d\n",fp->errcode);
+
+            if (statfilename) {
+                if (!write_fname) {
+                    //get input file timestamp
+                    if (!getfilespec(argv[optind], &filestat)) {
+                        //set output file timestamp
+                        if (setfilespec(statfilename, &filestat) < 0) {
+                            fprintf(stderr, "[bgzip] Failed to set file specification.\n");
+                        }
+                    }
+                    else {
+                        fprintf(stderr, "[bgzip] Failed to get file specification.\n");
+                    }
+                }
+
+                free(statfilename);
+            }
+
+            if (argc > optind && !pstdout && !test && !keep && !isstdin && !write_fname) unlink(argv[optind]);
+            if (!isstdin && !pstdout && !test && !write_fname) {
+                close(f_dst);                               //close output file when it is not stdout
+            }
+        }
+    } while (++optind < argc);
+
+    if (usedstdout && !reindex) {
+        //stdout in use, have to close explicitly to get any pending write errors
+        if (fclose(stdout) != 0 && errno != EBADF) {
+            fprintf(stderr, "[bgzip] Failed to close stdout, errno %d", errno);
+            ret = 1;
+        }
+    } else if (write_fname) {
+        if (compress == 1) { // close explicit output file (this is for compression)
+            if (index) {
+                if (index_fname) {
+                    if (bgzf_index_dump(fp, index_fname, NULL) < 0)
+                        error("Could not write index to '%s'\n", index_fname);
+                } else {
+                    if (bgzf_index_dump(fp, write_fname, ".gzi") < 0)
+                        error("Could not write index to '%s.gzi'\n", write_fname);
+                }
+            }
+
+            if (bgzf_close(fp) < 0)
+                error("Output close failed: Error %d\n", fp->errcode);
+        } else {
+            close(f_dst);
+        }
+    }
+
+
+    return ret;
+}
diff --git a/ext/htslib/builddir_vars.mk.in b/ext/htslib/builddir_vars.mk.in
new file mode 100644
index 0000000..09bb20f
--- /dev/null
+++ b/ext/htslib/builddir_vars.mk.in
@@ -0,0 +1,58 @@
+# Separate build directory Makefile overrides for htslib.
+#
+#    Copyright (C) 2021 University of Glasgow.
+#
+#    Author: John Marshall <jmarshall@hey.com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+# This is @configure_input@
+#
+# When building from a separate build directory, this file is included by
+# HTSlib's Makefile or htslib.mk instead of htslib_vars.mk. It adjusts
+# variables to account for a separate source directory and then includes
+# the real makefile fragment.
+
+ifneq "$(HTSPREFIX)" ""
+# When included externally via htslib.mk, just update $(HTSSRCDIR) and hence
+# $(HTSPREFIX) to point to the source directory (without using any extra
+# unprefixed variables, which would be in the external project's namespace).
+
+# Set to $(HTSDIR)/$(srcdir) (or just $(srcdir) if that's absolute)
+HTSSRCDIR = @HTSDIRslash_if_relsrcdir@@srcdir@
+
+include $(HTSSRCDIR)/htslib_vars.mk
+
+else
+# When included from HTSlib's Makefile, override $(srcdir) and set VPATH,
+# and make any other adjustments required. ($(HTSPREFIX) remains empty as
+# the items it prefixes will be found via VPATH instead.)
+
+srcdir = @srcdir@
+VPATH  = @srcdir@
+
+srcprefix = $(srcdir)/
+
+# Ensure that htscodecs.c can include its version.h. This -I option must come
+# before -I. so that these targets get this version.h rather than HTSlib's.
+htscodecs/htscodecs/htscodecs.o htscodecs/htscodecs/htscodecs.pico: ALL_CPPFLAGS = -Ihtscodecs/htscodecs -I. $(CPPFLAGS)
+
+include $(srcdir)/htslib_vars.mk
+
+endif
diff --git a/ext/htslib/config.h.in~ b/ext/htslib/config.h.in~
new file mode 100644
index 0000000..f9d38a4
--- /dev/null
+++ b/ext/htslib/config.h.in~
@@ -0,0 +1,180 @@
+/* config.h.in.  Generated from configure.ac by autoheader.  */
+
+/* If you use configure, this file provides #defines reflecting your
+   configuration choices.  If you have not run configure, suitable
+   conservative defaults will be used.
+
+   Autoheader adds a number of items to this template file that are not
+   used by HTSlib: STDC_HEADERS and most HAVE_*_H header file defines
+   are immaterial, as we assume standard ISO C headers and facilities;
+   the PACKAGE_* defines are unused and are overridden by the more
+   accurate PACKAGE_VERSION as computed by the Makefile.  */
+
+/* Define if HTSlib should enable GCS support. */
+#undef ENABLE_GCS
+
+/* Define if HTSlib should enable plugins. */
+#undef ENABLE_PLUGINS
+
+/* Define if HTSlib should enable S3 support. */
+#undef ENABLE_S3
+
+/* Define if __attribute__((constructor)) is available. */
+#undef HAVE_ATTRIBUTE_CONSTRUCTOR
+
+/* Define if __attribute__((target(...))) is available. */
+#undef HAVE_ATTRIBUTE_TARGET
+
+/* Defined to 1 if rANS source using AVX2 can be compiled. */
+#undef HAVE_AVX2
+
+/* Defined to 1 if rANS source using AVX512F can be compiled. */
+#undef HAVE_AVX512
+
+/* Defined to 1 if __builtin_cpu_supports("ssse3") works */
+#undef HAVE_BUILTIN_CPU_SUPPORT_SSSE3
+
+/* Define if clock_gettime exists and accepts CLOCK_PROCESS_CPUTIME_ID. */
+#undef HAVE_CLOCK_GETTIME_CPUTIME
+
+/* Define if you have the Common Crypto library. */
+#undef HAVE_COMMONCRYPTO
+
+/* Define to 1 if you have the declaration of '__cpuid_count', and to 0 if you
+   don't. */
+#undef HAVE_DECL___CPUID_COUNT
+
+/* Define to 1 if you have the declaration of '__get_cpuid_max', and to 0 if
+   you don't. */
+#undef HAVE_DECL___GET_CPUID_MAX
+
+/* Define to 1 if you have the 'drand48' function. */
+#undef HAVE_DRAND48
+
+/* Define if using an external libhtscodecs */
+#undef HAVE_EXTERNAL_LIBHTSCODECS
+
+/* Define to 1 if you have the 'fdatasync' function. */
+#undef HAVE_FDATASYNC
+
+/* Define to 1 if you have the 'fsync' function. */
+#undef HAVE_FSYNC
+
+/* Define to 1 if you have the 'getpagesize' function. */
+#undef HAVE_GETPAGESIZE
+
+/* Define to 1 if you have the 'gmtime_r' function. */
+#undef HAVE_GMTIME_R
+
+/* Define if you have libcrypto-style HMAC(). */
+#undef HAVE_HMAC
+
+/* Define to 1 if you have the <inttypes.h> header file. */
+#undef HAVE_INTTYPES_H
+
+/* Define to 1 if you have the 'bz2' library (-lbz2). */
+#undef HAVE_LIBBZ2
+
+/* Define if libcurl file access is enabled. */
+#undef HAVE_LIBCURL
+
+/* Define if libdeflate is available. */
+#undef HAVE_LIBDEFLATE
+
+/* Define to 1 if you have the 'lzma' library (-llzma). */
+#undef HAVE_LIBLZMA
+
+/* Define to 1 if you have the 'z' library (-lz). */
+#undef HAVE_LIBZ
+
+/* Define to 1 if you have the <lzma.h> header file. */
+#undef HAVE_LZMA_H
+
+/* Define to 1 if you have a working 'mmap' system call. */
+#undef HAVE_MMAP
+
+/* Defined to 1 if rANS source using popcnt can be compiled. */
+#undef HAVE_POPCNT
+
+/* Define to 1 if you have the 'srand48_deterministic' function. */
+#undef HAVE_SRAND48_DETERMINISTIC
+
+/* Defined to 1 if rANS source using SSE4.1 can be compiled. */
+#undef HAVE_SSE4_1
+
+/* Defined to 1 if rANS source using SSSE3 can be compiled. */
+#undef HAVE_SSSE3
+
+/* Define to 1 if you have the <stdint.h> header file. */
+#undef HAVE_STDINT_H
+
+/* Define to 1 if you have the <stdio.h> header file. */
+#undef HAVE_STDIO_H
+
+/* Define to 1 if you have the <stdlib.h> header file. */
+#undef HAVE_STDLIB_H
+
+/* Define to 1 if you have the <strings.h> header file. */
+#undef HAVE_STRINGS_H
+
+/* Define to 1 if you have the <string.h> header file. */
+#undef HAVE_STRING_H
+
+/* Define to 1 if you have the <sys/param.h> header file. */
+#undef HAVE_SYS_PARAM_H
+
+/* Define to 1 if you have the <sys/stat.h> header file. */
+#undef HAVE_SYS_STAT_H
+
+/* Define to 1 if you have the <sys/types.h> header file. */
+#undef HAVE_SYS_TYPES_H
+
+/* Define to 1 if you have the <unistd.h> header file. */
+#undef HAVE_UNISTD_H
+
+/* Define to the address where bug reports for this package should be sent. */
+#undef PACKAGE_BUGREPORT
+
+/* Define to the full name of this package. */
+#undef PACKAGE_NAME
+
+/* Define to the full name and version of this package. */
+#undef PACKAGE_STRING
+
+/* Define to the one symbol short name of this package. */
+#undef PACKAGE_TARNAME
+
+/* Define to the home page for this package. */
+#undef PACKAGE_URL
+
+/* Define to the version of this package. */
+#undef PACKAGE_VERSION
+
+/* Platform-dependent plugin filename extension. */
+#undef PLUGIN_EXT
+
+/* Define to 1 if all of the C89 standard headers exist (not just the ones
+   required in a freestanding environment). This macro is provided for
+   backward compatibility; new code need not use it. */
+#undef STDC_HEADERS
+
+
+/* Prevent unaligned access in htscodecs SSE4 rANS codec */
+#if defined(HTS_ALLOW_UNALIGNED) && HTS_ALLOW_UNALIGNED == 0
+#undef UBSAN
+#endif
+
+/* Number of bits in a file offset, on hosts where this is settable. */
+#undef _FILE_OFFSET_BITS
+
+/* Define to 1 on platforms where this makes off_t a 64-bit type. */
+#undef _LARGE_FILES
+
+/* Number of bits in time_t, on hosts where this is settable. */
+#undef _TIME_BITS
+
+/* Specify X/Open requirements */
+#undef _XOPEN_SOURCE
+
+/* Define to 1 on platforms where this makes time_t a 64-bit type. */
+#undef __MINGW_USE_VC2005_COMPAT
diff --git a/ext/htslib/config.mk.in b/ext/htslib/config.mk.in
new file mode 100644
index 0000000..59a121c
--- /dev/null
+++ b/ext/htslib/config.mk.in
@@ -0,0 +1,120 @@
+#  Optional configure Makefile overrides for htslib.
+#
+#    Copyright (C) 2015-2017, 2019, 2023 Genome Research Ltd.
+#
+#    Author: John Marshall <jm18@sanger.ac.uk>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+# This is @configure_input@
+#
+# If you use configure, this file overrides variables and augments rules
+# in the Makefile to reflect your configuration choices.  If you don't run
+# configure, the main Makefile contains suitable conservative defaults.
+
+prefix       = @prefix@
+exec_prefix  = @exec_prefix@
+bindir       = @bindir@
+includedir   = @includedir@
+libdir       = @libdir@
+libexecdir   = @libexecdir@
+datarootdir  = @datarootdir@
+mandir       = @mandir@
+
+CC     = @CC@
+RANLIB = @RANLIB@
+
+CPPFLAGS = @CPPFLAGS@
+CFLAGS   = @CFLAGS@
+LDFLAGS  = @LDFLAGS@
+VERSION_SCRIPT_LDFLAGS = @VERSION_SCRIPT_LDFLAGS@
+LIBS     = @LIBS@
+
+PLATFORM   = @PLATFORM@
+PLUGIN_EXT = @PLUGIN_EXT@
+
+# The default Makefile enables some of the optional files, but we blank
+# them so they can be controlled by configure instead.
+NONCONFIGURE_OBJS =
+
+# Lowercase here indicates these are "local" to config.mk
+plugin_OBJS =
+noplugin_LDFLAGS =
+noplugin_LIBS =
+
+# ifeq/.../endif, +=, and target-specific variables are GNU Make-specific.
+# If you don't have GNU Make, comment out this conditional and note that
+# to enable libcurl you will need to implement the following elsewhere.
+ifeq "libcurl-@libcurl@" "libcurl-enabled"
+
+LIBCURL_LIBS = -lcurl
+
+plugin_OBJS += hfile_libcurl.o
+
+hfile_libcurl$(PLUGIN_EXT): LIBS += $(LIBCURL_LIBS)
+
+noplugin_LIBS += $(LIBCURL_LIBS)
+
+endif
+
+ifeq "gcs-@gcs@" "gcs-enabled"
+plugin_OBJS += hfile_gcs.o
+endif
+
+ifeq "s3-@s3@" "s3-enabled"
+plugin_OBJS += hfile_s3.o
+plugin_OBJS += hfile_s3_write.o
+
+CRYPTO_LIBS = @CRYPTO_LIBS@
+noplugin_LIBS += $(CRYPTO_LIBS)
+hfile_s3$(PLUGIN_EXT): LIBS += $(CRYPTO_LIBS)
+hfile_s3_write$(PLUGIN_EXT): LIBS += $(CRYPTO_LIBS) $(LIBCURL_LIBS)
+endif
+
+ifeq "plugins-@enable_plugins@" "plugins-yes"
+
+plugindir  = @plugindir@
+pluginpath = @pluginpath@
+
+LIBHTS_OBJS += plugin.o
+PLUGIN_OBJS += $(plugin_OBJS)
+
+plugin.o plugin.pico: ALL_CPPFLAGS += -DPLUGINPATH=\"$(pluginpath)\"
+
+# When built as separate plugins, these record their version themselves.
+hfile_gcs.o hfile_gcs.pico: version.h
+hfile_libcurl.o hfile_libcurl.pico: version.h
+hfile_s3.o hfile_s3.pico: version.h
+hfile_s3_write.o hfile_s3_write.pico: version.h
+
+# Windows DLL plugins depend on the import library, built as a byproduct.
+$(plugin_OBJS:.o=.cygdll): cyghts-$(LIBHTS_SOVERSION).dll
+
+else
+
+LIBHTS_OBJS += $(plugin_OBJS)
+LDFLAGS += $(noplugin_LDFLAGS)
+LIBS += $(noplugin_LIBS)
+
+endif
+
+# Extra CFLAGS for specific files
+HTS_CFLAGS_AVX2 = @hts_cflags_avx2@
+HTS_CFLAGS_AVX512 = @hts_cflags_avx512@
+HTS_CFLAGS_SSE4 = @hts_cflags_sse4@
diff --git a/ext/htslib/configure.ac b/ext/htslib/configure.ac
new file mode 100644
index 0000000..87e928d
--- /dev/null
+++ b/ext/htslib/configure.ac
@@ -0,0 +1,675 @@
+# Configure script for htslib, a C library for high-throughput sequencing data.
+#
+#    Copyright (C) 2015-2024 Genome Research Ltd.
+#
+#    Author: John Marshall <jm18@sanger.ac.uk>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+dnl Process this file with autoconf to produce a configure script
+AC_INIT([HTSlib], m4_esyscmd_s([./version.sh 2>/dev/null]),
+        [samtools-help@lists.sourceforge.net], [], [http://www.htslib.org/])
+AC_PREREQ(2.63)  dnl This version introduced 4-argument AC_CHECK_HEADER
+AC_CONFIG_SRCDIR(hts.c)
+AC_CONFIG_HEADERS(config.h)
+
+m4_include([m4/hts_prog_cc_warnings.m4])
+m4_include([m4/hts_check_compile_flags_needed.m4])
+m4_include([m4/hts_hide_dynamic_syms.m4])
+m4_include([m4/pkg.m4])
+
+dnl Copyright notice to be copied into the generated configure script
+AC_COPYRIGHT([Portions copyright (C) 2020-2024 Genome Research Ltd.
+
+This configure script is free software: you are free to change and
+redistribute it.  There is NO WARRANTY, to the extent permitted by law.])
+
+dnl Notes to be copied (by autoheader) into the generated config.h.in
+AH_TOP([/* If you use configure, this file provides @%:@defines reflecting your
+   configuration choices.  If you have not run configure, suitable
+   conservative defaults will be used.
+
+   Autoheader adds a number of items to this template file that are not
+   used by HTSlib: STDC_HEADERS and most HAVE_*_H header file defines
+   are immaterial, as we assume standard ISO C headers and facilities;
+   the PACKAGE_* defines are unused and are overridden by the more
+   accurate PACKAGE_VERSION as computed by the Makefile.  */])
+
+dnl Variant of AC_MSG_ERROR that ensures subsequent make(1) invocations fail
+dnl until the configuration error is resolved and configure is run again.
+AC_DEFUN([MSG_ERROR],
+  [cat > config.mk <<'EOF'
+ifneq ($(MAKECMDGOALS),distclean)
+$(error Resolve configure error first)
+endif
+EOF
+   AC_MSG_ERROR([$1], [$2])])
+
+AC_PROG_CC
+AC_PROG_RANLIB
+
+dnl Turn on compiler warnings, if possible
+HTS_PROG_CC_WARNINGS
+dnl Flags to treat warnings as errors.  These need to be applied to CFLAGS
+dnl later as they can interfere with some of the tests (notably AC_SEARCH_LIBS)
+HTS_PROG_CC_WERROR(hts_late_cflags)
+
+# HTSlib uses X/Open-only facilities (M_SQRT2 etc, drand48() etc), and
+# various POSIX functions that are provided by various _POSIX_C_SOURCE values
+# or by _XOPEN_SOURCE >= 500. It also uses usleep(), which is removed when
+# _XOPEN_SOURCE >= 700. Additionally, some definitions may require
+# _XOPEN_SOURCE >= 600 on some platforms (snprintf on MinGW,
+# PTHREAD_MUTEX_RECURSIVE on some Linux distributions). Hence we set it to 600.
+
+# Define _XOPEN_SOURCE unless the user has already done so via $CPPFLAGS etc.
+AC_CHECK_DECL([_XOPEN_SOURCE], [],
+  [AC_DEFINE([_XOPEN_SOURCE], [600], [Specify X/Open requirements])],
+  [])
+
+dnl Check that we have cpuid, and if so run the x86 SIMD checks
+AC_CHECK_DECLS([__get_cpuid_max, __cpuid_count], [
+   hts_have_cpuid=yes
+], [
+   hts_have_cpuid=no
+], [[#include <cpuid.h>]])
+
+AS_IF(test "x$hts_have_cpuid" = "xyes", [
+dnl Options for rANS32x16 sse4.1 version - sse4.1
+HTS_CHECK_COMPILE_FLAGS_NEEDED([sse4.1], [-msse4.1 -mssse3 -mpopcnt],
+ [AC_LANG_PROGRAM([[
+    #ifdef __x86_64__
+    #include "x86intrin.h"
+    #endif
+  ]],[[
+    #ifdef __x86_64__
+    __m128i a = _mm_set_epi32(1, 2, 3, 4), b = _mm_set_epi32(4, 3, 2, 1);
+    __m128i c = _mm_shuffle_epi8(_mm_max_epu32(a, b), b);
+    return _mm_popcnt_u32(*((char *) &c));
+    #endif
+  ]])], [
+  hts_cflags_sse4="$flags_needed"
+  AC_DEFINE([HAVE_SSSE3],1,[Defined to 1 if rANS source using SSSE3 can be compiled.])
+  AC_DEFINE([HAVE_POPCNT],1,[Defined to 1 if rANS source using popcnt can be compiled.])
+  AC_DEFINE([HAVE_SSE4_1],1,[Defined to 1 if rANS source using SSE4.1 can be compiled.
+])
+
+dnl Propagate HTSlib's unaligned access preference to htscodecs
+  AH_VERBATIM([UBSAN],[
+/* Prevent unaligned access in htscodecs SSE4 rANS codec */
+#if defined(HTS_ALLOW_UNALIGNED) && HTS_ALLOW_UNALIGNED == 0
+#undef UBSAN
+#endif])
+  AC_DEFINE([UBSAN],1,[])
+])
+AC_SUBST([hts_cflags_sse4])
+
+dnl Options for rANS32x16 avx2 version
+HTS_CHECK_COMPILE_FLAGS_NEEDED([avx2], [-mavx2 -mpopcnt], [AC_LANG_PROGRAM([[
+    #ifdef __x86_64__
+    #include "x86intrin.h"
+    #endif
+  ]],[[
+    #ifdef __x86_64__
+    __m256i a = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8);
+    __m256i b = _mm256_add_epi32(a, a);
+    long long c = _mm256_extract_epi64(b, 0);
+    return _mm_popcnt_u32((int) c);
+    #endif
+  ]])], [
+  hts_cflags_avx2="$flags_needed"
+  AC_SUBST([hts_cflags_avx2])
+  AC_DEFINE([HAVE_POPCNT],1,[Defined to 1 if rANS source using popcnt can be compiled.])
+  AC_DEFINE([HAVE_AVX2],1,[Defined to 1 if rANS source using AVX2 can be compiled.])
+])
+
+dnl Options for rANS32x16 avx512 version
+HTS_CHECK_COMPILE_FLAGS_NEEDED([avx512f], [-mavx512f -mpopcnt],
+ [AC_LANG_PROGRAM([[
+    #ifdef __x86_64__
+    #include "x86intrin.h"
+    #endif
+  ]],[[
+    #ifdef __x86_64__
+    __m512i a = _mm512_set1_epi32(1);
+    __m512i b = _mm512_add_epi32(a, a);
+    __m256i c = _mm512_castsi512_si256(b);
+    __m256i d = _mm512_extracti64x4_epi64(a, 1);
+    return _mm_popcnt_u32(*((char *) &c)) + (*(char *) &d);
+    #endif
+  ]])], [
+  hts_cflags_avx512="$flags_needed"
+  AC_SUBST([hts_cflags_avx512])
+  AC_DEFINE([HAVE_POPCNT],1,[Defined to 1 if rANS source using popcnt can be compiled.])
+  AC_DEFINE([HAVE_AVX512],1,[Defined to 1 if rANS source using AVX512F can be compiled.])
+])
+
+dnl Check for working __builtin_cpu_supports (ssse3 is broken on some clangs)
+AC_MSG_CHECKING([for working __builtin_cpu_supports("ssse3")])
+AC_LINK_IFELSE([AC_LANG_PROGRAM([],[
+  if (__builtin_cpu_supports("ssse3")) {
+    return 0;
+  }
+])], [
+  AC_MSG_RESULT([yes])
+  AC_DEFINE([HAVE_BUILTIN_CPU_SUPPORT_SSSE3], 1,
+            [Defined to 1 if __builtin_cpu_supports("ssse3") works])
+], [
+  AC_MSG_RESULT([no])
+])
+
+dnl Check for function attribute used in conjunction with __builtin_cpu_supports
+AC_MSG_CHECKING([for __attribute__((target))])
+AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
+  __attribute__((target("ssse3")))
+  int zero(void) {
+    return 0;
+  }
+]], [[zero();]])], [
+  AC_MSG_RESULT([yes])
+  AC_DEFINE([HAVE_ATTRIBUTE_TARGET], 1,
+            [Define if __attribute__((target(...))) is available.])
+], [
+  AC_MSG_RESULT([no])
+])
+
+]) dnl End of AS_IF(hts_have_cpuid)
+
+dnl Avoid chicken-and-egg problem where pkg-config supplies the
+dnl PKG_PROG_PKG_CONFIG macro, but we want to use it to check
+dnl for pkg-config...
+m4_ifdef([PKG_PROG_PKG_CONFIG], [PKG_PROG_PKG_CONFIG], [PKG_CONFIG=""])
+
+need_crypto=no
+pc_requires=
+static_LDFLAGS=$LDFLAGS
+static_LIBS='-lpthread -lz -lm'
+private_LIBS=$LDFLAGS
+
+AC_ARG_ENABLE([versioned-symbols],
+  [AS_HELP_STRING([--disable-versioned-symbols],
+                  [disable versioned symbols in shared library])],
+  [], [enable_versioned_symbols=yes])
+
+AC_ARG_ENABLE([bz2],
+  [AS_HELP_STRING([--disable-bz2],
+                  [omit support for BZ2-compressed CRAM files])],
+  [], [enable_bz2=yes])
+
+AC_ARG_ENABLE([gcs],
+  [AS_HELP_STRING([--enable-gcs],
+                  [support Google Cloud Storage URLs])],
+  [], [enable_gcs=check])
+
+AC_SYS_LARGEFILE
+
+AC_ARG_ENABLE([libcurl],
+  [AS_HELP_STRING([--enable-libcurl],
+                  [enable libcurl-based support for http/https/etc URLs])],
+  [], [enable_libcurl=check])
+
+AC_ARG_ENABLE([lzma],
+  [AS_HELP_STRING([--disable-lzma],
+                  [omit support for LZMA-compressed CRAM files])],
+  [], [enable_lzma=yes])
+
+AC_ARG_ENABLE([plugins],
+  [AS_HELP_STRING([--enable-plugins],
+                  [enable separately-compiled plugins for file access])],
+  [], [enable_plugins=no])
+AC_SUBST(enable_plugins)
+
+AC_ARG_WITH([external-htscodecs],
+  [AS_HELP_STRING([--with-external-htscodecs],
+                  [get htscodecs functions from a shared library])],
+  [], [with_external_htscodecs=no])
+AC_SUBST(with_external_htscodecs)
+
+AC_ARG_WITH([libdeflate],
+  [AS_HELP_STRING([--with-libdeflate],
+                  [use libdeflate for faster crc and deflate algorithms])],
+  [], [with_libdeflate=check])
+
+AC_ARG_WITH([plugin-dir],
+  [AS_HELP_STRING([--with-plugin-dir=DIR],
+                  [plugin installation location [LIBEXECDIR/htslib]])],
+  [case $withval in
+     yes|no) MSG_ERROR([no directory specified for --with-plugin-dir]) ;;
+   esac],
+   [with_plugin_dir='$(libexecdir)/htslib'])
+AC_SUBST([plugindir], $with_plugin_dir)
+
+AC_ARG_WITH([plugin-path],
+  [AS_HELP_STRING([--with-plugin-path=PATH],
+                  [default HTS_PATH plugin search path [PLUGINDIR]])],
+  [case $withval in
+     yes) MSG_ERROR([no path specified for --with-plugin-path]) ;;
+     no)  with_plugin_path= ;;
+   esac],
+  [with_plugin_path=$with_plugin_dir])
+AC_SUBST([pluginpath], $with_plugin_path)
+
+AC_ARG_ENABLE([s3],
+  [AS_HELP_STRING([--enable-s3],
+                  [support Amazon AWS S3 URLs])],
+  [], [enable_s3=check])
+
+basic_host=${host_alias:-unknown-`uname -s`}
+AC_MSG_CHECKING([shared library type for $basic_host])
+case $basic_host in
+  *-cygwin* | *-CYGWIN*)
+    host_result="Cygwin DLL"
+    PLATFORM=CYGWIN
+    PLUGIN_EXT=.cygdll
+    ;;
+  *-darwin* | *-Darwin*)
+    host_result="Darwin dylib"
+    PLATFORM=Darwin
+    PLUGIN_EXT=.bundle
+    ;;
+  *-msys* | *-MSYS* | *-mingw* | *-MINGW*)
+    host_result="MSYS dll"
+    PLATFORM=MSYS
+    PLUGIN_EXT=.dll
+    # This also sets __USE_MINGW_ANSI_STDIO which in turn makes PRId64,
+    # %lld and %z printf formats work.  It also enforces the snprintf to
+    # be C99 compliant so it returns the correct values (in kstring.c).
+
+    # Now set by default, so no need to do it here.
+    # CPPFLAGS="$CPPFLAGS -D_XOPEN_SOURCE=600"
+    ;;
+  *)
+    host_result="plain .so"
+    PLATFORM=default
+    PLUGIN_EXT=.so
+    ;;
+esac
+AC_MSG_RESULT([$host_result])
+AC_SUBST([PLATFORM])
+
+dnl Check for versioned symbol support
+dnl Only try for .so shared libraries as other types won't work
+AS_IF([test x"$PLATFORM" = xdefault && test x"$enable_versioned_symbols" = xyes],
+  [AC_CACHE_CHECK([whether the linker supports versioned symbols],
+    [hts_cv_have_versioned_symbols], [
+      save_LDFLAGS=$LDFLAGS
+      LDFLAGS="-Wl,-version-script,$srcdir/htslib.map $LDFLAGS"
+      AC_LINK_IFELSE([AC_LANG_PROGRAM()],
+                     [hts_cv_have_versioned_symbols=yes],
+                     [hts_cv_have_versioned_symbols=no])
+      LDFLAGS=$save_LDFLAGS
+    ])
+   AS_IF([test "x$hts_cv_have_versioned_symbols" = xyes],[
+     VERSION_SCRIPT_LDFLAGS='-Wl,-version-script,$(srcprefix)htslib.map'
+     AC_SUBST([VERSION_SCRIPT_LDFLAGS])
+   ])
+])
+
+dnl Try to get more control over which symbols are exported in the shared
+dnl library.
+HTS_HIDE_DYNAMIC_SYMBOLS
+
+dnl FIXME This pulls in dozens of standard header checks
+AC_FUNC_MMAP
+AC_CHECK_FUNCS([gmtime_r fsync drand48 srand48_deterministic])
+
+# Darwin has a dubious fdatasync() symbol, but no declaration in <unistd.h>
+AC_CHECK_DECL([fdatasync(int)], [AC_CHECK_FUNCS(fdatasync)])
+
+AC_MSG_CHECKING([for __attribute__((constructor))])
+AC_LINK_IFELSE([AC_LANG_PROGRAM([[
+  static __attribute__((constructor)) void noop(void) {}
+]], [])], [
+  AC_MSG_RESULT([yes])
+  AC_DEFINE([HAVE_ATTRIBUTE_CONSTRUCTOR], 1,
+            [Define if __attribute__((constructor)) is available.])
+], [AC_MSG_RESULT([no])])
+
+AC_MSG_CHECKING([for clock_gettime with CLOCK_PROCESS_CPUTIME_ID])
+AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include <time.h>]], [[
+  struct timespec ts;
+  clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts);
+]])], [
+  AC_MSG_RESULT([yes])
+  AC_DEFINE([HAVE_CLOCK_GETTIME_CPUTIME], 1,
+            [Define if clock_gettime exists and accepts CLOCK_PROCESS_CPUTIME_ID.])
+], [AC_MSG_RESULT([no])])
+
+if test $enable_plugins != no; then
+  AC_SEARCH_LIBS([dlsym], [dl], [],
+    [MSG_ERROR([dlsym() not found
+
+Plugin support requires dynamic linking facilities from the operating system.
+Either configure with --disable-plugins or resolve this error to build HTSlib.])])
+  # Check if the compiler understands -rdynamic 
+  # TODO Test whether this is required and/or needs tweaking per-platform
+  HTS_TEST_CC_C_LD_FLAG([-rdynamic],[rdynamic_flag])
+  AS_IF([test x"$rdynamic_flag" != "xno"],
+    [LDFLAGS="$LDFLAGS $rdynamic_flag"
+     static_LDFLAGS="$static_LDFLAGS $rdynamic_flag"])
+  case "$ac_cv_search_dlsym" in
+    -l*) static_LIBS="$static_LIBS $ac_cv_search_dlsym" ;;
+  esac
+  AC_DEFINE([ENABLE_PLUGINS], 1, [Define if HTSlib should enable plugins.])
+  AC_SUBST([PLUGIN_EXT])
+  AC_DEFINE_UNQUOTED([PLUGIN_EXT], ["$PLUGIN_EXT"],
+                     [Platform-dependent plugin filename extension.])
+fi
+
+AC_SEARCH_LIBS([log], [m], [],
+  [MSG_ERROR([log() not found
+
+HTSLIB requires a working floating-point math library.
+FAILED.  This error must be resolved in order to build HTSlib successfully.])])
+
+zlib_devel=ok
+dnl Set a trivial non-empty INCLUDES to avoid excess default includes tests
+AC_CHECK_HEADER([zlib.h], [], [zlib_devel=missing], [;])
+AC_CHECK_LIB(z, inflate,  [], [zlib_devel=missing])
+
+if test $zlib_devel != ok; then
+  MSG_ERROR([zlib development files not found
+
+HTSlib uses compression routines from the zlib library <http://zlib.net>.
+Building HTSlib requires zlib development files to be installed on the build
+machine; you may need to ensure a package such as zlib1g-dev (on Debian or
+Ubuntu Linux) or zlib-devel (on RPM-based Linux distributions or Cygwin)
+is installed.
+
+FAILED.  This error must be resolved in order to build HTSlib successfully.])
+fi
+
+dnl connect() etc. fns are in libc on linux, but libsocket on illumos/Solaris
+AC_SEARCH_LIBS([recv], [socket ws2_32], [
+if test "$ac_cv_search_recv" != "none required"
+then
+  static_LIBS="$static_LIBS $ac_cv_search_recv"
+fi],
+  dnl on MinGW-i686, checking recv() linking requires an annotated declaration
+  [AC_MSG_CHECKING([for library containing recv using declaration])
+   LIBS="-lws2_32 $LIBS"
+   AC_LINK_IFELSE(
+     [AC_LANG_PROGRAM([[#include <winsock2.h>]], [[recv(0, 0, 0, 0);]])],
+     [AC_MSG_RESULT([-lws2_32])
+      static_LIBS="$static_LIBS -lws2_32"],
+     [AC_MSG_RESULT([no])
+      MSG_ERROR([unable to find the recv() function])])])
+
+if test "$enable_bz2" != no; then
+  bz2_devel=ok
+  AC_CHECK_HEADER([bzlib.h], [], [bz2_devel=missing], [;])
+  AC_CHECK_LIB([bz2], [BZ2_bzBuffToBuffCompress], [], [bz2_devel=missing])
+  if test $bz2_devel != ok; then
+    MSG_ERROR([libbzip2 development files not found
+
+The CRAM format may use bzip2 compression, which is implemented in HTSlib
+by using compression routines from libbzip2 <http://www.bzip.org/>.
+
+Building HTSlib requires libbzip2 development files to be installed on the
+build machine; you may need to ensure a package such as libbz2-dev (on Debian
+or Ubuntu Linux) or bzip2-devel (on RPM-based Linux distributions or Cygwin)
+is installed.
+
+Either configure with --disable-bz2 (which will make some CRAM files
+produced elsewhere unreadable) or resolve this error to build HTSlib.])
+  fi
+dnl Unfortunately the 'bzip2' package-cfg module is not standard.
+dnl Redhat/Fedora has it; Debian/Ubuntu does not.
+  if test -n "$PKG_CONFIG" && "$PKG_CONFIG" --exists bzip2; then
+     pc_requires="$pc_requires bzip2"
+  else
+     private_LIBS="$private_LIBS -lbz2"
+  fi
+  static_LIBS="$static_LIBS -lbz2"
+fi
+
+if test "$enable_lzma" != no; then
+  lzma_devel=ok
+  AC_CHECK_HEADERS([lzma.h], [], [lzma_devel=header-missing], [;])
+  AC_CHECK_LIB([lzma], [lzma_easy_buffer_encode], [], [lzma_devel=missing])
+  if test $lzma_devel = missing; then
+    MSG_ERROR([liblzma development files not found
+
+The CRAM format may use LZMA2 compression, which is implemented in HTSlib
+by using compression routines from liblzma <http://tukaani.org/xz/>.
+
+Building HTSlib requires liblzma development files to be installed on the
+build machine; you may need to ensure a package such as liblzma-dev (on Debian
+or Ubuntu Linux), xz-devel (on RPM-based Linux distributions or Cygwin), or
+xz (via Homebrew on macOS) is installed; or build XZ Utils from source.
+
+Either configure with --disable-lzma (which will make some CRAM files
+produced elsewhere unreadable) or resolve this error to build HTSlib.])
+  fi
+  pc_requires="$pc_requires liblzma"
+  static_LIBS="$static_LIBS -llzma"
+fi
+
+AS_IF([test "x$with_external_htscodecs" != "xno"],
+  [libhtscodecs=ok
+   AC_CHECK_HEADER([htscodecs/rANS_static4x16.h],[],
+                   [libhtscodecs='missing header'],[;])
+   AC_CHECK_LIB([htscodecs],[rans_compress_bound_4x16],
+                [:],[libhtscodecs='missing library'])
+   AS_IF([test "$libhtscodecs" = "ok"],
+     [AC_DEFINE([HAVE_EXTERNAL_LIBHTSCODECS], 1, [Define if using an external libhtscodecs])
+      LIBS="-lhtscodecs $LIBS"
+      private_LIBS="-lhtscodecs $private_LIBS"
+      static_LIBS="-lhtscodecs $static_LIBS"
+      selected_htscodecs_mk="htscodecs_external.mk"],
+      [MSG_ERROR([libhtscodecs development files not found: $libhtscodecs
+
+You asked to use an external htscodecs library, but do not have the
+required header / library files.  You either need to supply these and
+if necessary set CPPFLAGS and LDFLAGS so the compiler can find them;
+or configure using --without-external-htscodecs to build the required
+functions from the htscodecs submodule.
+])])],
+  [AC_MSG_CHECKING([whether htscodecs files are present])
+   AS_IF([test -e "$srcdir/htscodecs/htscodecs/rANS_static4x16.h"],
+     [AC_MSG_RESULT([yes])
+      selected_htscodecs_mk="htscodecs_bundled.mk"],
+     [AC_MSG_RESULT([no])
+      AS_IF([test -e "$srcdir/.git"],
+        [MSG_ERROR([htscodecs submodule files not present.
+
+HTSlib uses some functions from the htscodecs project, which is normally
+included as a submodule.  Try running:
+
+  git submodule update --init --recursive
+
+in  the top-level htslib directory to update it, and then re-run configure.
+])],
+        [MSG_ERROR([htscodecs submodule files not present.
+
+You have an incomplete distribution.  Please try downloading one of the
+official releases from https://www.htslib.org
+])])])])
+
+AS_IF([test "x$with_libdeflate" != "xno"],
+  [libdeflate=ok
+   AC_CHECK_HEADER([libdeflate.h],[],[libdeflate='missing header'],[;])
+   AC_CHECK_LIB([deflate], [libdeflate_deflate_compress],[:],[libdeflate='missing library'])
+   AS_IF([test "$libdeflate" = "ok"],
+    [AC_DEFINE([HAVE_LIBDEFLATE], 1, [Define if libdeflate is available.])
+     LIBS="-ldeflate $LIBS"
+     private_LIBS="$private_LIBS -ldeflate"
+     static_LIBS="$static_LIBS -ldeflate"],
+    [AS_IF([test "x$with_libdeflate" != "xcheck"],
+       [MSG_ERROR([libdeflate development files not found: $libdeflate
+
+You requested libdeflate, but do not have the required header / library
+files.  The source for libdeflate is available from
+<https://github.com/ebiggers/libdeflate>.  You may have to adjust
+search paths in CPPFLAGS and/or LDFLAGS if the header and library
+are not currently on them.
+
+Either configure with --without-libdeflate or resolve this error to build
+HTSlib.])])])])
+
+libcurl=disabled
+if test "$enable_libcurl" != no; then
+  libcurl_devel=ok
+  AC_CHECK_HEADER([curl/curl.h], [], [libcurl_devel="headers not found"], [;])
+  AC_CHECK_LIB([curl], [curl_easy_pause], [:],
+    [AC_CHECK_LIB([curl], [curl_easy_init],
+       [libcurl_devel="library is too old (7.18+ required)"],
+       [libcurl_devel="library not found"])])
+
+  if test "$libcurl_devel" = ok; then
+    AC_DEFINE([HAVE_LIBCURL], 1, [Define if libcurl file access is enabled.])
+    libcurl=enabled
+  elif test "$enable_libcurl" = check; then
+    AC_MSG_WARN([libcurl not enabled: $libcurl_devel])
+  else
+    MSG_ERROR([libcurl $libcurl_devel
+
+Support for HTTPS and other SSL-based URLs requires routines from the libcurl
+library <http://curl.se/libcurl/>.  Building HTSlib with libcurl enabled
+requires libcurl development files to be installed on the build machine; you
+may need to ensure a package such as libcurl4-{gnutls,nss,openssl}-dev (on
+Debian or Ubuntu Linux) or libcurl-devel (on RPM-based Linux distributions
+or Cygwin) is installed.
+
+Either configure with --disable-libcurl or resolve this error to build HTSlib.])
+  fi
+
+dnl -lcurl is only needed for static linking if hfile_libcurl is not a plugin
+  if test "$libcurl" = enabled ; then
+    if test "$enable_plugins" != yes ; then
+      static_LIBS="$static_LIBS -lcurl"
+    fi
+  fi
+fi
+AC_SUBST([libcurl])
+
+gcs=disabled
+if test "$enable_gcs" != no; then
+  if test $libcurl = enabled; then
+    AC_DEFINE([ENABLE_GCS], 1, [Define if HTSlib should enable GCS support.])
+    gcs=enabled
+  else
+    case "$enable_gcs" in
+      check) AC_MSG_WARN([GCS support not enabled: requires libcurl support]) ;;
+      *) MSG_ERROR([GCS support not enabled
+
+Support for Google Cloud Storage URLs requires libcurl support to be enabled
+in HTSlib.  Configure with --enable-libcurl in order to use GCS URLs.])
+      ;;
+    esac
+  fi
+fi
+AC_SUBST([gcs])
+
+s3=disabled
+if test "$enable_s3" != no; then
+  if test $libcurl = enabled; then
+    s3=enabled
+    need_crypto="$enable_s3"
+  else
+    case "$enable_s3" in
+      check) AC_MSG_WARN([S3 support not enabled: requires libcurl support]) ;;
+      *) MSG_ERROR([S3 support not enabled
+
+Support for Amazon AWS S3 URLs requires libcurl support to be enabled
+in HTSlib.  Configure with --enable-libcurl in order to use S3 URLs.])
+      ;;
+    esac
+  fi
+fi
+
+CRYPTO_LIBS=
+if test $need_crypto != no; then
+  AC_CHECK_FUNC([CCHmac],
+    [AC_DEFINE([HAVE_COMMONCRYPTO], 1,
+               [Define if you have the Common Crypto library.])],
+    [save_LIBS=$LIBS
+     AC_SEARCH_LIBS([HMAC], [crypto],
+       [AC_DEFINE([HAVE_HMAC], 1, [Define if you have libcrypto-style HMAC().])
+        case "$ac_cv_search_HMAC" in
+          -l*) CRYPTO_LIBS=$ac_cv_search_HMAC ;;
+        esac],
+     [case "$need_crypto" in
+     check) AC_MSG_WARN([S3 support not enabled: requires SSL development files])
+         s3=disabled ;;
+     *) MSG_ERROR([SSL development files not found
+
+Support for AWS S3 URLs requires routines from an SSL library.  Building
+HTSlib with libcurl enabled requires SSL development files to be installed
+on the build machine; you may need to ensure a package such as libgnutls-dev,
+libnss3-dev, or libssl-dev (on Debian or Ubuntu Linux, corresponding to the
+libcurl4-*-dev package installed), or openssl-devel (on RPM-based Linux
+distributions or Cygwin) is installed.
+
+Either configure with --disable-s3 or resolve this error to build HTSlib.]) ;;
+       esac])
+     LIBS=$save_LIBS])
+dnl Only need to add to static_LIBS if not building as a plugin
+  if test "$enable_plugins" != yes ; then
+     static_LIBS="$static_LIBS $CRYPTO_LIBS"
+  fi
+fi
+
+dnl Look for regcomp in various libraries (needed on windows/mingw).
+AC_SEARCH_LIBS(regcomp, regex, [libregex=needed], [])
+
+dnl Look for PTHREAD_MUTEX_RECURSIVE.
+dnl This is normally in pthread.h except on some broken glibc implementations.
+dnl Now set by default
+dnl AC_CHECK_DECL(PTHREAD_MUTEX_RECURSIVE, [], [AC_DEFINE([_XOPEN_SOURCE],[600], [Needed for PTHREAD_MUTEX_RECURSIVE])], [[#include <pthread.h>]])
+
+if test "$s3" = enabled ; then
+   AC_DEFINE([ENABLE_S3], 1, [Define if HTSlib should enable S3 support.])
+fi
+
+dnl Apply value from HTS_PROG_CC_WERROR (if set)
+AS_IF([test "x$hts_late_cflags" != x],[CFLAGS="$CFLAGS $hts_late_cflags"])
+
+AC_SUBST([s3])
+AC_SUBST([CRYPTO_LIBS])
+
+AC_SUBST([pc_requires])
+AC_SUBST([private_LIBS])
+AC_SUBST([static_LDFLAGS])
+AC_SUBST([static_LIBS])
+
+AC_CONFIG_FILES([config.mk htslib.pc.tmp:htslib.pc.in])
+AC_CONFIG_LINKS([htscodecs.mk:$selected_htscodecs_mk])
+
+if test "$srcdir" != .; then
+  # Set up for a separate build directory. As HTSlib uses a non-recursive
+  # makefile, we need to create additional build subdirectories explicitly.
+  AC_CONFIG_LINKS([Makefile:Makefile htslib.mk:htslib.mk])
+  AC_CONFIG_FILES([htslib_vars.mk:builddir_vars.mk.in])
+  AC_CONFIG_COMMANDS([mkdir],
+    [AS_MKDIR_P([cram])
+     AS_MKDIR_P([htscodecs/htscodecs])
+     AS_MKDIR_P([htscodecs/tests])
+     AS_MKDIR_P([test/fuzz])
+     AS_MKDIR_P([test/longrefs])
+     AS_MKDIR_P([test/tabix])])
+fi
+
+# @HTSDIRslash_if_relsrcdir@ will be empty when $srcdir is absolute
+case "$srcdir" in
+  /*) HTSDIRslash_if_relsrcdir= ;;
+  *)  HTSDIRslash_if_relsrcdir='$(HTSDIR)/' ;;
+esac
+AC_SUBST([HTSDIRslash_if_relsrcdir])
+
+AC_OUTPUT
diff --git a/ext/htslib/configure~ b/ext/htslib/configure~
new file mode 100755
index 0000000..7e99693
--- /dev/null
+++ b/ext/htslib/configure~
@@ -0,0 +1,8089 @@
+#! /bin/sh
+# Guess values for system-dependent variables and create Makefiles.
+# Generated by GNU Autoconf 2.71 for HTSlib 1.21.
+#
+# Report bugs to <samtools-help@lists.sourceforge.net>.
+#
+#
+# Copyright (C) 1992-1996, 1998-2017, 2020-2021 Free Software Foundation,
+# Inc.
+#
+#
+# This configure script is free software; the Free Software Foundation
+# gives unlimited permission to copy, distribute and modify it.
+#
+# Portions copyright (C) 2020-2024 Genome Research Ltd.
+#
+# This configure script is free software: you are free to change and
+# redistribute it.  There is NO WARRANTY, to the extent permitted by law.
+## -------------------- ##
+## M4sh Initialization. ##
+## -------------------- ##
+
+# Be more Bourne compatible
+DUALCASE=1; export DUALCASE # for MKS sh
+as_nop=:
+if test ${ZSH_VERSION+y} && (emulate sh) >/dev/null 2>&1
+then :
+  emulate sh
+  NULLCMD=:
+  # Pre-4.2 versions of Zsh do word splitting on ${1+"$@"}, which
+  # is contrary to our usage.  Disable this feature.
+  alias -g '${1+"$@"}'='"$@"'
+  setopt NO_GLOB_SUBST
+else $as_nop
+  case `(set -o) 2>/dev/null` in #(
+  *posix*) :
+    set -o posix ;; #(
+  *) :
+     ;;
+esac
+fi
+
+
+
+# Reset variables that may have inherited troublesome values from
+# the environment.
+
+# IFS needs to be set, to space, tab, and newline, in precisely that order.
+# (If _AS_PATH_WALK were called with IFS unset, it would have the
+# side effect of setting IFS to empty, thus disabling word splitting.)
+# Quoting is to prevent editors from complaining about space-tab.
+as_nl='
+'
+export as_nl
+IFS=" ""	$as_nl"
+
+PS1='$ '
+PS2='> '
+PS4='+ '
+
+# Ensure predictable behavior from utilities with locale-dependent output.
+LC_ALL=C
+export LC_ALL
+LANGUAGE=C
+export LANGUAGE
+
+# We cannot yet rely on "unset" to work, but we need these variables
+# to be unset--not just set to an empty or harmless value--now, to
+# avoid bugs in old shells (e.g. pre-3.0 UWIN ksh).  This construct
+# also avoids known problems related to "unset" and subshell syntax
+# in other old shells (e.g. bash 2.01 and pdksh 5.2.14).
+for as_var in BASH_ENV ENV MAIL MAILPATH CDPATH
+do eval test \${$as_var+y} \
+  && ( (unset $as_var) || exit 1) >/dev/null 2>&1 && unset $as_var || :
+done
+
+# Ensure that fds 0, 1, and 2 are open.
+if (exec 3>&0) 2>/dev/null; then :; else exec 0</dev/null; fi
+if (exec 3>&1) 2>/dev/null; then :; else exec 1>/dev/null; fi
+if (exec 3>&2)            ; then :; else exec 2>/dev/null; fi
+
+# The user is always right.
+if ${PATH_SEPARATOR+false} :; then
+  PATH_SEPARATOR=:
+  (PATH='/bin;/bin'; FPATH=$PATH; sh -c :) >/dev/null 2>&1 && {
+    (PATH='/bin:/bin'; FPATH=$PATH; sh -c :) >/dev/null 2>&1 ||
+      PATH_SEPARATOR=';'
+  }
+fi
+
+
+# Find who we are.  Look in the path if we contain no directory separator.
+as_myself=
+case $0 in #((
+  *[\\/]* ) as_myself=$0 ;;
+  *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  case $as_dir in #(((
+    '') as_dir=./ ;;
+    */) ;;
+    *) as_dir=$as_dir/ ;;
+  esac
+    test -r "$as_dir$0" && as_myself=$as_dir$0 && break
+  done
+IFS=$as_save_IFS
+
+     ;;
+esac
+# We did not find ourselves, most probably we were run as `sh COMMAND'
+# in which case we are not to be found in the path.
+if test "x$as_myself" = x; then
+  as_myself=$0
+fi
+if test ! -f "$as_myself"; then
+  printf "%s\n" "$as_myself: error: cannot find myself; rerun with an absolute file name" >&2
+  exit 1
+fi
+
+
+# Use a proper internal environment variable to ensure we don't fall
+  # into an infinite loop, continuously re-executing ourselves.
+  if test x"${_as_can_reexec}" != xno && test "x$CONFIG_SHELL" != x; then
+    _as_can_reexec=no; export _as_can_reexec;
+    # We cannot yet assume a decent shell, so we have to provide a
+# neutralization value for shells without unset; and this also
+# works around shells that cannot unset nonexistent variables.
+# Preserve -v and -x to the replacement shell.
+BASH_ENV=/dev/null
+ENV=/dev/null
+(unset BASH_ENV) >/dev/null 2>&1 && unset BASH_ENV ENV
+case $- in # ((((
+  *v*x* | *x*v* ) as_opts=-vx ;;
+  *v* ) as_opts=-v ;;
+  *x* ) as_opts=-x ;;
+  * ) as_opts= ;;
+esac
+exec $CONFIG_SHELL $as_opts "$as_myself" ${1+"$@"}
+# Admittedly, this is quite paranoid, since all the known shells bail
+# out after a failed `exec'.
+printf "%s\n" "$0: could not re-execute with $CONFIG_SHELL" >&2
+exit 255
+  fi
+  # We don't want this to propagate to other subprocesses.
+          { _as_can_reexec=; unset _as_can_reexec;}
+if test "x$CONFIG_SHELL" = x; then
+  as_bourne_compatible="as_nop=:
+if test \${ZSH_VERSION+y} && (emulate sh) >/dev/null 2>&1
+then :
+  emulate sh
+  NULLCMD=:
+  # Pre-4.2 versions of Zsh do word splitting on \${1+\"\$@\"}, which
+  # is contrary to our usage.  Disable this feature.
+  alias -g '\${1+\"\$@\"}'='\"\$@\"'
+  setopt NO_GLOB_SUBST
+else \$as_nop
+  case \`(set -o) 2>/dev/null\` in #(
+  *posix*) :
+    set -o posix ;; #(
+  *) :
+     ;;
+esac
+fi
+"
+  as_required="as_fn_return () { (exit \$1); }
+as_fn_success () { as_fn_return 0; }
+as_fn_failure () { as_fn_return 1; }
+as_fn_ret_success () { return 0; }
+as_fn_ret_failure () { return 1; }
+
+exitcode=0
+as_fn_success || { exitcode=1; echo as_fn_success failed.; }
+as_fn_failure && { exitcode=1; echo as_fn_failure succeeded.; }
+as_fn_ret_success || { exitcode=1; echo as_fn_ret_success failed.; }
+as_fn_ret_failure && { exitcode=1; echo as_fn_ret_failure succeeded.; }
+if ( set x; as_fn_ret_success y && test x = \"\$1\" )
+then :
+
+else \$as_nop
+  exitcode=1; echo positional parameters were not saved.
+fi
+test x\$exitcode = x0 || exit 1
+blah=\$(echo \$(echo blah))
+test x\"\$blah\" = xblah || exit 1
+test -x / || exit 1"
+  as_suggested="  as_lineno_1=";as_suggested=$as_suggested$LINENO;as_suggested=$as_suggested" as_lineno_1a=\$LINENO
+  as_lineno_2=";as_suggested=$as_suggested$LINENO;as_suggested=$as_suggested" as_lineno_2a=\$LINENO
+  eval 'test \"x\$as_lineno_1'\$as_run'\" != \"x\$as_lineno_2'\$as_run'\" &&
+  test \"x\`expr \$as_lineno_1'\$as_run' + 1\`\" = \"x\$as_lineno_2'\$as_run'\"' || exit 1
+test \$(( 1 + 1 )) = 2 || exit 1"
+  if (eval "$as_required") 2>/dev/null
+then :
+  as_have_required=yes
+else $as_nop
+  as_have_required=no
+fi
+  if test x$as_have_required = xyes && (eval "$as_suggested") 2>/dev/null
+then :
+
+else $as_nop
+  as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+as_found=false
+for as_dir in /bin$PATH_SEPARATOR/usr/bin$PATH_SEPARATOR$PATH
+do
+  IFS=$as_save_IFS
+  case $as_dir in #(((
+    '') as_dir=./ ;;
+    */) ;;
+    *) as_dir=$as_dir/ ;;
+  esac
+  as_found=:
+  case $as_dir in #(
+	 /*)
+	   for as_base in sh bash ksh sh5; do
+	     # Try only shells that exist, to save several forks.
+	     as_shell=$as_dir$as_base
+	     if { test -f "$as_shell" || test -f "$as_shell.exe"; } &&
+		    as_run=a "$as_shell" -c "$as_bourne_compatible""$as_required" 2>/dev/null
+then :
+  CONFIG_SHELL=$as_shell as_have_required=yes
+		   if as_run=a "$as_shell" -c "$as_bourne_compatible""$as_suggested" 2>/dev/null
+then :
+  break 2
+fi
+fi
+	   done;;
+       esac
+  as_found=false
+done
+IFS=$as_save_IFS
+if $as_found
+then :
+
+else $as_nop
+  if { test -f "$SHELL" || test -f "$SHELL.exe"; } &&
+	      as_run=a "$SHELL" -c "$as_bourne_compatible""$as_required" 2>/dev/null
+then :
+  CONFIG_SHELL=$SHELL as_have_required=yes
+fi
+fi
+
+
+      if test "x$CONFIG_SHELL" != x
+then :
+  export CONFIG_SHELL
+             # We cannot yet assume a decent shell, so we have to provide a
+# neutralization value for shells without unset; and this also
+# works around shells that cannot unset nonexistent variables.
+# Preserve -v and -x to the replacement shell.
+BASH_ENV=/dev/null
+ENV=/dev/null
+(unset BASH_ENV) >/dev/null 2>&1 && unset BASH_ENV ENV
+case $- in # ((((
+  *v*x* | *x*v* ) as_opts=-vx ;;
+  *v* ) as_opts=-v ;;
+  *x* ) as_opts=-x ;;
+  * ) as_opts= ;;
+esac
+exec $CONFIG_SHELL $as_opts "$as_myself" ${1+"$@"}
+# Admittedly, this is quite paranoid, since all the known shells bail
+# out after a failed `exec'.
+printf "%s\n" "$0: could not re-execute with $CONFIG_SHELL" >&2
+exit 255
+fi
+
+    if test x$as_have_required = xno
+then :
+  printf "%s\n" "$0: This script requires a shell more modern than all"
+  printf "%s\n" "$0: the shells that I found on your system."
+  if test ${ZSH_VERSION+y} ; then
+    printf "%s\n" "$0: In particular, zsh $ZSH_VERSION has bugs and should"
+    printf "%s\n" "$0: be upgraded to zsh 4.3.4 or later."
+  else
+    printf "%s\n" "$0: Please tell bug-autoconf@gnu.org and
+$0: samtools-help@lists.sourceforge.net about your system,
+$0: including any error possibly output before this
+$0: message. Then install a modern shell, or manually run
+$0: the script under such a shell if you do have one."
+  fi
+  exit 1
+fi
+fi
+fi
+SHELL=${CONFIG_SHELL-/bin/sh}
+export SHELL
+# Unset more variables known to interfere with behavior of common tools.
+CLICOLOR_FORCE= GREP_OPTIONS=
+unset CLICOLOR_FORCE GREP_OPTIONS
+
+## --------------------- ##
+## M4sh Shell Functions. ##
+## --------------------- ##
+# as_fn_unset VAR
+# ---------------
+# Portably unset VAR.
+as_fn_unset ()
+{
+  { eval $1=; unset $1;}
+}
+as_unset=as_fn_unset
+
+
+# as_fn_set_status STATUS
+# -----------------------
+# Set $? to STATUS, without forking.
+as_fn_set_status ()
+{
+  return $1
+} # as_fn_set_status
+
+# as_fn_exit STATUS
+# -----------------
+# Exit the shell with STATUS, even in a "trap 0" or "set -e" context.
+as_fn_exit ()
+{
+  set +e
+  as_fn_set_status $1
+  exit $1
+} # as_fn_exit
+# as_fn_nop
+# ---------
+# Do nothing but, unlike ":", preserve the value of $?.
+as_fn_nop ()
+{
+  return $?
+}
+as_nop=as_fn_nop
+
+# as_fn_mkdir_p
+# -------------
+# Create "$as_dir" as a directory, including parents if necessary.
+as_fn_mkdir_p ()
+{
+
+  case $as_dir in #(
+  -*) as_dir=./$as_dir;;
+  esac
+  test -d "$as_dir" || eval $as_mkdir_p || {
+    as_dirs=
+    while :; do
+      case $as_dir in #(
+      *\'*) as_qdir=`printf "%s\n" "$as_dir" | sed "s/'/'\\\\\\\\''/g"`;; #'(
+      *) as_qdir=$as_dir;;
+      esac
+      as_dirs="'$as_qdir' $as_dirs"
+      as_dir=`$as_dirname -- "$as_dir" ||
+$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$as_dir" : 'X\(//\)[^/]' \| \
+	 X"$as_dir" : 'X\(//\)$' \| \
+	 X"$as_dir" : 'X\(/\)' \| . 2>/dev/null ||
+printf "%s\n" X"$as_dir" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{
+	    s//\1/
+	    q
+	  }
+	  /^X\(\/\/\)[^/].*/{
+	    s//\1/
+	    q
+	  }
+	  /^X\(\/\/\)$/{
+	    s//\1/
+	    q
+	  }
+	  /^X\(\/\).*/{
+	    s//\1/
+	    q
+	  }
+	  s/.*/./; q'`
+      test -d "$as_dir" && break
+    done
+    test -z "$as_dirs" || eval "mkdir $as_dirs"
+  } || test -d "$as_dir" || as_fn_error $? "cannot create directory $as_dir"
+
+
+} # as_fn_mkdir_p
+
+# as_fn_executable_p FILE
+# -----------------------
+# Test if FILE is an executable regular file.
+as_fn_executable_p ()
+{
+  test -f "$1" && test -x "$1"
+} # as_fn_executable_p
+# as_fn_append VAR VALUE
+# ----------------------
+# Append the text in VALUE to the end of the definition contained in VAR. Take
+# advantage of any shell optimizations that allow amortized linear growth over
+# repeated appends, instead of the typical quadratic growth present in naive
+# implementations.
+if (eval "as_var=1; as_var+=2; test x\$as_var = x12") 2>/dev/null
+then :
+  eval 'as_fn_append ()
+  {
+    eval $1+=\$2
+  }'
+else $as_nop
+  as_fn_append ()
+  {
+    eval $1=\$$1\$2
+  }
+fi # as_fn_append
+
+# as_fn_arith ARG...
+# ------------------
+# Perform arithmetic evaluation on the ARGs, and store the result in the
+# global $as_val. Take advantage of shells that can avoid forks. The arguments
+# must be portable across $(()) and expr.
+if (eval "test \$(( 1 + 1 )) = 2") 2>/dev/null
+then :
+  eval 'as_fn_arith ()
+  {
+    as_val=$(( $* ))
+  }'
+else $as_nop
+  as_fn_arith ()
+  {
+    as_val=`expr "$@" || test $? -eq 1`
+  }
+fi # as_fn_arith
+
+# as_fn_nop
+# ---------
+# Do nothing but, unlike ":", preserve the value of $?.
+as_fn_nop ()
+{
+  return $?
+}
+as_nop=as_fn_nop
+
+# as_fn_error STATUS ERROR [LINENO LOG_FD]
+# ----------------------------------------
+# Output "`basename $0`: error: ERROR" to stderr. If LINENO and LOG_FD are
+# provided, also output the error to LOG_FD, referencing LINENO. Then exit the
+# script with STATUS, using 1 if that was 0.
+as_fn_error ()
+{
+  as_status=$1; test $as_status -eq 0 && as_status=1
+  if test "$4"; then
+    as_lineno=${as_lineno-"$3"} as_lineno_stack=as_lineno_stack=$as_lineno_stack
+    printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: $2" >&$4
+  fi
+  printf "%s\n" "$as_me: error: $2" >&2
+  as_fn_exit $as_status
+} # as_fn_error
+
+if expr a : '\(a\)' >/dev/null 2>&1 &&
+   test "X`expr 00001 : '.*\(...\)'`" = X001; then
+  as_expr=expr
+else
+  as_expr=false
+fi
+
+if (basename -- /) >/dev/null 2>&1 && test "X`basename -- / 2>&1`" = "X/"; then
+  as_basename=basename
+else
+  as_basename=false
+fi
+
+if (as_dir=`dirname -- /` && test "X$as_dir" = X/) >/dev/null 2>&1; then
+  as_dirname=dirname
+else
+  as_dirname=false
+fi
+
+as_me=`$as_basename -- "$0" ||
+$as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \
+	 X"$0" : 'X\(//\)$' \| \
+	 X"$0" : 'X\(/\)' \| . 2>/dev/null ||
+printf "%s\n" X/"$0" |
+    sed '/^.*\/\([^/][^/]*\)\/*$/{
+	    s//\1/
+	    q
+	  }
+	  /^X\/\(\/\/\)$/{
+	    s//\1/
+	    q
+	  }
+	  /^X\/\(\/\).*/{
+	    s//\1/
+	    q
+	  }
+	  s/.*/./; q'`
+
+# Avoid depending upon Character Ranges.
+as_cr_letters='abcdefghijklmnopqrstuvwxyz'
+as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+as_cr_Letters=$as_cr_letters$as_cr_LETTERS
+as_cr_digits='0123456789'
+as_cr_alnum=$as_cr_Letters$as_cr_digits
+
+
+  as_lineno_1=$LINENO as_lineno_1a=$LINENO
+  as_lineno_2=$LINENO as_lineno_2a=$LINENO
+  eval 'test "x$as_lineno_1'$as_run'" != "x$as_lineno_2'$as_run'" &&
+  test "x`expr $as_lineno_1'$as_run' + 1`" = "x$as_lineno_2'$as_run'"' || {
+  # Blame Lee E. McMahon (1931-1989) for sed's syntax.  :-)
+  sed -n '
+    p
+    /[$]LINENO/=
+  ' <$as_myself |
+    sed '
+      s/[$]LINENO.*/&-/
+      t lineno
+      b
+      :lineno
+      N
+      :loop
+      s/[$]LINENO\([^'$as_cr_alnum'_].*\n\)\(.*\)/\2\1\2/
+      t loop
+      s/-\n.*//
+    ' >$as_me.lineno &&
+  chmod +x "$as_me.lineno" ||
+    { printf "%s\n" "$as_me: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&2; as_fn_exit 1; }
+
+  # If we had to re-execute with $CONFIG_SHELL, we're ensured to have
+  # already done that, so ensure we don't try to do so again and fall
+  # in an infinite loop.  This has already happened in practice.
+  _as_can_reexec=no; export _as_can_reexec
+  # Don't try to exec as it changes $[0], causing all sort of problems
+  # (the dirname of $[0] is not the place where we might find the
+  # original and so on.  Autoconf is especially sensitive to this).
+  . "./$as_me.lineno"
+  # Exit status is that of the last command.
+  exit
+}
+
+
+# Determine whether it's possible to make 'echo' print without a newline.
+# These variables are no longer used directly by Autoconf, but are AC_SUBSTed
+# for compatibility with existing Makefiles.
+ECHO_C= ECHO_N= ECHO_T=
+case `echo -n x` in #(((((
+-n*)
+  case `echo 'xy\c'` in
+  *c*) ECHO_T='	';;	# ECHO_T is single tab character.
+  xy)  ECHO_C='\c';;
+  *)   echo `echo ksh88 bug on AIX 6.1` > /dev/null
+       ECHO_T='	';;
+  esac;;
+*)
+  ECHO_N='-n';;
+esac
+
+# For backward compatibility with old third-party macros, we provide
+# the shell variables $as_echo and $as_echo_n.  New code should use
+# AS_ECHO(["message"]) and AS_ECHO_N(["message"]), respectively.
+as_echo='printf %s\n'
+as_echo_n='printf %s'
+
+
+rm -f conf$$ conf$$.exe conf$$.file
+if test -d conf$$.dir; then
+  rm -f conf$$.dir/conf$$.file
+else
+  rm -f conf$$.dir
+  mkdir conf$$.dir 2>/dev/null
+fi
+if (echo >conf$$.file) 2>/dev/null; then
+  if ln -s conf$$.file conf$$ 2>/dev/null; then
+    as_ln_s='ln -s'
+    # ... but there are two gotchas:
+    # 1) On MSYS, both `ln -s file dir' and `ln file dir' fail.
+    # 2) DJGPP < 2.04 has no symlinks; `ln -s' creates a wrapper executable.
+    # In both cases, we have to default to `cp -pR'.
+    ln -s conf$$.file conf$$.dir 2>/dev/null && test ! -f conf$$.exe ||
+      as_ln_s='cp -pR'
+  elif ln conf$$.file conf$$ 2>/dev/null; then
+    as_ln_s=ln
+  else
+    as_ln_s='cp -pR'
+  fi
+else
+  as_ln_s='cp -pR'
+fi
+rm -f conf$$ conf$$.exe conf$$.dir/conf$$.file conf$$.file
+rmdir conf$$.dir 2>/dev/null
+
+if mkdir -p . 2>/dev/null; then
+  as_mkdir_p='mkdir -p "$as_dir"'
+else
+  test -d ./-p && rmdir ./-p
+  as_mkdir_p=false
+fi
+
+as_test_x='test -x'
+as_executable_p=as_fn_executable_p
+
+# Sed expression to map a string onto a valid CPP name.
+as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'"
+
+# Sed expression to map a string onto a valid variable name.
+as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'"
+
+
+test -n "$DJDIR" || exec 7<&0 </dev/null
+exec 6>&1
+
+# Name of the host.
+# hostname on some systems (SVR3.2, old GNU/Linux) returns a bogus exit status,
+# so uname gets run too.
+ac_hostname=`(hostname || uname -n) 2>/dev/null | sed 1q`
+
+#
+# Initializations.
+#
+ac_default_prefix=/usr/local
+ac_clean_files=
+ac_config_libobj_dir=.
+LIBOBJS=
+cross_compiling=no
+subdirs=
+MFLAGS=
+MAKEFLAGS=
+
+# Identity of this package.
+PACKAGE_NAME='HTSlib'
+PACKAGE_TARNAME='htslib'
+PACKAGE_VERSION='1.21'
+PACKAGE_STRING='HTSlib 1.21'
+PACKAGE_BUGREPORT='samtools-help@lists.sourceforge.net'
+PACKAGE_URL='http://www.htslib.org/'
+
+ac_unique_file="hts.c"
+# Factoring default headers for most tests.
+ac_includes_default="\
+#include <stddef.h>
+#ifdef HAVE_STDIO_H
+# include <stdio.h>
+#endif
+#ifdef HAVE_STDLIB_H
+# include <stdlib.h>
+#endif
+#ifdef HAVE_STRING_H
+# include <string.h>
+#endif
+#ifdef HAVE_INTTYPES_H
+# include <inttypes.h>
+#endif
+#ifdef HAVE_STDINT_H
+# include <stdint.h>
+#endif
+#ifdef HAVE_STRINGS_H
+# include <strings.h>
+#endif
+#ifdef HAVE_SYS_TYPES_H
+# include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+# include <sys/stat.h>
+#endif
+#ifdef HAVE_UNISTD_H
+# include <unistd.h>
+#endif"
+
+ac_header_c_list=
+ac_func_c_list=
+ac_subst_vars='LTLIBOBJS
+LIBOBJS
+HTSDIRslash_if_relsrcdir
+static_LIBS
+static_LDFLAGS
+private_LIBS
+pc_requires
+CRYPTO_LIBS
+s3
+gcs
+libcurl
+PLUGIN_EXT
+host_os
+host_vendor
+host_cpu
+host
+build_os
+build_vendor
+build_cpu
+build
+VERSION_SCRIPT_LDFLAGS
+PLATFORM
+pluginpath
+plugindir
+with_external_htscodecs
+enable_plugins
+PKG_CONFIG_LIBDIR
+PKG_CONFIG_PATH
+PKG_CONFIG
+hts_cflags_avx512
+hts_cflags_avx2
+hts_cflags_sse4
+GREP
+RANLIB
+OBJEXT
+EXEEXT
+ac_ct_CC
+CPPFLAGS
+LDFLAGS
+CFLAGS
+CC
+target_alias
+host_alias
+build_alias
+LIBS
+ECHO_T
+ECHO_N
+ECHO_C
+DEFS
+mandir
+localedir
+libdir
+psdir
+pdfdir
+dvidir
+htmldir
+infodir
+docdir
+oldincludedir
+includedir
+runstatedir
+localstatedir
+sharedstatedir
+sysconfdir
+datadir
+datarootdir
+libexecdir
+sbindir
+bindir
+program_transform_name
+prefix
+exec_prefix
+PACKAGE_URL
+PACKAGE_BUGREPORT
+PACKAGE_STRING
+PACKAGE_VERSION
+PACKAGE_TARNAME
+PACKAGE_NAME
+PATH_SEPARATOR
+SHELL'
+ac_subst_files=''
+ac_user_opts='
+enable_option_checking
+enable_warnings
+enable_werror
+enable_versioned_symbols
+enable_bz2
+enable_gcs
+enable_largefile
+enable_libcurl
+enable_lzma
+enable_plugins
+with_external_htscodecs
+with_libdeflate
+with_plugin_dir
+with_plugin_path
+enable_s3
+'
+      ac_precious_vars='build_alias
+host_alias
+target_alias
+CC
+CFLAGS
+LDFLAGS
+LIBS
+CPPFLAGS
+PKG_CONFIG
+PKG_CONFIG_PATH
+PKG_CONFIG_LIBDIR'
+
+
+# Initialize some variables set by options.
+ac_init_help=
+ac_init_version=false
+ac_unrecognized_opts=
+ac_unrecognized_sep=
+# The variables have the same names as the options, with
+# dashes changed to underlines.
+cache_file=/dev/null
+exec_prefix=NONE
+no_create=
+no_recursion=
+prefix=NONE
+program_prefix=NONE
+program_suffix=NONE
+program_transform_name=s,x,x,
+silent=
+site=
+srcdir=
+verbose=
+x_includes=NONE
+x_libraries=NONE
+
+# Installation directory options.
+# These are left unexpanded so users can "make install exec_prefix=/foo"
+# and all the variables that are supposed to be based on exec_prefix
+# by default will actually change.
+# Use braces instead of parens because sh, perl, etc. also accept them.
+# (The list follows the same order as the GNU Coding Standards.)
+bindir='${exec_prefix}/bin'
+sbindir='${exec_prefix}/sbin'
+libexecdir='${exec_prefix}/libexec'
+datarootdir='${prefix}/share'
+datadir='${datarootdir}'
+sysconfdir='${prefix}/etc'
+sharedstatedir='${prefix}/com'
+localstatedir='${prefix}/var'
+runstatedir='${localstatedir}/run'
+includedir='${prefix}/include'
+oldincludedir='/usr/include'
+docdir='${datarootdir}/doc/${PACKAGE_TARNAME}'
+infodir='${datarootdir}/info'
+htmldir='${docdir}'
+dvidir='${docdir}'
+pdfdir='${docdir}'
+psdir='${docdir}'
+libdir='${exec_prefix}/lib'
+localedir='${datarootdir}/locale'
+mandir='${datarootdir}/man'
+
+ac_prev=
+ac_dashdash=
+for ac_option
+do
+  # If the previous option needs an argument, assign it.
+  if test -n "$ac_prev"; then
+    eval $ac_prev=\$ac_option
+    ac_prev=
+    continue
+  fi
+
+  case $ac_option in
+  *=?*) ac_optarg=`expr "X$ac_option" : '[^=]*=\(.*\)'` ;;
+  *=)   ac_optarg= ;;
+  *)    ac_optarg=yes ;;
+  esac
+
+  case $ac_dashdash$ac_option in
+  --)
+    ac_dashdash=yes ;;
+
+  -bindir | --bindir | --bindi | --bind | --bin | --bi)
+    ac_prev=bindir ;;
+  -bindir=* | --bindir=* | --bindi=* | --bind=* | --bin=* | --bi=*)
+    bindir=$ac_optarg ;;
+
+  -build | --build | --buil | --bui | --bu)
+    ac_prev=build_alias ;;
+  -build=* | --build=* | --buil=* | --bui=* | --bu=*)
+    build_alias=$ac_optarg ;;
+
+  -cache-file | --cache-file | --cache-fil | --cache-fi \
+  | --cache-f | --cache- | --cache | --cach | --cac | --ca | --c)
+    ac_prev=cache_file ;;
+  -cache-file=* | --cache-file=* | --cache-fil=* | --cache-fi=* \
+  | --cache-f=* | --cache-=* | --cache=* | --cach=* | --cac=* | --ca=* | --c=*)
+    cache_file=$ac_optarg ;;
+
+  --config-cache | -C)
+    cache_file=config.cache ;;
+
+  -datadir | --datadir | --datadi | --datad)
+    ac_prev=datadir ;;
+  -datadir=* | --datadir=* | --datadi=* | --datad=*)
+    datadir=$ac_optarg ;;
+
+  -datarootdir | --datarootdir | --datarootdi | --datarootd | --dataroot \
+  | --dataroo | --dataro | --datar)
+    ac_prev=datarootdir ;;
+  -datarootdir=* | --datarootdir=* | --datarootdi=* | --datarootd=* \
+  | --dataroot=* | --dataroo=* | --dataro=* | --datar=*)
+    datarootdir=$ac_optarg ;;
+
+  -disable-* | --disable-*)
+    ac_useropt=`expr "x$ac_option" : 'x-*disable-\(.*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null &&
+      as_fn_error $? "invalid feature name: \`$ac_useropt'"
+    ac_useropt_orig=$ac_useropt
+    ac_useropt=`printf "%s\n" "$ac_useropt" | sed 's/[-+.]/_/g'`
+    case $ac_user_opts in
+      *"
+"enable_$ac_useropt"
+"*) ;;
+      *) ac_unrecognized_opts="$ac_unrecognized_opts$ac_unrecognized_sep--disable-$ac_useropt_orig"
+	 ac_unrecognized_sep=', ';;
+    esac
+    eval enable_$ac_useropt=no ;;
+
+  -docdir | --docdir | --docdi | --doc | --do)
+    ac_prev=docdir ;;
+  -docdir=* | --docdir=* | --docdi=* | --doc=* | --do=*)
+    docdir=$ac_optarg ;;
+
+  -dvidir | --dvidir | --dvidi | --dvid | --dvi | --dv)
+    ac_prev=dvidir ;;
+  -dvidir=* | --dvidir=* | --dvidi=* | --dvid=* | --dvi=* | --dv=*)
+    dvidir=$ac_optarg ;;
+
+  -enable-* | --enable-*)
+    ac_useropt=`expr "x$ac_option" : 'x-*enable-\([^=]*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null &&
+      as_fn_error $? "invalid feature name: \`$ac_useropt'"
+    ac_useropt_orig=$ac_useropt
+    ac_useropt=`printf "%s\n" "$ac_useropt" | sed 's/[-+.]/_/g'`
+    case $ac_user_opts in
+      *"
+"enable_$ac_useropt"
+"*) ;;
+      *) ac_unrecognized_opts="$ac_unrecognized_opts$ac_unrecognized_sep--enable-$ac_useropt_orig"
+	 ac_unrecognized_sep=', ';;
+    esac
+    eval enable_$ac_useropt=\$ac_optarg ;;
+
+  -exec-prefix | --exec_prefix | --exec-prefix | --exec-prefi \
+  | --exec-pref | --exec-pre | --exec-pr | --exec-p | --exec- \
+  | --exec | --exe | --ex)
+    ac_prev=exec_prefix ;;
+  -exec-prefix=* | --exec_prefix=* | --exec-prefix=* | --exec-prefi=* \
+  | --exec-pref=* | --exec-pre=* | --exec-pr=* | --exec-p=* | --exec-=* \
+  | --exec=* | --exe=* | --ex=*)
+    exec_prefix=$ac_optarg ;;
+
+  -gas | --gas | --ga | --g)
+    # Obsolete; use --with-gas.
+    with_gas=yes ;;
+
+  -help | --help | --hel | --he | -h)
+    ac_init_help=long ;;
+  -help=r* | --help=r* | --hel=r* | --he=r* | -hr*)
+    ac_init_help=recursive ;;
+  -help=s* | --help=s* | --hel=s* | --he=s* | -hs*)
+    ac_init_help=short ;;
+
+  -host | --host | --hos | --ho)
+    ac_prev=host_alias ;;
+  -host=* | --host=* | --hos=* | --ho=*)
+    host_alias=$ac_optarg ;;
+
+  -htmldir | --htmldir | --htmldi | --htmld | --html | --htm | --ht)
+    ac_prev=htmldir ;;
+  -htmldir=* | --htmldir=* | --htmldi=* | --htmld=* | --html=* | --htm=* \
+  | --ht=*)
+    htmldir=$ac_optarg ;;
+
+  -includedir | --includedir | --includedi | --included | --include \
+  | --includ | --inclu | --incl | --inc)
+    ac_prev=includedir ;;
+  -includedir=* | --includedir=* | --includedi=* | --included=* | --include=* \
+  | --includ=* | --inclu=* | --incl=* | --inc=*)
+    includedir=$ac_optarg ;;
+
+  -infodir | --infodir | --infodi | --infod | --info | --inf)
+    ac_prev=infodir ;;
+  -infodir=* | --infodir=* | --infodi=* | --infod=* | --info=* | --inf=*)
+    infodir=$ac_optarg ;;
+
+  -libdir | --libdir | --libdi | --libd)
+    ac_prev=libdir ;;
+  -libdir=* | --libdir=* | --libdi=* | --libd=*)
+    libdir=$ac_optarg ;;
+
+  -libexecdir | --libexecdir | --libexecdi | --libexecd | --libexec \
+  | --libexe | --libex | --libe)
+    ac_prev=libexecdir ;;
+  -libexecdir=* | --libexecdir=* | --libexecdi=* | --libexecd=* | --libexec=* \
+  | --libexe=* | --libex=* | --libe=*)
+    libexecdir=$ac_optarg ;;
+
+  -localedir | --localedir | --localedi | --localed | --locale)
+    ac_prev=localedir ;;
+  -localedir=* | --localedir=* | --localedi=* | --localed=* | --locale=*)
+    localedir=$ac_optarg ;;
+
+  -localstatedir | --localstatedir | --localstatedi | --localstated \
+  | --localstate | --localstat | --localsta | --localst | --locals)
+    ac_prev=localstatedir ;;
+  -localstatedir=* | --localstatedir=* | --localstatedi=* | --localstated=* \
+  | --localstate=* | --localstat=* | --localsta=* | --localst=* | --locals=*)
+    localstatedir=$ac_optarg ;;
+
+  -mandir | --mandir | --mandi | --mand | --man | --ma | --m)
+    ac_prev=mandir ;;
+  -mandir=* | --mandir=* | --mandi=* | --mand=* | --man=* | --ma=* | --m=*)
+    mandir=$ac_optarg ;;
+
+  -nfp | --nfp | --nf)
+    # Obsolete; use --without-fp.
+    with_fp=no ;;
+
+  -no-create | --no-create | --no-creat | --no-crea | --no-cre \
+  | --no-cr | --no-c | -n)
+    no_create=yes ;;
+
+  -no-recursion | --no-recursion | --no-recursio | --no-recursi \
+  | --no-recurs | --no-recur | --no-recu | --no-rec | --no-re | --no-r)
+    no_recursion=yes ;;
+
+  -oldincludedir | --oldincludedir | --oldincludedi | --oldincluded \
+  | --oldinclude | --oldinclud | --oldinclu | --oldincl | --oldinc \
+  | --oldin | --oldi | --old | --ol | --o)
+    ac_prev=oldincludedir ;;
+  -oldincludedir=* | --oldincludedir=* | --oldincludedi=* | --oldincluded=* \
+  | --oldinclude=* | --oldinclud=* | --oldinclu=* | --oldincl=* | --oldinc=* \
+  | --oldin=* | --oldi=* | --old=* | --ol=* | --o=*)
+    oldincludedir=$ac_optarg ;;
+
+  -prefix | --prefix | --prefi | --pref | --pre | --pr | --p)
+    ac_prev=prefix ;;
+  -prefix=* | --prefix=* | --prefi=* | --pref=* | --pre=* | --pr=* | --p=*)
+    prefix=$ac_optarg ;;
+
+  -program-prefix | --program-prefix | --program-prefi | --program-pref \
+  | --program-pre | --program-pr | --program-p)
+    ac_prev=program_prefix ;;
+  -program-prefix=* | --program-prefix=* | --program-prefi=* \
+  | --program-pref=* | --program-pre=* | --program-pr=* | --program-p=*)
+    program_prefix=$ac_optarg ;;
+
+  -program-suffix | --program-suffix | --program-suffi | --program-suff \
+  | --program-suf | --program-su | --program-s)
+    ac_prev=program_suffix ;;
+  -program-suffix=* | --program-suffix=* | --program-suffi=* \
+  | --program-suff=* | --program-suf=* | --program-su=* | --program-s=*)
+    program_suffix=$ac_optarg ;;
+
+  -program-transform-name | --program-transform-name \
+  | --program-transform-nam | --program-transform-na \
+  | --program-transform-n | --program-transform- \
+  | --program-transform | --program-transfor \
+  | --program-transfo | --program-transf \
+  | --program-trans | --program-tran \
+  | --progr-tra | --program-tr | --program-t)
+    ac_prev=program_transform_name ;;
+  -program-transform-name=* | --program-transform-name=* \
+  | --program-transform-nam=* | --program-transform-na=* \
+  | --program-transform-n=* | --program-transform-=* \
+  | --program-transform=* | --program-transfor=* \
+  | --program-transfo=* | --program-transf=* \
+  | --program-trans=* | --program-tran=* \
+  | --progr-tra=* | --program-tr=* | --program-t=*)
+    program_transform_name=$ac_optarg ;;
+
+  -pdfdir | --pdfdir | --pdfdi | --pdfd | --pdf | --pd)
+    ac_prev=pdfdir ;;
+  -pdfdir=* | --pdfdir=* | --pdfdi=* | --pdfd=* | --pdf=* | --pd=*)
+    pdfdir=$ac_optarg ;;
+
+  -psdir | --psdir | --psdi | --psd | --ps)
+    ac_prev=psdir ;;
+  -psdir=* | --psdir=* | --psdi=* | --psd=* | --ps=*)
+    psdir=$ac_optarg ;;
+
+  -q | -quiet | --quiet | --quie | --qui | --qu | --q \
+  | -silent | --silent | --silen | --sile | --sil)
+    silent=yes ;;
+
+  -runstatedir | --runstatedir | --runstatedi | --runstated \
+  | --runstate | --runstat | --runsta | --runst | --runs \
+  | --run | --ru | --r)
+    ac_prev=runstatedir ;;
+  -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \
+  | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \
+  | --run=* | --ru=* | --r=*)
+    runstatedir=$ac_optarg ;;
+
+  -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
+    ac_prev=sbindir ;;
+  -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
+  | --sbi=* | --sb=*)
+    sbindir=$ac_optarg ;;
+
+  -sharedstatedir | --sharedstatedir | --sharedstatedi \
+  | --sharedstated | --sharedstate | --sharedstat | --sharedsta \
+  | --sharedst | --shareds | --shared | --share | --shar \
+  | --sha | --sh)
+    ac_prev=sharedstatedir ;;
+  -sharedstatedir=* | --sharedstatedir=* | --sharedstatedi=* \
+  | --sharedstated=* | --sharedstate=* | --sharedstat=* | --sharedsta=* \
+  | --sharedst=* | --shareds=* | --shared=* | --share=* | --shar=* \
+  | --sha=* | --sh=*)
+    sharedstatedir=$ac_optarg ;;
+
+  -site | --site | --sit)
+    ac_prev=site ;;
+  -site=* | --site=* | --sit=*)
+    site=$ac_optarg ;;
+
+  -srcdir | --srcdir | --srcdi | --srcd | --src | --sr)
+    ac_prev=srcdir ;;
+  -srcdir=* | --srcdir=* | --srcdi=* | --srcd=* | --src=* | --sr=*)
+    srcdir=$ac_optarg ;;
+
+  -sysconfdir | --sysconfdir | --sysconfdi | --sysconfd | --sysconf \
+  | --syscon | --sysco | --sysc | --sys | --sy)
+    ac_prev=sysconfdir ;;
+  -sysconfdir=* | --sysconfdir=* | --sysconfdi=* | --sysconfd=* | --sysconf=* \
+  | --syscon=* | --sysco=* | --sysc=* | --sys=* | --sy=*)
+    sysconfdir=$ac_optarg ;;
+
+  -target | --target | --targe | --targ | --tar | --ta | --t)
+    ac_prev=target_alias ;;
+  -target=* | --target=* | --targe=* | --targ=* | --tar=* | --ta=* | --t=*)
+    target_alias=$ac_optarg ;;
+
+  -v | -verbose | --verbose | --verbos | --verbo | --verb)
+    verbose=yes ;;
+
+  -version | --version | --versio | --versi | --vers | -V)
+    ac_init_version=: ;;
+
+  -with-* | --with-*)
+    ac_useropt=`expr "x$ac_option" : 'x-*with-\([^=]*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null &&
+      as_fn_error $? "invalid package name: \`$ac_useropt'"
+    ac_useropt_orig=$ac_useropt
+    ac_useropt=`printf "%s\n" "$ac_useropt" | sed 's/[-+.]/_/g'`
+    case $ac_user_opts in
+      *"
+"with_$ac_useropt"
+"*) ;;
+      *) ac_unrecognized_opts="$ac_unrecognized_opts$ac_unrecognized_sep--with-$ac_useropt_orig"
+	 ac_unrecognized_sep=', ';;
+    esac
+    eval with_$ac_useropt=\$ac_optarg ;;
+
+  -without-* | --without-*)
+    ac_useropt=`expr "x$ac_option" : 'x-*without-\(.*\)'`
+    # Reject names that are not valid shell variable names.
+    expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null &&
+      as_fn_error $? "invalid package name: \`$ac_useropt'"
+    ac_useropt_orig=$ac_useropt
+    ac_useropt=`printf "%s\n" "$ac_useropt" | sed 's/[-+.]/_/g'`
+    case $ac_user_opts in
+      *"
+"with_$ac_useropt"
+"*) ;;
+      *) ac_unrecognized_opts="$ac_unrecognized_opts$ac_unrecognized_sep--without-$ac_useropt_orig"
+	 ac_unrecognized_sep=', ';;
+    esac
+    eval with_$ac_useropt=no ;;
+
+  --x)
+    # Obsolete; use --with-x.
+    with_x=yes ;;
+
+  -x-includes | --x-includes | --x-include | --x-includ | --x-inclu \
+  | --x-incl | --x-inc | --x-in | --x-i)
+    ac_prev=x_includes ;;
+  -x-includes=* | --x-includes=* | --x-include=* | --x-includ=* | --x-inclu=* \
+  | --x-incl=* | --x-inc=* | --x-in=* | --x-i=*)
+    x_includes=$ac_optarg ;;
+
+  -x-libraries | --x-libraries | --x-librarie | --x-librari \
+  | --x-librar | --x-libra | --x-libr | --x-lib | --x-li | --x-l)
+    ac_prev=x_libraries ;;
+  -x-libraries=* | --x-libraries=* | --x-librarie=* | --x-librari=* \
+  | --x-librar=* | --x-libra=* | --x-libr=* | --x-lib=* | --x-li=* | --x-l=*)
+    x_libraries=$ac_optarg ;;
+
+  -*) as_fn_error $? "unrecognized option: \`$ac_option'
+Try \`$0 --help' for more information"
+    ;;
+
+  *=*)
+    ac_envvar=`expr "x$ac_option" : 'x\([^=]*\)='`
+    # Reject names that are not valid shell variable names.
+    case $ac_envvar in #(
+      '' | [0-9]* | *[!_$as_cr_alnum]* )
+      as_fn_error $? "invalid variable name: \`$ac_envvar'" ;;
+    esac
+    eval $ac_envvar=\$ac_optarg
+    export $ac_envvar ;;
+
+  *)
+    # FIXME: should be removed in autoconf 3.0.
+    printf "%s\n" "$as_me: WARNING: you should use --build, --host, --target" >&2
+    expr "x$ac_option" : ".*[^-._$as_cr_alnum]" >/dev/null &&
+      printf "%s\n" "$as_me: WARNING: invalid host type: $ac_option" >&2
+    : "${build_alias=$ac_option} ${host_alias=$ac_option} ${target_alias=$ac_option}"
+    ;;
+
+  esac
+done
+
+if test -n "$ac_prev"; then
+  ac_option=--`echo $ac_prev | sed 's/_/-/g'`
+  as_fn_error $? "missing argument to $ac_option"
+fi
+
+if test -n "$ac_unrecognized_opts"; then
+  case $enable_option_checking in
+    no) ;;
+    fatal) as_fn_error $? "unrecognized options: $ac_unrecognized_opts" ;;
+    *)     printf "%s\n" "$as_me: WARNING: unrecognized options: $ac_unrecognized_opts" >&2 ;;
+  esac
+fi
+
+# Check all directory arguments for consistency.
+for ac_var in	exec_prefix prefix bindir sbindir libexecdir datarootdir \
+		datadir sysconfdir sharedstatedir localstatedir includedir \
+		oldincludedir docdir infodir htmldir dvidir pdfdir psdir \
+		libdir localedir mandir runstatedir
+do
+  eval ac_val=\$$ac_var
+  # Remove trailing slashes.
+  case $ac_val in
+    */ )
+      ac_val=`expr "X$ac_val" : 'X\(.*[^/]\)' \| "X$ac_val" : 'X\(.*\)'`
+      eval $ac_var=\$ac_val;;
+  esac
+  # Be sure to have absolute directory names.
+  case $ac_val in
+    [\\/$]* | ?:[\\/]* )  continue;;
+    NONE | '' ) case $ac_var in *prefix ) continue;; esac;;
+  esac
+  as_fn_error $? "expected an absolute directory name for --$ac_var: $ac_val"
+done
+
+# There might be people who depend on the old broken behavior: `$host'
+# used to hold the argument of --host etc.
+# FIXME: To remove some day.
+build=$build_alias
+host=$host_alias
+target=$target_alias
+
+# FIXME: To remove some day.
+if test "x$host_alias" != x; then
+  if test "x$build_alias" = x; then
+    cross_compiling=maybe
+  elif test "x$build_alias" != "x$host_alias"; then
+    cross_compiling=yes
+  fi
+fi
+
+ac_tool_prefix=
+test -n "$host_alias" && ac_tool_prefix=$host_alias-
+
+test "$silent" = yes && exec 6>/dev/null
+
+
+ac_pwd=`pwd` && test -n "$ac_pwd" &&
+ac_ls_di=`ls -di .` &&
+ac_pwd_ls_di=`cd "$ac_pwd" && ls -di .` ||
+  as_fn_error $? "working directory cannot be determined"
+test "X$ac_ls_di" = "X$ac_pwd_ls_di" ||
+  as_fn_error $? "pwd does not report name of working directory"
+
+
+# Find the source files, if location was not specified.
+if test -z "$srcdir"; then
+  ac_srcdir_defaulted=yes
+  # Try the directory containing this script, then the parent directory.
+  ac_confdir=`$as_dirname -- "$as_myself" ||
+$as_expr X"$as_myself" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$as_myself" : 'X\(//\)[^/]' \| \
+	 X"$as_myself" : 'X\(//\)$' \| \
+	 X"$as_myself" : 'X\(/\)' \| . 2>/dev/null ||
+printf "%s\n" X"$as_myself" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{
+	    s//\1/
+	    q
+	  }
+	  /^X\(\/\/\)[^/].*/{
+	    s//\1/
+	    q
+	  }
+	  /^X\(\/\/\)$/{
+	    s//\1/
+	    q
+	  }
+	  /^X\(\/\).*/{
+	    s//\1/
+	    q
+	  }
+	  s/.*/./; q'`
+  srcdir=$ac_confdir
+  if test ! -r "$srcdir/$ac_unique_file"; then
+    srcdir=..
+  fi
+else
+  ac_srcdir_defaulted=no
+fi
+if test ! -r "$srcdir/$ac_unique_file"; then
+  test "$ac_srcdir_defaulted" = yes && srcdir="$ac_confdir or .."
+  as_fn_error $? "cannot find sources ($ac_unique_file) in $srcdir"
+fi
+ac_msg="sources are in $srcdir, but \`cd $srcdir' does not work"
+ac_abs_confdir=`(
+	cd "$srcdir" && test -r "./$ac_unique_file" || as_fn_error $? "$ac_msg"
+	pwd)`
+# When building in place, set srcdir=.
+if test "$ac_abs_confdir" = "$ac_pwd"; then
+  srcdir=.
+fi
+# Remove unnecessary trailing slashes from srcdir.
+# Double slashes in file names in object file debugging info
+# mess up M-x gdb in Emacs.
+case $srcdir in
+*/) srcdir=`expr "X$srcdir" : 'X\(.*[^/]\)' \| "X$srcdir" : 'X\(.*\)'`;;
+esac
+for ac_var in $ac_precious_vars; do
+  eval ac_env_${ac_var}_set=\${${ac_var}+set}
+  eval ac_env_${ac_var}_value=\$${ac_var}
+  eval ac_cv_env_${ac_var}_set=\${${ac_var}+set}
+  eval ac_cv_env_${ac_var}_value=\$${ac_var}
+done
+
+#
+# Report the --help message.
+#
+if test "$ac_init_help" = "long"; then
+  # Omit some internal or obsolete options to make the list less imposing.
+  # This message is too long to be a string in the A/UX 3.1 sh.
+  cat <<_ACEOF
+\`configure' configures HTSlib 1.21 to adapt to many kinds of systems.
+
+Usage: $0 [OPTION]... [VAR=VALUE]...
+
+To assign environment variables (e.g., CC, CFLAGS...), specify them as
+VAR=VALUE.  See below for descriptions of some of the useful variables.
+
+Defaults for the options are specified in brackets.
+
+Configuration:
+  -h, --help              display this help and exit
+      --help=short        display options specific to this package
+      --help=recursive    display the short help of all the included packages
+  -V, --version           display version information and exit
+  -q, --quiet, --silent   do not print \`checking ...' messages
+      --cache-file=FILE   cache test results in FILE [disabled]
+  -C, --config-cache      alias for \`--cache-file=config.cache'
+  -n, --no-create         do not create output files
+      --srcdir=DIR        find the sources in DIR [configure dir or \`..']
+
+Installation directories:
+  --prefix=PREFIX         install architecture-independent files in PREFIX
+                          [$ac_default_prefix]
+  --exec-prefix=EPREFIX   install architecture-dependent files in EPREFIX
+                          [PREFIX]
+
+By default, \`make install' will install all the files in
+\`$ac_default_prefix/bin', \`$ac_default_prefix/lib' etc.  You can specify
+an installation prefix other than \`$ac_default_prefix' using \`--prefix',
+for instance \`--prefix=\$HOME'.
+
+For better control, use the options below.
+
+Fine tuning of the installation directories:
+  --bindir=DIR            user executables [EPREFIX/bin]
+  --sbindir=DIR           system admin executables [EPREFIX/sbin]
+  --libexecdir=DIR        program executables [EPREFIX/libexec]
+  --sysconfdir=DIR        read-only single-machine data [PREFIX/etc]
+  --sharedstatedir=DIR    modifiable architecture-independent data [PREFIX/com]
+  --localstatedir=DIR     modifiable single-machine data [PREFIX/var]
+  --runstatedir=DIR       modifiable per-process data [LOCALSTATEDIR/run]
+  --libdir=DIR            object code libraries [EPREFIX/lib]
+  --includedir=DIR        C header files [PREFIX/include]
+  --oldincludedir=DIR     C header files for non-gcc [/usr/include]
+  --datarootdir=DIR       read-only arch.-independent data root [PREFIX/share]
+  --datadir=DIR           read-only architecture-independent data [DATAROOTDIR]
+  --infodir=DIR           info documentation [DATAROOTDIR/info]
+  --localedir=DIR         locale-dependent data [DATAROOTDIR/locale]
+  --mandir=DIR            man documentation [DATAROOTDIR/man]
+  --docdir=DIR            documentation root [DATAROOTDIR/doc/htslib]
+  --htmldir=DIR           html documentation [DOCDIR]
+  --dvidir=DIR            dvi documentation [DOCDIR]
+  --pdfdir=DIR            pdf documentation [DOCDIR]
+  --psdir=DIR             ps documentation [DOCDIR]
+_ACEOF
+
+  cat <<\_ACEOF
+
+System types:
+  --build=BUILD     configure for building on BUILD [guessed]
+  --host=HOST       cross-compile to build programs to run on HOST [BUILD]
+_ACEOF
+fi
+
+if test -n "$ac_init_help"; then
+  case $ac_init_help in
+     short | recursive ) echo "Configuration of HTSlib 1.21:";;
+   esac
+  cat <<\_ACEOF
+
+Optional Features:
+  --disable-option-checking  ignore unrecognized --enable/--with options
+  --disable-FEATURE       do not include FEATURE (same as --enable-FEATURE=no)
+  --enable-FEATURE[=ARG]  include FEATURE [ARG=yes]
+  --disable-warnings      turn off compiler warnings
+  --enable-werror         change warnings into errors, where supported
+  --disable-versioned-symbols
+                          disable versioned symbols in shared library
+  --disable-bz2           omit support for BZ2-compressed CRAM files
+  --enable-gcs            support Google Cloud Storage URLs
+  --disable-largefile     omit support for large files
+  --enable-libcurl        enable libcurl-based support for http/https/etc URLs
+  --disable-lzma          omit support for LZMA-compressed CRAM files
+  --enable-plugins        enable separately-compiled plugins for file access
+  --enable-s3             support Amazon AWS S3 URLs
+
+Optional Packages:
+  --with-PACKAGE[=ARG]    use PACKAGE [ARG=yes]
+  --without-PACKAGE       do not use PACKAGE (same as --with-PACKAGE=no)
+  --with-external-htscodecs
+                          get htscodecs functions from a shared library
+  --with-libdeflate       use libdeflate for faster crc and deflate algorithms
+  --with-plugin-dir=DIR   plugin installation location [LIBEXECDIR/htslib]
+  --with-plugin-path=PATH default HTS_PATH plugin search path [PLUGINDIR]
+
+Some influential environment variables:
+  CC          C compiler command
+  CFLAGS      C compiler flags
+  LDFLAGS     linker flags, e.g. -L<lib dir> if you have libraries in a
+              nonstandard directory <lib dir>
+  LIBS        libraries to pass to the linker, e.g. -l<library>
+  CPPFLAGS    (Objective) C/C++ preprocessor flags, e.g. -I<include dir> if
+              you have headers in a nonstandard directory <include dir>
+  PKG_CONFIG  path to pkg-config utility
+  PKG_CONFIG_PATH
+              directories to add to pkg-config's search path
+  PKG_CONFIG_LIBDIR
+              path overriding pkg-config's built-in search path
+
+Use these variables to override the choices made by `configure' or to help
+it to find libraries and programs with nonstandard names/locations.
+
+Report bugs to <samtools-help@lists.sourceforge.net>.
+HTSlib home page: <http://www.htslib.org/>.
+_ACEOF
+ac_status=$?
+fi
+
+if test "$ac_init_help" = "recursive"; then
+  # If there are subdirs, report their specific --help.
+  for ac_dir in : $ac_subdirs_all; do test "x$ac_dir" = x: && continue
+    test -d "$ac_dir" ||
+      { cd "$srcdir" && ac_pwd=`pwd` && srcdir=. && test -d "$ac_dir"; } ||
+      continue
+    ac_builddir=.
+
+case "$ac_dir" in
+.) ac_dir_suffix= ac_top_builddir_sub=. ac_top_build_prefix= ;;
+*)
+  ac_dir_suffix=/`printf "%s\n" "$ac_dir" | sed 's|^\.[\\/]||'`
+  # A ".." for each directory in $ac_dir_suffix.
+  ac_top_builddir_sub=`printf "%s\n" "$ac_dir_suffix" | sed 's|/[^\\/]*|/..|g;s|/||'`
+  case $ac_top_builddir_sub in
+  "") ac_top_builddir_sub=. ac_top_build_prefix= ;;
+  *)  ac_top_build_prefix=$ac_top_builddir_sub/ ;;
+  esac ;;
+esac
+ac_abs_top_builddir=$ac_pwd
+ac_abs_builddir=$ac_pwd$ac_dir_suffix
+# for backward compatibility:
+ac_top_builddir=$ac_top_build_prefix
+
+case $srcdir in
+  .)  # We are building in place.
+    ac_srcdir=.
+    ac_top_srcdir=$ac_top_builddir_sub
+    ac_abs_top_srcdir=$ac_pwd ;;
+  [\\/]* | ?:[\\/]* )  # Absolute name.
+    ac_srcdir=$srcdir$ac_dir_suffix;
+    ac_top_srcdir=$srcdir
+    ac_abs_top_srcdir=$srcdir ;;
+  *) # Relative name.
+    ac_srcdir=$ac_top_build_prefix$srcdir$ac_dir_suffix
+    ac_top_srcdir=$ac_top_build_prefix$srcdir
+    ac_abs_top_srcdir=$ac_pwd/$srcdir ;;
+esac
+ac_abs_srcdir=$ac_abs_top_srcdir$ac_dir_suffix
+
+    cd "$ac_dir" || { ac_status=$?; continue; }
+    # Check for configure.gnu first; this name is used for a wrapper for
+    # Metaconfig's "Configure" on case-insensitive file systems.
+    if test -f "$ac_srcdir/configure.gnu"; then
+      echo &&
+      $SHELL "$ac_srcdir/configure.gnu" --help=recursive
+    elif test -f "$ac_srcdir/configure"; then
+      echo &&
+      $SHELL "$ac_srcdir/configure" --help=recursive
+    else
+      printf "%s\n" "$as_me: WARNING: no configuration information is in $ac_dir" >&2
+    fi || ac_status=$?
+    cd "$ac_pwd" || { ac_status=$?; break; }
+  done
+fi
+
+test -n "$ac_init_help" && exit $ac_status
+if $ac_init_version; then
+  cat <<\_ACEOF
+HTSlib configure 1.21
+generated by GNU Autoconf 2.71
+
+Copyright (C) 2021 Free Software Foundation, Inc.
+This configure script is free software; the Free Software Foundation
+gives unlimited permission to copy, distribute and modify it.
+
+Portions copyright (C) 2020-2024 Genome Research Ltd.
+
+This configure script is free software: you are free to change and
+redistribute it.  There is NO WARRANTY, to the extent permitted by law.
+_ACEOF
+  exit
+fi
+
+## ------------------------ ##
+## Autoconf initialization. ##
+## ------------------------ ##
+
+# ac_fn_c_try_compile LINENO
+# --------------------------
+# Try to compile conftest.$ac_ext, and return whether this succeeded.
+ac_fn_c_try_compile ()
+{
+  as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack
+  rm -f conftest.$ac_objext conftest.beam
+  if { { ac_try="$ac_compile"
+case "(($ac_try" in
+  *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+  *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
+printf "%s\n" "$ac_try_echo"; } >&5
+  (eval "$ac_compile") 2>conftest.err
+  ac_status=$?
+  if test -s conftest.err; then
+    grep -v '^ *+' conftest.err >conftest.er1
+    cat conftest.er1 >&5
+    mv -f conftest.er1 conftest.err
+  fi
+  printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; } && {
+	 test -z "$ac_c_werror_flag" ||
+	 test ! -s conftest.err
+       } && test -s conftest.$ac_objext
+then :
+  ac_retval=0
+else $as_nop
+  printf "%s\n" "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+	ac_retval=1
+fi
+  eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno
+  as_fn_set_status $ac_retval
+
+} # ac_fn_c_try_compile
+
+# ac_fn_check_decl LINENO SYMBOL VAR INCLUDES EXTRA-OPTIONS FLAG-VAR
+# ------------------------------------------------------------------
+# Tests whether SYMBOL is declared in INCLUDES, setting cache variable VAR
+# accordingly. Pass EXTRA-OPTIONS to the compiler, using FLAG-VAR.
+ac_fn_check_decl ()
+{
+  as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack
+  as_decl_name=`echo $2|sed 's/ *(.*//'`
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether $as_decl_name is declared" >&5
+printf %s "checking whether $as_decl_name is declared... " >&6; }
+if eval test \${$3+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  as_decl_use=`echo $2|sed -e 's/(/((/' -e 's/)/) 0&/' -e 's/,/) 0& (/g'`
+  eval ac_save_FLAGS=\$$6
+  as_fn_append $6 " $5"
+  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+$4
+int
+main (void)
+{
+#ifndef $as_decl_name
+#ifdef __cplusplus
+  (void) $as_decl_use;
+#else
+  (void) $as_decl_name;
+#endif
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"
+then :
+  eval "$3=yes"
+else $as_nop
+  eval "$3=no"
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext
+  eval $6=\$ac_save_FLAGS
+
+fi
+eval ac_res=\$$3
+	       { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5
+printf "%s\n" "$ac_res" >&6; }
+  eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno
+
+} # ac_fn_check_decl
+
+# ac_fn_c_check_header_compile LINENO HEADER VAR INCLUDES
+# -------------------------------------------------------
+# Tests whether HEADER exists and can be compiled using the include files in
+# INCLUDES, setting the cache variable VAR accordingly.
+ac_fn_c_check_header_compile ()
+{
+  as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $2" >&5
+printf %s "checking for $2... " >&6; }
+if eval test \${$3+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+$4
+#include <$2>
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"
+then :
+  eval "$3=yes"
+else $as_nop
+  eval "$3=no"
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext
+fi
+eval ac_res=\$$3
+	       { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5
+printf "%s\n" "$ac_res" >&6; }
+  eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno
+
+} # ac_fn_c_check_header_compile
+
+# ac_fn_c_try_link LINENO
+# -----------------------
+# Try to link conftest.$ac_ext, and return whether this succeeded.
+ac_fn_c_try_link ()
+{
+  as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack
+  rm -f conftest.$ac_objext conftest.beam conftest$ac_exeext
+  if { { ac_try="$ac_link"
+case "(($ac_try" in
+  *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+  *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
+printf "%s\n" "$ac_try_echo"; } >&5
+  (eval "$ac_link") 2>conftest.err
+  ac_status=$?
+  if test -s conftest.err; then
+    grep -v '^ *+' conftest.err >conftest.er1
+    cat conftest.er1 >&5
+    mv -f conftest.er1 conftest.err
+  fi
+  printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; } && {
+	 test -z "$ac_c_werror_flag" ||
+	 test ! -s conftest.err
+       } && test -s conftest$ac_exeext && {
+	 test "$cross_compiling" = yes ||
+	 test -x conftest$ac_exeext
+       }
+then :
+  ac_retval=0
+else $as_nop
+  printf "%s\n" "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+	ac_retval=1
+fi
+  # Delete the IPA/IPO (Inter Procedural Analysis/Optimization) information
+  # created by the PGI compiler (conftest_ipa8_conftest.oo), as it would
+  # interfere with the next link command; also delete a directory that is
+  # left behind by Apple's compiler.  We do this before executing the actions.
+  rm -rf conftest.dSYM conftest_ipa8_conftest.oo
+  eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno
+  as_fn_set_status $ac_retval
+
+} # ac_fn_c_try_link
+
+# ac_fn_c_check_func LINENO FUNC VAR
+# ----------------------------------
+# Tests whether FUNC exists, setting the cache variable VAR accordingly
+ac_fn_c_check_func ()
+{
+  as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $2" >&5
+printf %s "checking for $2... " >&6; }
+if eval test \${$3+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+/* Define $2 to an innocuous variant, in case <limits.h> declares $2.
+   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
+#define $2 innocuous_$2
+
+/* System header to define __stub macros and hopefully few prototypes,
+   which can conflict with char $2 (); below.  */
+
+#include <limits.h>
+#undef $2
+
+/* Override any GCC internal prototype to avoid an error.
+   Use char because int might match the return type of a GCC
+   builtin and then its argument prototype would still apply.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+char $2 ();
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined __stub_$2 || defined __stub___$2
+choke me
+#endif
+
+int
+main (void)
+{
+return $2 ();
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"
+then :
+  eval "$3=yes"
+else $as_nop
+  eval "$3=no"
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam \
+    conftest$ac_exeext conftest.$ac_ext
+fi
+eval ac_res=\$$3
+	       { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5
+printf "%s\n" "$ac_res" >&6; }
+  eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno
+
+} # ac_fn_c_check_func
+
+# ac_fn_c_try_run LINENO
+# ----------------------
+# Try to run conftest.$ac_ext, and return whether this succeeded. Assumes that
+# executables *can* be run.
+ac_fn_c_try_run ()
+{
+  as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack
+  if { { ac_try="$ac_link"
+case "(($ac_try" in
+  *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+  *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
+printf "%s\n" "$ac_try_echo"; } >&5
+  (eval "$ac_link") 2>&5
+  ac_status=$?
+  printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; } && { ac_try='./conftest$ac_exeext'
+  { { case "(($ac_try" in
+  *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+  *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
+printf "%s\n" "$ac_try_echo"; } >&5
+  (eval "$ac_try") 2>&5
+  ac_status=$?
+  printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; }
+then :
+  ac_retval=0
+else $as_nop
+  printf "%s\n" "$as_me: program exited with status $ac_status" >&5
+       printf "%s\n" "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+       ac_retval=$ac_status
+fi
+  rm -rf conftest.dSYM conftest_ipa8_conftest.oo
+  eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno
+  as_fn_set_status $ac_retval
+
+} # ac_fn_c_try_run
+ac_configure_args_raw=
+for ac_arg
+do
+  case $ac_arg in
+  *\'*)
+    ac_arg=`printf "%s\n" "$ac_arg" | sed "s/'/'\\\\\\\\''/g"` ;;
+  esac
+  as_fn_append ac_configure_args_raw " '$ac_arg'"
+done
+
+case $ac_configure_args_raw in
+  *$as_nl*)
+    ac_safe_unquote= ;;
+  *)
+    ac_unsafe_z='|&;<>()$`\\"*?[ ''	' # This string ends in space, tab.
+    ac_unsafe_a="$ac_unsafe_z#~"
+    ac_safe_unquote="s/ '\\([^$ac_unsafe_a][^$ac_unsafe_z]*\\)'/ \\1/g"
+    ac_configure_args_raw=`      printf "%s\n" "$ac_configure_args_raw" | sed "$ac_safe_unquote"`;;
+esac
+
+cat >config.log <<_ACEOF
+This file contains any messages produced by compilers while
+running configure, to aid debugging if configure makes a mistake.
+
+It was created by HTSlib $as_me 1.21, which was
+generated by GNU Autoconf 2.71.  Invocation command line was
+
+  $ $0$ac_configure_args_raw
+
+_ACEOF
+exec 5>>config.log
+{
+cat <<_ASUNAME
+## --------- ##
+## Platform. ##
+## --------- ##
+
+hostname = `(hostname || uname -n) 2>/dev/null | sed 1q`
+uname -m = `(uname -m) 2>/dev/null || echo unknown`
+uname -r = `(uname -r) 2>/dev/null || echo unknown`
+uname -s = `(uname -s) 2>/dev/null || echo unknown`
+uname -v = `(uname -v) 2>/dev/null || echo unknown`
+
+/usr/bin/uname -p = `(/usr/bin/uname -p) 2>/dev/null || echo unknown`
+/bin/uname -X     = `(/bin/uname -X) 2>/dev/null     || echo unknown`
+
+/bin/arch              = `(/bin/arch) 2>/dev/null              || echo unknown`
+/usr/bin/arch -k       = `(/usr/bin/arch -k) 2>/dev/null       || echo unknown`
+/usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null || echo unknown`
+/usr/bin/hostinfo      = `(/usr/bin/hostinfo) 2>/dev/null      || echo unknown`
+/bin/machine           = `(/bin/machine) 2>/dev/null           || echo unknown`
+/usr/bin/oslevel       = `(/usr/bin/oslevel) 2>/dev/null       || echo unknown`
+/bin/universe          = `(/bin/universe) 2>/dev/null          || echo unknown`
+
+_ASUNAME
+
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  case $as_dir in #(((
+    '') as_dir=./ ;;
+    */) ;;
+    *) as_dir=$as_dir/ ;;
+  esac
+    printf "%s\n" "PATH: $as_dir"
+  done
+IFS=$as_save_IFS
+
+} >&5
+
+cat >&5 <<_ACEOF
+
+
+## ----------- ##
+## Core tests. ##
+## ----------- ##
+
+_ACEOF
+
+
+# Keep a trace of the command line.
+# Strip out --no-create and --no-recursion so they do not pile up.
+# Strip out --silent because we don't want to record it for future runs.
+# Also quote any args containing shell meta-characters.
+# Make two passes to allow for proper duplicate-argument suppression.
+ac_configure_args=
+ac_configure_args0=
+ac_configure_args1=
+ac_must_keep_next=false
+for ac_pass in 1 2
+do
+  for ac_arg
+  do
+    case $ac_arg in
+    -no-create | --no-c* | -n | -no-recursion | --no-r*) continue ;;
+    -q | -quiet | --quiet | --quie | --qui | --qu | --q \
+    | -silent | --silent | --silen | --sile | --sil)
+      continue ;;
+    *\'*)
+      ac_arg=`printf "%s\n" "$ac_arg" | sed "s/'/'\\\\\\\\''/g"` ;;
+    esac
+    case $ac_pass in
+    1) as_fn_append ac_configure_args0 " '$ac_arg'" ;;
+    2)
+      as_fn_append ac_configure_args1 " '$ac_arg'"
+      if test $ac_must_keep_next = true; then
+	ac_must_keep_next=false # Got value, back to normal.
+      else
+	case $ac_arg in
+	  *=* | --config-cache | -C | -disable-* | --disable-* \
+	  | -enable-* | --enable-* | -gas | --g* | -nfp | --nf* \
+	  | -q | -quiet | --q* | -silent | --sil* | -v | -verb* \
+	  | -with-* | --with-* | -without-* | --without-* | --x)
+	    case "$ac_configure_args0 " in
+	      "$ac_configure_args1"*" '$ac_arg' "* ) continue ;;
+	    esac
+	    ;;
+	  -* ) ac_must_keep_next=true ;;
+	esac
+      fi
+      as_fn_append ac_configure_args " '$ac_arg'"
+      ;;
+    esac
+  done
+done
+{ ac_configure_args0=; unset ac_configure_args0;}
+{ ac_configure_args1=; unset ac_configure_args1;}
+
+# When interrupted or exit'd, cleanup temporary files, and complete
+# config.log.  We remove comments because anyway the quotes in there
+# would cause problems or look ugly.
+# WARNING: Use '\'' to represent an apostrophe within the trap.
+# WARNING: Do not start the trap code with a newline, due to a FreeBSD 4.0 bug.
+trap 'exit_status=$?
+  # Sanitize IFS.
+  IFS=" ""	$as_nl"
+  # Save into config.log some information that might help in debugging.
+  {
+    echo
+
+    printf "%s\n" "## ---------------- ##
+## Cache variables. ##
+## ---------------- ##"
+    echo
+    # The following way of writing the cache mishandles newlines in values,
+(
+  for ac_var in `(set) 2>&1 | sed -n '\''s/^\([a-zA-Z_][a-zA-Z0-9_]*\)=.*/\1/p'\''`; do
+    eval ac_val=\$$ac_var
+    case $ac_val in #(
+    *${as_nl}*)
+      case $ac_var in #(
+      *_cv_*) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: cache variable $ac_var contains a newline" >&5
+printf "%s\n" "$as_me: WARNING: cache variable $ac_var contains a newline" >&2;} ;;
+      esac
+      case $ac_var in #(
+      _ | IFS | as_nl) ;; #(
+      BASH_ARGV | BASH_SOURCE) eval $ac_var= ;; #(
+      *) { eval $ac_var=; unset $ac_var;} ;;
+      esac ;;
+    esac
+  done
+  (set) 2>&1 |
+    case $as_nl`(ac_space='\'' '\''; set) 2>&1` in #(
+    *${as_nl}ac_space=\ *)
+      sed -n \
+	"s/'\''/'\''\\\\'\'''\''/g;
+	  s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='\''\\2'\''/p"
+      ;; #(
+    *)
+      sed -n "/^[_$as_cr_alnum]*_cv_[_$as_cr_alnum]*=/p"
+      ;;
+    esac |
+    sort
+)
+    echo
+
+    printf "%s\n" "## ----------------- ##
+## Output variables. ##
+## ----------------- ##"
+    echo
+    for ac_var in $ac_subst_vars
+    do
+      eval ac_val=\$$ac_var
+      case $ac_val in
+      *\'\''*) ac_val=`printf "%s\n" "$ac_val" | sed "s/'\''/'\''\\\\\\\\'\'''\''/g"`;;
+      esac
+      printf "%s\n" "$ac_var='\''$ac_val'\''"
+    done | sort
+    echo
+
+    if test -n "$ac_subst_files"; then
+      printf "%s\n" "## ------------------- ##
+## File substitutions. ##
+## ------------------- ##"
+      echo
+      for ac_var in $ac_subst_files
+      do
+	eval ac_val=\$$ac_var
+	case $ac_val in
+	*\'\''*) ac_val=`printf "%s\n" "$ac_val" | sed "s/'\''/'\''\\\\\\\\'\'''\''/g"`;;
+	esac
+	printf "%s\n" "$ac_var='\''$ac_val'\''"
+      done | sort
+      echo
+    fi
+
+    if test -s confdefs.h; then
+      printf "%s\n" "## ----------- ##
+## confdefs.h. ##
+## ----------- ##"
+      echo
+      cat confdefs.h
+      echo
+    fi
+    test "$ac_signal" != 0 &&
+      printf "%s\n" "$as_me: caught signal $ac_signal"
+    printf "%s\n" "$as_me: exit $exit_status"
+  } >&5
+  rm -f core *.core core.conftest.* &&
+    rm -f -r conftest* confdefs* conf$$* $ac_clean_files &&
+    exit $exit_status
+' 0
+for ac_signal in 1 2 13 15; do
+  trap 'ac_signal='$ac_signal'; as_fn_exit 1' $ac_signal
+done
+ac_signal=0
+
+# confdefs.h avoids OS command line length limits that DEFS can exceed.
+rm -f -r conftest* confdefs.h
+
+printf "%s\n" "/* confdefs.h */" > confdefs.h
+
+# Predefined preprocessor variables.
+
+printf "%s\n" "#define PACKAGE_NAME \"$PACKAGE_NAME\"" >>confdefs.h
+
+printf "%s\n" "#define PACKAGE_TARNAME \"$PACKAGE_TARNAME\"" >>confdefs.h
+
+printf "%s\n" "#define PACKAGE_VERSION \"$PACKAGE_VERSION\"" >>confdefs.h
+
+printf "%s\n" "#define PACKAGE_STRING \"$PACKAGE_STRING\"" >>confdefs.h
+
+printf "%s\n" "#define PACKAGE_BUGREPORT \"$PACKAGE_BUGREPORT\"" >>confdefs.h
+
+printf "%s\n" "#define PACKAGE_URL \"$PACKAGE_URL\"" >>confdefs.h
+
+
+# Let the site file select an alternate cache file if it wants to.
+# Prefer an explicitly selected file to automatically selected ones.
+if test -n "$CONFIG_SITE"; then
+  ac_site_files="$CONFIG_SITE"
+elif test "x$prefix" != xNONE; then
+  ac_site_files="$prefix/share/config.site $prefix/etc/config.site"
+else
+  ac_site_files="$ac_default_prefix/share/config.site $ac_default_prefix/etc/config.site"
+fi
+
+for ac_site_file in $ac_site_files
+do
+  case $ac_site_file in #(
+  */*) :
+     ;; #(
+  *) :
+    ac_site_file=./$ac_site_file ;;
+esac
+  if test -f "$ac_site_file" && test -r "$ac_site_file"; then
+    { printf "%s\n" "$as_me:${as_lineno-$LINENO}: loading site script $ac_site_file" >&5
+printf "%s\n" "$as_me: loading site script $ac_site_file" >&6;}
+    sed 's/^/| /' "$ac_site_file" >&5
+    . "$ac_site_file" \
+      || { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "failed to load site script $ac_site_file
+See \`config.log' for more details" "$LINENO" 5; }
+  fi
+done
+
+if test -r "$cache_file"; then
+  # Some versions of bash will fail to source /dev/null (special files
+  # actually), so we avoid doing that.  DJGPP emulates it as a regular file.
+  if test /dev/null != "$cache_file" && test -f "$cache_file"; then
+    { printf "%s\n" "$as_me:${as_lineno-$LINENO}: loading cache $cache_file" >&5
+printf "%s\n" "$as_me: loading cache $cache_file" >&6;}
+    case $cache_file in
+      [\\/]* | ?:[\\/]* ) . "$cache_file";;
+      *)                      . "./$cache_file";;
+    esac
+  fi
+else
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: creating cache $cache_file" >&5
+printf "%s\n" "$as_me: creating cache $cache_file" >&6;}
+  >$cache_file
+fi
+
+# Test code for whether the C compiler supports C89 (global declarations)
+ac_c_conftest_c89_globals='
+/* Does the compiler advertise C89 conformance?
+   Do not test the value of __STDC__, because some compilers set it to 0
+   while being otherwise adequately conformant. */
+#if !defined __STDC__
+# error "Compiler does not advertise C89 conformance"
+#endif
+
+#include <stddef.h>
+#include <stdarg.h>
+struct stat;
+/* Most of the following tests are stolen from RCS 5.7 src/conf.sh.  */
+struct buf { int x; };
+struct buf * (*rcsopen) (struct buf *, struct stat *, int);
+static char *e (p, i)
+     char **p;
+     int i;
+{
+  return p[i];
+}
+static char *f (char * (*g) (char **, int), char **p, ...)
+{
+  char *s;
+  va_list v;
+  va_start (v,p);
+  s = g (p, va_arg (v,int));
+  va_end (v);
+  return s;
+}
+
+/* OSF 4.0 Compaq cc is some sort of almost-ANSI by default.  It has
+   function prototypes and stuff, but not \xHH hex character constants.
+   These do not provoke an error unfortunately, instead are silently treated
+   as an "x".  The following induces an error, until -std is added to get
+   proper ANSI mode.  Curiously \x00 != x always comes out true, for an
+   array size at least.  It is necessary to write \x00 == 0 to get something
+   that is true only with -std.  */
+int osf4_cc_array ['\''\x00'\'' == 0 ? 1 : -1];
+
+/* IBM C 6 for AIX is almost-ANSI by default, but it replaces macro parameters
+   inside strings and character constants.  */
+#define FOO(x) '\''x'\''
+int xlc6_cc_array[FOO(a) == '\''x'\'' ? 1 : -1];
+
+int test (int i, double x);
+struct s1 {int (*f) (int a);};
+struct s2 {int (*f) (double a);};
+int pairnames (int, char **, int *(*)(struct buf *, struct stat *, int),
+               int, int);'
+
+# Test code for whether the C compiler supports C89 (body of main).
+ac_c_conftest_c89_main='
+ok |= (argc == 0 || f (e, argv, 0) != argv[0] || f (e, argv, 1) != argv[1]);
+'
+
+# Test code for whether the C compiler supports C99 (global declarations)
+ac_c_conftest_c99_globals='
+// Does the compiler advertise C99 conformance?
+#if !defined __STDC_VERSION__ || __STDC_VERSION__ < 199901L
+# error "Compiler does not advertise C99 conformance"
+#endif
+
+#include <stdbool.h>
+extern int puts (const char *);
+extern int printf (const char *, ...);
+extern int dprintf (int, const char *, ...);
+extern void *malloc (size_t);
+
+// Check varargs macros.  These examples are taken from C99 6.10.3.5.
+// dprintf is used instead of fprintf to avoid needing to declare
+// FILE and stderr.
+#define debug(...) dprintf (2, __VA_ARGS__)
+#define showlist(...) puts (#__VA_ARGS__)
+#define report(test,...) ((test) ? puts (#test) : printf (__VA_ARGS__))
+static void
+test_varargs_macros (void)
+{
+  int x = 1234;
+  int y = 5678;
+  debug ("Flag");
+  debug ("X = %d\n", x);
+  showlist (The first, second, and third items.);
+  report (x>y, "x is %d but y is %d", x, y);
+}
+
+// Check long long types.
+#define BIG64 18446744073709551615ull
+#define BIG32 4294967295ul
+#define BIG_OK (BIG64 / BIG32 == 4294967297ull && BIG64 % BIG32 == 0)
+#if !BIG_OK
+  #error "your preprocessor is broken"
+#endif
+#if BIG_OK
+#else
+  #error "your preprocessor is broken"
+#endif
+static long long int bignum = -9223372036854775807LL;
+static unsigned long long int ubignum = BIG64;
+
+struct incomplete_array
+{
+  int datasize;
+  double data[];
+};
+
+struct named_init {
+  int number;
+  const wchar_t *name;
+  double average;
+};
+
+typedef const char *ccp;
+
+static inline int
+test_restrict (ccp restrict text)
+{
+  // See if C++-style comments work.
+  // Iterate through items via the restricted pointer.
+  // Also check for declarations in for loops.
+  for (unsigned int i = 0; *(text+i) != '\''\0'\''; ++i)
+    continue;
+  return 0;
+}
+
+// Check varargs and va_copy.
+static bool
+test_varargs (const char *format, ...)
+{
+  va_list args;
+  va_start (args, format);
+  va_list args_copy;
+  va_copy (args_copy, args);
+
+  const char *str = "";
+  int number = 0;
+  float fnumber = 0;
+
+  while (*format)
+    {
+      switch (*format++)
+	{
+	case '\''s'\'': // string
+	  str = va_arg (args_copy, const char *);
+	  break;
+	case '\''d'\'': // int
+	  number = va_arg (args_copy, int);
+	  break;
+	case '\''f'\'': // float
+	  fnumber = va_arg (args_copy, double);
+	  break;
+	default:
+	  break;
+	}
+    }
+  va_end (args_copy);
+  va_end (args);
+
+  return *str && number && fnumber;
+}
+'
+
+# Test code for whether the C compiler supports C99 (body of main).
+ac_c_conftest_c99_main='
+  // Check bool.
+  _Bool success = false;
+  success |= (argc != 0);
+
+  // Check restrict.
+  if (test_restrict ("String literal") == 0)
+    success = true;
+  char *restrict newvar = "Another string";
+
+  // Check varargs.
+  success &= test_varargs ("s, d'\'' f .", "string", 65, 34.234);
+  test_varargs_macros ();
+
+  // Check flexible array members.
+  struct incomplete_array *ia =
+    malloc (sizeof (struct incomplete_array) + (sizeof (double) * 10));
+  ia->datasize = 10;
+  for (int i = 0; i < ia->datasize; ++i)
+    ia->data[i] = i * 1.234;
+
+  // Check named initializers.
+  struct named_init ni = {
+    .number = 34,
+    .name = L"Test wide string",
+    .average = 543.34343,
+  };
+
+  ni.number = 58;
+
+  int dynamic_array[ni.number];
+  dynamic_array[0] = argv[0][0];
+  dynamic_array[ni.number - 1] = 543;
+
+  // work around unused variable warnings
+  ok |= (!success || bignum == 0LL || ubignum == 0uLL || newvar[0] == '\''x'\''
+	 || dynamic_array[ni.number - 1] != 543);
+'
+
+# Test code for whether the C compiler supports C11 (global declarations)
+ac_c_conftest_c11_globals='
+// Does the compiler advertise C11 conformance?
+#if !defined __STDC_VERSION__ || __STDC_VERSION__ < 201112L
+# error "Compiler does not advertise C11 conformance"
+#endif
+
+// Check _Alignas.
+char _Alignas (double) aligned_as_double;
+char _Alignas (0) no_special_alignment;
+extern char aligned_as_int;
+char _Alignas (0) _Alignas (int) aligned_as_int;
+
+// Check _Alignof.
+enum
+{
+  int_alignment = _Alignof (int),
+  int_array_alignment = _Alignof (int[100]),
+  char_alignment = _Alignof (char)
+};
+_Static_assert (0 < -_Alignof (int), "_Alignof is signed");
+
+// Check _Noreturn.
+int _Noreturn does_not_return (void) { for (;;) continue; }
+
+// Check _Static_assert.
+struct test_static_assert
+{
+  int x;
+  _Static_assert (sizeof (int) <= sizeof (long int),
+                  "_Static_assert does not work in struct");
+  long int y;
+};
+
+// Check UTF-8 literals.
+#define u8 syntax error!
+char const utf8_literal[] = u8"happens to be ASCII" "another string";
+
+// Check duplicate typedefs.
+typedef long *long_ptr;
+typedef long int *long_ptr;
+typedef long_ptr long_ptr;
+
+// Anonymous structures and unions -- taken from C11 6.7.2.1 Example 1.
+struct anonymous
+{
+  union {
+    struct { int i; int j; };
+    struct { int k; long int l; } w;
+  };
+  int m;
+} v1;
+'
+
+# Test code for whether the C compiler supports C11 (body of main).
+ac_c_conftest_c11_main='
+  _Static_assert ((offsetof (struct anonymous, i)
+		   == offsetof (struct anonymous, w.k)),
+		  "Anonymous union alignment botch");
+  v1.i = 2;
+  v1.w.k = 5;
+  ok |= v1.i != 5;
+'
+
+# Test code for whether the C compiler supports C11 (complete).
+ac_c_conftest_c11_program="${ac_c_conftest_c89_globals}
+${ac_c_conftest_c99_globals}
+${ac_c_conftest_c11_globals}
+
+int
+main (int argc, char **argv)
+{
+  int ok = 0;
+  ${ac_c_conftest_c89_main}
+  ${ac_c_conftest_c99_main}
+  ${ac_c_conftest_c11_main}
+  return ok;
+}
+"
+
+# Test code for whether the C compiler supports C99 (complete).
+ac_c_conftest_c99_program="${ac_c_conftest_c89_globals}
+${ac_c_conftest_c99_globals}
+
+int
+main (int argc, char **argv)
+{
+  int ok = 0;
+  ${ac_c_conftest_c89_main}
+  ${ac_c_conftest_c99_main}
+  return ok;
+}
+"
+
+# Test code for whether the C compiler supports C89 (complete).
+ac_c_conftest_c89_program="${ac_c_conftest_c89_globals}
+
+int
+main (int argc, char **argv)
+{
+  int ok = 0;
+  ${ac_c_conftest_c89_main}
+  return ok;
+}
+"
+
+as_fn_append ac_header_c_list " stdio.h stdio_h HAVE_STDIO_H"
+as_fn_append ac_header_c_list " stdlib.h stdlib_h HAVE_STDLIB_H"
+as_fn_append ac_header_c_list " string.h string_h HAVE_STRING_H"
+as_fn_append ac_header_c_list " inttypes.h inttypes_h HAVE_INTTYPES_H"
+as_fn_append ac_header_c_list " stdint.h stdint_h HAVE_STDINT_H"
+as_fn_append ac_header_c_list " strings.h strings_h HAVE_STRINGS_H"
+as_fn_append ac_header_c_list " sys/stat.h sys_stat_h HAVE_SYS_STAT_H"
+as_fn_append ac_header_c_list " sys/types.h sys_types_h HAVE_SYS_TYPES_H"
+as_fn_append ac_header_c_list " unistd.h unistd_h HAVE_UNISTD_H"
+as_fn_append ac_header_c_list " sys/param.h sys_param_h HAVE_SYS_PARAM_H"
+as_fn_append ac_func_c_list " getpagesize HAVE_GETPAGESIZE"
+
+# Auxiliary files required by this configure script.
+ac_aux_files="config.guess config.sub"
+
+# Locations in which to look for auxiliary files.
+ac_aux_dir_candidates="${srcdir}${PATH_SEPARATOR}${srcdir}/..${PATH_SEPARATOR}${srcdir}/../.."
+
+# Search for a directory containing all of the required auxiliary files,
+# $ac_aux_files, from the $PATH-style list $ac_aux_dir_candidates.
+# If we don't find one directory that contains all the files we need,
+# we report the set of missing files from the *first* directory in
+# $ac_aux_dir_candidates and give up.
+ac_missing_aux_files=""
+ac_first_candidate=:
+printf "%s\n" "$as_me:${as_lineno-$LINENO}: looking for aux files: $ac_aux_files" >&5
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+as_found=false
+for as_dir in $ac_aux_dir_candidates
+do
+  IFS=$as_save_IFS
+  case $as_dir in #(((
+    '') as_dir=./ ;;
+    */) ;;
+    *) as_dir=$as_dir/ ;;
+  esac
+  as_found=:
+
+  printf "%s\n" "$as_me:${as_lineno-$LINENO}:  trying $as_dir" >&5
+  ac_aux_dir_found=yes
+  ac_install_sh=
+  for ac_aux in $ac_aux_files
+  do
+    # As a special case, if "install-sh" is required, that requirement
+    # can be satisfied by any of "install-sh", "install.sh", or "shtool",
+    # and $ac_install_sh is set appropriately for whichever one is found.
+    if test x"$ac_aux" = x"install-sh"
+    then
+      if test -f "${as_dir}install-sh"; then
+        printf "%s\n" "$as_me:${as_lineno-$LINENO}:   ${as_dir}install-sh found" >&5
+        ac_install_sh="${as_dir}install-sh -c"
+      elif test -f "${as_dir}install.sh"; then
+        printf "%s\n" "$as_me:${as_lineno-$LINENO}:   ${as_dir}install.sh found" >&5
+        ac_install_sh="${as_dir}install.sh -c"
+      elif test -f "${as_dir}shtool"; then
+        printf "%s\n" "$as_me:${as_lineno-$LINENO}:   ${as_dir}shtool found" >&5
+        ac_install_sh="${as_dir}shtool install -c"
+      else
+        ac_aux_dir_found=no
+        if $ac_first_candidate; then
+          ac_missing_aux_files="${ac_missing_aux_files} install-sh"
+        else
+          break
+        fi
+      fi
+    else
+      if test -f "${as_dir}${ac_aux}"; then
+        printf "%s\n" "$as_me:${as_lineno-$LINENO}:   ${as_dir}${ac_aux} found" >&5
+      else
+        ac_aux_dir_found=no
+        if $ac_first_candidate; then
+          ac_missing_aux_files="${ac_missing_aux_files} ${ac_aux}"
+        else
+          break
+        fi
+      fi
+    fi
+  done
+  if test "$ac_aux_dir_found" = yes; then
+    ac_aux_dir="$as_dir"
+    break
+  fi
+  ac_first_candidate=false
+
+  as_found=false
+done
+IFS=$as_save_IFS
+if $as_found
+then :
+
+else $as_nop
+  as_fn_error $? "cannot find required auxiliary files:$ac_missing_aux_files" "$LINENO" 5
+fi
+
+
+# These three variables are undocumented and unsupported,
+# and are intended to be withdrawn in a future Autoconf release.
+# They can cause serious problems if a builder's source tree is in a directory
+# whose full name contains unusual characters.
+if test -f "${ac_aux_dir}config.guess"; then
+  ac_config_guess="$SHELL ${ac_aux_dir}config.guess"
+fi
+if test -f "${ac_aux_dir}config.sub"; then
+  ac_config_sub="$SHELL ${ac_aux_dir}config.sub"
+fi
+if test -f "$ac_aux_dir/configure"; then
+  ac_configure="$SHELL ${ac_aux_dir}configure"
+fi
+
+# Check that the precious variables saved in the cache have kept the same
+# value.
+ac_cache_corrupted=false
+for ac_var in $ac_precious_vars; do
+  eval ac_old_set=\$ac_cv_env_${ac_var}_set
+  eval ac_new_set=\$ac_env_${ac_var}_set
+  eval ac_old_val=\$ac_cv_env_${ac_var}_value
+  eval ac_new_val=\$ac_env_${ac_var}_value
+  case $ac_old_set,$ac_new_set in
+    set,)
+      { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&5
+printf "%s\n" "$as_me: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&2;}
+      ac_cache_corrupted=: ;;
+    ,set)
+      { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: \`$ac_var' was not set in the previous run" >&5
+printf "%s\n" "$as_me: error: \`$ac_var' was not set in the previous run" >&2;}
+      ac_cache_corrupted=: ;;
+    ,);;
+    *)
+      if test "x$ac_old_val" != "x$ac_new_val"; then
+	# differences in whitespace do not lead to failure.
+	ac_old_val_w=`echo x $ac_old_val`
+	ac_new_val_w=`echo x $ac_new_val`
+	if test "$ac_old_val_w" != "$ac_new_val_w"; then
+	  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: \`$ac_var' has changed since the previous run:" >&5
+printf "%s\n" "$as_me: error: \`$ac_var' has changed since the previous run:" >&2;}
+	  ac_cache_corrupted=:
+	else
+	  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: warning: ignoring whitespace changes in \`$ac_var' since the previous run:" >&5
+printf "%s\n" "$as_me: warning: ignoring whitespace changes in \`$ac_var' since the previous run:" >&2;}
+	  eval $ac_var=\$ac_old_val
+	fi
+	{ printf "%s\n" "$as_me:${as_lineno-$LINENO}:   former value:  \`$ac_old_val'" >&5
+printf "%s\n" "$as_me:   former value:  \`$ac_old_val'" >&2;}
+	{ printf "%s\n" "$as_me:${as_lineno-$LINENO}:   current value: \`$ac_new_val'" >&5
+printf "%s\n" "$as_me:   current value: \`$ac_new_val'" >&2;}
+      fi;;
+  esac
+  # Pass precious variables to config.status.
+  if test "$ac_new_set" = set; then
+    case $ac_new_val in
+    *\'*) ac_arg=$ac_var=`printf "%s\n" "$ac_new_val" | sed "s/'/'\\\\\\\\''/g"` ;;
+    *) ac_arg=$ac_var=$ac_new_val ;;
+    esac
+    case " $ac_configure_args " in
+      *" '$ac_arg' "*) ;; # Avoid dups.  Use of quotes ensures accuracy.
+      *) as_fn_append ac_configure_args " '$ac_arg'" ;;
+    esac
+  fi
+done
+if $ac_cache_corrupted; then
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;}
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: changes in the environment can compromise the build" >&5
+printf "%s\n" "$as_me: error: changes in the environment can compromise the build" >&2;}
+  as_fn_error $? "run \`${MAKE-make} distclean' and/or \`rm $cache_file'
+	    and start over" "$LINENO" 5
+fi
+## -------------------- ##
+## Main body of script. ##
+## -------------------- ##
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+
+
+ac_config_headers="$ac_config_headers config.h"
+
+
+
+
+# SYNOPSIS
+#
+# HTS_PROG_CC_WERROR(FLAGS_VAR)
+#
+# Set FLAGS_VAR to the flags needed to make the C compiler treat warnings
+# as errors.
+
+
+# hts_check_compile_flags_needed.m4
+#
+# SYNOPSIS
+#
+#   HTS_CHECK_COMPILE_FLAGS_NEEDED(FEATURE, FLAGS, [INPUT], [ACTION-SUCCESS], [ACTION-FAILURE], [EXTRA-FLAGS])
+#
+# DESCRIPTION
+#
+#   Check whether the given FLAGS are required to build and link INPUT with
+#   the current language's compiler.  Compilation and linking are first
+#   tries without FLAGS.  If that fails it then tries to compile and
+#   link again with FLAGS.
+#
+#   FEATURE describes the feature being tested, and is used when printing
+#   messages and to name the cache entry (along with the tested flags).
+#
+#   ACTION-SUCCESS/ACTION-FAILURE are shell commands to execute on
+#   success/failure.  In ACTION-SUCCESS, $flags_needed will be set to
+#   either an empty string or FLAGS depending on the test results.
+#
+#   If EXTRA-FLAGS is defined, it is added to the current language's default
+#   flags (e.g. CFLAGS) when the check is done.  The check is thus made with
+#   the flags: "CFLAGS EXTRA-FLAGS FLAG".  This can for example be used to
+#   force the compiler to issue an error when a bad flag is given.
+#
+#   If omitted, INPUT defaults to AC_LANG_PROGRAM(), although that probably
+#   isn't very useful.
+#
+#   NOTE: Implementation based on AX_CHECK_COMPILE_FLAG.
+#
+# LICENSE
+#
+#   Copyright (c) 2008 Guido U. Draheim <guidod@gmx.de>
+#   Copyright (c) 2011 Maarten Bosmans <mkbosmans@gmail.com>
+#   Copyright (c) 2023 Robert Davies <rmd@sanger.ac.uk>
+#
+#   Copying and distribution of this file, with or without modification, are
+#   permitted in any medium without royalty provided the copyright notice
+#   and this notice are preserved.  This file is offered as-is, without any
+#   warranty.
+
+#   HTS_CHECK_COMPILE_FLAGS_NEEDED(FEATURE, FLAGS, [INPUT], [ACTION-SUCCESS], [ACTION-FAILURE], [EXTRA-FLAGS])
+
+
+
+# SYNOPSIS
+#
+# HTS_TEST_CC_C_LD_FLAG(FLAG, FOUND_VAR)
+#
+# Test if FLAG can be used on both CFLAGS and LDFLAGS.  It it works,
+# variable FOUND_VAR is set to FLAG.
+
+
+
+
+
+# pkg.m4 - Macros to locate and use pkg-config.   -*- Autoconf -*-
+# serial 12 (pkg-config-0.29.2)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+if test -n "$ac_tool_prefix"; then
+  # Extract the first word of "${ac_tool_prefix}gcc", so it can be a program name with args.
+set dummy ${ac_tool_prefix}gcc; ac_word=$2
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+printf %s "checking for $ac_word... " >&6; }
+if test ${ac_cv_prog_CC+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  case $as_dir in #(((
+    '') as_dir=./ ;;
+    */) ;;
+    *) as_dir=$as_dir/ ;;
+  esac
+    for ac_exec_ext in '' $ac_executable_extensions; do
+  if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then
+    ac_cv_prog_CC="${ac_tool_prefix}gcc"
+    printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+  done
+IFS=$as_save_IFS
+
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $CC" >&5
+printf "%s\n" "$CC" >&6; }
+else
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5
+printf "%s\n" "no" >&6; }
+fi
+
+
+fi
+if test -z "$ac_cv_prog_CC"; then
+  ac_ct_CC=$CC
+  # Extract the first word of "gcc", so it can be a program name with args.
+set dummy gcc; ac_word=$2
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+printf %s "checking for $ac_word... " >&6; }
+if test ${ac_cv_prog_ac_ct_CC+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  if test -n "$ac_ct_CC"; then
+  ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  case $as_dir in #(((
+    '') as_dir=./ ;;
+    */) ;;
+    *) as_dir=$as_dir/ ;;
+  esac
+    for ac_exec_ext in '' $ac_executable_extensions; do
+  if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_CC="gcc"
+    printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+  done
+IFS=$as_save_IFS
+
+fi
+fi
+ac_ct_CC=$ac_cv_prog_ac_ct_CC
+if test -n "$ac_ct_CC"; then
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_ct_CC" >&5
+printf "%s\n" "$ac_ct_CC" >&6; }
+else
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5
+printf "%s\n" "no" >&6; }
+fi
+
+  if test "x$ac_ct_CC" = x; then
+    CC=""
+  else
+    case $cross_compiling:$ac_tool_warned in
+yes:)
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5
+printf "%s\n" "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;}
+ac_tool_warned=yes ;;
+esac
+    CC=$ac_ct_CC
+  fi
+else
+  CC="$ac_cv_prog_CC"
+fi
+
+if test -z "$CC"; then
+          if test -n "$ac_tool_prefix"; then
+    # Extract the first word of "${ac_tool_prefix}cc", so it can be a program name with args.
+set dummy ${ac_tool_prefix}cc; ac_word=$2
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+printf %s "checking for $ac_word... " >&6; }
+if test ${ac_cv_prog_CC+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  case $as_dir in #(((
+    '') as_dir=./ ;;
+    */) ;;
+    *) as_dir=$as_dir/ ;;
+  esac
+    for ac_exec_ext in '' $ac_executable_extensions; do
+  if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then
+    ac_cv_prog_CC="${ac_tool_prefix}cc"
+    printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+  done
+IFS=$as_save_IFS
+
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $CC" >&5
+printf "%s\n" "$CC" >&6; }
+else
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5
+printf "%s\n" "no" >&6; }
+fi
+
+
+  fi
+fi
+if test -z "$CC"; then
+  # Extract the first word of "cc", so it can be a program name with args.
+set dummy cc; ac_word=$2
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+printf %s "checking for $ac_word... " >&6; }
+if test ${ac_cv_prog_CC+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+  ac_prog_rejected=no
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  case $as_dir in #(((
+    '') as_dir=./ ;;
+    */) ;;
+    *) as_dir=$as_dir/ ;;
+  esac
+    for ac_exec_ext in '' $ac_executable_extensions; do
+  if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then
+    if test "$as_dir$ac_word$ac_exec_ext" = "/usr/ucb/cc"; then
+       ac_prog_rejected=yes
+       continue
+     fi
+    ac_cv_prog_CC="cc"
+    printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+  done
+IFS=$as_save_IFS
+
+if test $ac_prog_rejected = yes; then
+  # We found a bogon in the path, so make sure we never use it.
+  set dummy $ac_cv_prog_CC
+  shift
+  if test $# != 0; then
+    # We chose a different compiler from the bogus one.
+    # However, it has the same basename, so the bogon will be chosen
+    # first if we set CC to just the basename; use the full file name.
+    shift
+    ac_cv_prog_CC="$as_dir$ac_word${1+' '}$@"
+  fi
+fi
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $CC" >&5
+printf "%s\n" "$CC" >&6; }
+else
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5
+printf "%s\n" "no" >&6; }
+fi
+
+
+fi
+if test -z "$CC"; then
+  if test -n "$ac_tool_prefix"; then
+  for ac_prog in cl.exe
+  do
+    # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args.
+set dummy $ac_tool_prefix$ac_prog; ac_word=$2
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+printf %s "checking for $ac_word... " >&6; }
+if test ${ac_cv_prog_CC+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  case $as_dir in #(((
+    '') as_dir=./ ;;
+    */) ;;
+    *) as_dir=$as_dir/ ;;
+  esac
+    for ac_exec_ext in '' $ac_executable_extensions; do
+  if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then
+    ac_cv_prog_CC="$ac_tool_prefix$ac_prog"
+    printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+  done
+IFS=$as_save_IFS
+
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $CC" >&5
+printf "%s\n" "$CC" >&6; }
+else
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5
+printf "%s\n" "no" >&6; }
+fi
+
+
+    test -n "$CC" && break
+  done
+fi
+if test -z "$CC"; then
+  ac_ct_CC=$CC
+  for ac_prog in cl.exe
+do
+  # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+printf %s "checking for $ac_word... " >&6; }
+if test ${ac_cv_prog_ac_ct_CC+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  if test -n "$ac_ct_CC"; then
+  ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  case $as_dir in #(((
+    '') as_dir=./ ;;
+    */) ;;
+    *) as_dir=$as_dir/ ;;
+  esac
+    for ac_exec_ext in '' $ac_executable_extensions; do
+  if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_CC="$ac_prog"
+    printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+  done
+IFS=$as_save_IFS
+
+fi
+fi
+ac_ct_CC=$ac_cv_prog_ac_ct_CC
+if test -n "$ac_ct_CC"; then
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_ct_CC" >&5
+printf "%s\n" "$ac_ct_CC" >&6; }
+else
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5
+printf "%s\n" "no" >&6; }
+fi
+
+
+  test -n "$ac_ct_CC" && break
+done
+
+  if test "x$ac_ct_CC" = x; then
+    CC=""
+  else
+    case $cross_compiling:$ac_tool_warned in
+yes:)
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5
+printf "%s\n" "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;}
+ac_tool_warned=yes ;;
+esac
+    CC=$ac_ct_CC
+  fi
+fi
+
+fi
+if test -z "$CC"; then
+  if test -n "$ac_tool_prefix"; then
+  # Extract the first word of "${ac_tool_prefix}clang", so it can be a program name with args.
+set dummy ${ac_tool_prefix}clang; ac_word=$2
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+printf %s "checking for $ac_word... " >&6; }
+if test ${ac_cv_prog_CC+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  case $as_dir in #(((
+    '') as_dir=./ ;;
+    */) ;;
+    *) as_dir=$as_dir/ ;;
+  esac
+    for ac_exec_ext in '' $ac_executable_extensions; do
+  if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then
+    ac_cv_prog_CC="${ac_tool_prefix}clang"
+    printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+  done
+IFS=$as_save_IFS
+
+fi
+fi
+CC=$ac_cv_prog_CC
+if test -n "$CC"; then
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $CC" >&5
+printf "%s\n" "$CC" >&6; }
+else
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5
+printf "%s\n" "no" >&6; }
+fi
+
+
+fi
+if test -z "$ac_cv_prog_CC"; then
+  ac_ct_CC=$CC
+  # Extract the first word of "clang", so it can be a program name with args.
+set dummy clang; ac_word=$2
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+printf %s "checking for $ac_word... " >&6; }
+if test ${ac_cv_prog_ac_ct_CC+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  if test -n "$ac_ct_CC"; then
+  ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  case $as_dir in #(((
+    '') as_dir=./ ;;
+    */) ;;
+    *) as_dir=$as_dir/ ;;
+  esac
+    for ac_exec_ext in '' $ac_executable_extensions; do
+  if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_CC="clang"
+    printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+  done
+IFS=$as_save_IFS
+
+fi
+fi
+ac_ct_CC=$ac_cv_prog_ac_ct_CC
+if test -n "$ac_ct_CC"; then
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_ct_CC" >&5
+printf "%s\n" "$ac_ct_CC" >&6; }
+else
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5
+printf "%s\n" "no" >&6; }
+fi
+
+  if test "x$ac_ct_CC" = x; then
+    CC=""
+  else
+    case $cross_compiling:$ac_tool_warned in
+yes:)
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5
+printf "%s\n" "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;}
+ac_tool_warned=yes ;;
+esac
+    CC=$ac_ct_CC
+  fi
+else
+  CC="$ac_cv_prog_CC"
+fi
+
+fi
+
+
+test -z "$CC" && { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "no acceptable C compiler found in \$PATH
+See \`config.log' for more details" "$LINENO" 5; }
+
+# Provide some information about the compiler.
+printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for C compiler version" >&5
+set X $ac_compile
+ac_compiler=$2
+for ac_option in --version -v -V -qversion -version; do
+  { { ac_try="$ac_compiler $ac_option >&5"
+case "(($ac_try" in
+  *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+  *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
+printf "%s\n" "$ac_try_echo"; } >&5
+  (eval "$ac_compiler $ac_option >&5") 2>conftest.err
+  ac_status=$?
+  if test -s conftest.err; then
+    sed '10a\
+... rest of stderr output deleted ...
+         10q' conftest.err >conftest.er1
+    cat conftest.er1 >&5
+  fi
+  rm -f conftest.er1 conftest.err
+  printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }
+done
+
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+int
+main (void)
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+ac_clean_files_save=$ac_clean_files
+ac_clean_files="$ac_clean_files a.out a.out.dSYM a.exe b.out"
+# Try to create an executable without -o first, disregard a.out.
+# It will help us diagnose broken compilers, and finding out an intuition
+# of exeext.
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether the C compiler works" >&5
+printf %s "checking whether the C compiler works... " >&6; }
+ac_link_default=`printf "%s\n" "$ac_link" | sed 's/ -o *conftest[^ ]*//'`
+
+# The possible output files:
+ac_files="a.out conftest.exe conftest a.exe a_out.exe b.out conftest.*"
+
+ac_rmfiles=
+for ac_file in $ac_files
+do
+  case $ac_file in
+    *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM | *.o | *.obj ) ;;
+    * ) ac_rmfiles="$ac_rmfiles $ac_file";;
+  esac
+done
+rm -f $ac_rmfiles
+
+if { { ac_try="$ac_link_default"
+case "(($ac_try" in
+  *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+  *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
+printf "%s\n" "$ac_try_echo"; } >&5
+  (eval "$ac_link_default") 2>&5
+  ac_status=$?
+  printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }
+then :
+  # Autoconf-2.13 could set the ac_cv_exeext variable to `no'.
+# So ignore a value of `no', otherwise this would lead to `EXEEXT = no'
+# in a Makefile.  We should not override ac_cv_exeext if it was cached,
+# so that the user can short-circuit this test for compilers unknown to
+# Autoconf.
+for ac_file in $ac_files ''
+do
+  test -f "$ac_file" || continue
+  case $ac_file in
+    *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM | *.o | *.obj )
+	;;
+    [ab].out )
+	# We found the default executable, but exeext='' is most
+	# certainly right.
+	break;;
+    *.* )
+	if test ${ac_cv_exeext+y} && test "$ac_cv_exeext" != no;
+	then :; else
+	   ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'`
+	fi
+	# We set ac_cv_exeext here because the later test for it is not
+	# safe: cross compilers may not add the suffix if given an `-o'
+	# argument, so we may need to know it at that point already.
+	# Even if this section looks crufty: it has the advantage of
+	# actually working.
+	break;;
+    * )
+	break;;
+  esac
+done
+test "$ac_cv_exeext" = no && ac_cv_exeext=
+
+else $as_nop
+  ac_file=''
+fi
+if test -z "$ac_file"
+then :
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5
+printf "%s\n" "no" >&6; }
+printf "%s\n" "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+{ { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error 77 "C compiler cannot create executables
+See \`config.log' for more details" "$LINENO" 5; }
+else $as_nop
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+printf "%s\n" "yes" >&6; }
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for C compiler default output file name" >&5
+printf %s "checking for C compiler default output file name... " >&6; }
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_file" >&5
+printf "%s\n" "$ac_file" >&6; }
+ac_exeext=$ac_cv_exeext
+
+rm -f -r a.out a.out.dSYM a.exe conftest$ac_cv_exeext b.out
+ac_clean_files=$ac_clean_files_save
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for suffix of executables" >&5
+printf %s "checking for suffix of executables... " >&6; }
+if { { ac_try="$ac_link"
+case "(($ac_try" in
+  *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+  *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
+printf "%s\n" "$ac_try_echo"; } >&5
+  (eval "$ac_link") 2>&5
+  ac_status=$?
+  printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }
+then :
+  # If both `conftest.exe' and `conftest' are `present' (well, observable)
+# catch `conftest.exe'.  For instance with Cygwin, `ls conftest' will
+# work properly (i.e., refer to `conftest.exe'), while it won't with
+# `rm'.
+for ac_file in conftest.exe conftest conftest.*; do
+  test -f "$ac_file" || continue
+  case $ac_file in
+    *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM | *.o | *.obj ) ;;
+    *.* ) ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'`
+	  break;;
+    * ) break;;
+  esac
+done
+else $as_nop
+  { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "cannot compute suffix of executables: cannot compile and link
+See \`config.log' for more details" "$LINENO" 5; }
+fi
+rm -f conftest conftest$ac_cv_exeext
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_exeext" >&5
+printf "%s\n" "$ac_cv_exeext" >&6; }
+
+rm -f conftest.$ac_ext
+EXEEXT=$ac_cv_exeext
+ac_exeext=$EXEEXT
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+#include <stdio.h>
+int
+main (void)
+{
+FILE *f = fopen ("conftest.out", "w");
+ return ferror (f) || fclose (f) != 0;
+
+  ;
+  return 0;
+}
+_ACEOF
+ac_clean_files="$ac_clean_files conftest.out"
+# Check that the compiler produces executables we can run.  If not, either
+# the compiler is broken, or we cross compile.
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether we are cross compiling" >&5
+printf %s "checking whether we are cross compiling... " >&6; }
+if test "$cross_compiling" != yes; then
+  { { ac_try="$ac_link"
+case "(($ac_try" in
+  *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+  *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
+printf "%s\n" "$ac_try_echo"; } >&5
+  (eval "$ac_link") 2>&5
+  ac_status=$?
+  printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }
+  if { ac_try='./conftest$ac_cv_exeext'
+  { { case "(($ac_try" in
+  *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+  *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
+printf "%s\n" "$ac_try_echo"; } >&5
+  (eval "$ac_try") 2>&5
+  ac_status=$?
+  printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; }; then
+    cross_compiling=no
+  else
+    if test "$cross_compiling" = maybe; then
+	cross_compiling=yes
+    else
+	{ { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error 77 "cannot run C compiled programs.
+If you meant to cross compile, use \`--host'.
+See \`config.log' for more details" "$LINENO" 5; }
+    fi
+  fi
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $cross_compiling" >&5
+printf "%s\n" "$cross_compiling" >&6; }
+
+rm -f conftest.$ac_ext conftest$ac_cv_exeext conftest.out
+ac_clean_files=$ac_clean_files_save
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for suffix of object files" >&5
+printf %s "checking for suffix of object files... " >&6; }
+if test ${ac_cv_objext+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+int
+main (void)
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.o conftest.obj
+if { { ac_try="$ac_compile"
+case "(($ac_try" in
+  *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+  *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\""
+printf "%s\n" "$ac_try_echo"; } >&5
+  (eval "$ac_compile") 2>&5
+  ac_status=$?
+  printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }
+then :
+  for ac_file in conftest.o conftest.obj conftest.*; do
+  test -f "$ac_file" || continue;
+  case $ac_file in
+    *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM ) ;;
+    *) ac_cv_objext=`expr "$ac_file" : '.*\.\(.*\)'`
+       break;;
+  esac
+done
+else $as_nop
+  printf "%s\n" "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+{ { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "cannot compute suffix of object files: cannot compile
+See \`config.log' for more details" "$LINENO" 5; }
+fi
+rm -f conftest.$ac_cv_objext conftest.$ac_ext
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_objext" >&5
+printf "%s\n" "$ac_cv_objext" >&6; }
+OBJEXT=$ac_cv_objext
+ac_objext=$OBJEXT
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether the compiler supports GNU C" >&5
+printf %s "checking whether the compiler supports GNU C... " >&6; }
+if test ${ac_cv_c_compiler_gnu+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+int
+main (void)
+{
+#ifndef __GNUC__
+       choke me
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"
+then :
+  ac_compiler_gnu=yes
+else $as_nop
+  ac_compiler_gnu=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext
+ac_cv_c_compiler_gnu=$ac_compiler_gnu
+
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_compiler_gnu" >&5
+printf "%s\n" "$ac_cv_c_compiler_gnu" >&6; }
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+if test $ac_compiler_gnu = yes; then
+  GCC=yes
+else
+  GCC=
+fi
+ac_test_CFLAGS=${CFLAGS+y}
+ac_save_CFLAGS=$CFLAGS
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether $CC accepts -g" >&5
+printf %s "checking whether $CC accepts -g... " >&6; }
+if test ${ac_cv_prog_cc_g+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  ac_save_c_werror_flag=$ac_c_werror_flag
+   ac_c_werror_flag=yes
+   ac_cv_prog_cc_g=no
+   CFLAGS="-g"
+   cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+int
+main (void)
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"
+then :
+  ac_cv_prog_cc_g=yes
+else $as_nop
+  CFLAGS=""
+      cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+int
+main (void)
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"
+then :
+
+else $as_nop
+  ac_c_werror_flag=$ac_save_c_werror_flag
+	 CFLAGS="-g"
+	 cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+int
+main (void)
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"
+then :
+  ac_cv_prog_cc_g=yes
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext
+   ac_c_werror_flag=$ac_save_c_werror_flag
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_g" >&5
+printf "%s\n" "$ac_cv_prog_cc_g" >&6; }
+if test $ac_test_CFLAGS; then
+  CFLAGS=$ac_save_CFLAGS
+elif test $ac_cv_prog_cc_g = yes; then
+  if test "$GCC" = yes; then
+    CFLAGS="-g -O2"
+  else
+    CFLAGS="-g"
+  fi
+else
+  if test "$GCC" = yes; then
+    CFLAGS="-O2"
+  else
+    CFLAGS=
+  fi
+fi
+ac_prog_cc_stdc=no
+if test x$ac_prog_cc_stdc = xno
+then :
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $CC option to enable C11 features" >&5
+printf %s "checking for $CC option to enable C11 features... " >&6; }
+if test ${ac_cv_prog_cc_c11+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  ac_cv_prog_cc_c11=no
+ac_save_CC=$CC
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+$ac_c_conftest_c11_program
+_ACEOF
+for ac_arg in '' -std=gnu11
+do
+  CC="$ac_save_CC $ac_arg"
+  if ac_fn_c_try_compile "$LINENO"
+then :
+  ac_cv_prog_cc_c11=$ac_arg
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam
+  test "x$ac_cv_prog_cc_c11" != "xno" && break
+done
+rm -f conftest.$ac_ext
+CC=$ac_save_CC
+fi
+
+if test "x$ac_cv_prog_cc_c11" = xno
+then :
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: unsupported" >&5
+printf "%s\n" "unsupported" >&6; }
+else $as_nop
+  if test "x$ac_cv_prog_cc_c11" = x
+then :
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: none needed" >&5
+printf "%s\n" "none needed" >&6; }
+else $as_nop
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_c11" >&5
+printf "%s\n" "$ac_cv_prog_cc_c11" >&6; }
+     CC="$CC $ac_cv_prog_cc_c11"
+fi
+  ac_cv_prog_cc_stdc=$ac_cv_prog_cc_c11
+  ac_prog_cc_stdc=c11
+fi
+fi
+if test x$ac_prog_cc_stdc = xno
+then :
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $CC option to enable C99 features" >&5
+printf %s "checking for $CC option to enable C99 features... " >&6; }
+if test ${ac_cv_prog_cc_c99+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  ac_cv_prog_cc_c99=no
+ac_save_CC=$CC
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+$ac_c_conftest_c99_program
+_ACEOF
+for ac_arg in '' -std=gnu99 -std=c99 -c99 -qlanglvl=extc1x -qlanglvl=extc99 -AC99 -D_STDC_C99=
+do
+  CC="$ac_save_CC $ac_arg"
+  if ac_fn_c_try_compile "$LINENO"
+then :
+  ac_cv_prog_cc_c99=$ac_arg
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam
+  test "x$ac_cv_prog_cc_c99" != "xno" && break
+done
+rm -f conftest.$ac_ext
+CC=$ac_save_CC
+fi
+
+if test "x$ac_cv_prog_cc_c99" = xno
+then :
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: unsupported" >&5
+printf "%s\n" "unsupported" >&6; }
+else $as_nop
+  if test "x$ac_cv_prog_cc_c99" = x
+then :
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: none needed" >&5
+printf "%s\n" "none needed" >&6; }
+else $as_nop
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_c99" >&5
+printf "%s\n" "$ac_cv_prog_cc_c99" >&6; }
+     CC="$CC $ac_cv_prog_cc_c99"
+fi
+  ac_cv_prog_cc_stdc=$ac_cv_prog_cc_c99
+  ac_prog_cc_stdc=c99
+fi
+fi
+if test x$ac_prog_cc_stdc = xno
+then :
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $CC option to enable C89 features" >&5
+printf %s "checking for $CC option to enable C89 features... " >&6; }
+if test ${ac_cv_prog_cc_c89+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  ac_cv_prog_cc_c89=no
+ac_save_CC=$CC
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+$ac_c_conftest_c89_program
+_ACEOF
+for ac_arg in '' -qlanglvl=extc89 -qlanglvl=ansi -std -Ae "-Aa -D_HPUX_SOURCE" "-Xc -D__EXTENSIONS__"
+do
+  CC="$ac_save_CC $ac_arg"
+  if ac_fn_c_try_compile "$LINENO"
+then :
+  ac_cv_prog_cc_c89=$ac_arg
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam
+  test "x$ac_cv_prog_cc_c89" != "xno" && break
+done
+rm -f conftest.$ac_ext
+CC=$ac_save_CC
+fi
+
+if test "x$ac_cv_prog_cc_c89" = xno
+then :
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: unsupported" >&5
+printf "%s\n" "unsupported" >&6; }
+else $as_nop
+  if test "x$ac_cv_prog_cc_c89" = x
+then :
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: none needed" >&5
+printf "%s\n" "none needed" >&6; }
+else $as_nop
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_c89" >&5
+printf "%s\n" "$ac_cv_prog_cc_c89" >&6; }
+     CC="$CC $ac_cv_prog_cc_c89"
+fi
+  ac_cv_prog_cc_stdc=$ac_cv_prog_cc_c89
+  ac_prog_cc_stdc=c89
+fi
+fi
+
+ac_ext=c
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_c_compiler_gnu
+
+if test -n "$ac_tool_prefix"; then
+  # Extract the first word of "${ac_tool_prefix}ranlib", so it can be a program name with args.
+set dummy ${ac_tool_prefix}ranlib; ac_word=$2
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+printf %s "checking for $ac_word... " >&6; }
+if test ${ac_cv_prog_RANLIB+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  if test -n "$RANLIB"; then
+  ac_cv_prog_RANLIB="$RANLIB" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  case $as_dir in #(((
+    '') as_dir=./ ;;
+    */) ;;
+    *) as_dir=$as_dir/ ;;
+  esac
+    for ac_exec_ext in '' $ac_executable_extensions; do
+  if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then
+    ac_cv_prog_RANLIB="${ac_tool_prefix}ranlib"
+    printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+  done
+IFS=$as_save_IFS
+
+fi
+fi
+RANLIB=$ac_cv_prog_RANLIB
+if test -n "$RANLIB"; then
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $RANLIB" >&5
+printf "%s\n" "$RANLIB" >&6; }
+else
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5
+printf "%s\n" "no" >&6; }
+fi
+
+
+fi
+if test -z "$ac_cv_prog_RANLIB"; then
+  ac_ct_RANLIB=$RANLIB
+  # Extract the first word of "ranlib", so it can be a program name with args.
+set dummy ranlib; ac_word=$2
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+printf %s "checking for $ac_word... " >&6; }
+if test ${ac_cv_prog_ac_ct_RANLIB+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  if test -n "$ac_ct_RANLIB"; then
+  ac_cv_prog_ac_ct_RANLIB="$ac_ct_RANLIB" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  case $as_dir in #(((
+    '') as_dir=./ ;;
+    */) ;;
+    *) as_dir=$as_dir/ ;;
+  esac
+    for ac_exec_ext in '' $ac_executable_extensions; do
+  if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then
+    ac_cv_prog_ac_ct_RANLIB="ranlib"
+    printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+  done
+IFS=$as_save_IFS
+
+fi
+fi
+ac_ct_RANLIB=$ac_cv_prog_ac_ct_RANLIB
+if test -n "$ac_ct_RANLIB"; then
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_ct_RANLIB" >&5
+printf "%s\n" "$ac_ct_RANLIB" >&6; }
+else
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5
+printf "%s\n" "no" >&6; }
+fi
+
+  if test "x$ac_ct_RANLIB" = x; then
+    RANLIB=":"
+  else
+    case $cross_compiling:$ac_tool_warned in
+yes:)
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5
+printf "%s\n" "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;}
+ac_tool_warned=yes ;;
+esac
+    RANLIB=$ac_ct_RANLIB
+  fi
+else
+  RANLIB="$ac_cv_prog_RANLIB"
+fi
+
+
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for grep that handles long lines and -e" >&5
+printf %s "checking for grep that handles long lines and -e... " >&6; }
+if test ${ac_cv_path_GREP+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  if test -z "$GREP"; then
+  ac_path_GREP_found=false
+  # Loop through the user's path and test for each of PROGNAME-LIST
+  as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH$PATH_SEPARATOR/usr/xpg4/bin
+do
+  IFS=$as_save_IFS
+  case $as_dir in #(((
+    '') as_dir=./ ;;
+    */) ;;
+    *) as_dir=$as_dir/ ;;
+  esac
+    for ac_prog in grep ggrep
+   do
+    for ac_exec_ext in '' $ac_executable_extensions; do
+      ac_path_GREP="$as_dir$ac_prog$ac_exec_ext"
+      as_fn_executable_p "$ac_path_GREP" || continue
+# Check for GNU ac_path_GREP and select it if it is found.
+  # Check for GNU $ac_path_GREP
+case `"$ac_path_GREP" --version 2>&1` in
+*GNU*)
+  ac_cv_path_GREP="$ac_path_GREP" ac_path_GREP_found=:;;
+*)
+  ac_count=0
+  printf %s 0123456789 >"conftest.in"
+  while :
+  do
+    cat "conftest.in" "conftest.in" >"conftest.tmp"
+    mv "conftest.tmp" "conftest.in"
+    cp "conftest.in" "conftest.nl"
+    printf "%s\n" 'GREP' >> "conftest.nl"
+    "$ac_path_GREP" -e 'GREP$' -e '-(cannot match)-' < "conftest.nl" >"conftest.out" 2>/dev/null || break
+    diff "conftest.out" "conftest.nl" >/dev/null 2>&1 || break
+    as_fn_arith $ac_count + 1 && ac_count=$as_val
+    if test $ac_count -gt ${ac_path_GREP_max-0}; then
+      # Best one so far, save it but keep looking for a better one
+      ac_cv_path_GREP="$ac_path_GREP"
+      ac_path_GREP_max=$ac_count
+    fi
+    # 10*(2^10) chars as input seems more than enough
+    test $ac_count -gt 10 && break
+  done
+  rm -f conftest.in conftest.tmp conftest.nl conftest.out;;
+esac
+
+      $ac_path_GREP_found && break 3
+    done
+  done
+  done
+IFS=$as_save_IFS
+  if test -z "$ac_cv_path_GREP"; then
+    as_fn_error $? "no acceptable grep could be found in $PATH$PATH_SEPARATOR/usr/xpg4/bin" "$LINENO" 5
+  fi
+else
+  ac_cv_path_GREP=$GREP
+fi
+
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_path_GREP" >&5
+printf "%s\n" "$ac_cv_path_GREP" >&6; }
+ GREP="$ac_cv_path_GREP"
+
+
+
+  # Check whether --enable-warnings was given.
+if test ${enable_warnings+y}
+then :
+  enableval=$enable_warnings;
+else $as_nop
+  enable_warnings=yes
+fi
+
+
+  if test "x$enable_warnings" != xno
+then :
+
+
+
+    ansi=""
+    if test "x$ansi" = "x"
+then :
+  msg="for C compiler warning flags"
+else $as_nop
+  msg="for C compiler warning and ANSI conformance flags"
+fi
+
+    { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking $msg" >&5
+printf %s "checking $msg... " >&6; }
+    if test ${hts_cv_prog_cc_warnings+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+        hts_cv_prog_cc_warnings=""
+      if test "x$CC" != "x"
+then :
+
+        cat > conftest.c <<EOF
+int main(int argc, char **argv) { return 0; }
+EOF
+
+
+
+        # Tests for flags to enable C compiler warnings
+        # GCC compatible
+        if test "x$GCC" = "xyes" &&
+               "$CC" -c -Wall conftest.c > /dev/null 2>&1 &&
+               test -f conftest.o
+then :
+            if test "x$ansi" = "x"
+then :
+  hts_cv_prog_cc_warnings="-Wall"
+else $as_nop
+  hts_cv_prog_cc_warnings="-Wall -ansi -pedantic"
+fi
+
+elif # Sun Studio or Solaris C compiler
+        "$CC" -V 2>&1 | $GREP -i -E "WorkShop|Sun C" > /dev/null 2>&1 &&
+         "$CC" -c -v -Xc conftest.c > /dev/null 2>&1 &&
+         test -f conftest.o
+then :
+          if test "x$ansi" = "x"
+then :
+  hts_cv_prog_cc_warnings="-v"
+else $as_nop
+  hts_cv_prog_cc_warnings="-v -Xc"
+fi
+
+elif # Digital Unix C compiler
+        "$CC" -V 2>&1 | $GREP -i "Digital UNIX Compiler" > /dev/null 2>&1 &&
+         "$CC" -c -verbose -w0 -warnprotos -std1 conftest.c > /dev/null 2>&1 &&
+         test -f conftest.o
+then :
+             if test "x$ansi" = "x"
+then :
+  hts_cv_prog_cc_warnings="-verbose -w0 -warnprotos"
+else $as_nop
+  hts_cv_prog_cc_warnings="-verbose -w0 -warnprotos -std1"
+fi
+
+elif # C for AIX Compiler
+        "$CC" 2>&1 | $GREP -i "C for AIX Compiler" > /dev/null 2>&1 &&
+         "$CC" -c -qlanglvl=ansi -qinfo=all conftest.c > /dev/null 2>&1 &&
+         test -f conftest.o
+then :
+          if test "x$ansi" = "x"
+then :
+  hts_cv_prog_cc_warnings="-qsrcmsg -qinfo=all:noppt:noppc:noobs:nocnd"
+else $as_nop
+  hts_cv_prog_cc_warnings="-qsrcmsg -qinfo=all:noppt:noppc:noobs:nocnd -qlanglvl=ansi"
+fi
+
+elif # IRIX C compiler
+        "$CC" -version 2>&1 | $GREP -i "MIPSpro Compilers" > /dev/null 2>&1 &&
+         "$CC" -c -fullwarn -ansi -ansiE conftest.c > /dev/null 2>&1 &&
+         test -f conftest.o
+then :
+             if test "x$ansi" = "x"
+then :
+  hts_cv_prog_cc_warnings="-fullwarn"
+else $as_nop
+  hts_cv_prog_cc_warnings="-fullwarn -ansi -ansiE"
+fi
+
+elif # HP-UX C compiler
+        what "$CC" 2>&1 | $GREP -i "HP C Compiler" > /dev/null 2>&1 &&
+         "$CC" -c -Aa +w1 conftest.c > /dev/null 2>&1 &&
+         test -f conftest.o
+then :
+          if test "x$ansi" = "x"
+then :
+  hts_cv_prog_cc_warnings="+w1"
+else $as_nop
+  hts_cv_prog_cc_warnings="+w1 -Aa"
+fi
+
+elif # The NEC SX series (Super-UX 10) C compiler
+        "$CC" -V 2>&1 | $GREP "/SX" > /dev/null 2>&1 &&
+         "$CC" -c -pvctl,fullmsg -Xc conftest.c > /dev/null 2>&1 &&
+         test -f conftest.o
+then :
+
+        if test "x$ansi" = "x"
+then :
+  hts_cv_prog_cc_warnings="-pvctl,fullmsg"
+else $as_nop
+  hts_cv_prog_cc_warnings="-pvctl,fullmsg -Xc"
+fi
+
+elif # The Cray C compiler (Unicos)
+        "$CC" -V 2>&1 | $GREP -i "Cray" > /dev/null 2>&1 &&
+         "$CC" -c -h msglevel_2 conftest.c > /dev/null 2>&1 &&
+         test -f conftest.o
+then :
+          if test "x$ansi" = "x"
+then :
+  hts_cv_prog_cc_warnings="-h#msglevel_2"
+else $as_nop
+  hts_cv_prog_cc_warnings="-h#msglevel_2,conform"
+fi
+
+elif # The Tiny C Compiler
+        "$CC" -v 2>&1 | $GREP "tcc version" > /dev/null &&
+         "$CC" -Wall -c conftest.c > /dev/null 2>&1 &&
+         test -f conftest.o
+then :
+           hts_cv_prog_cc_warnings="-Wall"
+
+fi
+        rm -f conftest.*
+
+fi
+
+fi
+
+
+    if test "x$hts_cv_prog_cc_warnings" != "x"
+then :
+
+ac_arg_result=`echo "$hts_cv_prog_cc_warnings" | tr '#' ' '`
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_arg_result" >&5
+printf "%s\n" "$ac_arg_result" >&6; }
+
+ac_arg_needed=""
+for ac_arg in $hts_cv_prog_cc_warnings
+do
+  ac_arg_sp=`echo "$ac_arg" | tr '#' ' '`
+  case " $CFLAGS " in #(
+  *" $ac_arg_sp "*) :
+     ;; #(
+  *) :
+    ac_arg_needed="$ac_arg_all $ac_arg_sp" ;;
+esac
+done
+CFLAGS="$ac_arg_needed $CFLAGS"
+else $as_nop
+        { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: unknown" >&5
+printf "%s\n" "unknown" >&6; }
+
+fi
+
+fi
+
+
+  # Check whether --enable-werror was given.
+if test ${enable_werror+y}
+then :
+  enableval=$enable_werror;
+else $as_nop
+  enable_werror=no
+fi
+
+
+  if test "x$enable_werror" != xno
+then :
+
+    { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for C compiler flags to error on warnings" >&5
+printf %s "checking for C compiler flags to error on warnings... " >&6; }
+    if test ${hts_cv_prog_cc_werror+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+        hts_cv_prog_cc_werror=""
+      if test "x$CC" != "x"
+then :
+
+        cat > conftest.c <<EOF
+int main(int argc, char **argv) { return 0; }
+EOF
+
+        if 	 # Tests for flags to make the C compiler treat warnings as errors
+         # GCC compatible
+         test "x$GCC" = "xyes" &&
+          "$CC" -c -Werror conftest.c > /dev/null 2>&1 &&
+          test -f conftest.o
+then :
+  hts_cv_prog_cc_werror="-Werror"
+elif # Sun Studio or Solaris C compiler
+         "$CC" -V 2>&1 | $GREP -i -E "WorkShop|Sun C" > /dev/null 2>&1 &&
+          "$CC" -c -errwarn=%all conftest.c > /dev/null 2>&1 &&
+          test -f conftest.o
+then :
+  hts_cv_prog_cc_werror="-errwarn=%all"
+elif # The Tiny C Compiler
+         "$CC" -v 2>&1 | $GREP "tcc version" > /dev/null &&
+          "$CC" -Wall -c conftest.c > /dev/null 2>&1 &&
+          test -f conftest.o
+then :
+  hts_cv_prog_cc_werror="-Werror"
+
+fi
+        rm -f conftest.*
+
+fi
+
+fi
+
+    if test "x$hts_cv_prog_cc_werror" != x
+then :
+
+      { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $hts_cv_prog_cc_werror" >&5
+printf "%s\n" "$hts_cv_prog_cc_werror" >&6; }
+      if test "xhts_late_cflags" != x
+then :
+  eval hts_late_cflags="$hts_cv_prog_cc_werror"
+fi
+
+else $as_nop
+        { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: unknown" >&5
+printf "%s\n" "unknown" >&6; }
+
+fi
+
+fi
+
+
+# HTSlib uses X/Open-only facilities (M_SQRT2 etc, drand48() etc), and
+# various POSIX functions that are provided by various _POSIX_C_SOURCE values
+# or by _XOPEN_SOURCE >= 500. It also uses usleep(), which is removed when
+# _XOPEN_SOURCE >= 700. Additionally, some definitions may require
+# _XOPEN_SOURCE >= 600 on some platforms (snprintf on MinGW,
+# PTHREAD_MUTEX_RECURSIVE on some Linux distributions). Hence we set it to 600.
+
+# Define _XOPEN_SOURCE unless the user has already done so via $CPPFLAGS etc.
+
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $CC options needed to detect all undeclared functions" >&5
+printf %s "checking for $CC options needed to detect all undeclared functions... " >&6; }
+if test ${ac_cv_c_undeclared_builtin_options+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  ac_save_CFLAGS=$CFLAGS
+   ac_cv_c_undeclared_builtin_options='cannot detect'
+   for ac_arg in '' -fno-builtin; do
+     CFLAGS="$ac_save_CFLAGS $ac_arg"
+     # This test program should *not* compile successfully.
+     cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+int
+main (void)
+{
+(void) strchr;
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"
+then :
+
+else $as_nop
+  # This test program should compile successfully.
+        # No library function is consistently available on
+        # freestanding implementations, so test against a dummy
+        # declaration.  Include always-available headers on the
+        # off chance that they somehow elicit warnings.
+        cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+#include <float.h>
+#include <limits.h>
+#include <stdarg.h>
+#include <stddef.h>
+extern void ac_decl (int, char *);
+
+int
+main (void)
+{
+(void) ac_decl (0, (char *) 0);
+  (void) ac_decl;
+
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"
+then :
+  if test x"$ac_arg" = x
+then :
+  ac_cv_c_undeclared_builtin_options='none needed'
+else $as_nop
+  ac_cv_c_undeclared_builtin_options=$ac_arg
+fi
+          break
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext
+    done
+    CFLAGS=$ac_save_CFLAGS
+
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_undeclared_builtin_options" >&5
+printf "%s\n" "$ac_cv_c_undeclared_builtin_options" >&6; }
+  case $ac_cv_c_undeclared_builtin_options in #(
+  'cannot detect') :
+    { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "cannot make $CC report undeclared builtins
+See \`config.log' for more details" "$LINENO" 5; } ;; #(
+  'none needed') :
+    ac_c_undeclared_builtin_options='' ;; #(
+  *) :
+    ac_c_undeclared_builtin_options=$ac_cv_c_undeclared_builtin_options ;;
+esac
+
+ac_header= ac_cache=
+for ac_item in $ac_header_c_list
+do
+  if test $ac_cache; then
+    ac_fn_c_check_header_compile "$LINENO" $ac_header ac_cv_header_$ac_cache "$ac_includes_default"
+    if eval test \"x\$ac_cv_header_$ac_cache\" = xyes; then
+      printf "%s\n" "#define $ac_item 1" >> confdefs.h
+    fi
+    ac_header= ac_cache=
+  elif test $ac_header; then
+    ac_cache=$ac_item
+  else
+    ac_header=$ac_item
+  fi
+done
+
+
+
+
+
+
+
+
+if test $ac_cv_header_stdlib_h = yes && test $ac_cv_header_string_h = yes
+then :
+
+printf "%s\n" "#define STDC_HEADERS 1" >>confdefs.h
+
+fi
+ac_fn_check_decl "$LINENO" "_XOPEN_SOURCE" "ac_cv_have_decl__XOPEN_SOURCE" "$ac_includes_default" "$ac_c_undeclared_builtin_options" "CFLAGS"
+if test "x$ac_cv_have_decl__XOPEN_SOURCE" = xyes
+then :
+
+else $as_nop
+
+printf "%s\n" "#define _XOPEN_SOURCE 600" >>confdefs.h
+
+fi
+
+ac_fn_check_decl "$LINENO" "__get_cpuid_max" "ac_cv_have_decl___get_cpuid_max" "#include <cpuid.h>
+" "$ac_c_undeclared_builtin_options" "CFLAGS"
+if test "x$ac_cv_have_decl___get_cpuid_max" = xyes
+then :
+  ac_have_decl=1
+else $as_nop
+  ac_have_decl=0
+fi
+printf "%s\n" "#define HAVE_DECL___GET_CPUID_MAX $ac_have_decl" >>confdefs.h
+if test $ac_have_decl = 1
+then :
+
+   hts_have_cpuid=yes
+
+else $as_nop
+
+   hts_have_cpuid=no
+
+fi
+ac_fn_check_decl "$LINENO" "__cpuid_count" "ac_cv_have_decl___cpuid_count" "#include <cpuid.h>
+" "$ac_c_undeclared_builtin_options" "CFLAGS"
+if test "x$ac_cv_have_decl___cpuid_count" = xyes
+then :
+  ac_have_decl=1
+else $as_nop
+  ac_have_decl=0
+fi
+printf "%s\n" "#define HAVE_DECL___CPUID_COUNT $ac_have_decl" >>confdefs.h
+if test $ac_have_decl = 1
+then :
+
+   hts_have_cpuid=yes
+
+else $as_nop
+
+   hts_have_cpuid=no
+
+fi
+
+
+if test "x$hts_have_cpuid" = "xyes"
+then :
+
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking C compiler flags needed for sse4.1" >&5
+printf %s "checking C compiler flags needed for sse4.1... " >&6; }
+if test ${hts_cv_check_cflags_needed_sse4_1___msse4_1__mssse3__mpopcnt+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+
+  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+    #ifdef __x86_64__
+    #include "x86intrin.h"
+    #endif
+
+int
+main (void)
+{
+
+    #ifdef __x86_64__
+    __m128i a = _mm_set_epi32(1, 2, 3, 4), b = _mm_set_epi32(4, 3, 2, 1);
+    __m128i c = _mm_shuffle_epi8(_mm_max_epu32(a, b), b);
+    return _mm_popcnt_u32(*((char *) &c));
+    #endif
+
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"
+then :
+  hts_cv_check_cflags_needed_sse4_1___msse4_1__mssse3__mpopcnt=none
+else $as_nop
+  ax_check_save_flags=$CFLAGS
+     CFLAGS="$CFLAGS  -msse4.1 -mssse3 -mpopcnt"
+     cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+    #ifdef __x86_64__
+    #include "x86intrin.h"
+    #endif
+
+int
+main (void)
+{
+
+    #ifdef __x86_64__
+    __m128i a = _mm_set_epi32(1, 2, 3, 4), b = _mm_set_epi32(4, 3, 2, 1);
+    __m128i c = _mm_shuffle_epi8(_mm_max_epu32(a, b), b);
+    return _mm_popcnt_u32(*((char *) &c));
+    #endif
+
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"
+then :
+  hts_cv_check_cflags_needed_sse4_1___msse4_1__mssse3__mpopcnt="-msse4.1 -mssse3 -mpopcnt"
+else $as_nop
+  hts_cv_check_cflags_needed_sse4_1___msse4_1__mssse3__mpopcnt=unsupported
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam \
+    conftest$ac_exeext conftest.$ac_ext
+     CFLAGS=$ax_check_save_flags
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam \
+    conftest$ac_exeext conftest.$ac_ext
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $hts_cv_check_cflags_needed_sse4_1___msse4_1__mssse3__mpopcnt" >&5
+printf "%s\n" "$hts_cv_check_cflags_needed_sse4_1___msse4_1__mssse3__mpopcnt" >&6; }
+if test "x$hts_cv_check_cflags_needed_sse4_1___msse4_1__mssse3__mpopcnt" = xunsupported
+then :
+
+  :
+
+else $as_nop
+
+  if test "x$hts_cv_check_cflags_needed_sse4_1___msse4_1__mssse3__mpopcnt" = xnone
+then :
+  flags_needed=""
+else $as_nop
+  flags_needed="$hts_cv_check_cflags_needed_sse4_1___msse4_1__mssse3__mpopcnt"
+fi
+
+  hts_cflags_sse4="$flags_needed"
+
+printf "%s\n" "#define HAVE_SSSE3 1" >>confdefs.h
+
+
+printf "%s\n" "#define HAVE_POPCNT 1" >>confdefs.h
+
+
+printf "%s\n" "#define HAVE_SSE4_1 1" >>confdefs.h
+
+
+
+  printf "%s\n" "#define UBSAN 1" >>confdefs.h
+
+
+
+fi
+
+
+
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking C compiler flags needed for avx2" >&5
+printf %s "checking C compiler flags needed for avx2... " >&6; }
+if test ${hts_cv_check_cflags_needed_avx2___mavx2__mpopcnt+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+
+  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+    #ifdef __x86_64__
+    #include "x86intrin.h"
+    #endif
+
+int
+main (void)
+{
+
+    #ifdef __x86_64__
+    __m256i a = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8);
+    __m256i b = _mm256_add_epi32(a, a);
+    long long c = _mm256_extract_epi64(b, 0);
+    return _mm_popcnt_u32((int) c);
+    #endif
+
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"
+then :
+  hts_cv_check_cflags_needed_avx2___mavx2__mpopcnt=none
+else $as_nop
+  ax_check_save_flags=$CFLAGS
+     CFLAGS="$CFLAGS  -mavx2 -mpopcnt"
+     cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+    #ifdef __x86_64__
+    #include "x86intrin.h"
+    #endif
+
+int
+main (void)
+{
+
+    #ifdef __x86_64__
+    __m256i a = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8);
+    __m256i b = _mm256_add_epi32(a, a);
+    long long c = _mm256_extract_epi64(b, 0);
+    return _mm_popcnt_u32((int) c);
+    #endif
+
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"
+then :
+  hts_cv_check_cflags_needed_avx2___mavx2__mpopcnt="-mavx2 -mpopcnt"
+else $as_nop
+  hts_cv_check_cflags_needed_avx2___mavx2__mpopcnt=unsupported
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam \
+    conftest$ac_exeext conftest.$ac_ext
+     CFLAGS=$ax_check_save_flags
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam \
+    conftest$ac_exeext conftest.$ac_ext
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $hts_cv_check_cflags_needed_avx2___mavx2__mpopcnt" >&5
+printf "%s\n" "$hts_cv_check_cflags_needed_avx2___mavx2__mpopcnt" >&6; }
+if test "x$hts_cv_check_cflags_needed_avx2___mavx2__mpopcnt" = xunsupported
+then :
+
+  :
+
+else $as_nop
+
+  if test "x$hts_cv_check_cflags_needed_avx2___mavx2__mpopcnt" = xnone
+then :
+  flags_needed=""
+else $as_nop
+  flags_needed="$hts_cv_check_cflags_needed_avx2___mavx2__mpopcnt"
+fi
+
+  hts_cflags_avx2="$flags_needed"
+
+
+printf "%s\n" "#define HAVE_POPCNT 1" >>confdefs.h
+
+
+printf "%s\n" "#define HAVE_AVX2 1" >>confdefs.h
+
+
+
+fi
+
+
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking C compiler flags needed for avx512f" >&5
+printf %s "checking C compiler flags needed for avx512f... " >&6; }
+if test ${hts_cv_check_cflags_needed_avx512f___mavx512f__mpopcnt+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+
+  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+    #ifdef __x86_64__
+    #include "x86intrin.h"
+    #endif
+
+int
+main (void)
+{
+
+    #ifdef __x86_64__
+    __m512i a = _mm512_set1_epi32(1);
+    __m512i b = _mm512_add_epi32(a, a);
+    __m256i c = _mm512_castsi512_si256(b);
+    __m256i d = _mm512_extracti64x4_epi64(a, 1);
+    return _mm_popcnt_u32(*((char *) &c)) + (*(char *) &d);
+    #endif
+
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"
+then :
+  hts_cv_check_cflags_needed_avx512f___mavx512f__mpopcnt=none
+else $as_nop
+  ax_check_save_flags=$CFLAGS
+     CFLAGS="$CFLAGS  -mavx512f -mpopcnt"
+     cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+    #ifdef __x86_64__
+    #include "x86intrin.h"
+    #endif
+
+int
+main (void)
+{
+
+    #ifdef __x86_64__
+    __m512i a = _mm512_set1_epi32(1);
+    __m512i b = _mm512_add_epi32(a, a);
+    __m256i c = _mm512_castsi512_si256(b);
+    __m256i d = _mm512_extracti64x4_epi64(a, 1);
+    return _mm_popcnt_u32(*((char *) &c)) + (*(char *) &d);
+    #endif
+
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"
+then :
+  hts_cv_check_cflags_needed_avx512f___mavx512f__mpopcnt="-mavx512f -mpopcnt"
+else $as_nop
+  hts_cv_check_cflags_needed_avx512f___mavx512f__mpopcnt=unsupported
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam \
+    conftest$ac_exeext conftest.$ac_ext
+     CFLAGS=$ax_check_save_flags
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam \
+    conftest$ac_exeext conftest.$ac_ext
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $hts_cv_check_cflags_needed_avx512f___mavx512f__mpopcnt" >&5
+printf "%s\n" "$hts_cv_check_cflags_needed_avx512f___mavx512f__mpopcnt" >&6; }
+if test "x$hts_cv_check_cflags_needed_avx512f___mavx512f__mpopcnt" = xunsupported
+then :
+
+  :
+
+else $as_nop
+
+  if test "x$hts_cv_check_cflags_needed_avx512f___mavx512f__mpopcnt" = xnone
+then :
+  flags_needed=""
+else $as_nop
+  flags_needed="$hts_cv_check_cflags_needed_avx512f___mavx512f__mpopcnt"
+fi
+
+  hts_cflags_avx512="$flags_needed"
+
+
+printf "%s\n" "#define HAVE_POPCNT 1" >>confdefs.h
+
+
+printf "%s\n" "#define HAVE_AVX512 1" >>confdefs.h
+
+
+
+fi
+
+
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for working __builtin_cpu_supports(\"ssse3\")" >&5
+printf %s "checking for working __builtin_cpu_supports(\"ssse3\")... " >&6; }
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+int
+main (void)
+{
+
+  if (__builtin_cpu_supports("ssse3")) {
+    return 0;
+  }
+
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"
+then :
+
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+printf "%s\n" "yes" >&6; }
+
+printf "%s\n" "#define HAVE_BUILTIN_CPU_SUPPORT_SSSE3 1" >>confdefs.h
+
+
+else $as_nop
+
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5
+printf "%s\n" "no" >&6; }
+
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam \
+    conftest$ac_exeext conftest.$ac_ext
+
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for __attribute__((target))" >&5
+printf %s "checking for __attribute__((target))... " >&6; }
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+  __attribute__((target("ssse3")))
+  int zero(void) {
+    return 0;
+  }
+
+int
+main (void)
+{
+zero();
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"
+then :
+
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+printf "%s\n" "yes" >&6; }
+
+printf "%s\n" "#define HAVE_ATTRIBUTE_TARGET 1" >>confdefs.h
+
+
+else $as_nop
+
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5
+printf "%s\n" "no" >&6; }
+
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext
+
+
+fi
+
+
+
+
+
+
+
+if test "x$ac_cv_env_PKG_CONFIG_set" != "xset"; then
+	if test -n "$ac_tool_prefix"; then
+  # Extract the first word of "${ac_tool_prefix}pkg-config", so it can be a program name with args.
+set dummy ${ac_tool_prefix}pkg-config; ac_word=$2
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+printf %s "checking for $ac_word... " >&6; }
+if test ${ac_cv_path_PKG_CONFIG+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  case $PKG_CONFIG in
+  [\\/]* | ?:[\\/]*)
+  ac_cv_path_PKG_CONFIG="$PKG_CONFIG" # Let the user override the test with a path.
+  ;;
+  *)
+  as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  case $as_dir in #(((
+    '') as_dir=./ ;;
+    */) ;;
+    *) as_dir=$as_dir/ ;;
+  esac
+    for ac_exec_ext in '' $ac_executable_extensions; do
+  if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then
+    ac_cv_path_PKG_CONFIG="$as_dir$ac_word$ac_exec_ext"
+    printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+  done
+IFS=$as_save_IFS
+
+  ;;
+esac
+fi
+PKG_CONFIG=$ac_cv_path_PKG_CONFIG
+if test -n "$PKG_CONFIG"; then
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $PKG_CONFIG" >&5
+printf "%s\n" "$PKG_CONFIG" >&6; }
+else
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5
+printf "%s\n" "no" >&6; }
+fi
+
+
+fi
+if test -z "$ac_cv_path_PKG_CONFIG"; then
+  ac_pt_PKG_CONFIG=$PKG_CONFIG
+  # Extract the first word of "pkg-config", so it can be a program name with args.
+set dummy pkg-config; ac_word=$2
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+printf %s "checking for $ac_word... " >&6; }
+if test ${ac_cv_path_ac_pt_PKG_CONFIG+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  case $ac_pt_PKG_CONFIG in
+  [\\/]* | ?:[\\/]*)
+  ac_cv_path_ac_pt_PKG_CONFIG="$ac_pt_PKG_CONFIG" # Let the user override the test with a path.
+  ;;
+  *)
+  as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  case $as_dir in #(((
+    '') as_dir=./ ;;
+    */) ;;
+    *) as_dir=$as_dir/ ;;
+  esac
+    for ac_exec_ext in '' $ac_executable_extensions; do
+  if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then
+    ac_cv_path_ac_pt_PKG_CONFIG="$as_dir$ac_word$ac_exec_ext"
+    printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+  done
+IFS=$as_save_IFS
+
+  ;;
+esac
+fi
+ac_pt_PKG_CONFIG=$ac_cv_path_ac_pt_PKG_CONFIG
+if test -n "$ac_pt_PKG_CONFIG"; then
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_pt_PKG_CONFIG" >&5
+printf "%s\n" "$ac_pt_PKG_CONFIG" >&6; }
+else
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5
+printf "%s\n" "no" >&6; }
+fi
+
+  if test "x$ac_pt_PKG_CONFIG" = x; then
+    PKG_CONFIG=""
+  else
+    case $cross_compiling:$ac_tool_warned in
+yes:)
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5
+printf "%s\n" "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;}
+ac_tool_warned=yes ;;
+esac
+    PKG_CONFIG=$ac_pt_PKG_CONFIG
+  fi
+else
+  PKG_CONFIG="$ac_cv_path_PKG_CONFIG"
+fi
+
+fi
+if test -n "$PKG_CONFIG"; then
+	_pkg_min_version=0.9.0
+	{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking pkg-config is at least version $_pkg_min_version" >&5
+printf %s "checking pkg-config is at least version $_pkg_min_version... " >&6; }
+	if $PKG_CONFIG --atleast-pkgconfig-version $_pkg_min_version; then
+		{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+printf "%s\n" "yes" >&6; }
+	else
+		{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5
+printf "%s\n" "no" >&6; }
+		PKG_CONFIG=""
+	fi
+fi
+
+need_crypto=no
+pc_requires=
+static_LDFLAGS=$LDFLAGS
+static_LIBS='-lpthread -lz -lm'
+private_LIBS=$LDFLAGS
+
+# Check whether --enable-versioned-symbols was given.
+if test ${enable_versioned_symbols+y}
+then :
+  enableval=$enable_versioned_symbols;
+else $as_nop
+  enable_versioned_symbols=yes
+fi
+
+
+# Check whether --enable-bz2 was given.
+if test ${enable_bz2+y}
+then :
+  enableval=$enable_bz2;
+else $as_nop
+  enable_bz2=yes
+fi
+
+
+# Check whether --enable-gcs was given.
+if test ${enable_gcs+y}
+then :
+  enableval=$enable_gcs;
+else $as_nop
+  enable_gcs=check
+fi
+
+
+# Check whether --enable-largefile was given.
+if test ${enable_largefile+y}
+then :
+  enableval=$enable_largefile;
+fi
+
+if test "$enable_largefile" != no; then
+
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for special C compiler options needed for large files" >&5
+printf %s "checking for special C compiler options needed for large files... " >&6; }
+if test ${ac_cv_sys_largefile_CC+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  ac_cv_sys_largefile_CC=no
+     if test "$GCC" != yes; then
+       ac_save_CC=$CC
+       while :; do
+	 # IRIX 6.2 and later do not support large files by default,
+	 # so use the C compiler's -n32 option if that helps.
+	 cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+#include <sys/types.h>
+ /* Check that off_t can represent 2**63 - 1 correctly.
+    We can't simply define LARGE_OFF_T to be 9223372036854775807,
+    since some C++ compilers masquerading as C compilers
+    incorrectly reject 9223372036854775807.  */
+#define LARGE_OFF_T (((off_t) 1 << 31 << 31) - 1 + ((off_t) 1 << 31 << 31))
+  int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
+		       && LARGE_OFF_T % 2147483647 == 1)
+		      ? 1 : -1];
+int
+main (void)
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+	 if ac_fn_c_try_compile "$LINENO"
+then :
+  break
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam
+	 CC="$CC -n32"
+	 if ac_fn_c_try_compile "$LINENO"
+then :
+  ac_cv_sys_largefile_CC=' -n32'; break
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam
+	 break
+       done
+       CC=$ac_save_CC
+       rm -f conftest.$ac_ext
+    fi
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sys_largefile_CC" >&5
+printf "%s\n" "$ac_cv_sys_largefile_CC" >&6; }
+  if test "$ac_cv_sys_largefile_CC" != no; then
+    CC=$CC$ac_cv_sys_largefile_CC
+  fi
+
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for _FILE_OFFSET_BITS value needed for large files" >&5
+printf %s "checking for _FILE_OFFSET_BITS value needed for large files... " >&6; }
+if test ${ac_cv_sys_file_offset_bits+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  while :; do
+  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+#include <sys/types.h>
+ /* Check that off_t can represent 2**63 - 1 correctly.
+    We can't simply define LARGE_OFF_T to be 9223372036854775807,
+    since some C++ compilers masquerading as C compilers
+    incorrectly reject 9223372036854775807.  */
+#define LARGE_OFF_T (((off_t) 1 << 31 << 31) - 1 + ((off_t) 1 << 31 << 31))
+  int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
+		       && LARGE_OFF_T % 2147483647 == 1)
+		      ? 1 : -1];
+int
+main (void)
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"
+then :
+  ac_cv_sys_file_offset_bits=no; break
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext
+  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+#define _FILE_OFFSET_BITS 64
+#include <sys/types.h>
+ /* Check that off_t can represent 2**63 - 1 correctly.
+    We can't simply define LARGE_OFF_T to be 9223372036854775807,
+    since some C++ compilers masquerading as C compilers
+    incorrectly reject 9223372036854775807.  */
+#define LARGE_OFF_T (((off_t) 1 << 31 << 31) - 1 + ((off_t) 1 << 31 << 31))
+  int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
+		       && LARGE_OFF_T % 2147483647 == 1)
+		      ? 1 : -1];
+int
+main (void)
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"
+then :
+  ac_cv_sys_file_offset_bits=64; break
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext
+  ac_cv_sys_file_offset_bits=unknown
+  break
+done
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sys_file_offset_bits" >&5
+printf "%s\n" "$ac_cv_sys_file_offset_bits" >&6; }
+case $ac_cv_sys_file_offset_bits in #(
+  no | unknown) ;;
+  *)
+printf "%s\n" "#define _FILE_OFFSET_BITS $ac_cv_sys_file_offset_bits" >>confdefs.h
+;;
+esac
+rm -rf conftest*
+  if test $ac_cv_sys_file_offset_bits = unknown; then
+    { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for _LARGE_FILES value needed for large files" >&5
+printf %s "checking for _LARGE_FILES value needed for large files... " >&6; }
+if test ${ac_cv_sys_large_files+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  while :; do
+  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+#include <sys/types.h>
+ /* Check that off_t can represent 2**63 - 1 correctly.
+    We can't simply define LARGE_OFF_T to be 9223372036854775807,
+    since some C++ compilers masquerading as C compilers
+    incorrectly reject 9223372036854775807.  */
+#define LARGE_OFF_T (((off_t) 1 << 31 << 31) - 1 + ((off_t) 1 << 31 << 31))
+  int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
+		       && LARGE_OFF_T % 2147483647 == 1)
+		      ? 1 : -1];
+int
+main (void)
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"
+then :
+  ac_cv_sys_large_files=no; break
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext
+  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+#define _LARGE_FILES 1
+#include <sys/types.h>
+ /* Check that off_t can represent 2**63 - 1 correctly.
+    We can't simply define LARGE_OFF_T to be 9223372036854775807,
+    since some C++ compilers masquerading as C compilers
+    incorrectly reject 9223372036854775807.  */
+#define LARGE_OFF_T (((off_t) 1 << 31 << 31) - 1 + ((off_t) 1 << 31 << 31))
+  int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
+		       && LARGE_OFF_T % 2147483647 == 1)
+		      ? 1 : -1];
+int
+main (void)
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"
+then :
+  ac_cv_sys_large_files=1; break
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext
+  ac_cv_sys_large_files=unknown
+  break
+done
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sys_large_files" >&5
+printf "%s\n" "$ac_cv_sys_large_files" >&6; }
+case $ac_cv_sys_large_files in #(
+  no | unknown) ;;
+  *)
+printf "%s\n" "#define _LARGE_FILES $ac_cv_sys_large_files" >>confdefs.h
+;;
+esac
+rm -rf conftest*
+  fi
+fi
+
+
+# Check whether --enable-libcurl was given.
+if test ${enable_libcurl+y}
+then :
+  enableval=$enable_libcurl;
+else $as_nop
+  enable_libcurl=check
+fi
+
+
+# Check whether --enable-lzma was given.
+if test ${enable_lzma+y}
+then :
+  enableval=$enable_lzma;
+else $as_nop
+  enable_lzma=yes
+fi
+
+
+# Check whether --enable-plugins was given.
+if test ${enable_plugins+y}
+then :
+  enableval=$enable_plugins;
+else $as_nop
+  enable_plugins=no
+fi
+
+
+
+
+# Check whether --with-external-htscodecs was given.
+if test ${with_external_htscodecs+y}
+then :
+  withval=$with_external_htscodecs;
+else $as_nop
+  with_external_htscodecs=no
+fi
+
+
+
+
+# Check whether --with-libdeflate was given.
+if test ${with_libdeflate+y}
+then :
+  withval=$with_libdeflate;
+else $as_nop
+  with_libdeflate=check
+fi
+
+
+
+# Check whether --with-plugin-dir was given.
+if test ${with_plugin_dir+y}
+then :
+  withval=$with_plugin_dir; case $withval in
+     yes|no) cat > config.mk <<'EOF'
+ifneq ($(MAKECMDGOALS),distclean)
+$(error Resolve configure error first)
+endif
+EOF
+   as_fn_error $? "no directory specified for --with-plugin-dir" "$LINENO" 5 ;;
+   esac
+else $as_nop
+  with_plugin_dir='$(libexecdir)/htslib'
+fi
+
+plugindir=$with_plugin_dir
+
+
+
+# Check whether --with-plugin-path was given.
+if test ${with_plugin_path+y}
+then :
+  withval=$with_plugin_path; case $withval in
+     yes) cat > config.mk <<'EOF'
+ifneq ($(MAKECMDGOALS),distclean)
+$(error Resolve configure error first)
+endif
+EOF
+   as_fn_error $? "no path specified for --with-plugin-path" "$LINENO" 5 ;;
+     no)  with_plugin_path= ;;
+   esac
+else $as_nop
+  with_plugin_path=$with_plugin_dir
+fi
+
+pluginpath=$with_plugin_path
+
+
+# Check whether --enable-s3 was given.
+if test ${enable_s3+y}
+then :
+  enableval=$enable_s3;
+else $as_nop
+  enable_s3=check
+fi
+
+
+basic_host=${host_alias:-unknown-`uname -s`}
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking shared library type for $basic_host" >&5
+printf %s "checking shared library type for $basic_host... " >&6; }
+case $basic_host in
+  *-cygwin* | *-CYGWIN*)
+    host_result="Cygwin DLL"
+    PLATFORM=CYGWIN
+    PLUGIN_EXT=.cygdll
+    ;;
+  *-darwin* | *-Darwin*)
+    host_result="Darwin dylib"
+    PLATFORM=Darwin
+    PLUGIN_EXT=.bundle
+    ;;
+  *-msys* | *-MSYS* | *-mingw* | *-MINGW*)
+    host_result="MSYS dll"
+    PLATFORM=MSYS
+    PLUGIN_EXT=.dll
+    # This also sets __USE_MINGW_ANSI_STDIO which in turn makes PRId64,
+    # %lld and %z printf formats work.  It also enforces the snprintf to
+    # be C99 compliant so it returns the correct values (in kstring.c).
+
+    # Now set by default, so no need to do it here.
+    # CPPFLAGS="$CPPFLAGS -D_XOPEN_SOURCE=600"
+    ;;
+  *)
+    host_result="plain .so"
+    PLATFORM=default
+    PLUGIN_EXT=.so
+    ;;
+esac
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $host_result" >&5
+printf "%s\n" "$host_result" >&6; }
+
+
+if test x"$PLATFORM" = xdefault && test x"$enable_versioned_symbols" = xyes
+then :
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether the linker supports versioned symbols" >&5
+printf %s "checking whether the linker supports versioned symbols... " >&6; }
+if test ${hts_cv_have_versioned_symbols+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+
+      save_LDFLAGS=$LDFLAGS
+      LDFLAGS="-Wl,-version-script,$srcdir/htslib.map $LDFLAGS"
+      cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+int
+main (void)
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"
+then :
+  hts_cv_have_versioned_symbols=yes
+else $as_nop
+  hts_cv_have_versioned_symbols=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam \
+    conftest$ac_exeext conftest.$ac_ext
+      LDFLAGS=$save_LDFLAGS
+
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $hts_cv_have_versioned_symbols" >&5
+printf "%s\n" "$hts_cv_have_versioned_symbols" >&6; }
+   if test "x$hts_cv_have_versioned_symbols" = xyes
+then :
+
+     VERSION_SCRIPT_LDFLAGS='-Wl,-version-script,$(srcprefix)htslib.map'
+
+
+fi
+
+fi
+
+
+  # Test for flags to set default shared library visibility to hidden
+  # -fvisibility=hidden : GCC compatible
+  # -xldscope=hidden    : SunStudio
+  ac_opt_found=no
+  if test "x$ac_opt_found" = "xno"
+then :
+    { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether the compiler accepts -fvisibility=hidden" >&5
+printf %s "checking whether the compiler accepts -fvisibility=hidden... " >&6; }
+if test ${hts_cv_check__fvisibility_hidden+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  ac_check_save_cflags=$CFLAGS
+    ac_check_save_ldflags=$LDFLAGS
+    CFLAGS="$CFLAGS -fvisibility=hidden"
+    LDFLAGS="$LDFLAGS -fvisibility=hidden"
+    cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+int
+main (void)
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"
+then :
+  hts_cv_check__fvisibility_hidden=yes
+       if test "xac_opt_found" != x
+then :
+  eval ac_opt_found="-fvisibility=hidden"
+fi
+else $as_nop
+  hts_cv_check__fvisibility_hidden=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam \
+    conftest$ac_exeext conftest.$ac_ext
+    CFLAGS=$ac_check_save_cflags
+    LDFLAGS=$ac_check_save_ldflags
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $hts_cv_check__fvisibility_hidden" >&5
+printf "%s\n" "$hts_cv_check__fvisibility_hidden" >&6; }
+
+fi
+   if test "x$ac_opt_found" = "xno"
+then :
+    { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether the compiler accepts -xldscope=hidden" >&5
+printf %s "checking whether the compiler accepts -xldscope=hidden... " >&6; }
+if test ${hts_cv_check__xldscope_hidden+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  ac_check_save_cflags=$CFLAGS
+    ac_check_save_ldflags=$LDFLAGS
+    CFLAGS="$CFLAGS -xldscope=hidden"
+    LDFLAGS="$LDFLAGS -xldscope=hidden"
+    cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+int
+main (void)
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"
+then :
+  hts_cv_check__xldscope_hidden=yes
+       if test "xac_opt_found" != x
+then :
+  eval ac_opt_found="-xldscope=hidden"
+fi
+else $as_nop
+  hts_cv_check__xldscope_hidden=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam \
+    conftest$ac_exeext conftest.$ac_ext
+    CFLAGS=$ac_check_save_cflags
+    LDFLAGS=$ac_check_save_ldflags
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $hts_cv_check__xldscope_hidden" >&5
+printf "%s\n" "$hts_cv_check__xldscope_hidden" >&6; }
+
+fi
+
+  if test "x$ac_opt_found" != "xno"
+then :
+  CFLAGS="$CFLAGS $ac_opt_found"
+    LDFLAGS="$LDFLAGS $ac_opt_found"
+fi
+
+
+
+
+
+  # Make sure we can run config.sub.
+$SHELL "${ac_aux_dir}config.sub" sun4 >/dev/null 2>&1 ||
+  as_fn_error $? "cannot run $SHELL ${ac_aux_dir}config.sub" "$LINENO" 5
+
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking build system type" >&5
+printf %s "checking build system type... " >&6; }
+if test ${ac_cv_build+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  ac_build_alias=$build_alias
+test "x$ac_build_alias" = x &&
+  ac_build_alias=`$SHELL "${ac_aux_dir}config.guess"`
+test "x$ac_build_alias" = x &&
+  as_fn_error $? "cannot guess build type; you must specify one" "$LINENO" 5
+ac_cv_build=`$SHELL "${ac_aux_dir}config.sub" $ac_build_alias` ||
+  as_fn_error $? "$SHELL ${ac_aux_dir}config.sub $ac_build_alias failed" "$LINENO" 5
+
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_build" >&5
+printf "%s\n" "$ac_cv_build" >&6; }
+case $ac_cv_build in
+*-*-*) ;;
+*) as_fn_error $? "invalid value of canonical build" "$LINENO" 5;;
+esac
+build=$ac_cv_build
+ac_save_IFS=$IFS; IFS='-'
+set x $ac_cv_build
+shift
+build_cpu=$1
+build_vendor=$2
+shift; shift
+# Remember, the first character of IFS is used to create $*,
+# except with old shells:
+build_os=$*
+IFS=$ac_save_IFS
+case $build_os in *\ *) build_os=`echo "$build_os" | sed 's/ /-/g'`;; esac
+
+
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking host system type" >&5
+printf %s "checking host system type... " >&6; }
+if test ${ac_cv_host+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  if test "x$host_alias" = x; then
+  ac_cv_host=$ac_cv_build
+else
+  ac_cv_host=`$SHELL "${ac_aux_dir}config.sub" $host_alias` ||
+    as_fn_error $? "$SHELL ${ac_aux_dir}config.sub $host_alias failed" "$LINENO" 5
+fi
+
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_host" >&5
+printf "%s\n" "$ac_cv_host" >&6; }
+case $ac_cv_host in
+*-*-*) ;;
+*) as_fn_error $? "invalid value of canonical host" "$LINENO" 5;;
+esac
+host=$ac_cv_host
+ac_save_IFS=$IFS; IFS='-'
+set x $ac_cv_host
+shift
+host_cpu=$1
+host_vendor=$2
+shift; shift
+# Remember, the first character of IFS is used to create $*,
+# except with old shells:
+host_os=$*
+IFS=$ac_save_IFS
+case $host_os in *\ *) host_os=`echo "$host_os" | sed 's/ /-/g'`;; esac
+
+
+
+ac_func=
+for ac_item in $ac_func_c_list
+do
+  if test $ac_func; then
+    ac_fn_c_check_func "$LINENO" $ac_func ac_cv_func_$ac_func
+    if eval test \"x\$ac_cv_func_$ac_func\" = xyes; then
+      echo "#define $ac_item 1" >> confdefs.h
+    fi
+    ac_func=
+  else
+    ac_func=$ac_item
+  fi
+done
+
+
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for working mmap" >&5
+printf %s "checking for working mmap... " >&6; }
+if test ${ac_cv_func_mmap_fixed_mapped+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  if test "$cross_compiling" = yes
+then :
+  case "$host_os" in # ((
+			  # Guess yes on platforms where we know the result.
+		  linux*) ac_cv_func_mmap_fixed_mapped=yes ;;
+			  # If we don't know, assume the worst.
+		  *)      ac_cv_func_mmap_fixed_mapped=no ;;
+		esac
+else $as_nop
+  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+$ac_includes_default
+/* malloc might have been renamed as rpl_malloc. */
+#undef malloc
+
+/* Thanks to Mike Haertel and Jim Avera for this test.
+   Here is a matrix of mmap possibilities:
+	mmap private not fixed
+	mmap private fixed at somewhere currently unmapped
+	mmap private fixed at somewhere already mapped
+	mmap shared not fixed
+	mmap shared fixed at somewhere currently unmapped
+	mmap shared fixed at somewhere already mapped
+   For private mappings, we should verify that changes cannot be read()
+   back from the file, nor mmap's back from the file at a different
+   address.  (There have been systems where private was not correctly
+   implemented like the infamous i386 svr4.0, and systems where the
+   VM page cache was not coherent with the file system buffer cache
+   like early versions of FreeBSD and possibly contemporary NetBSD.)
+   For shared mappings, we should conversely verify that changes get
+   propagated back to all the places they're supposed to be.
+
+   Grep wants private fixed already mapped.
+   The main things grep needs to know about mmap are:
+   * does it exist and is it safe to write into the mmap'd area
+   * how to use it (BSD variants)  */
+
+#include <fcntl.h>
+#include <sys/mman.h>
+
+/* This mess was copied from the GNU getpagesize.h.  */
+#ifndef HAVE_GETPAGESIZE
+# ifdef _SC_PAGESIZE
+#  define getpagesize() sysconf(_SC_PAGESIZE)
+# else /* no _SC_PAGESIZE */
+#  ifdef HAVE_SYS_PARAM_H
+#   include <sys/param.h>
+#   ifdef EXEC_PAGESIZE
+#    define getpagesize() EXEC_PAGESIZE
+#   else /* no EXEC_PAGESIZE */
+#    ifdef NBPG
+#     define getpagesize() NBPG * CLSIZE
+#     ifndef CLSIZE
+#      define CLSIZE 1
+#     endif /* no CLSIZE */
+#    else /* no NBPG */
+#     ifdef NBPC
+#      define getpagesize() NBPC
+#     else /* no NBPC */
+#      ifdef PAGESIZE
+#       define getpagesize() PAGESIZE
+#      endif /* PAGESIZE */
+#     endif /* no NBPC */
+#    endif /* no NBPG */
+#   endif /* no EXEC_PAGESIZE */
+#  else /* no HAVE_SYS_PARAM_H */
+#   define getpagesize() 8192	/* punt totally */
+#  endif /* no HAVE_SYS_PARAM_H */
+# endif /* no _SC_PAGESIZE */
+
+#endif /* no HAVE_GETPAGESIZE */
+
+int
+main (void)
+{
+  char *data, *data2, *data3;
+  const char *cdata2;
+  int i, pagesize;
+  int fd, fd2;
+
+  pagesize = getpagesize ();
+
+  /* First, make a file with some known garbage in it. */
+  data = (char *) malloc (pagesize);
+  if (!data)
+    return 1;
+  for (i = 0; i < pagesize; ++i)
+    *(data + i) = rand ();
+  umask (0);
+  fd = creat ("conftest.mmap", 0600);
+  if (fd < 0)
+    return 2;
+  if (write (fd, data, pagesize) != pagesize)
+    return 3;
+  close (fd);
+
+  /* Next, check that the tail of a page is zero-filled.  File must have
+     non-zero length, otherwise we risk SIGBUS for entire page.  */
+  fd2 = open ("conftest.txt", O_RDWR | O_CREAT | O_TRUNC, 0600);
+  if (fd2 < 0)
+    return 4;
+  cdata2 = "";
+  if (write (fd2, cdata2, 1) != 1)
+    return 5;
+  data2 = (char *) mmap (0, pagesize, PROT_READ | PROT_WRITE, MAP_SHARED, fd2, 0L);
+  if (data2 == MAP_FAILED)
+    return 6;
+  for (i = 0; i < pagesize; ++i)
+    if (*(data2 + i))
+      return 7;
+  close (fd2);
+  if (munmap (data2, pagesize))
+    return 8;
+
+  /* Next, try to mmap the file at a fixed address which already has
+     something else allocated at it.  If we can, also make sure that
+     we see the same garbage.  */
+  fd = open ("conftest.mmap", O_RDWR);
+  if (fd < 0)
+    return 9;
+  if (data2 != mmap (data2, pagesize, PROT_READ | PROT_WRITE,
+		     MAP_PRIVATE | MAP_FIXED, fd, 0L))
+    return 10;
+  for (i = 0; i < pagesize; ++i)
+    if (*(data + i) != *(data2 + i))
+      return 11;
+
+  /* Finally, make sure that changes to the mapped area do not
+     percolate back to the file as seen by read().  (This is a bug on
+     some variants of i386 svr4.0.)  */
+  for (i = 0; i < pagesize; ++i)
+    *(data2 + i) = *(data2 + i) + 1;
+  data3 = (char *) malloc (pagesize);
+  if (!data3)
+    return 12;
+  if (read (fd, data3, pagesize) != pagesize)
+    return 13;
+  for (i = 0; i < pagesize; ++i)
+    if (*(data + i) != *(data3 + i))
+      return 14;
+  close (fd);
+  free (data);
+  free (data3);
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_run "$LINENO"
+then :
+  ac_cv_func_mmap_fixed_mapped=yes
+else $as_nop
+  ac_cv_func_mmap_fixed_mapped=no
+fi
+rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \
+  conftest.$ac_objext conftest.beam conftest.$ac_ext
+fi
+
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_mmap_fixed_mapped" >&5
+printf "%s\n" "$ac_cv_func_mmap_fixed_mapped" >&6; }
+if test $ac_cv_func_mmap_fixed_mapped = yes; then
+
+printf "%s\n" "#define HAVE_MMAP 1" >>confdefs.h
+
+fi
+rm -f conftest.mmap conftest.txt
+
+ac_fn_c_check_func "$LINENO" "gmtime_r" "ac_cv_func_gmtime_r"
+if test "x$ac_cv_func_gmtime_r" = xyes
+then :
+  printf "%s\n" "#define HAVE_GMTIME_R 1" >>confdefs.h
+
+fi
+ac_fn_c_check_func "$LINENO" "fsync" "ac_cv_func_fsync"
+if test "x$ac_cv_func_fsync" = xyes
+then :
+  printf "%s\n" "#define HAVE_FSYNC 1" >>confdefs.h
+
+fi
+ac_fn_c_check_func "$LINENO" "drand48" "ac_cv_func_drand48"
+if test "x$ac_cv_func_drand48" = xyes
+then :
+  printf "%s\n" "#define HAVE_DRAND48 1" >>confdefs.h
+
+fi
+ac_fn_c_check_func "$LINENO" "srand48_deterministic" "ac_cv_func_srand48_deterministic"
+if test "x$ac_cv_func_srand48_deterministic" = xyes
+then :
+  printf "%s\n" "#define HAVE_SRAND48_DETERMINISTIC 1" >>confdefs.h
+
+fi
+
+
+# Darwin has a dubious fdatasync() symbol, but no declaration in <unistd.h>
+as_ac_Symbol=`printf "%s\n" "ac_cv_have_decl_fdatasync(int)" | $as_tr_sh`
+ac_fn_check_decl "$LINENO" "fdatasync(int)" "$as_ac_Symbol" "$ac_includes_default" "$ac_c_undeclared_builtin_options" "CFLAGS"
+if eval test \"x\$"$as_ac_Symbol"\" = x"yes"
+then :
+  ac_fn_c_check_func "$LINENO" "fdatasync" "ac_cv_func_fdatasync"
+if test "x$ac_cv_func_fdatasync" = xyes
+then :
+  printf "%s\n" "#define HAVE_FDATASYNC 1" >>confdefs.h
+
+fi
+
+fi
+
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for __attribute__((constructor))" >&5
+printf %s "checking for __attribute__((constructor))... " >&6; }
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+  static __attribute__((constructor)) void noop(void) {}
+
+int
+main (void)
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"
+then :
+
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+printf "%s\n" "yes" >&6; }
+
+printf "%s\n" "#define HAVE_ATTRIBUTE_CONSTRUCTOR 1" >>confdefs.h
+
+
+else $as_nop
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5
+printf "%s\n" "no" >&6; }
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam \
+    conftest$ac_exeext conftest.$ac_ext
+
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for clock_gettime with CLOCK_PROCESS_CPUTIME_ID" >&5
+printf %s "checking for clock_gettime with CLOCK_PROCESS_CPUTIME_ID... " >&6; }
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+#include <time.h>
+int
+main (void)
+{
+
+  struct timespec ts;
+  clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts);
+
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"
+then :
+
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+printf "%s\n" "yes" >&6; }
+
+printf "%s\n" "#define HAVE_CLOCK_GETTIME_CPUTIME 1" >>confdefs.h
+
+
+else $as_nop
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5
+printf "%s\n" "no" >&6; }
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam \
+    conftest$ac_exeext conftest.$ac_ext
+
+if test $enable_plugins != no; then
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for library containing dlsym" >&5
+printf %s "checking for library containing dlsym... " >&6; }
+if test ${ac_cv_search_dlsym+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  ac_func_search_save_LIBS=$LIBS
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+/* Override any GCC internal prototype to avoid an error.
+   Use char because int might match the return type of a GCC
+   builtin and then its argument prototype would still apply.  */
+char dlsym ();
+int
+main (void)
+{
+return dlsym ();
+  ;
+  return 0;
+}
+_ACEOF
+for ac_lib in '' dl
+do
+  if test -z "$ac_lib"; then
+    ac_res="none required"
+  else
+    ac_res=-l$ac_lib
+    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
+  fi
+  if ac_fn_c_try_link "$LINENO"
+then :
+  ac_cv_search_dlsym=$ac_res
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam \
+    conftest$ac_exeext
+  if test ${ac_cv_search_dlsym+y}
+then :
+  break
+fi
+done
+if test ${ac_cv_search_dlsym+y}
+then :
+
+else $as_nop
+  ac_cv_search_dlsym=no
+fi
+rm conftest.$ac_ext
+LIBS=$ac_func_search_save_LIBS
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_dlsym" >&5
+printf "%s\n" "$ac_cv_search_dlsym" >&6; }
+ac_res=$ac_cv_search_dlsym
+if test "$ac_res" != no
+then :
+  test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
+
+else $as_nop
+  cat > config.mk <<'EOF'
+ifneq ($(MAKECMDGOALS),distclean)
+$(error Resolve configure error first)
+endif
+EOF
+   as_fn_error $? "dlsym() not found
+
+Plugin support requires dynamic linking facilities from the operating system.
+Either configure with --disable-plugins or resolve this error to build HTSlib." "$LINENO" 5
+fi
+
+  # Check if the compiler understands -rdynamic
+  # TODO Test whether this is required and/or needs tweaking per-platform
+    { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether the compiler accepts -rdynamic" >&5
+printf %s "checking whether the compiler accepts -rdynamic... " >&6; }
+if test ${hts_cv_check__rdynamic+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  ac_check_save_cflags=$CFLAGS
+    ac_check_save_ldflags=$LDFLAGS
+    CFLAGS="$CFLAGS -rdynamic"
+    LDFLAGS="$LDFLAGS -rdynamic"
+    cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+int
+main (void)
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"
+then :
+  hts_cv_check__rdynamic=yes
+       if test "xrdynamic_flag" != x
+then :
+  eval rdynamic_flag="-rdynamic"
+fi
+else $as_nop
+  hts_cv_check__rdynamic=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam \
+    conftest$ac_exeext conftest.$ac_ext
+    CFLAGS=$ac_check_save_cflags
+    LDFLAGS=$ac_check_save_ldflags
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $hts_cv_check__rdynamic" >&5
+printf "%s\n" "$hts_cv_check__rdynamic" >&6; }
+
+  if test x"$rdynamic_flag" != "xno"
+then :
+  LDFLAGS="$LDFLAGS $rdynamic_flag"
+     static_LDFLAGS="$static_LDFLAGS $rdynamic_flag"
+fi
+  case "$ac_cv_search_dlsym" in
+    -l*) static_LIBS="$static_LIBS $ac_cv_search_dlsym" ;;
+  esac
+
+printf "%s\n" "#define ENABLE_PLUGINS 1" >>confdefs.h
+
+
+
+printf "%s\n" "#define PLUGIN_EXT \"$PLUGIN_EXT\"" >>confdefs.h
+
+fi
+
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for library containing log" >&5
+printf %s "checking for library containing log... " >&6; }
+if test ${ac_cv_search_log+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  ac_func_search_save_LIBS=$LIBS
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+/* Override any GCC internal prototype to avoid an error.
+   Use char because int might match the return type of a GCC
+   builtin and then its argument prototype would still apply.  */
+char log ();
+int
+main (void)
+{
+return log ();
+  ;
+  return 0;
+}
+_ACEOF
+for ac_lib in '' m
+do
+  if test -z "$ac_lib"; then
+    ac_res="none required"
+  else
+    ac_res=-l$ac_lib
+    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
+  fi
+  if ac_fn_c_try_link "$LINENO"
+then :
+  ac_cv_search_log=$ac_res
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam \
+    conftest$ac_exeext
+  if test ${ac_cv_search_log+y}
+then :
+  break
+fi
+done
+if test ${ac_cv_search_log+y}
+then :
+
+else $as_nop
+  ac_cv_search_log=no
+fi
+rm conftest.$ac_ext
+LIBS=$ac_func_search_save_LIBS
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_log" >&5
+printf "%s\n" "$ac_cv_search_log" >&6; }
+ac_res=$ac_cv_search_log
+if test "$ac_res" != no
+then :
+  test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
+
+else $as_nop
+  cat > config.mk <<'EOF'
+ifneq ($(MAKECMDGOALS),distclean)
+$(error Resolve configure error first)
+endif
+EOF
+   as_fn_error $? "log() not found
+
+HTSLIB requires a working floating-point math library.
+FAILED.  This error must be resolved in order to build HTSlib successfully." "$LINENO" 5
+fi
+
+
+zlib_devel=ok
+ac_fn_c_check_header_compile "$LINENO" "zlib.h" "ac_cv_header_zlib_h" ";
+"
+if test "x$ac_cv_header_zlib_h" = xyes
+then :
+
+else $as_nop
+  zlib_devel=missing
+fi
+
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for inflate in -lz" >&5
+printf %s "checking for inflate in -lz... " >&6; }
+if test ${ac_cv_lib_z_inflate+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lz  $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+/* Override any GCC internal prototype to avoid an error.
+   Use char because int might match the return type of a GCC
+   builtin and then its argument prototype would still apply.  */
+char inflate ();
+int
+main (void)
+{
+return inflate ();
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"
+then :
+  ac_cv_lib_z_inflate=yes
+else $as_nop
+  ac_cv_lib_z_inflate=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam \
+    conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_z_inflate" >&5
+printf "%s\n" "$ac_cv_lib_z_inflate" >&6; }
+if test "x$ac_cv_lib_z_inflate" = xyes
+then :
+  printf "%s\n" "#define HAVE_LIBZ 1" >>confdefs.h
+
+  LIBS="-lz $LIBS"
+
+else $as_nop
+  zlib_devel=missing
+fi
+
+
+if test $zlib_devel != ok; then
+  cat > config.mk <<'EOF'
+ifneq ($(MAKECMDGOALS),distclean)
+$(error Resolve configure error first)
+endif
+EOF
+   as_fn_error $? "zlib development files not found
+
+HTSlib uses compression routines from the zlib library <http://zlib.net>.
+Building HTSlib requires zlib development files to be installed on the build
+machine; you may need to ensure a package such as zlib1g-dev (on Debian or
+Ubuntu Linux) or zlib-devel (on RPM-based Linux distributions or Cygwin)
+is installed.
+
+FAILED.  This error must be resolved in order to build HTSlib successfully." "$LINENO" 5
+fi
+
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for library containing recv" >&5
+printf %s "checking for library containing recv... " >&6; }
+if test ${ac_cv_search_recv+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  ac_func_search_save_LIBS=$LIBS
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+/* Override any GCC internal prototype to avoid an error.
+   Use char because int might match the return type of a GCC
+   builtin and then its argument prototype would still apply.  */
+char recv ();
+int
+main (void)
+{
+return recv ();
+  ;
+  return 0;
+}
+_ACEOF
+for ac_lib in '' socket ws2_32
+do
+  if test -z "$ac_lib"; then
+    ac_res="none required"
+  else
+    ac_res=-l$ac_lib
+    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
+  fi
+  if ac_fn_c_try_link "$LINENO"
+then :
+  ac_cv_search_recv=$ac_res
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam \
+    conftest$ac_exeext
+  if test ${ac_cv_search_recv+y}
+then :
+  break
+fi
+done
+if test ${ac_cv_search_recv+y}
+then :
+
+else $as_nop
+  ac_cv_search_recv=no
+fi
+rm conftest.$ac_ext
+LIBS=$ac_func_search_save_LIBS
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_recv" >&5
+printf "%s\n" "$ac_cv_search_recv" >&6; }
+ac_res=$ac_cv_search_recv
+if test "$ac_res" != no
+then :
+  test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
+
+if test "$ac_cv_search_recv" != "none required"
+then
+  static_LIBS="$static_LIBS $ac_cv_search_recv"
+fi
+else $as_nop
+    { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for library containing recv using declaration" >&5
+printf %s "checking for library containing recv using declaration... " >&6; }
+   LIBS="-lws2_32 $LIBS"
+   cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+#include <winsock2.h>
+int
+main (void)
+{
+recv(0, 0, 0, 0);
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"
+then :
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: -lws2_32" >&5
+printf "%s\n" "-lws2_32" >&6; }
+      static_LIBS="$static_LIBS -lws2_32"
+else $as_nop
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5
+printf "%s\n" "no" >&6; }
+      cat > config.mk <<'EOF'
+ifneq ($(MAKECMDGOALS),distclean)
+$(error Resolve configure error first)
+endif
+EOF
+   as_fn_error $? "unable to find the recv() function" "$LINENO" 5
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam \
+    conftest$ac_exeext conftest.$ac_ext
+fi
+
+
+if test "$enable_bz2" != no; then
+  bz2_devel=ok
+  ac_fn_c_check_header_compile "$LINENO" "bzlib.h" "ac_cv_header_bzlib_h" ";
+"
+if test "x$ac_cv_header_bzlib_h" = xyes
+then :
+
+else $as_nop
+  bz2_devel=missing
+fi
+
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for BZ2_bzBuffToBuffCompress in -lbz2" >&5
+printf %s "checking for BZ2_bzBuffToBuffCompress in -lbz2... " >&6; }
+if test ${ac_cv_lib_bz2_BZ2_bzBuffToBuffCompress+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lbz2  $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+/* Override any GCC internal prototype to avoid an error.
+   Use char because int might match the return type of a GCC
+   builtin and then its argument prototype would still apply.  */
+char BZ2_bzBuffToBuffCompress ();
+int
+main (void)
+{
+return BZ2_bzBuffToBuffCompress ();
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"
+then :
+  ac_cv_lib_bz2_BZ2_bzBuffToBuffCompress=yes
+else $as_nop
+  ac_cv_lib_bz2_BZ2_bzBuffToBuffCompress=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam \
+    conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_bz2_BZ2_bzBuffToBuffCompress" >&5
+printf "%s\n" "$ac_cv_lib_bz2_BZ2_bzBuffToBuffCompress" >&6; }
+if test "x$ac_cv_lib_bz2_BZ2_bzBuffToBuffCompress" = xyes
+then :
+  printf "%s\n" "#define HAVE_LIBBZ2 1" >>confdefs.h
+
+  LIBS="-lbz2 $LIBS"
+
+else $as_nop
+  bz2_devel=missing
+fi
+
+  if test $bz2_devel != ok; then
+    cat > config.mk <<'EOF'
+ifneq ($(MAKECMDGOALS),distclean)
+$(error Resolve configure error first)
+endif
+EOF
+   as_fn_error $? "libbzip2 development files not found
+
+The CRAM format may use bzip2 compression, which is implemented in HTSlib
+by using compression routines from libbzip2 <http://www.bzip.org/>.
+
+Building HTSlib requires libbzip2 development files to be installed on the
+build machine; you may need to ensure a package such as libbz2-dev (on Debian
+or Ubuntu Linux) or bzip2-devel (on RPM-based Linux distributions or Cygwin)
+is installed.
+
+Either configure with --disable-bz2 (which will make some CRAM files
+produced elsewhere unreadable) or resolve this error to build HTSlib." "$LINENO" 5
+  fi
+  if test -n "$PKG_CONFIG" && "$PKG_CONFIG" --exists bzip2; then
+     pc_requires="$pc_requires bzip2"
+  else
+     private_LIBS="$private_LIBS -lbz2"
+  fi
+  static_LIBS="$static_LIBS -lbz2"
+fi
+
+if test "$enable_lzma" != no; then
+  lzma_devel=ok
+         for ac_header in lzma.h
+do :
+  ac_fn_c_check_header_compile "$LINENO" "lzma.h" "ac_cv_header_lzma_h" ";
+"
+if test "x$ac_cv_header_lzma_h" = xyes
+then :
+  printf "%s\n" "#define HAVE_LZMA_H 1" >>confdefs.h
+
+else $as_nop
+  lzma_devel=header-missing
+fi
+
+done
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for lzma_easy_buffer_encode in -llzma" >&5
+printf %s "checking for lzma_easy_buffer_encode in -llzma... " >&6; }
+if test ${ac_cv_lib_lzma_lzma_easy_buffer_encode+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-llzma  $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+/* Override any GCC internal prototype to avoid an error.
+   Use char because int might match the return type of a GCC
+   builtin and then its argument prototype would still apply.  */
+char lzma_easy_buffer_encode ();
+int
+main (void)
+{
+return lzma_easy_buffer_encode ();
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"
+then :
+  ac_cv_lib_lzma_lzma_easy_buffer_encode=yes
+else $as_nop
+  ac_cv_lib_lzma_lzma_easy_buffer_encode=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam \
+    conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_lzma_lzma_easy_buffer_encode" >&5
+printf "%s\n" "$ac_cv_lib_lzma_lzma_easy_buffer_encode" >&6; }
+if test "x$ac_cv_lib_lzma_lzma_easy_buffer_encode" = xyes
+then :
+  printf "%s\n" "#define HAVE_LIBLZMA 1" >>confdefs.h
+
+  LIBS="-llzma $LIBS"
+
+else $as_nop
+  lzma_devel=missing
+fi
+
+  if test $lzma_devel = missing; then
+    cat > config.mk <<'EOF'
+ifneq ($(MAKECMDGOALS),distclean)
+$(error Resolve configure error first)
+endif
+EOF
+   as_fn_error $? "liblzma development files not found
+
+The CRAM format may use LZMA2 compression, which is implemented in HTSlib
+by using compression routines from liblzma <http://tukaani.org/xz/>.
+
+Building HTSlib requires liblzma development files to be installed on the
+build machine; you may need to ensure a package such as liblzma-dev (on Debian
+or Ubuntu Linux), xz-devel (on RPM-based Linux distributions or Cygwin), or
+xz (via Homebrew on macOS) is installed; or build XZ Utils from source.
+
+Either configure with --disable-lzma (which will make some CRAM files
+produced elsewhere unreadable) or resolve this error to build HTSlib." "$LINENO" 5
+  fi
+  pc_requires="$pc_requires liblzma"
+  static_LIBS="$static_LIBS -llzma"
+fi
+
+if test "x$with_external_htscodecs" != "xno"
+then :
+  libhtscodecs=ok
+   ac_fn_c_check_header_compile "$LINENO" "htscodecs/rANS_static4x16.h" "ac_cv_header_htscodecs_rANS_static4x16_h" ";
+"
+if test "x$ac_cv_header_htscodecs_rANS_static4x16_h" = xyes
+then :
+
+else $as_nop
+  libhtscodecs='missing header'
+fi
+
+   { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for rans_compress_bound_4x16 in -lhtscodecs" >&5
+printf %s "checking for rans_compress_bound_4x16 in -lhtscodecs... " >&6; }
+if test ${ac_cv_lib_htscodecs_rans_compress_bound_4x16+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lhtscodecs  $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+/* Override any GCC internal prototype to avoid an error.
+   Use char because int might match the return type of a GCC
+   builtin and then its argument prototype would still apply.  */
+char rans_compress_bound_4x16 ();
+int
+main (void)
+{
+return rans_compress_bound_4x16 ();
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"
+then :
+  ac_cv_lib_htscodecs_rans_compress_bound_4x16=yes
+else $as_nop
+  ac_cv_lib_htscodecs_rans_compress_bound_4x16=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam \
+    conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_htscodecs_rans_compress_bound_4x16" >&5
+printf "%s\n" "$ac_cv_lib_htscodecs_rans_compress_bound_4x16" >&6; }
+if test "x$ac_cv_lib_htscodecs_rans_compress_bound_4x16" = xyes
+then :
+  :
+else $as_nop
+  libhtscodecs='missing library'
+fi
+
+   if test "$libhtscodecs" = "ok"
+then :
+
+printf "%s\n" "#define HAVE_EXTERNAL_LIBHTSCODECS 1" >>confdefs.h
+
+      LIBS="-lhtscodecs $LIBS"
+      private_LIBS="-lhtscodecs $private_LIBS"
+      static_LIBS="-lhtscodecs $static_LIBS"
+      selected_htscodecs_mk="htscodecs_external.mk"
+else $as_nop
+  cat > config.mk <<'EOF'
+ifneq ($(MAKECMDGOALS),distclean)
+$(error Resolve configure error first)
+endif
+EOF
+   as_fn_error $? "libhtscodecs development files not found: $libhtscodecs
+
+You asked to use an external htscodecs library, but do not have the
+required header / library files.  You either need to supply these and
+if necessary set CPPFLAGS and LDFLAGS so the compiler can find them;
+or configure using --without-external-htscodecs to build the required
+functions from the htscodecs submodule.
+" "$LINENO" 5
+fi
+else $as_nop
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether htscodecs files are present" >&5
+printf %s "checking whether htscodecs files are present... " >&6; }
+   if test -e "$srcdir/htscodecs/htscodecs/rANS_static4x16.h"
+then :
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+printf "%s\n" "yes" >&6; }
+      selected_htscodecs_mk="htscodecs_bundled.mk"
+else $as_nop
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5
+printf "%s\n" "no" >&6; }
+      if test -e "$srcdir/.git"
+then :
+  cat > config.mk <<'EOF'
+ifneq ($(MAKECMDGOALS),distclean)
+$(error Resolve configure error first)
+endif
+EOF
+   as_fn_error $? "htscodecs submodule files not present.
+
+HTSlib uses some functions from the htscodecs project, which is normally
+included as a submodule.  Try running:
+
+  git submodule update --init --recursive
+
+in  the top-level htslib directory to update it, and then re-run configure.
+" "$LINENO" 5
+else $as_nop
+  cat > config.mk <<'EOF'
+ifneq ($(MAKECMDGOALS),distclean)
+$(error Resolve configure error first)
+endif
+EOF
+   as_fn_error $? "htscodecs submodule files not present.
+
+You have an incomplete distribution.  Please try downloading one of the
+official releases from https://www.htslib.org
+" "$LINENO" 5
+fi
+fi
+fi
+
+if test "x$with_libdeflate" != "xno"
+then :
+  libdeflate=ok
+   ac_fn_c_check_header_compile "$LINENO" "libdeflate.h" "ac_cv_header_libdeflate_h" ";
+"
+if test "x$ac_cv_header_libdeflate_h" = xyes
+then :
+
+else $as_nop
+  libdeflate='missing header'
+fi
+
+   { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for libdeflate_deflate_compress in -ldeflate" >&5
+printf %s "checking for libdeflate_deflate_compress in -ldeflate... " >&6; }
+if test ${ac_cv_lib_deflate_libdeflate_deflate_compress+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-ldeflate  $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+/* Override any GCC internal prototype to avoid an error.
+   Use char because int might match the return type of a GCC
+   builtin and then its argument prototype would still apply.  */
+char libdeflate_deflate_compress ();
+int
+main (void)
+{
+return libdeflate_deflate_compress ();
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"
+then :
+  ac_cv_lib_deflate_libdeflate_deflate_compress=yes
+else $as_nop
+  ac_cv_lib_deflate_libdeflate_deflate_compress=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam \
+    conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_deflate_libdeflate_deflate_compress" >&5
+printf "%s\n" "$ac_cv_lib_deflate_libdeflate_deflate_compress" >&6; }
+if test "x$ac_cv_lib_deflate_libdeflate_deflate_compress" = xyes
+then :
+  :
+else $as_nop
+  libdeflate='missing library'
+fi
+
+   if test "$libdeflate" = "ok"
+then :
+
+printf "%s\n" "#define HAVE_LIBDEFLATE 1" >>confdefs.h
+
+     LIBS="-ldeflate $LIBS"
+     private_LIBS="$private_LIBS -ldeflate"
+     static_LIBS="$static_LIBS -ldeflate"
+else $as_nop
+  if test "x$with_libdeflate" != "xcheck"
+then :
+  cat > config.mk <<'EOF'
+ifneq ($(MAKECMDGOALS),distclean)
+$(error Resolve configure error first)
+endif
+EOF
+   as_fn_error $? "libdeflate development files not found: $libdeflate
+
+You requested libdeflate, but do not have the required header / library
+files.  The source for libdeflate is available from
+<https://github.com/ebiggers/libdeflate>.  You may have to adjust
+search paths in CPPFLAGS and/or LDFLAGS if the header and library
+are not currently on them.
+
+Either configure with --without-libdeflate or resolve this error to build
+HTSlib." "$LINENO" 5
+fi
+fi
+fi
+
+libcurl=disabled
+if test "$enable_libcurl" != no; then
+  libcurl_devel=ok
+  ac_fn_c_check_header_compile "$LINENO" "curl/curl.h" "ac_cv_header_curl_curl_h" ";
+"
+if test "x$ac_cv_header_curl_curl_h" = xyes
+then :
+
+else $as_nop
+  libcurl_devel="headers not found"
+fi
+
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curl_easy_pause in -lcurl" >&5
+printf %s "checking for curl_easy_pause in -lcurl... " >&6; }
+if test ${ac_cv_lib_curl_curl_easy_pause+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lcurl  $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+/* Override any GCC internal prototype to avoid an error.
+   Use char because int might match the return type of a GCC
+   builtin and then its argument prototype would still apply.  */
+char curl_easy_pause ();
+int
+main (void)
+{
+return curl_easy_pause ();
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"
+then :
+  ac_cv_lib_curl_curl_easy_pause=yes
+else $as_nop
+  ac_cv_lib_curl_curl_easy_pause=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam \
+    conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curl_curl_easy_pause" >&5
+printf "%s\n" "$ac_cv_lib_curl_curl_easy_pause" >&6; }
+if test "x$ac_cv_lib_curl_curl_easy_pause" = xyes
+then :
+  :
+else $as_nop
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curl_easy_init in -lcurl" >&5
+printf %s "checking for curl_easy_init in -lcurl... " >&6; }
+if test ${ac_cv_lib_curl_curl_easy_init+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lcurl  $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+/* Override any GCC internal prototype to avoid an error.
+   Use char because int might match the return type of a GCC
+   builtin and then its argument prototype would still apply.  */
+char curl_easy_init ();
+int
+main (void)
+{
+return curl_easy_init ();
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"
+then :
+  ac_cv_lib_curl_curl_easy_init=yes
+else $as_nop
+  ac_cv_lib_curl_curl_easy_init=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam \
+    conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curl_curl_easy_init" >&5
+printf "%s\n" "$ac_cv_lib_curl_curl_easy_init" >&6; }
+if test "x$ac_cv_lib_curl_curl_easy_init" = xyes
+then :
+  libcurl_devel="library is too old (7.18+ required)"
+else $as_nop
+  libcurl_devel="library not found"
+fi
+
+fi
+
+
+  if test "$libcurl_devel" = ok; then
+
+printf "%s\n" "#define HAVE_LIBCURL 1" >>confdefs.h
+
+    libcurl=enabled
+  elif test "$enable_libcurl" = check; then
+    { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: libcurl not enabled: $libcurl_devel" >&5
+printf "%s\n" "$as_me: WARNING: libcurl not enabled: $libcurl_devel" >&2;}
+  else
+    cat > config.mk <<'EOF'
+ifneq ($(MAKECMDGOALS),distclean)
+$(error Resolve configure error first)
+endif
+EOF
+   as_fn_error $? "libcurl $libcurl_devel
+
+Support for HTTPS and other SSL-based URLs requires routines from the libcurl
+library <http://curl.se/libcurl/>.  Building HTSlib with libcurl enabled
+requires libcurl development files to be installed on the build machine; you
+may need to ensure a package such as libcurl4-{gnutls,nss,openssl}-dev (on
+Debian or Ubuntu Linux) or libcurl-devel (on RPM-based Linux distributions
+or Cygwin) is installed.
+
+Either configure with --disable-libcurl or resolve this error to build HTSlib." "$LINENO" 5
+  fi
+
+  if test "$libcurl" = enabled ; then
+    if test "$enable_plugins" != yes ; then
+      static_LIBS="$static_LIBS -lcurl"
+    fi
+  fi
+fi
+
+
+gcs=disabled
+if test "$enable_gcs" != no; then
+  if test $libcurl = enabled; then
+
+printf "%s\n" "#define ENABLE_GCS 1" >>confdefs.h
+
+    gcs=enabled
+  else
+    case "$enable_gcs" in
+      check) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: GCS support not enabled: requires libcurl support" >&5
+printf "%s\n" "$as_me: WARNING: GCS support not enabled: requires libcurl support" >&2;} ;;
+      *) cat > config.mk <<'EOF'
+ifneq ($(MAKECMDGOALS),distclean)
+$(error Resolve configure error first)
+endif
+EOF
+   as_fn_error $? "GCS support not enabled
+
+Support for Google Cloud Storage URLs requires libcurl support to be enabled
+in HTSlib.  Configure with --enable-libcurl in order to use GCS URLs." "$LINENO" 5
+      ;;
+    esac
+  fi
+fi
+
+
+s3=disabled
+if test "$enable_s3" != no; then
+  if test $libcurl = enabled; then
+    s3=enabled
+    need_crypto="$enable_s3"
+  else
+    case "$enable_s3" in
+      check) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: S3 support not enabled: requires libcurl support" >&5
+printf "%s\n" "$as_me: WARNING: S3 support not enabled: requires libcurl support" >&2;} ;;
+      *) cat > config.mk <<'EOF'
+ifneq ($(MAKECMDGOALS),distclean)
+$(error Resolve configure error first)
+endif
+EOF
+   as_fn_error $? "S3 support not enabled
+
+Support for Amazon AWS S3 URLs requires libcurl support to be enabled
+in HTSlib.  Configure with --enable-libcurl in order to use S3 URLs." "$LINENO" 5
+      ;;
+    esac
+  fi
+fi
+
+CRYPTO_LIBS=
+if test $need_crypto != no; then
+  ac_fn_c_check_func "$LINENO" "CCHmac" "ac_cv_func_CCHmac"
+if test "x$ac_cv_func_CCHmac" = xyes
+then :
+
+printf "%s\n" "#define HAVE_COMMONCRYPTO 1" >>confdefs.h
+
+else $as_nop
+  save_LIBS=$LIBS
+     { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for library containing HMAC" >&5
+printf %s "checking for library containing HMAC... " >&6; }
+if test ${ac_cv_search_HMAC+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  ac_func_search_save_LIBS=$LIBS
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+/* Override any GCC internal prototype to avoid an error.
+   Use char because int might match the return type of a GCC
+   builtin and then its argument prototype would still apply.  */
+char HMAC ();
+int
+main (void)
+{
+return HMAC ();
+  ;
+  return 0;
+}
+_ACEOF
+for ac_lib in '' crypto
+do
+  if test -z "$ac_lib"; then
+    ac_res="none required"
+  else
+    ac_res=-l$ac_lib
+    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
+  fi
+  if ac_fn_c_try_link "$LINENO"
+then :
+  ac_cv_search_HMAC=$ac_res
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam \
+    conftest$ac_exeext
+  if test ${ac_cv_search_HMAC+y}
+then :
+  break
+fi
+done
+if test ${ac_cv_search_HMAC+y}
+then :
+
+else $as_nop
+  ac_cv_search_HMAC=no
+fi
+rm conftest.$ac_ext
+LIBS=$ac_func_search_save_LIBS
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_HMAC" >&5
+printf "%s\n" "$ac_cv_search_HMAC" >&6; }
+ac_res=$ac_cv_search_HMAC
+if test "$ac_res" != no
+then :
+  test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
+
+printf "%s\n" "#define HAVE_HMAC 1" >>confdefs.h
+
+        case "$ac_cv_search_HMAC" in
+          -l*) CRYPTO_LIBS=$ac_cv_search_HMAC ;;
+        esac
+else $as_nop
+  case "$need_crypto" in
+     check) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: S3 support not enabled: requires SSL development files" >&5
+printf "%s\n" "$as_me: WARNING: S3 support not enabled: requires SSL development files" >&2;}
+         s3=disabled ;;
+     *) cat > config.mk <<'EOF'
+ifneq ($(MAKECMDGOALS),distclean)
+$(error Resolve configure error first)
+endif
+EOF
+   as_fn_error $? "SSL development files not found
+
+Support for AWS S3 URLs requires routines from an SSL library.  Building
+HTSlib with libcurl enabled requires SSL development files to be installed
+on the build machine; you may need to ensure a package such as libgnutls-dev,
+libnss3-dev, or libssl-dev (on Debian or Ubuntu Linux, corresponding to the
+libcurl4-*-dev package installed), or openssl-devel (on RPM-based Linux
+distributions or Cygwin) is installed.
+
+Either configure with --disable-s3 or resolve this error to build HTSlib." "$LINENO" 5 ;;
+       esac
+fi
+
+     LIBS=$save_LIBS
+fi
+
+  if test "$enable_plugins" != yes ; then
+     static_LIBS="$static_LIBS $CRYPTO_LIBS"
+  fi
+fi
+
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for library containing regcomp" >&5
+printf %s "checking for library containing regcomp... " >&6; }
+if test ${ac_cv_search_regcomp+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  ac_func_search_save_LIBS=$LIBS
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+/* Override any GCC internal prototype to avoid an error.
+   Use char because int might match the return type of a GCC
+   builtin and then its argument prototype would still apply.  */
+char regcomp ();
+int
+main (void)
+{
+return regcomp ();
+  ;
+  return 0;
+}
+_ACEOF
+for ac_lib in '' regex
+do
+  if test -z "$ac_lib"; then
+    ac_res="none required"
+  else
+    ac_res=-l$ac_lib
+    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
+  fi
+  if ac_fn_c_try_link "$LINENO"
+then :
+  ac_cv_search_regcomp=$ac_res
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam \
+    conftest$ac_exeext
+  if test ${ac_cv_search_regcomp+y}
+then :
+  break
+fi
+done
+if test ${ac_cv_search_regcomp+y}
+then :
+
+else $as_nop
+  ac_cv_search_regcomp=no
+fi
+rm conftest.$ac_ext
+LIBS=$ac_func_search_save_LIBS
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_regcomp" >&5
+printf "%s\n" "$ac_cv_search_regcomp" >&6; }
+ac_res=$ac_cv_search_regcomp
+if test "$ac_res" != no
+then :
+  test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
+  libregex=needed
+fi
+
+
+
+if test "$s3" = enabled ; then
+
+printf "%s\n" "#define ENABLE_S3 1" >>confdefs.h
+
+fi
+
+if test "x$hts_late_cflags" != x
+then :
+  CFLAGS="$CFLAGS $hts_late_cflags"
+fi
+
+
+
+
+
+
+
+
+
+ac_config_files="$ac_config_files config.mk htslib.pc.tmp:htslib.pc.in"
+
+ac_config_links="$ac_config_links htscodecs.mk:$selected_htscodecs_mk"
+
+
+if test "$srcdir" != .; then
+  # Set up for a separate build directory. As HTSlib uses a non-recursive
+  # makefile, we need to create additional build subdirectories explicitly.
+  ac_config_links="$ac_config_links Makefile:Makefile htslib.mk:htslib.mk"
+
+  ac_config_files="$ac_config_files htslib_vars.mk:builddir_vars.mk.in"
+
+  ac_config_commands="$ac_config_commands mkdir"
+
+fi
+
+# @HTSDIRslash_if_relsrcdir@ will be empty when $srcdir is absolute
+case "$srcdir" in
+  /*) HTSDIRslash_if_relsrcdir= ;;
+  *)  HTSDIRslash_if_relsrcdir='$(HTSDIR)/' ;;
+esac
+
+
+cat >confcache <<\_ACEOF
+# This file is a shell script that caches the results of configure
+# tests run on this system so they can be shared between configure
+# scripts and configure runs, see configure's option --config-cache.
+# It is not useful on other systems.  If it contains results you don't
+# want to keep, you may remove or edit it.
+#
+# config.status only pays attention to the cache file if you give it
+# the --recheck option to rerun configure.
+#
+# `ac_cv_env_foo' variables (set or unset) will be overridden when
+# loading this file, other *unset* `ac_cv_foo' will be assigned the
+# following values.
+
+_ACEOF
+
+# The following way of writing the cache mishandles newlines in values,
+# but we know of no workaround that is simple, portable, and efficient.
+# So, we kill variables containing newlines.
+# Ultrix sh set writes to stderr and can't be redirected directly,
+# and sets the high bit in the cache file unless we assign to the vars.
+(
+  for ac_var in `(set) 2>&1 | sed -n 's/^\([a-zA-Z_][a-zA-Z0-9_]*\)=.*/\1/p'`; do
+    eval ac_val=\$$ac_var
+    case $ac_val in #(
+    *${as_nl}*)
+      case $ac_var in #(
+      *_cv_*) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: cache variable $ac_var contains a newline" >&5
+printf "%s\n" "$as_me: WARNING: cache variable $ac_var contains a newline" >&2;} ;;
+      esac
+      case $ac_var in #(
+      _ | IFS | as_nl) ;; #(
+      BASH_ARGV | BASH_SOURCE) eval $ac_var= ;; #(
+      *) { eval $ac_var=; unset $ac_var;} ;;
+      esac ;;
+    esac
+  done
+
+  (set) 2>&1 |
+    case $as_nl`(ac_space=' '; set) 2>&1` in #(
+    *${as_nl}ac_space=\ *)
+      # `set' does not quote correctly, so add quotes: double-quote
+      # substitution turns \\\\ into \\, and sed turns \\ into \.
+      sed -n \
+	"s/'/'\\\\''/g;
+	  s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='\\2'/p"
+      ;; #(
+    *)
+      # `set' quotes correctly as required by POSIX, so do not add quotes.
+      sed -n "/^[_$as_cr_alnum]*_cv_[_$as_cr_alnum]*=/p"
+      ;;
+    esac |
+    sort
+) |
+  sed '
+     /^ac_cv_env_/b end
+     t clear
+     :clear
+     s/^\([^=]*\)=\(.*[{}].*\)$/test ${\1+y} || &/
+     t end
+     s/^\([^=]*\)=\(.*\)$/\1=${\1=\2}/
+     :end' >>confcache
+if diff "$cache_file" confcache >/dev/null 2>&1; then :; else
+  if test -w "$cache_file"; then
+    if test "x$cache_file" != "x/dev/null"; then
+      { printf "%s\n" "$as_me:${as_lineno-$LINENO}: updating cache $cache_file" >&5
+printf "%s\n" "$as_me: updating cache $cache_file" >&6;}
+      if test ! -f "$cache_file" || test -h "$cache_file"; then
+	cat confcache >"$cache_file"
+      else
+        case $cache_file in #(
+        */* | ?:*)
+	  mv -f confcache "$cache_file"$$ &&
+	  mv -f "$cache_file"$$ "$cache_file" ;; #(
+        *)
+	  mv -f confcache "$cache_file" ;;
+	esac
+      fi
+    fi
+  else
+    { printf "%s\n" "$as_me:${as_lineno-$LINENO}: not updating unwritable cache $cache_file" >&5
+printf "%s\n" "$as_me: not updating unwritable cache $cache_file" >&6;}
+  fi
+fi
+rm -f confcache
+
+test "x$prefix" = xNONE && prefix=$ac_default_prefix
+# Let make expand exec_prefix.
+test "x$exec_prefix" = xNONE && exec_prefix='${prefix}'
+
+DEFS=-DHAVE_CONFIG_H
+
+ac_libobjs=
+ac_ltlibobjs=
+U=
+for ac_i in : $LIBOBJS; do test "x$ac_i" = x: && continue
+  # 1. Remove the extension, and $U if already installed.
+  ac_script='s/\$U\././;s/\.o$//;s/\.obj$//'
+  ac_i=`printf "%s\n" "$ac_i" | sed "$ac_script"`
+  # 2. Prepend LIBOBJDIR.  When used with automake>=1.10 LIBOBJDIR
+  #    will be set to the directory where LIBOBJS objects are built.
+  as_fn_append ac_libobjs " \${LIBOBJDIR}$ac_i\$U.$ac_objext"
+  as_fn_append ac_ltlibobjs " \${LIBOBJDIR}$ac_i"'$U.lo'
+done
+LIBOBJS=$ac_libobjs
+
+LTLIBOBJS=$ac_ltlibobjs
+
+
+
+: "${CONFIG_STATUS=./config.status}"
+ac_write_fail=0
+ac_clean_files_save=$ac_clean_files
+ac_clean_files="$ac_clean_files $CONFIG_STATUS"
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: creating $CONFIG_STATUS" >&5
+printf "%s\n" "$as_me: creating $CONFIG_STATUS" >&6;}
+as_write_fail=0
+cat >$CONFIG_STATUS <<_ASEOF || as_write_fail=1
+#! $SHELL
+# Generated by $as_me.
+# Run this file to recreate the current configuration.
+# Compiler output produced by configure, useful for debugging
+# configure, is in config.log if it exists.
+
+debug=false
+ac_cs_recheck=false
+ac_cs_silent=false
+
+SHELL=\${CONFIG_SHELL-$SHELL}
+export SHELL
+_ASEOF
+cat >>$CONFIG_STATUS <<\_ASEOF || as_write_fail=1
+## -------------------- ##
+## M4sh Initialization. ##
+## -------------------- ##
+
+# Be more Bourne compatible
+DUALCASE=1; export DUALCASE # for MKS sh
+as_nop=:
+if test ${ZSH_VERSION+y} && (emulate sh) >/dev/null 2>&1
+then :
+  emulate sh
+  NULLCMD=:
+  # Pre-4.2 versions of Zsh do word splitting on ${1+"$@"}, which
+  # is contrary to our usage.  Disable this feature.
+  alias -g '${1+"$@"}'='"$@"'
+  setopt NO_GLOB_SUBST
+else $as_nop
+  case `(set -o) 2>/dev/null` in #(
+  *posix*) :
+    set -o posix ;; #(
+  *) :
+     ;;
+esac
+fi
+
+
+
+# Reset variables that may have inherited troublesome values from
+# the environment.
+
+# IFS needs to be set, to space, tab, and newline, in precisely that order.
+# (If _AS_PATH_WALK were called with IFS unset, it would have the
+# side effect of setting IFS to empty, thus disabling word splitting.)
+# Quoting is to prevent editors from complaining about space-tab.
+as_nl='
+'
+export as_nl
+IFS=" ""	$as_nl"
+
+PS1='$ '
+PS2='> '
+PS4='+ '
+
+# Ensure predictable behavior from utilities with locale-dependent output.
+LC_ALL=C
+export LC_ALL
+LANGUAGE=C
+export LANGUAGE
+
+# We cannot yet rely on "unset" to work, but we need these variables
+# to be unset--not just set to an empty or harmless value--now, to
+# avoid bugs in old shells (e.g. pre-3.0 UWIN ksh).  This construct
+# also avoids known problems related to "unset" and subshell syntax
+# in other old shells (e.g. bash 2.01 and pdksh 5.2.14).
+for as_var in BASH_ENV ENV MAIL MAILPATH CDPATH
+do eval test \${$as_var+y} \
+  && ( (unset $as_var) || exit 1) >/dev/null 2>&1 && unset $as_var || :
+done
+
+# Ensure that fds 0, 1, and 2 are open.
+if (exec 3>&0) 2>/dev/null; then :; else exec 0</dev/null; fi
+if (exec 3>&1) 2>/dev/null; then :; else exec 1>/dev/null; fi
+if (exec 3>&2)            ; then :; else exec 2>/dev/null; fi
+
+# The user is always right.
+if ${PATH_SEPARATOR+false} :; then
+  PATH_SEPARATOR=:
+  (PATH='/bin;/bin'; FPATH=$PATH; sh -c :) >/dev/null 2>&1 && {
+    (PATH='/bin:/bin'; FPATH=$PATH; sh -c :) >/dev/null 2>&1 ||
+      PATH_SEPARATOR=';'
+  }
+fi
+
+
+# Find who we are.  Look in the path if we contain no directory separator.
+as_myself=
+case $0 in #((
+  *[\\/]* ) as_myself=$0 ;;
+  *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  case $as_dir in #(((
+    '') as_dir=./ ;;
+    */) ;;
+    *) as_dir=$as_dir/ ;;
+  esac
+    test -r "$as_dir$0" && as_myself=$as_dir$0 && break
+  done
+IFS=$as_save_IFS
+
+     ;;
+esac
+# We did not find ourselves, most probably we were run as `sh COMMAND'
+# in which case we are not to be found in the path.
+if test "x$as_myself" = x; then
+  as_myself=$0
+fi
+if test ! -f "$as_myself"; then
+  printf "%s\n" "$as_myself: error: cannot find myself; rerun with an absolute file name" >&2
+  exit 1
+fi
+
+
+
+# as_fn_error STATUS ERROR [LINENO LOG_FD]
+# ----------------------------------------
+# Output "`basename $0`: error: ERROR" to stderr. If LINENO and LOG_FD are
+# provided, also output the error to LOG_FD, referencing LINENO. Then exit the
+# script with STATUS, using 1 if that was 0.
+as_fn_error ()
+{
+  as_status=$1; test $as_status -eq 0 && as_status=1
+  if test "$4"; then
+    as_lineno=${as_lineno-"$3"} as_lineno_stack=as_lineno_stack=$as_lineno_stack
+    printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: $2" >&$4
+  fi
+  printf "%s\n" "$as_me: error: $2" >&2
+  as_fn_exit $as_status
+} # as_fn_error
+
+
+
+# as_fn_set_status STATUS
+# -----------------------
+# Set $? to STATUS, without forking.
+as_fn_set_status ()
+{
+  return $1
+} # as_fn_set_status
+
+# as_fn_exit STATUS
+# -----------------
+# Exit the shell with STATUS, even in a "trap 0" or "set -e" context.
+as_fn_exit ()
+{
+  set +e
+  as_fn_set_status $1
+  exit $1
+} # as_fn_exit
+
+# as_fn_unset VAR
+# ---------------
+# Portably unset VAR.
+as_fn_unset ()
+{
+  { eval $1=; unset $1;}
+}
+as_unset=as_fn_unset
+
+# as_fn_append VAR VALUE
+# ----------------------
+# Append the text in VALUE to the end of the definition contained in VAR. Take
+# advantage of any shell optimizations that allow amortized linear growth over
+# repeated appends, instead of the typical quadratic growth present in naive
+# implementations.
+if (eval "as_var=1; as_var+=2; test x\$as_var = x12") 2>/dev/null
+then :
+  eval 'as_fn_append ()
+  {
+    eval $1+=\$2
+  }'
+else $as_nop
+  as_fn_append ()
+  {
+    eval $1=\$$1\$2
+  }
+fi # as_fn_append
+
+# as_fn_arith ARG...
+# ------------------
+# Perform arithmetic evaluation on the ARGs, and store the result in the
+# global $as_val. Take advantage of shells that can avoid forks. The arguments
+# must be portable across $(()) and expr.
+if (eval "test \$(( 1 + 1 )) = 2") 2>/dev/null
+then :
+  eval 'as_fn_arith ()
+  {
+    as_val=$(( $* ))
+  }'
+else $as_nop
+  as_fn_arith ()
+  {
+    as_val=`expr "$@" || test $? -eq 1`
+  }
+fi # as_fn_arith
+
+
+if expr a : '\(a\)' >/dev/null 2>&1 &&
+   test "X`expr 00001 : '.*\(...\)'`" = X001; then
+  as_expr=expr
+else
+  as_expr=false
+fi
+
+if (basename -- /) >/dev/null 2>&1 && test "X`basename -- / 2>&1`" = "X/"; then
+  as_basename=basename
+else
+  as_basename=false
+fi
+
+if (as_dir=`dirname -- /` && test "X$as_dir" = X/) >/dev/null 2>&1; then
+  as_dirname=dirname
+else
+  as_dirname=false
+fi
+
+as_me=`$as_basename -- "$0" ||
+$as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \
+	 X"$0" : 'X\(//\)$' \| \
+	 X"$0" : 'X\(/\)' \| . 2>/dev/null ||
+printf "%s\n" X/"$0" |
+    sed '/^.*\/\([^/][^/]*\)\/*$/{
+	    s//\1/
+	    q
+	  }
+	  /^X\/\(\/\/\)$/{
+	    s//\1/
+	    q
+	  }
+	  /^X\/\(\/\).*/{
+	    s//\1/
+	    q
+	  }
+	  s/.*/./; q'`
+
+# Avoid depending upon Character Ranges.
+as_cr_letters='abcdefghijklmnopqrstuvwxyz'
+as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+as_cr_Letters=$as_cr_letters$as_cr_LETTERS
+as_cr_digits='0123456789'
+as_cr_alnum=$as_cr_Letters$as_cr_digits
+
+
+# Determine whether it's possible to make 'echo' print without a newline.
+# These variables are no longer used directly by Autoconf, but are AC_SUBSTed
+# for compatibility with existing Makefiles.
+ECHO_C= ECHO_N= ECHO_T=
+case `echo -n x` in #(((((
+-n*)
+  case `echo 'xy\c'` in
+  *c*) ECHO_T='	';;	# ECHO_T is single tab character.
+  xy)  ECHO_C='\c';;
+  *)   echo `echo ksh88 bug on AIX 6.1` > /dev/null
+       ECHO_T='	';;
+  esac;;
+*)
+  ECHO_N='-n';;
+esac
+
+# For backward compatibility with old third-party macros, we provide
+# the shell variables $as_echo and $as_echo_n.  New code should use
+# AS_ECHO(["message"]) and AS_ECHO_N(["message"]), respectively.
+as_echo='printf %s\n'
+as_echo_n='printf %s'
+
+rm -f conf$$ conf$$.exe conf$$.file
+if test -d conf$$.dir; then
+  rm -f conf$$.dir/conf$$.file
+else
+  rm -f conf$$.dir
+  mkdir conf$$.dir 2>/dev/null
+fi
+if (echo >conf$$.file) 2>/dev/null; then
+  if ln -s conf$$.file conf$$ 2>/dev/null; then
+    as_ln_s='ln -s'
+    # ... but there are two gotchas:
+    # 1) On MSYS, both `ln -s file dir' and `ln file dir' fail.
+    # 2) DJGPP < 2.04 has no symlinks; `ln -s' creates a wrapper executable.
+    # In both cases, we have to default to `cp -pR'.
+    ln -s conf$$.file conf$$.dir 2>/dev/null && test ! -f conf$$.exe ||
+      as_ln_s='cp -pR'
+  elif ln conf$$.file conf$$ 2>/dev/null; then
+    as_ln_s=ln
+  else
+    as_ln_s='cp -pR'
+  fi
+else
+  as_ln_s='cp -pR'
+fi
+rm -f conf$$ conf$$.exe conf$$.dir/conf$$.file conf$$.file
+rmdir conf$$.dir 2>/dev/null
+
+
+# as_fn_mkdir_p
+# -------------
+# Create "$as_dir" as a directory, including parents if necessary.
+as_fn_mkdir_p ()
+{
+
+  case $as_dir in #(
+  -*) as_dir=./$as_dir;;
+  esac
+  test -d "$as_dir" || eval $as_mkdir_p || {
+    as_dirs=
+    while :; do
+      case $as_dir in #(
+      *\'*) as_qdir=`printf "%s\n" "$as_dir" | sed "s/'/'\\\\\\\\''/g"`;; #'(
+      *) as_qdir=$as_dir;;
+      esac
+      as_dirs="'$as_qdir' $as_dirs"
+      as_dir=`$as_dirname -- "$as_dir" ||
+$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$as_dir" : 'X\(//\)[^/]' \| \
+	 X"$as_dir" : 'X\(//\)$' \| \
+	 X"$as_dir" : 'X\(/\)' \| . 2>/dev/null ||
+printf "%s\n" X"$as_dir" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{
+	    s//\1/
+	    q
+	  }
+	  /^X\(\/\/\)[^/].*/{
+	    s//\1/
+	    q
+	  }
+	  /^X\(\/\/\)$/{
+	    s//\1/
+	    q
+	  }
+	  /^X\(\/\).*/{
+	    s//\1/
+	    q
+	  }
+	  s/.*/./; q'`
+      test -d "$as_dir" && break
+    done
+    test -z "$as_dirs" || eval "mkdir $as_dirs"
+  } || test -d "$as_dir" || as_fn_error $? "cannot create directory $as_dir"
+
+
+} # as_fn_mkdir_p
+if mkdir -p . 2>/dev/null; then
+  as_mkdir_p='mkdir -p "$as_dir"'
+else
+  test -d ./-p && rmdir ./-p
+  as_mkdir_p=false
+fi
+
+
+# as_fn_executable_p FILE
+# -----------------------
+# Test if FILE is an executable regular file.
+as_fn_executable_p ()
+{
+  test -f "$1" && test -x "$1"
+} # as_fn_executable_p
+as_test_x='test -x'
+as_executable_p=as_fn_executable_p
+
+# Sed expression to map a string onto a valid CPP name.
+as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'"
+
+# Sed expression to map a string onto a valid variable name.
+as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'"
+
+
+exec 6>&1
+## ----------------------------------- ##
+## Main body of $CONFIG_STATUS script. ##
+## ----------------------------------- ##
+_ASEOF
+test $as_write_fail = 0 && chmod +x $CONFIG_STATUS || ac_write_fail=1
+
+cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
+# Save the log message, to keep $0 and so on meaningful, and to
+# report actual input values of CONFIG_FILES etc. instead of their
+# values after options handling.
+ac_log="
+This file was extended by HTSlib $as_me 1.21, which was
+generated by GNU Autoconf 2.71.  Invocation command line was
+
+  CONFIG_FILES    = $CONFIG_FILES
+  CONFIG_HEADERS  = $CONFIG_HEADERS
+  CONFIG_LINKS    = $CONFIG_LINKS
+  CONFIG_COMMANDS = $CONFIG_COMMANDS
+  $ $0 $@
+
+on `(hostname || uname -n) 2>/dev/null | sed 1q`
+"
+
+_ACEOF
+
+case $ac_config_files in *"
+"*) set x $ac_config_files; shift; ac_config_files=$*;;
+esac
+
+case $ac_config_headers in *"
+"*) set x $ac_config_headers; shift; ac_config_headers=$*;;
+esac
+
+
+cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
+# Files that config.status was made for.
+config_files="$ac_config_files"
+config_headers="$ac_config_headers"
+config_links="$ac_config_links"
+config_commands="$ac_config_commands"
+
+_ACEOF
+
+cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
+ac_cs_usage="\
+\`$as_me' instantiates files and other configuration actions
+from templates according to the current configuration.  Unless the files
+and actions are specified as TAGs, all are instantiated by default.
+
+Usage: $0 [OPTION]... [TAG]...
+
+  -h, --help       print this help, then exit
+  -V, --version    print version number and configuration settings, then exit
+      --config     print configuration, then exit
+  -q, --quiet, --silent
+                   do not print progress messages
+  -d, --debug      don't remove temporary files
+      --recheck    update $as_me by reconfiguring in the same conditions
+      --file=FILE[:TEMPLATE]
+                   instantiate the configuration file FILE
+      --header=FILE[:TEMPLATE]
+                   instantiate the configuration header FILE
+
+Configuration files:
+$config_files
+
+Configuration headers:
+$config_headers
+
+Configuration links:
+$config_links
+
+Configuration commands:
+$config_commands
+
+Report bugs to <samtools-help@lists.sourceforge.net>.
+HTSlib home page: <http://www.htslib.org/>."
+
+_ACEOF
+ac_cs_config=`printf "%s\n" "$ac_configure_args" | sed "$ac_safe_unquote"`
+ac_cs_config_escaped=`printf "%s\n" "$ac_cs_config" | sed "s/^ //; s/'/'\\\\\\\\''/g"`
+cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
+ac_cs_config='$ac_cs_config_escaped'
+ac_cs_version="\\
+HTSlib config.status 1.21
+configured by $0, generated by GNU Autoconf 2.71,
+  with options \\"\$ac_cs_config\\"
+
+Copyright (C) 2021 Free Software Foundation, Inc.
+This config.status script is free software; the Free Software Foundation
+gives unlimited permission to copy, distribute and modify it."
+
+ac_pwd='$ac_pwd'
+srcdir='$srcdir'
+test -n "\$AWK" || AWK=awk
+_ACEOF
+
+cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
+# The default lists apply if the user does not specify any file.
+ac_need_defaults=:
+while test $# != 0
+do
+  case $1 in
+  --*=?*)
+    ac_option=`expr "X$1" : 'X\([^=]*\)='`
+    ac_optarg=`expr "X$1" : 'X[^=]*=\(.*\)'`
+    ac_shift=:
+    ;;
+  --*=)
+    ac_option=`expr "X$1" : 'X\([^=]*\)='`
+    ac_optarg=
+    ac_shift=:
+    ;;
+  *)
+    ac_option=$1
+    ac_optarg=$2
+    ac_shift=shift
+    ;;
+  esac
+
+  case $ac_option in
+  # Handling of the options.
+  -recheck | --recheck | --rechec | --reche | --rech | --rec | --re | --r)
+    ac_cs_recheck=: ;;
+  --version | --versio | --versi | --vers | --ver | --ve | --v | -V )
+    printf "%s\n" "$ac_cs_version"; exit ;;
+  --config | --confi | --conf | --con | --co | --c )
+    printf "%s\n" "$ac_cs_config"; exit ;;
+  --debug | --debu | --deb | --de | --d | -d )
+    debug=: ;;
+  --file | --fil | --fi | --f )
+    $ac_shift
+    case $ac_optarg in
+    *\'*) ac_optarg=`printf "%s\n" "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"` ;;
+    '') as_fn_error $? "missing file argument" ;;
+    esac
+    as_fn_append CONFIG_FILES " '$ac_optarg'"
+    ac_need_defaults=false;;
+  --header | --heade | --head | --hea )
+    $ac_shift
+    case $ac_optarg in
+    *\'*) ac_optarg=`printf "%s\n" "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"` ;;
+    esac
+    as_fn_append CONFIG_HEADERS " '$ac_optarg'"
+    ac_need_defaults=false;;
+  --he | --h)
+    # Conflict between --help and --header
+    as_fn_error $? "ambiguous option: \`$1'
+Try \`$0 --help' for more information.";;
+  --help | --hel | -h )
+    printf "%s\n" "$ac_cs_usage"; exit ;;
+  -q | -quiet | --quiet | --quie | --qui | --qu | --q \
+  | -silent | --silent | --silen | --sile | --sil | --si | --s)
+    ac_cs_silent=: ;;
+
+  # This is an error.
+  -*) as_fn_error $? "unrecognized option: \`$1'
+Try \`$0 --help' for more information." ;;
+
+  *) as_fn_append ac_config_targets " $1"
+     ac_need_defaults=false ;;
+
+  esac
+  shift
+done
+
+ac_configure_extra_args=
+
+if $ac_cs_silent; then
+  exec 6>/dev/null
+  ac_configure_extra_args="$ac_configure_extra_args --silent"
+fi
+
+_ACEOF
+cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
+if \$ac_cs_recheck; then
+  set X $SHELL '$0' $ac_configure_args \$ac_configure_extra_args --no-create --no-recursion
+  shift
+  \printf "%s\n" "running CONFIG_SHELL=$SHELL \$*" >&6
+  CONFIG_SHELL='$SHELL'
+  export CONFIG_SHELL
+  exec "\$@"
+fi
+
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
+exec 5>>config.log
+{
+  echo
+  sed 'h;s/./-/g;s/^.../## /;s/...$/ ##/;p;x;p;x' <<_ASBOX
+## Running $as_me. ##
+_ASBOX
+  printf "%s\n" "$ac_log"
+} >&5
+
+_ACEOF
+cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
+_ACEOF
+
+cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
+
+# Handling of arguments.
+for ac_config_target in $ac_config_targets
+do
+  case $ac_config_target in
+    "config.h") CONFIG_HEADERS="$CONFIG_HEADERS config.h" ;;
+    "config.mk") CONFIG_FILES="$CONFIG_FILES config.mk" ;;
+    "htslib.pc.tmp") CONFIG_FILES="$CONFIG_FILES htslib.pc.tmp:htslib.pc.in" ;;
+    "htscodecs.mk") CONFIG_LINKS="$CONFIG_LINKS htscodecs.mk:$selected_htscodecs_mk" ;;
+    "Makefile") CONFIG_LINKS="$CONFIG_LINKS Makefile:Makefile" ;;
+    "htslib.mk") CONFIG_LINKS="$CONFIG_LINKS htslib.mk:htslib.mk" ;;
+    "htslib_vars.mk") CONFIG_FILES="$CONFIG_FILES htslib_vars.mk:builddir_vars.mk.in" ;;
+    "mkdir") CONFIG_COMMANDS="$CONFIG_COMMANDS mkdir" ;;
+
+  *) as_fn_error $? "invalid argument: \`$ac_config_target'" "$LINENO" 5;;
+  esac
+done
+
+
+# If the user did not use the arguments to specify the items to instantiate,
+# then the envvar interface is used.  Set only those that are not.
+# We use the long form for the default assignment because of an extremely
+# bizarre bug on SunOS 4.1.3.
+if $ac_need_defaults; then
+  test ${CONFIG_FILES+y} || CONFIG_FILES=$config_files
+  test ${CONFIG_HEADERS+y} || CONFIG_HEADERS=$config_headers
+  test ${CONFIG_LINKS+y} || CONFIG_LINKS=$config_links
+  test ${CONFIG_COMMANDS+y} || CONFIG_COMMANDS=$config_commands
+fi
+
+# Have a temporary directory for convenience.  Make it in the build tree
+# simply because there is no reason against having it here, and in addition,
+# creating and moving files from /tmp can sometimes cause problems.
+# Hook for its removal unless debugging.
+# Note that there is a small window in which the directory will not be cleaned:
+# after its creation but before its name has been assigned to `$tmp'.
+$debug ||
+{
+  tmp= ac_tmp=
+  trap 'exit_status=$?
+  : "${ac_tmp:=$tmp}"
+  { test ! -d "$ac_tmp" || rm -fr "$ac_tmp"; } && exit $exit_status
+' 0
+  trap 'as_fn_exit 1' 1 2 13 15
+}
+# Create a (secure) tmp directory for tmp files.
+
+{
+  tmp=`(umask 077 && mktemp -d "./confXXXXXX") 2>/dev/null` &&
+  test -d "$tmp"
+}  ||
+{
+  tmp=./conf$$-$RANDOM
+  (umask 077 && mkdir "$tmp")
+} || as_fn_error $? "cannot create a temporary directory in ." "$LINENO" 5
+ac_tmp=$tmp
+
+# Set up the scripts for CONFIG_FILES section.
+# No need to generate them if there are no CONFIG_FILES.
+# This happens for instance with `./config.status config.h'.
+if test -n "$CONFIG_FILES"; then
+
+
+ac_cr=`echo X | tr X '\015'`
+# On cygwin, bash can eat \r inside `` if the user requested igncr.
+# But we know of no other shell where ac_cr would be empty at this
+# point, so we can use a bashism as a fallback.
+if test "x$ac_cr" = x; then
+  eval ac_cr=\$\'\\r\'
+fi
+ac_cs_awk_cr=`$AWK 'BEGIN { print "a\rb" }' </dev/null 2>/dev/null`
+if test "$ac_cs_awk_cr" = "a${ac_cr}b"; then
+  ac_cs_awk_cr='\\r'
+else
+  ac_cs_awk_cr=$ac_cr
+fi
+
+echo 'BEGIN {' >"$ac_tmp/subs1.awk" &&
+_ACEOF
+
+
+{
+  echo "cat >conf$$subs.awk <<_ACEOF" &&
+  echo "$ac_subst_vars" | sed 's/.*/&!$&$ac_delim/' &&
+  echo "_ACEOF"
+} >conf$$subs.sh ||
+  as_fn_error $? "could not make $CONFIG_STATUS" "$LINENO" 5
+ac_delim_num=`echo "$ac_subst_vars" | grep -c '^'`
+ac_delim='%!_!# '
+for ac_last_try in false false false false false :; do
+  . ./conf$$subs.sh ||
+    as_fn_error $? "could not make $CONFIG_STATUS" "$LINENO" 5
+
+  ac_delim_n=`sed -n "s/.*$ac_delim\$/X/p" conf$$subs.awk | grep -c X`
+  if test $ac_delim_n = $ac_delim_num; then
+    break
+  elif $ac_last_try; then
+    as_fn_error $? "could not make $CONFIG_STATUS" "$LINENO" 5
+  else
+    ac_delim="$ac_delim!$ac_delim _$ac_delim!! "
+  fi
+done
+rm -f conf$$subs.sh
+
+cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
+cat >>"\$ac_tmp/subs1.awk" <<\\_ACAWK &&
+_ACEOF
+sed -n '
+h
+s/^/S["/; s/!.*/"]=/
+p
+g
+s/^[^!]*!//
+:repl
+t repl
+s/'"$ac_delim"'$//
+t delim
+:nl
+h
+s/\(.\{148\}\)..*/\1/
+t more1
+s/["\\]/\\&/g; s/^/"/; s/$/\\n"\\/
+p
+n
+b repl
+:more1
+s/["\\]/\\&/g; s/^/"/; s/$/"\\/
+p
+g
+s/.\{148\}//
+t nl
+:delim
+h
+s/\(.\{148\}\)..*/\1/
+t more2
+s/["\\]/\\&/g; s/^/"/; s/$/"/
+p
+b
+:more2
+s/["\\]/\\&/g; s/^/"/; s/$/"\\/
+p
+g
+s/.\{148\}//
+t delim
+' <conf$$subs.awk | sed '
+/^[^""]/{
+  N
+  s/\n//
+}
+' >>$CONFIG_STATUS || ac_write_fail=1
+rm -f conf$$subs.awk
+cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
+_ACAWK
+cat >>"\$ac_tmp/subs1.awk" <<_ACAWK &&
+  for (key in S) S_is_set[key] = 1
+  FS = ""
+
+}
+{
+  line = $ 0
+  nfields = split(line, field, "@")
+  substed = 0
+  len = length(field[1])
+  for (i = 2; i < nfields; i++) {
+    key = field[i]
+    keylen = length(key)
+    if (S_is_set[key]) {
+      value = S[key]
+      line = substr(line, 1, len) "" value "" substr(line, len + keylen + 3)
+      len += length(value) + length(field[++i])
+      substed = 1
+    } else
+      len += 1 + keylen
+  }
+
+  print line
+}
+
+_ACAWK
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
+if sed "s/$ac_cr//" < /dev/null > /dev/null 2>&1; then
+  sed "s/$ac_cr\$//; s/$ac_cr/$ac_cs_awk_cr/g"
+else
+  cat
+fi < "$ac_tmp/subs1.awk" > "$ac_tmp/subs.awk" \
+  || as_fn_error $? "could not setup config files machinery" "$LINENO" 5
+_ACEOF
+
+# VPATH may cause trouble with some makes, so we remove sole $(srcdir),
+# ${srcdir} and @srcdir@ entries from VPATH if srcdir is ".", strip leading and
+# trailing colons and then remove the whole line if VPATH becomes empty
+# (actually we leave an empty line to preserve line numbers).
+if test "x$srcdir" = x.; then
+  ac_vpsub='/^[	 ]*VPATH[	 ]*=[	 ]*/{
+h
+s///
+s/^/:/
+s/[	 ]*$/:/
+s/:\$(srcdir):/:/g
+s/:\${srcdir}:/:/g
+s/:@srcdir@:/:/g
+s/^:*//
+s/:*$//
+x
+s/\(=[	 ]*\).*/\1/
+G
+s/\n//
+s/^[^=]*=[	 ]*$//
+}'
+fi
+
+cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
+fi # test -n "$CONFIG_FILES"
+
+# Set up the scripts for CONFIG_HEADERS section.
+# No need to generate them if there are no CONFIG_HEADERS.
+# This happens for instance with `./config.status Makefile'.
+if test -n "$CONFIG_HEADERS"; then
+cat >"$ac_tmp/defines.awk" <<\_ACAWK ||
+BEGIN {
+_ACEOF
+
+# Transform confdefs.h into an awk script `defines.awk', embedded as
+# here-document in config.status, that substitutes the proper values into
+# config.h.in to produce config.h.
+
+# Create a delimiter string that does not exist in confdefs.h, to ease
+# handling of long lines.
+ac_delim='%!_!# '
+for ac_last_try in false false :; do
+  ac_tt=`sed -n "/$ac_delim/p" confdefs.h`
+  if test -z "$ac_tt"; then
+    break
+  elif $ac_last_try; then
+    as_fn_error $? "could not make $CONFIG_HEADERS" "$LINENO" 5
+  else
+    ac_delim="$ac_delim!$ac_delim _$ac_delim!! "
+  fi
+done
+
+# For the awk script, D is an array of macro values keyed by name,
+# likewise P contains macro parameters if any.  Preserve backslash
+# newline sequences.
+
+ac_word_re=[_$as_cr_Letters][_$as_cr_alnum]*
+sed -n '
+s/.\{148\}/&'"$ac_delim"'/g
+t rset
+:rset
+s/^[	 ]*#[	 ]*define[	 ][	 ]*/ /
+t def
+d
+:def
+s/\\$//
+t bsnl
+s/["\\]/\\&/g
+s/^ \('"$ac_word_re"'\)\(([^()]*)\)[	 ]*\(.*\)/P["\1"]="\2"\
+D["\1"]=" \3"/p
+s/^ \('"$ac_word_re"'\)[	 ]*\(.*\)/D["\1"]=" \2"/p
+d
+:bsnl
+s/["\\]/\\&/g
+s/^ \('"$ac_word_re"'\)\(([^()]*)\)[	 ]*\(.*\)/P["\1"]="\2"\
+D["\1"]=" \3\\\\\\n"\\/p
+t cont
+s/^ \('"$ac_word_re"'\)[	 ]*\(.*\)/D["\1"]=" \2\\\\\\n"\\/p
+t cont
+d
+:cont
+n
+s/.\{148\}/&'"$ac_delim"'/g
+t clear
+:clear
+s/\\$//
+t bsnlc
+s/["\\]/\\&/g; s/^/"/; s/$/"/p
+d
+:bsnlc
+s/["\\]/\\&/g; s/^/"/; s/$/\\\\\\n"\\/p
+b cont
+' <confdefs.h | sed '
+s/'"$ac_delim"'/"\\\
+"/g' >>$CONFIG_STATUS || ac_write_fail=1
+
+cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
+  for (key in D) D_is_set[key] = 1
+  FS = ""
+}
+/^[\t ]*#[\t ]*(define|undef)[\t ]+$ac_word_re([\t (]|\$)/ {
+  line = \$ 0
+  split(line, arg, " ")
+  if (arg[1] == "#") {
+    defundef = arg[2]
+    mac1 = arg[3]
+  } else {
+    defundef = substr(arg[1], 2)
+    mac1 = arg[2]
+  }
+  split(mac1, mac2, "(") #)
+  macro = mac2[1]
+  prefix = substr(line, 1, index(line, defundef) - 1)
+  if (D_is_set[macro]) {
+    # Preserve the white space surrounding the "#".
+    print prefix "define", macro P[macro] D[macro]
+    next
+  } else {
+    # Replace #undef with comments.  This is necessary, for example,
+    # in the case of _POSIX_SOURCE, which is predefined and required
+    # on some systems where configure will not decide to define it.
+    if (defundef == "undef") {
+      print "/*", prefix defundef, macro, "*/"
+      next
+    }
+  }
+}
+{ print }
+_ACAWK
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
+  as_fn_error $? "could not setup config headers machinery" "$LINENO" 5
+fi # test -n "$CONFIG_HEADERS"
+
+
+eval set X "  :F $CONFIG_FILES  :H $CONFIG_HEADERS  :L $CONFIG_LINKS  :C $CONFIG_COMMANDS"
+shift
+for ac_tag
+do
+  case $ac_tag in
+  :[FHLC]) ac_mode=$ac_tag; continue;;
+  esac
+  case $ac_mode$ac_tag in
+  :[FHL]*:*);;
+  :L* | :C*:*) as_fn_error $? "invalid tag \`$ac_tag'" "$LINENO" 5;;
+  :[FH]-) ac_tag=-:-;;
+  :[FH]*) ac_tag=$ac_tag:$ac_tag.in;;
+  esac
+  ac_save_IFS=$IFS
+  IFS=:
+  set x $ac_tag
+  IFS=$ac_save_IFS
+  shift
+  ac_file=$1
+  shift
+
+  case $ac_mode in
+  :L) ac_source=$1;;
+  :[FH])
+    ac_file_inputs=
+    for ac_f
+    do
+      case $ac_f in
+      -) ac_f="$ac_tmp/stdin";;
+      *) # Look for the file first in the build tree, then in the source tree
+	 # (if the path is not absolute).  The absolute path cannot be DOS-style,
+	 # because $ac_f cannot contain `:'.
+	 test -f "$ac_f" ||
+	   case $ac_f in
+	   [\\/$]*) false;;
+	   *) test -f "$srcdir/$ac_f" && ac_f="$srcdir/$ac_f";;
+	   esac ||
+	   as_fn_error 1 "cannot find input file: \`$ac_f'" "$LINENO" 5;;
+      esac
+      case $ac_f in *\'*) ac_f=`printf "%s\n" "$ac_f" | sed "s/'/'\\\\\\\\''/g"`;; esac
+      as_fn_append ac_file_inputs " '$ac_f'"
+    done
+
+    # Let's still pretend it is `configure' which instantiates (i.e., don't
+    # use $as_me), people would be surprised to read:
+    #    /* config.h.  Generated by config.status.  */
+    configure_input='Generated from '`
+	  printf "%s\n" "$*" | sed 's|^[^:]*/||;s|:[^:]*/|, |g'
+	`' by configure.'
+    if test x"$ac_file" != x-; then
+      configure_input="$ac_file.  $configure_input"
+      { printf "%s\n" "$as_me:${as_lineno-$LINENO}: creating $ac_file" >&5
+printf "%s\n" "$as_me: creating $ac_file" >&6;}
+    fi
+    # Neutralize special characters interpreted by sed in replacement strings.
+    case $configure_input in #(
+    *\&* | *\|* | *\\* )
+       ac_sed_conf_input=`printf "%s\n" "$configure_input" |
+       sed 's/[\\\\&|]/\\\\&/g'`;; #(
+    *) ac_sed_conf_input=$configure_input;;
+    esac
+
+    case $ac_tag in
+    *:-:* | *:-) cat >"$ac_tmp/stdin" \
+      || as_fn_error $? "could not create $ac_file" "$LINENO" 5 ;;
+    esac
+    ;;
+  esac
+
+  ac_dir=`$as_dirname -- "$ac_file" ||
+$as_expr X"$ac_file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
+	 X"$ac_file" : 'X\(//\)[^/]' \| \
+	 X"$ac_file" : 'X\(//\)$' \| \
+	 X"$ac_file" : 'X\(/\)' \| . 2>/dev/null ||
+printf "%s\n" X"$ac_file" |
+    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{
+	    s//\1/
+	    q
+	  }
+	  /^X\(\/\/\)[^/].*/{
+	    s//\1/
+	    q
+	  }
+	  /^X\(\/\/\)$/{
+	    s//\1/
+	    q
+	  }
+	  /^X\(\/\).*/{
+	    s//\1/
+	    q
+	  }
+	  s/.*/./; q'`
+  as_dir="$ac_dir"; as_fn_mkdir_p
+  ac_builddir=.
+
+case "$ac_dir" in
+.) ac_dir_suffix= ac_top_builddir_sub=. ac_top_build_prefix= ;;
+*)
+  ac_dir_suffix=/`printf "%s\n" "$ac_dir" | sed 's|^\.[\\/]||'`
+  # A ".." for each directory in $ac_dir_suffix.
+  ac_top_builddir_sub=`printf "%s\n" "$ac_dir_suffix" | sed 's|/[^\\/]*|/..|g;s|/||'`
+  case $ac_top_builddir_sub in
+  "") ac_top_builddir_sub=. ac_top_build_prefix= ;;
+  *)  ac_top_build_prefix=$ac_top_builddir_sub/ ;;
+  esac ;;
+esac
+ac_abs_top_builddir=$ac_pwd
+ac_abs_builddir=$ac_pwd$ac_dir_suffix
+# for backward compatibility:
+ac_top_builddir=$ac_top_build_prefix
+
+case $srcdir in
+  .)  # We are building in place.
+    ac_srcdir=.
+    ac_top_srcdir=$ac_top_builddir_sub
+    ac_abs_top_srcdir=$ac_pwd ;;
+  [\\/]* | ?:[\\/]* )  # Absolute name.
+    ac_srcdir=$srcdir$ac_dir_suffix;
+    ac_top_srcdir=$srcdir
+    ac_abs_top_srcdir=$srcdir ;;
+  *) # Relative name.
+    ac_srcdir=$ac_top_build_prefix$srcdir$ac_dir_suffix
+    ac_top_srcdir=$ac_top_build_prefix$srcdir
+    ac_abs_top_srcdir=$ac_pwd/$srcdir ;;
+esac
+ac_abs_srcdir=$ac_abs_top_srcdir$ac_dir_suffix
+
+
+  case $ac_mode in
+  :F)
+  #
+  # CONFIG_FILE
+  #
+
+_ACEOF
+
+cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
+# If the template does not know about datarootdir, expand it.
+# FIXME: This hack should be removed a few years after 2.60.
+ac_datarootdir_hack=; ac_datarootdir_seen=
+ac_sed_dataroot='
+/datarootdir/ {
+  p
+  q
+}
+/@datadir@/p
+/@docdir@/p
+/@infodir@/p
+/@localedir@/p
+/@mandir@/p'
+case `eval "sed -n \"\$ac_sed_dataroot\" $ac_file_inputs"` in
+*datarootdir*) ac_datarootdir_seen=yes;;
+*@datadir@*|*@docdir@*|*@infodir@*|*@localedir@*|*@mandir@*)
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: $ac_file_inputs seems to ignore the --datarootdir setting" >&5
+printf "%s\n" "$as_me: WARNING: $ac_file_inputs seems to ignore the --datarootdir setting" >&2;}
+_ACEOF
+cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
+  ac_datarootdir_hack='
+  s&@datadir@&$datadir&g
+  s&@docdir@&$docdir&g
+  s&@infodir@&$infodir&g
+  s&@localedir@&$localedir&g
+  s&@mandir@&$mandir&g
+  s&\\\${datarootdir}&$datarootdir&g' ;;
+esac
+_ACEOF
+
+# Neutralize VPATH when `$srcdir' = `.'.
+# Shell code in configure.ac might set extrasub.
+# FIXME: do we really want to maintain this feature?
+cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
+ac_sed_extra="$ac_vpsub
+$extrasub
+_ACEOF
+cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
+:t
+/@[a-zA-Z_][a-zA-Z_0-9]*@/!b
+s|@configure_input@|$ac_sed_conf_input|;t t
+s&@top_builddir@&$ac_top_builddir_sub&;t t
+s&@top_build_prefix@&$ac_top_build_prefix&;t t
+s&@srcdir@&$ac_srcdir&;t t
+s&@abs_srcdir@&$ac_abs_srcdir&;t t
+s&@top_srcdir@&$ac_top_srcdir&;t t
+s&@abs_top_srcdir@&$ac_abs_top_srcdir&;t t
+s&@builddir@&$ac_builddir&;t t
+s&@abs_builddir@&$ac_abs_builddir&;t t
+s&@abs_top_builddir@&$ac_abs_top_builddir&;t t
+$ac_datarootdir_hack
+"
+eval sed \"\$ac_sed_extra\" "$ac_file_inputs" | $AWK -f "$ac_tmp/subs.awk" \
+  >$ac_tmp/out || as_fn_error $? "could not create $ac_file" "$LINENO" 5
+
+test -z "$ac_datarootdir_hack$ac_datarootdir_seen" &&
+  { ac_out=`sed -n '/\${datarootdir}/p' "$ac_tmp/out"`; test -n "$ac_out"; } &&
+  { ac_out=`sed -n '/^[	 ]*datarootdir[	 ]*:*=/p' \
+      "$ac_tmp/out"`; test -z "$ac_out"; } &&
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: $ac_file contains a reference to the variable \`datarootdir'
+which seems to be undefined.  Please make sure it is defined" >&5
+printf "%s\n" "$as_me: WARNING: $ac_file contains a reference to the variable \`datarootdir'
+which seems to be undefined.  Please make sure it is defined" >&2;}
+
+  rm -f "$ac_tmp/stdin"
+  case $ac_file in
+  -) cat "$ac_tmp/out" && rm -f "$ac_tmp/out";;
+  *) rm -f "$ac_file" && mv "$ac_tmp/out" "$ac_file";;
+  esac \
+  || as_fn_error $? "could not create $ac_file" "$LINENO" 5
+ ;;
+  :H)
+  #
+  # CONFIG_HEADER
+  #
+  if test x"$ac_file" != x-; then
+    {
+      printf "%s\n" "/* $configure_input  */" >&1 \
+      && eval '$AWK -f "$ac_tmp/defines.awk"' "$ac_file_inputs"
+    } >"$ac_tmp/config.h" \
+      || as_fn_error $? "could not create $ac_file" "$LINENO" 5
+    if diff "$ac_file" "$ac_tmp/config.h" >/dev/null 2>&1; then
+      { printf "%s\n" "$as_me:${as_lineno-$LINENO}: $ac_file is unchanged" >&5
+printf "%s\n" "$as_me: $ac_file is unchanged" >&6;}
+    else
+      rm -f "$ac_file"
+      mv "$ac_tmp/config.h" "$ac_file" \
+	|| as_fn_error $? "could not create $ac_file" "$LINENO" 5
+    fi
+  else
+    printf "%s\n" "/* $configure_input  */" >&1 \
+      && eval '$AWK -f "$ac_tmp/defines.awk"' "$ac_file_inputs" \
+      || as_fn_error $? "could not create -" "$LINENO" 5
+  fi
+ ;;
+  :L)
+  #
+  # CONFIG_LINK
+  #
+
+  if test "$ac_source" = "$ac_file" && test "$srcdir" = '.'; then
+    :
+  else
+    # Prefer the file from the source tree if names are identical.
+    if test "$ac_source" = "$ac_file" || test ! -r "$ac_source"; then
+      ac_source=$srcdir/$ac_source
+    fi
+
+    { printf "%s\n" "$as_me:${as_lineno-$LINENO}: linking $ac_source to $ac_file" >&5
+printf "%s\n" "$as_me: linking $ac_source to $ac_file" >&6;}
+
+    if test ! -r "$ac_source"; then
+      as_fn_error $? "$ac_source: file not found" "$LINENO" 5
+    fi
+    rm -f "$ac_file"
+
+    # Try a relative symlink, then a hard link, then a copy.
+    case $ac_source in
+    [\\/$]* | ?:[\\/]* ) ac_rel_source=$ac_source ;;
+	*) ac_rel_source=$ac_top_build_prefix$ac_source ;;
+    esac
+    ln -s "$ac_rel_source" "$ac_file" 2>/dev/null ||
+      ln "$ac_source" "$ac_file" 2>/dev/null ||
+      cp -p "$ac_source" "$ac_file" ||
+      as_fn_error $? "cannot link or copy $ac_source to $ac_file" "$LINENO" 5
+  fi
+ ;;
+  :C)  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: executing $ac_file commands" >&5
+printf "%s\n" "$as_me: executing $ac_file commands" >&6;}
+ ;;
+  esac
+
+
+  case $ac_file$ac_mode in
+    "mkdir":C) as_dir=cram; as_fn_mkdir_p
+     as_dir=htscodecs/htscodecs; as_fn_mkdir_p
+     as_dir=htscodecs/tests; as_fn_mkdir_p
+     as_dir=test/fuzz; as_fn_mkdir_p
+     as_dir=test/longrefs; as_fn_mkdir_p
+     as_dir=test/tabix; as_fn_mkdir_p ;;
+
+  esac
+done # for ac_tag
+
+
+as_fn_exit 0
+_ACEOF
+ac_clean_files=$ac_clean_files_save
+
+test $ac_write_fail = 0 ||
+  as_fn_error $? "write failure creating $CONFIG_STATUS" "$LINENO" 5
+
+
+# configure is writing to config.log, and then calls config.status.
+# config.status does its own redirection, appending to config.log.
+# Unfortunately, on DOS this fails, as config.log is still kept open
+# by configure, so config.status won't be able to write to it; its
+# output is simply discarded.  So we exec the FD to /dev/null,
+# effectively closing config.log, so it can be properly (re)opened and
+# appended to by config.status.  When coming back to configure, we
+# need to make the FD available again.
+if test "$no_create" != yes; then
+  ac_cs_success=:
+  ac_config_status_args=
+  test "$silent" = yes &&
+    ac_config_status_args="$ac_config_status_args --quiet"
+  exec 5>/dev/null
+  $SHELL $CONFIG_STATUS $ac_config_status_args || ac_cs_success=false
+  exec 5>>config.log
+  # Use ||, not &&, to avoid exiting from the if with $? = 1, which
+  # would make configure fail if this is the last instruction.
+  $ac_cs_success || as_fn_exit 1
+fi
+if test -n "$ac_unrecognized_opts" && test "$enable_option_checking" != no; then
+  { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: unrecognized options: $ac_unrecognized_opts" >&5
+printf "%s\n" "$as_me: WARNING: unrecognized options: $ac_unrecognized_opts" >&2;}
+fi
+
+
diff --git a/ext/htslib/cram/README b/ext/htslib/cram/README
new file mode 100644
index 0000000..1354382
--- /dev/null
+++ b/ext/htslib/cram/README
@@ -0,0 +1,214 @@
+CRAM encoding internals
+=======================
+
+A quick summary of functions involved.
+
+The encoder works by accumulating a bunch of BAM records (via the
+cram_put_bam_seq function), and at a certain point (eg counter of
+records, or switching reference) the array of BAM records it turned
+into a container, which in turn creates slices, holding CRAM
+data-series in blocks.  The function that turns an array of BAM
+objects into the container is below.
+
+cram_encode_container func:
+    Validate references MD5 against header, unless no_ref mode
+    If embed_ref <= 1, fetch ref
+        Switch to embed_ref=2 if failed
+
+    Foreach slice:
+        If embed_ref == 2
+	    call cram_generate_reference
+	        if failed switch to no_ref mode
+	Foreach sequence
+	    call process_one_read to append BAM onto each data series (DS)
+	        call cram_stats_add for each DS to gather metrics
+		call cram_encode_aux
+
+    # We now have cram DS, per slice
+    call cram_encoder_init, per DS (based on cram_stats_add data)
+
+    Foreach slice:
+        call cram_encode_slice to turn DS to blocks
+	    call cram_compess_slice
+
+    call cram_encode_compression_header
+
+Threading
+---------
+
+CRAM can be multi-threaded, but this brings complications.
+
+The above function is the main CPU user, so it is this bit which can
+be executed in parallel from multiple threads.  To understand this we
+need to now look at how the primary loop works when writing a CRAM:
+
+Encoding main thread:
+    repeatedly calls cram_put_bam_seq
+        calls cram_new_container on first time through to initialise
+	calls cram_next_container when current is full or we need to flush
+	    calls cram_flush_container_mt to flush last container
+        pushes BAM object onto current container
+
+If non-threaded, cram_flush_container_mt does:
+    call cram_flush_container
+        call cram_encode_container to go from BAM to CRAM data-series
+ 	call cram_flush_container2 (writes it out)
+
+If threaded, cram_flush_container_mt does:
+    Main: Dispatch cram_flush_thread job
+        Thread: call cram_encode_container to go from BAM to CRAM data-series
+    Main: Call cram_flush_result to drain queue of encoded containers
+        Main: Call cram_flush_container2 (writes it out);
+
+
+
+Decisions on when to create new containers, detection of sorted vs unsorted,
+switching to multi-seq mode, etc occur at the main thread in
+cram_put_bam_seq.
+
+We can change our mind on container parameters at any point up until
+the cram_encode_container call.  At that point these parameters get
+baked into a container compression header and all data-series
+generated need to be in sync with the parameters.
+
+It is possible that some parameter changes can get detected while
+encoding the container, as it is there where we fetch references.  Eg
+the need to enable embedded reference or switch to non-ref mode.
+
+While encoding a container, we can change the parameters for *this*
+container, and we can also set the default parameter for subsequent
+new parameters via the global cram fd to avoid spamming attempts to
+load a reference which doesn't exist, but we cannot change other
+containers that are being processed in parallel.  They'll fend for
+themselves.
+
+References
+----------
+
+To avoid spamming the reference servers, there is a shared cache of
+references being currently used by all the worker threads (leading to
+confusing terminology of reference-counting of references).  So each
+container fetches its section of reference, but the memory for that is
+handled via its own layer.
+
+The shared references and ref meta-data is held in cram_fd -> refs (a
+refs_t pointer):
+
+    // References structure.
+    struct refs_t {
+        string_alloc_t *pool;  // String pool for holding filenames and SN vals
+    
+        khash_t(refs) *h_meta; // ref_entry*, index by name
+        ref_entry **ref_id;    // ref_entry*, index by ID
+        int nref;              // number of ref_entry
+    
+        char *fn;              // current file opened
+        BGZF *fp;              // and the hFILE* to go with it.
+    
+        int count;             // how many cram_fd sharing this refs struct
+    
+        pthread_mutex_t lock;  // Mutex for multi-threaded updating
+        ref_entry *last;       // Last queried sequence
+        int last_id;           // Used in cram_ref_decr_locked to delay free
+    };
+
+Within this, ref_entry is the per-reference information:
+
+    typedef struct ref_entry {
+        char *name;
+        char *fn;
+        int64_t length;
+        int64_t offset;
+        int bases_per_line;
+        int line_length;
+        int64_t count;     // for shared references so we know to dealloc seq
+        char *seq;
+        mFILE *mf;
+        int is_md5;        // Reference comes from a raw seq found by MD5
+        int validated_md5;
+    } ref_entry;
+
+Sharing of references to track use between threads is via
+cram_ref_incr* and cram_ref_decr* (which locked and unlocked
+variants).  We free a reference when the usage count hits zero.  To
+avoid spamming discard and reload in single-thread creation of a
+pos-sorted CRAM, we keep track of the last reference in cram_fd and
+delay discard by one loop iteration.
+
+There are complexities here around whether the references come from a
+single ref.fa file, are from a local MD5sum cache with one file per
+reference (mmapped), or whether they're fetched from some remote
+REF_PATH query such as the EBI.  (This later case typically downloads
+to a local md5 based ref-cache first and mmaps from there.)
+
+The refs struct start off by being populated from the SAM header.  We
+have M5 tag and name known, maybe a filename, but length is 0 and seq
+is NULL.  This is done by cram_load_reference:
+
+cram_load_reference (cram_fd, filename):
+    if filename non-NULL
+        call refs_load_fai
+	    Populates ref_entry with filename, name, length, line-len, etc
+    	sanitise_SQ_lines
+    If no refs loaded
+        call refs_from_header
+	    populates ref_entry with name.
+	    Sets length=0 as marker for not-yet-loaded
+
+The main interface used from the code is cram_get_ref().  It takes a
+reference ID, start and end coordinate and returns a pointer to the
+relevant sub-sequence.
+
+cram_get_ref:
+    r = fd->refs->ref_id[id];    // current ref
+    call cram_populate_ref if stored length is 0 (ie ref.fa set)
+        search REF_PATH / REF_CACHE
+	call bgzf_open if local_path
+	call open_path_mfile otherwise
+	copy to local REF_CACHE if required (eg remote fetch)
+
+    If start = 1 and end = ref-length
+       If ref seq unknown
+           call cram_ref_load to load entire ref and use that
+
+    If ref seq now known, return it
+
+    // Otherwise known via .fai or we've errored by now.
+    call load_ref_portion to return a sub-seq from index fasta
+
+The encoder asks for the entire reference rather than a small portion
+of it as we're usually encoding a large amount.  The decoder may be
+dealing with small range queries, so it only asks for the relevant
+sub-section of reference as specified in the cram slice headers.
+
+
+TODO
+====
+
+- Multi-ref mode is enabled when we have too many small containers in
+  a row.
+
+  Instead of firing off new containers when we switch reference, we
+  could always make a new container after N records, separating off
+  M <= N to make the container such that all M are the same reference,
+  and shuffling any remaining N-M down as the start of the next.
+
+  This means we can detect how many new containers we would create,
+  and enable multi-ref mode straight away rather than keeping a recent
+  history of how many small containers we've emitted.
+
+- The cache of references currently being used is a better place to
+  track the global embed-ref and non-ref logic.  Better than cram_fd.
+  Cram_fd is a one-way change, as once we enable non-ref we'll stick
+  with it.
+
+  However if it was per-ref in the ref-cache then we'd probe and try
+  each reference once, and then all new containers for that ref would
+  honour the per-ref parameters.  So a single missing reference in the
+  middle of a large file wouldn't change behaviour for all subsequence
+  references.
+
+  Optionally we could still do meta-analysis on how many references
+  are failing, and switch the global cram_fd params to avoid repeated
+  testing of reference availability if it's becoming obvious that none
+  of them are known.
diff --git a/ext/htslib/cram/cram.h b/ext/htslib/cram/cram.h
new file mode 100644
index 0000000..ba7b130
--- /dev/null
+++ b/ext/htslib/cram/cram.h
@@ -0,0 +1,61 @@
+/*
+Copyright (c) 2012-2013, 2015, 2018 Genome Research Ltd.
+Author: James Bonfield <jkb@sanger.ac.uk>
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+Institute nor the names of its contributors may be used to endorse or promote
+products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+/*! \file
+ * CRAM interface.
+ *
+ * Consider using the higher level hts_*() API for programs that wish to
+ * be file format agnostic (see htslib/hts.h).
+ *
+ * This API should be used for CRAM specific code. The specifics of the
+ * public API are implemented in cram_io.h, cram_encode.h and cram_decode.h
+ * although these should not be included directly (use this file instead).
+ */
+
+#ifndef CRAM_ALL_H
+#define CRAM_ALL_H
+
+#include "cram_samtools.h"
+#include "../header.h"
+#include "cram_structs.h"
+#include "cram_io.h"
+#include "cram_encode.h"
+#include "cram_decode.h"
+#include "cram_stats.h"
+#include "cram_codecs.h"
+#include "cram_index.h"
+
+// Validate against the external cram.h,
+//
+// This contains duplicated portions from cram_io.h and cram_structs.h,
+// so we want to ensure that the prototypes match.
+#include "../htslib/cram.h"
+
+#endif
diff --git a/ext/htslib/cram/cram_codecs.c b/ext/htslib/cram/cram_codecs.c
new file mode 100644
index 0000000..a72419e
--- /dev/null
+++ b/ext/htslib/cram/cram_codecs.c
@@ -0,0 +1,4160 @@
+/*
+Copyright (c) 2012-2021,2023 Genome Research Ltd.
+Author: James Bonfield <jkb@sanger.ac.uk>
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+Institute nor the names of its contributors may be used to endorse or promote
+products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+/*
+ * FIXME: add checking of cram_external_type to return NULL on unsupported
+ * {codec,type} tuples.
+ */
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include <limits.h>
+#include <stdint.h>
+#include <errno.h>
+#include <stddef.h>
+
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+#include "../fuzz_settings.h"
+#endif
+
+#include "../htslib/hts_endian.h"
+
+#if defined(HAVE_EXTERNAL_LIBHTSCODECS)
+#include <htscodecs/varint.h>
+#include <htscodecs/pack.h>
+#include <htscodecs/rle.h>
+#else
+#include "../htscodecs/htscodecs/varint.h"
+#include "../htscodecs/htscodecs/pack.h"
+#include "../htscodecs/htscodecs/rle.h"
+#endif
+
+#include "cram.h"
+
+/*
+ * ---------------------------------------------------------------------------
+ * Block bit-level I/O functions.
+ * All defined static here to promote easy inlining by the compiler.
+ */
+
+#if 0
+/* Get a single bit, MSB first */
+static signed int get_bit_MSB(cram_block *block) {
+    unsigned int val;
+
+    if (block->byte > block->alloc)
+        return -1;
+
+    val = block->data[block->byte] >> block->bit;
+    if (--block->bit == -1) {
+        block->bit = 7;
+        block->byte++;
+        //printf("(%02X)", block->data[block->byte]);
+    }
+
+    //printf("-B%d-", val&1);
+
+    return val & 1;
+}
+#endif
+
+/*
+ * Count number of successive 0 and 1 bits
+ */
+static int get_one_bits_MSB(cram_block *block) {
+    int n = 0, b;
+    if (block->byte >= block->uncomp_size)
+        return -1;
+    do {
+        b = block->data[block->byte] >> block->bit;
+        if (--block->bit == -1) {
+            block->bit = 7;
+            block->byte++;
+            if (block->byte == block->uncomp_size && (b&1))
+                return -1;
+        }
+        n++;
+    } while (b&1);
+
+    return n-1;
+}
+
+static int get_zero_bits_MSB(cram_block *block) {
+    int n = 0, b;
+    if (block->byte >= block->uncomp_size)
+        return -1;
+    do {
+        b = block->data[block->byte] >> block->bit;
+        if (--block->bit == -1) {
+            block->bit = 7;
+            block->byte++;
+            if (block->byte == block->uncomp_size && !(b&1))
+                return -1;
+        }
+        n++;
+    } while (!(b&1));
+
+    return n-1;
+}
+
+#if 0
+/* Stores a single bit */
+static void store_bit_MSB(cram_block *block, unsigned int bit) {
+    if (block->byte >= block->alloc) {
+        block->alloc = block->alloc ? block->alloc*2 : 1024;
+        block->data = realloc(block->data, block->alloc);
+    }
+
+    if (bit)
+        block->data[block->byte] |= (1 << block->bit);
+
+    if (--block->bit == -1) {
+        block->bit = 7;
+        block->byte++;
+        block->data[block->byte] = 0;
+    }
+}
+#endif
+
+#if 0
+/* Rounds to the next whole byte boundary first */
+static void store_bytes_MSB(cram_block *block, char *bytes, int len) {
+    if (block->bit != 7) {
+        block->bit = 7;
+        block->byte++;
+    }
+
+    while (block->byte + len >= block->alloc) {
+        block->alloc = block->alloc ? block->alloc*2 : 1024;
+        block->data = realloc(block->data, block->alloc);
+    }
+
+    memcpy(&block->data[block->byte], bytes, len);
+    block->byte += len;
+}
+#endif
+
+/* Local optimised copy for inlining */
+static inline int64_t get_bits_MSB(cram_block *block, int nbits) {
+    uint64_t val = 0;
+    int i;
+
+#if 0
+    // Fits within the current byte */
+    if (nbits <= block->bit+1) {
+        val = (block->data[block->byte]>>(block->bit-(nbits-1))) & ((1<<nbits)-1);
+        if ((block->bit -= nbits) == -1) {
+            block->bit = 7;
+            block->byte++;
+        }
+        return val;
+    }
+
+    // partial first byte
+    val = block->data[block->byte] & ((1<<(block->bit+1))-1);
+    nbits -= block->bit+1;
+    block->bit = 7;
+    block->byte++;
+
+    // whole middle bytes
+    while (nbits >= 8) {
+        val = (val << 8) | block->data[block->byte++];
+        nbits -= 8;
+    }
+
+    val <<= nbits;
+    val |= (block->data[block->byte]>>(block->bit-(nbits-1))) & ((1<<nbits)-1);
+    block->bit -= nbits;
+    return val;
+#endif
+
+#if 0
+    /* Inefficient implementation! */
+    //printf("{");
+    for (i = 0; i < nbits; i++)
+        //val = (val << 1) | get_bit_MSB(block);
+        GET_BIT_MSB(block, val);
+#endif
+
+#if 1
+    /* Combination of 1st two methods */
+    if (nbits <= block->bit+1) {
+        val = (block->data[block->byte]>>(block->bit-(nbits-1))) & ((1<<nbits)-1);
+        if ((block->bit -= nbits) == -1) {
+            block->bit = 7;
+            block->byte++;
+        }
+        return val;
+    }
+
+    switch(nbits) {
+//  case 15: GET_BIT_MSB(block, val); // fall through
+//  case 14: GET_BIT_MSB(block, val); // fall through
+//  case 13: GET_BIT_MSB(block, val); // fall through
+//  case 12: GET_BIT_MSB(block, val); // fall through
+//  case 11: GET_BIT_MSB(block, val); // fall through
+//  case 10: GET_BIT_MSB(block, val); // fall through
+//  case  9: GET_BIT_MSB(block, val); // fall through
+    case  8: GET_BIT_MSB(block, val); // fall through
+    case  7: GET_BIT_MSB(block, val); // fall through
+    case  6: GET_BIT_MSB(block, val); // fall through
+    case  5: GET_BIT_MSB(block, val); // fall through
+    case  4: GET_BIT_MSB(block, val); // fall through
+    case  3: GET_BIT_MSB(block, val); // fall through
+    case  2: GET_BIT_MSB(block, val); // fall through
+    case  1: GET_BIT_MSB(block, val);
+        break;
+
+    default:
+        for (i = 0; i < nbits; i++)
+            //val = (val << 1) | get_bit_MSB(block);
+            GET_BIT_MSB(block, val);
+    }
+#endif
+
+    //printf("=0x%x}", val);
+
+    return val;
+}
+
+/*
+ * Can store up to 24-bits worth of data encoded in an integer value
+ * Possibly we'd want to have a less optimal store_bits function when dealing
+ * with nbits > 24, but for now we assume the codes generated are never
+ * that big. (Given this is only possible with 121392 or more
+ * characters with exactly the correct frequency distribution we check
+ * for it elsewhere.)
+ */
+static int store_bits_MSB(cram_block *block, uint64_t val, int nbits) {
+    //fprintf(stderr, " store_bits: %02x %d\n", val, nbits);
+
+    /*
+     * Use slow mode until we tweak the huffman generator to never generate
+     * codes longer than 24-bits.
+     */
+    unsigned int mask;
+
+    if (block->byte+8 >= block->alloc) {
+        if (block->byte) {
+            block->alloc *= 2;
+            block->data = realloc(block->data, block->alloc + 8);
+            if (!block->data)
+                return -1;
+        } else {
+            block->alloc = 1024;
+            block->data = realloc(block->data, block->alloc + 8);
+            if (!block->data)
+                return -1;
+            block->data[0] = 0; // initialise first byte of buffer
+        }
+    }
+
+    /* fits in current bit-field */
+    if (nbits <= block->bit+1) {
+        block->data[block->byte] |= (val << (block->bit+1-nbits));
+        if ((block->bit-=nbits) == -1) {
+            block->bit = 7;
+            block->byte++;
+            block->data[block->byte] = 0;
+        }
+        return 0;
+    }
+
+    block->data[block->byte] |= (val >> (nbits -= block->bit+1));
+    block->bit = 7;
+    block->byte++;
+    block->data[block->byte] = 0;
+
+    mask = 1<<(nbits-1);
+    do {
+        if (val & mask)
+            block->data[block->byte] |= (1 << block->bit);
+        if (--block->bit == -1) {
+            block->bit = 7;
+            block->byte++;
+            block->data[block->byte] = 0;
+        }
+        mask >>= 1;
+    } while(--nbits);
+
+    return 0;
+}
+
+/*
+ * Returns the next 'size' bytes from a block, or NULL if insufficient
+ * data left.This is just a pointer into the block data and not an
+ * allocated object, so do not free the result.
+ */
+static char *cram_extract_block(cram_block *b, int size) {
+    char *cp = (char *)b->data + b->idx;
+    b->idx += size;
+    if (b->idx > b->uncomp_size)
+        return NULL;
+
+    return cp;
+}
+
+/*
+ * ---------------------------------------------------------------------------
+ * EXTERNAL
+ *
+ * In CRAM 3.0 and earlier, E_EXTERNAL use the data type to determine the
+ * size of the object being returned.  This type is hard coded in the
+ * spec document (changing from uint32 to uint64 requires a spec change)
+ * and there is no data format introspection so implementations have
+ * to determine which size to use based on version numbers.   It also
+ * doesn't support signed data.
+ *
+ * With CRAM 4.0 onwards the size and sign of the data is no longer stated
+ * explicitly in the specification.  Instead EXTERNAL is replaced by three
+ * new encodings, for bytes and signed / unsigned integers which used a
+ * variable sized encoding.
+ *
+ * For simplicity we use the same encode and decode functions for
+ * bytes (CRAM4) and external (CRAM3). Given we already had code to
+ * replace codec + type into a function pointer it makes little
+ * difference how we ended up at that function.  However we disallow
+ * this codec to operate on integer data for CRAM4 onwards.
+ */
+int cram_external_decode_int(cram_slice *slice, cram_codec *c,
+                             cram_block *in, char *out, int *out_size) {
+    char *cp;
+    cram_block *b;
+
+    /* Find the external block */
+    b = cram_get_block_by_id(slice, c->u.external.content_id);
+    if (!b)
+        return *out_size?-1:0;
+
+    cp = (char *)b->data + b->idx;
+    // E_INT and E_LONG are guaranteed single item queries
+    int err = 0;
+    *(int32_t *)out = c->vv->varint_get32(&cp, (char *)b->data + b->uncomp_size, &err);
+    b->idx = cp - (char *)b->data;
+    *out_size = 1;
+
+    return err ? -1 : 0;
+}
+
+int cram_external_decode_long(cram_slice *slice, cram_codec *c,
+                              cram_block *in, char *out, int *out_size) {
+    char *cp;
+    cram_block *b;
+
+    /* Find the external block */
+    b = cram_get_block_by_id(slice, c->u.external.content_id);
+    if (!b)
+        return *out_size?-1:0;
+
+    cp = (char *)b->data + b->idx;
+    // E_INT and E_LONG are guaranteed single item queries
+    int err = 0;
+    *(int64_t *)out = c->vv->varint_get64(&cp, (char *)b->data + b->uncomp_size, &err);
+    b->idx = cp - (char *)b->data;
+    *out_size = 1;
+
+    return err ? -1 : 0;
+}
+
+int cram_external_decode_char(cram_slice *slice, cram_codec *c,
+                              cram_block *in, char *out,
+                              int *out_size) {
+    char *cp;
+    cram_block *b;
+
+    /* Find the external block */
+    b = cram_get_block_by_id(slice, c->u.external.content_id);
+    if (!b)
+        return *out_size?-1:0;
+
+    cp = cram_extract_block(b, *out_size);
+    if (!cp)
+        return -1;
+
+    if (out)
+        memcpy(out, cp, *out_size);
+    return 0;
+}
+
+static int cram_external_decode_block(cram_slice *slice, cram_codec *c,
+                                      cram_block *in, char *out_,
+                                      int *out_size) {
+    char *cp;
+    cram_block *out = (cram_block *)out_;
+    cram_block *b = NULL;
+
+    /* Find the external block */
+    b = cram_get_block_by_id(slice, c->u.external.content_id);
+    if (!b)
+        return *out_size?-1:0;
+
+    cp = cram_extract_block(b, *out_size);
+    if (!cp)
+        return -1;
+
+    BLOCK_APPEND(out, cp, *out_size);
+    return 0;
+
+ block_err:
+    return -1;
+}
+
+void cram_external_decode_free(cram_codec *c) {
+    if (c)
+        free(c);
+}
+
+
+int cram_external_decode_size(cram_slice *slice, cram_codec *c) {
+    cram_block *b;
+
+    /* Find the external block */
+    b = cram_get_block_by_id(slice, c->u.external.content_id);
+    if (!b)
+        return -1;
+
+    return b->uncomp_size;
+}
+
+cram_block *cram_external_get_block(cram_slice *slice, cram_codec *c) {
+    return cram_get_block_by_id(slice, c->u.external.content_id);
+}
+
+int cram_external_describe(cram_codec *c, kstring_t *ks) {
+    return ksprintf(ks, "EXTERNAL(id=%d)",
+                    c->u.external.content_id) < 0 ? -1 : 0;
+}
+
+cram_codec *cram_external_decode_init(cram_block_compression_hdr *hdr,
+                                      char *data, int size,
+                                      enum cram_encoding codec,
+                                      enum cram_external_type option,
+                                      int version, varint_vec *vv) {
+    cram_codec *c = NULL;
+    char *cp = data;
+
+    if (size < 1)
+        goto malformed;
+
+    if (!(c = malloc(sizeof(*c))))
+        return NULL;
+
+    c->codec  = E_EXTERNAL;
+    if (CRAM_MAJOR_VERS(version) >= 4) {
+        // Version 4 does not permit integer data to be encoded as a
+        // series of bytes.  This is used purely for bytes, either
+        // singular or declared as arrays
+        switch (codec) {
+        case E_EXTERNAL:
+            if (option == E_BYTE_ARRAY_BLOCK)
+                c->decode = cram_external_decode_block;
+            else if (option == E_BYTE || option == E_BYTE_ARRAY)
+                c->decode = cram_external_decode_char;
+            else
+                goto malformed;
+            break;
+        default:
+            goto malformed;
+        }
+    } else {
+        // CRAM 3 and earlier encodes integers as EXTERNAL.  We need
+        // use the option field to indicate the input data format so
+        // we know which serialisation format to use.
+        if (option == E_INT)
+            c->decode = cram_external_decode_int;
+        else if (option == E_LONG)
+            c->decode = cram_external_decode_long;
+        else if (option == E_BYTE_ARRAY || option == E_BYTE)
+            c->decode = cram_external_decode_char;
+        else
+            c->decode = cram_external_decode_block;
+    }
+    c->free   = cram_external_decode_free;
+    c->size   = cram_external_decode_size;
+    c->get_block = cram_external_get_block;
+    c->describe = cram_external_describe;
+
+    c->u.external.content_id = vv->varint_get32(&cp, data+size, NULL);
+
+    if (cp - data != size)
+        goto malformed;
+
+    c->u.external.type = option;
+
+    return c;
+
+ malformed:
+    hts_log_error("Malformed external header stream");
+    free(c);
+    return NULL;
+}
+
+int cram_external_encode_int(cram_slice *slice, cram_codec *c,
+                             char *in, int in_size) {
+    uint32_t *i32 = (uint32_t *)in;
+    return c->vv->varint_put32_blk(c->out, *i32) >= 0 ? 0 : -1;
+}
+
+int cram_external_encode_sint(cram_slice *slice, cram_codec *c,
+                             char *in, int in_size) {
+    int32_t *i32 = (int32_t *)in;
+    return c->vv->varint_put32s_blk(c->out, *i32) >= 0 ? 0 : -1;
+}
+
+int cram_external_encode_long(cram_slice *slice, cram_codec *c,
+                             char *in, int in_size) {
+    uint64_t *i64 = (uint64_t *)in;
+    return c->vv->varint_put64_blk(c->out, *i64) >= 0 ? 0 : -1;
+}
+
+int cram_external_encode_slong(cram_slice *slice, cram_codec *c,
+                               char *in, int in_size) {
+    int64_t *i64 = (int64_t *)in;
+    return c->vv->varint_put64s_blk(c->out, *i64) >= 0 ? 0 : -1;
+}
+
+int cram_external_encode_char(cram_slice *slice, cram_codec *c,
+                              char *in, int in_size) {
+    BLOCK_APPEND(c->out, in, in_size);
+    return 0;
+
+ block_err:
+    return -1;
+}
+
+void cram_external_encode_free(cram_codec *c) {
+    if (!c)
+        return;
+    free(c);
+}
+
+int cram_external_encode_store(cram_codec *c, cram_block *b, char *prefix,
+                               int version) {
+    char tmp[99], *tp = tmp, *tpend = tmp+99;
+    int len = 0, r = 0, n;
+
+    if (prefix) {
+        size_t l = strlen(prefix);
+        BLOCK_APPEND(b, prefix, l);
+        len += l;
+    }
+
+    tp += c->vv->varint_put32(tp, tpend, c->u.e_external.content_id);
+    len += (n = c->vv->varint_put32_blk(b, c->codec)); r |= n;
+    len += (n = c->vv->varint_put32_blk(b, tp-tmp));   r |= n;
+    BLOCK_APPEND(b, tmp, tp-tmp);
+    len += tp-tmp;
+
+    if (r > 0)
+        return len;
+
+ block_err:
+    return -1;
+}
+
+cram_codec *cram_external_encode_init(cram_stats *st,
+                                      enum cram_encoding codec,
+                                      enum cram_external_type option,
+                                      void *dat,
+                                      int version, varint_vec *vv) {
+    cram_codec *c;
+
+    c = malloc(sizeof(*c));
+    if (!c)
+        return NULL;
+    c->codec = E_EXTERNAL;
+    c->free = cram_external_encode_free;
+    if (CRAM_MAJOR_VERS(version) >= 4) {
+        // Version 4 does not permit integer data to be encoded as a
+        // series of bytes.  This is used purely for bytes, either
+        // singular or declared as arrays
+        switch (codec) {
+        case E_EXTERNAL:
+            if (option != E_BYTE && option != E_BYTE_ARRAY)
+                return NULL;
+            c->encode = cram_external_encode_char;
+            break;
+        default:
+            return NULL;
+        }
+    } else {
+        // CRAM 3 and earlier encodes integers as EXTERNAL.  We need
+        // use the option field to indicate the input data format so
+        // we know which serialisation format to use.
+        if (option == E_INT)
+            c->encode = cram_external_encode_int;
+        else if (option == E_LONG)
+            c->encode = cram_external_encode_long;
+        else if (option == E_BYTE_ARRAY || option == E_BYTE)
+            c->encode = cram_external_encode_char;
+        else
+            abort();
+    }
+    c->store = cram_external_encode_store;
+    c->flush = NULL;
+
+    c->u.e_external.content_id = (size_t)dat;
+
+    return c;
+}
+
+/*
+ * ---------------------------------------------------------------------------
+ * VARINT
+ *
+ * In CRAM 3.0 and earlier, E_EXTERNAL stored both integers in ITF8
+ * format as well as bytes.  In CRAM 4 EXTERNAL is only for bytes and
+ * byte arrays, with two dedicated encodings for integers:
+ * VARINT_SIGNED and VARINT_UNSIGNED.  These also differ a little to
+ * EXTERNAL with the addition of an offset field, meaning we can store
+ * values in, say, the range -2 to 1 million without needing to use
+ * a signed zig-zag transformation.
+ */
+int cram_varint_decode_int(cram_slice *slice, cram_codec *c,
+                           cram_block *in, char *out, int *out_size) {
+    char *cp;
+    cram_block *b;
+
+    /* Find the data block */
+    b = cram_get_block_by_id(slice, c->u.varint.content_id);
+    if (!b)
+        return *out_size?-1:0;
+
+    cp = (char *)b->data + b->idx;
+    // E_INT and E_LONG are guaranteed single item queries
+    int err = 0;
+    *(int32_t *)out = c->vv->varint_get32(&cp,
+                                          (char *)b->data + b->uncomp_size,
+                                          &err) + c->u.varint.offset;
+    b->idx = cp - (char *)b->data;
+    *out_size = 1;
+
+    return err ? -1 : 0;
+}
+
+int cram_varint_decode_sint(cram_slice *slice, cram_codec *c,
+                            cram_block *in, char *out, int *out_size) {
+    char *cp;
+    cram_block *b;
+
+    /* Find the data block */
+    b = cram_get_block_by_id(slice, c->u.varint.content_id);
+    if (!b)
+        return *out_size?-1:0;
+
+    cp = (char *)b->data + b->idx;
+    // E_INT and E_LONG are guaranteed single item queries
+    int err = 0;
+    *(int32_t *)out = c->vv->varint_get32s(&cp,
+                                           (char *)b->data + b->uncomp_size,
+                                           &err) + c->u.varint.offset;
+    b->idx = cp - (char *)b->data;
+    *out_size = 1;
+
+    return err ? -1 : 0;
+}
+
+int cram_varint_decode_long(cram_slice *slice, cram_codec *c,
+                            cram_block *in, char *out, int *out_size) {
+    char *cp;
+    cram_block *b;
+
+    /* Find the data block */
+    b = cram_get_block_by_id(slice, c->u.varint.content_id);
+    if (!b)
+        return *out_size?-1:0;
+
+    cp = (char *)b->data + b->idx;
+    // E_INT and E_LONG are guaranteed single item queries
+    int err = 0;
+    *(int64_t *)out = c->vv->varint_get64(&cp,
+                                          (char *)b->data + b->uncomp_size,
+                                          &err) + c->u.varint.offset;
+    b->idx = cp - (char *)b->data;
+    *out_size = 1;
+
+    return err ? -1 : 0;
+}
+
+int cram_varint_decode_slong(cram_slice *slice, cram_codec *c,
+                             cram_block *in, char *out, int *out_size) {
+    char *cp;
+    cram_block *b;
+
+    /* Find the data block */
+    b = cram_get_block_by_id(slice, c->u.varint.content_id);
+    if (!b)
+        return *out_size?-1:0;
+
+    cp = (char *)b->data + b->idx;
+    // E_INT and E_LONG are guaranteed single item queries
+    int err = 0;
+    *(int64_t *)out = c->vv->varint_get64s(&cp,
+                                           (char *)b->data + b->uncomp_size,
+                                           &err) + c->u.varint.offset;
+    b->idx = cp - (char *)b->data;
+    *out_size = 1;
+
+    return err ? -1 : 0;
+}
+
+void cram_varint_decode_free(cram_codec *c) {
+    if (c)
+        free(c);
+}
+
+int cram_varint_decode_size(cram_slice *slice, cram_codec *c) {
+    cram_block *b;
+
+    /* Find the data block */
+    b = cram_get_block_by_id(slice, c->u.varint.content_id);
+    if (!b)
+        return -1;
+
+    return b->uncomp_size;
+}
+
+cram_block *cram_varint_get_block(cram_slice *slice, cram_codec *c) {
+    return cram_get_block_by_id(slice, c->u.varint.content_id);
+}
+
+int cram_varint_describe(cram_codec *c, kstring_t *ks) {
+    return ksprintf(ks, "VARINT(id=%d,offset=%"PRId64",type=%d)",
+                    c->u.varint.content_id,
+                    c->u.varint.offset,
+                    c->u.varint.type)
+        < 0 ? -1 : 0;
+}
+
+cram_codec *cram_varint_decode_init(cram_block_compression_hdr *hdr,
+                                    char *data, int size,
+                                    enum cram_encoding codec,
+                                    enum cram_external_type option,
+                                    int version, varint_vec *vv) {
+    cram_codec *c;
+    char *cp = data, *cp_end = data+size;
+
+    if (!(c = malloc(sizeof(*c))))
+        return NULL;
+
+    c->codec  = codec;
+
+    // Function pointer choice is theoretically by codec type.
+    // Given we have some vars as int32 and some as int64 we
+    // use option too for sizing, although on disk format
+    // does not change.
+    switch(codec) {
+    case E_VARINT_UNSIGNED:
+        c->decode = (option == E_INT)
+            ? cram_varint_decode_int
+            : cram_varint_decode_long;
+        break;
+    case E_VARINT_SIGNED:
+        c->decode = (option == E_INT)
+            ? cram_varint_decode_sint
+            : cram_varint_decode_slong;
+        break;
+    default:
+        return NULL;
+    }
+
+    c->free   = cram_varint_decode_free;
+    c->size   = cram_varint_decode_size;
+    c->get_block = cram_varint_get_block;
+    c->describe = cram_varint_describe;
+
+    c->u.varint.content_id = vv->varint_get32 (&cp, cp_end, NULL);
+    c->u.varint.offset     = vv->varint_get64s(&cp, cp_end, NULL);
+
+    if (cp - data != size) {
+        fprintf(stderr, "Malformed varint header stream\n");
+        free(c);
+        return NULL;
+    }
+
+    c->u.varint.type = option;
+
+    return c;
+}
+
+int cram_varint_encode_int(cram_slice *slice, cram_codec *c,
+                           char *in, int in_size) {
+    uint32_t *i32 = (uint32_t *)in;
+    return c->vv->varint_put32_blk(c->out, *i32 - c->u.varint.offset) >= 0
+        ? 0 : -1;
+}
+
+int cram_varint_encode_sint(cram_slice *slice, cram_codec *c,
+                            char *in, int in_size) {
+    int32_t *i32 = (int32_t *)in;
+    return c->vv->varint_put32s_blk(c->out, *i32 - c->u.varint.offset) >= 0
+        ? 0 : -1;
+}
+
+int cram_varint_encode_long(cram_slice *slice, cram_codec *c,
+                            char *in, int in_size) {
+    uint64_t *i64 = (uint64_t *)in;
+    return c->vv->varint_put64_blk(c->out, *i64 - c->u.varint.offset) >= 0
+        ? 0 : -1;
+}
+
+int cram_varint_encode_slong(cram_slice *slice, cram_codec *c,
+                             char *in, int in_size) {
+    int64_t *i64 = (int64_t *)in;
+    return c->vv->varint_put64s_blk(c->out, *i64 - c->u.varint.offset) >= 0
+        ? 0 : -1;
+}
+
+void cram_varint_encode_free(cram_codec *c) {
+    if (!c)
+        return;
+    free(c);
+}
+
+int cram_varint_encode_store(cram_codec *c, cram_block *b, char *prefix,
+                             int version) {
+    char tmp[99], *tp = tmp;
+    int len = 0;
+
+    if (prefix) {
+        size_t l = strlen(prefix);
+        BLOCK_APPEND(b, prefix, l);
+        len += l;
+    }
+
+    tp += c->vv->varint_put32 (tp, NULL, c->u.e_varint.content_id);
+    tp += c->vv->varint_put64s(tp, NULL, c->u.e_varint.offset);
+    len += c->vv->varint_put32_blk(b, c->codec);
+    len += c->vv->varint_put32_blk(b, tp-tmp);
+    BLOCK_APPEND(b, tmp, tp-tmp);
+    len += tp-tmp;
+
+    return len;
+
+ block_err:
+    return -1;
+}
+
+cram_codec *cram_varint_encode_init(cram_stats *st,
+                                    enum cram_encoding codec,
+                                    enum cram_external_type option,
+                                    void *dat,
+                                    int version, varint_vec *vv) {
+    cram_codec *c;
+
+    if (!(c = malloc(sizeof(*c))))
+        return NULL;
+
+    c->u.e_varint.offset = 0;
+    if (st) {
+        // Marginal difference so far! Not worth the hassle?
+        if (st->min_val < 0 && st->min_val >= -127
+            && st->max_val / -st->min_val > 100) {
+            c->u.e_varint.offset = -st->min_val;
+            codec = E_VARINT_UNSIGNED;
+        } else if (st->min_val > 0) {
+            c->u.e_varint.offset = -st->min_val;
+        }
+    }
+
+    c->codec = codec;
+    c->free = cram_varint_encode_free;
+
+    // Function pointer choice is theoretically by codec type.
+    // Given we have some vars as int32 and some as int64 we
+    // use option too for sizing, although on disk format
+    // does not change.
+    switch (codec) {
+    case E_VARINT_UNSIGNED:
+        c->encode = (option == E_INT)
+            ? cram_varint_encode_int
+            : cram_varint_encode_long;
+        break;
+    case E_VARINT_SIGNED:
+        c->encode = (option == E_INT)
+            ? cram_varint_encode_sint
+            : cram_varint_encode_slong;
+        break;
+    default:
+        return NULL;
+    }
+    c->store = cram_varint_encode_store;
+    c->flush = NULL;
+
+    c->u.e_varint.content_id = (size_t)dat;
+
+    return c;
+}
+/*
+ * ---------------------------------------------------------------------------
+ * CONST_BYTE and CONST_INT
+ */
+int cram_const_decode_byte(cram_slice *slice, cram_codec *c,
+                           cram_block *in, char *out, int *out_size) {
+    int i, n;
+
+    for (i = 0, n = *out_size; i < n; i++)
+        out[i] = c->u.xconst.val;
+
+    return 0;
+}
+
+int cram_const_decode_int(cram_slice *slice, cram_codec *c,
+                          cram_block *in, char *out, int *out_size) {
+    int32_t *out_i = (int32_t *)out;
+    int i, n;
+
+    for (i = 0, n = *out_size; i < n; i++)
+        out_i[i] = c->u.xconst.val;
+
+    return 0;
+}
+
+int cram_const_decode_long(cram_slice *slice, cram_codec *c,
+                           cram_block *in, char *out, int *out_size) {
+    int64_t *out_i = (int64_t *)out;
+    int i, n;
+
+    for (i = 0, n = *out_size; i < n; i++)
+        out_i[i] = c->u.xconst.val;
+
+    return 0;
+}
+
+void cram_const_decode_free(cram_codec *c) {
+    if (c)
+        free(c);
+}
+
+int cram_const_decode_size(cram_slice *slice, cram_codec *c) {
+    return 0;
+}
+
+int cram_const_describe(cram_codec *c, kstring_t *ks) {
+    return ksprintf(ks, "CONST(val=%"PRId64")",
+                    c->u.xconst.val) < 0 ? -1 : 0;
+}
+
+cram_codec *cram_const_decode_init(cram_block_compression_hdr *hdr,
+                                   char *data, int size,
+                                   enum cram_encoding codec,
+                                   enum cram_external_type option,
+                                   int version, varint_vec *vv) {
+    cram_codec *c;
+    char *cp = data;
+
+    if (!(c = malloc(sizeof(*c))))
+        return NULL;
+
+    c->codec  = codec;
+    if (codec == E_CONST_BYTE)
+        c->decode = cram_const_decode_byte;
+    else if (option == E_INT)
+        c->decode = cram_const_decode_int;
+    else
+        c->decode = cram_const_decode_long;
+    c->free   = cram_const_decode_free;
+    c->size   = cram_const_decode_size;
+    c->get_block = NULL;
+    c->describe = cram_const_describe;
+
+    c->u.xconst.val = vv->varint_get64s(&cp, data+size, NULL);
+
+    if (cp - data != size) {
+        fprintf(stderr, "Malformed const header stream\n");
+        free(c);
+        return NULL;
+    }
+
+    return c;
+}
+
+int cram_const_encode(cram_slice *slice, cram_codec *c,
+                      char *in, int in_size) {
+    return 0;
+}
+
+int cram_const_encode_store(cram_codec *c, cram_block *b, char *prefix,
+                            int version) {
+    char tmp[99], *tp = tmp;
+    int len = 0;
+
+    if (prefix) {
+        size_t l = strlen(prefix);
+        BLOCK_APPEND(b, prefix, l);
+        len += l;
+    }
+
+    tp += c->vv->varint_put64s(tp, NULL, c->u.xconst.val);
+    len += c->vv->varint_put32_blk(b, c->codec);
+    len += c->vv->varint_put32_blk(b, tp-tmp);
+    BLOCK_APPEND(b, tmp, tp-tmp);
+    len += tp-tmp;
+
+    return len;
+
+ block_err:
+    return -1;
+}
+
+cram_codec *cram_const_encode_init(cram_stats *st,
+                                   enum cram_encoding codec,
+                                   enum cram_external_type option,
+                                   void *dat,
+                                   int version, varint_vec *vv) {
+    cram_codec *c;
+
+    if (!(c = malloc(sizeof(*c))))
+        return NULL;
+
+    c->codec = codec;
+    c->free = cram_const_decode_free; // as as decode
+    c->encode = cram_const_encode; // a nop
+    c->store = cram_const_encode_store;
+    c->flush = NULL;
+    c->u.e_xconst.val = st->min_val;
+
+    return c;
+}
+
+/*
+ * ---------------------------------------------------------------------------
+ * BETA
+ */
+int cram_beta_decode_long(cram_slice *slice, cram_codec *c, cram_block *in, char *out, int *out_size) {
+    int64_t *out_i = (int64_t *)out;
+    int i, n = *out_size;
+
+    if (c->u.beta.nbits) {
+        if (cram_not_enough_bits(in, c->u.beta.nbits * n))
+            return -1;
+
+        for (i = 0; i < n; i++)
+            out_i[i] = get_bits_MSB(in, c->u.beta.nbits) - c->u.beta.offset;
+    } else {
+        for (i = 0; i < n; i++)
+            out_i[i] = -c->u.beta.offset;
+    }
+
+    return 0;
+}
+
+int cram_beta_decode_int(cram_slice *slice, cram_codec *c, cram_block *in, char *out, int *out_size) {
+    int32_t *out_i = (int32_t *)out;
+    int i, n = *out_size;
+
+    if (c->u.beta.nbits) {
+        if (cram_not_enough_bits(in, c->u.beta.nbits * n))
+            return -1;
+
+        for (i = 0; i < n; i++)
+            out_i[i] = get_bits_MSB(in, c->u.beta.nbits) - c->u.beta.offset;
+    } else {
+        for (i = 0; i < n; i++)
+            out_i[i] = -c->u.beta.offset;
+    }
+
+    return 0;
+}
+
+int cram_beta_decode_char(cram_slice *slice, cram_codec *c, cram_block *in, char *out, int *out_size) {
+    int i, n = *out_size;
+
+
+    if (c->u.beta.nbits) {
+        if (cram_not_enough_bits(in, c->u.beta.nbits * n))
+            return -1;
+
+        if (out)
+            for (i = 0; i < n; i++)
+                out[i] = get_bits_MSB(in, c->u.beta.nbits) - c->u.beta.offset;
+        else
+            for (i = 0; i < n; i++)
+                get_bits_MSB(in, c->u.beta.nbits);
+    } else {
+        if (out)
+            for (i = 0; i < n; i++)
+                out[i] = -c->u.beta.offset;
+    }
+
+    return 0;
+}
+
+void cram_beta_decode_free(cram_codec *c) {
+    if (c)
+        free(c);
+}
+
+int cram_beta_describe(cram_codec *c, kstring_t *ks) {
+    return ksprintf(ks, "BETA(offset=%d, nbits=%d)",
+                    c->u.beta.offset, c->u.beta.nbits)
+        < 0 ? -1 : 0;
+}
+
+cram_codec *cram_beta_decode_init(cram_block_compression_hdr *hdr,
+                                  char *data, int size,
+                                  enum cram_encoding codec,
+                                  enum cram_external_type option,
+                                  int version, varint_vec *vv) {
+    cram_codec *c;
+    char *cp = data;
+
+    if (!(c = malloc(sizeof(*c))))
+        return NULL;
+
+    c->codec  = E_BETA;
+    if (option == E_INT || option == E_SINT)
+        c->decode = cram_beta_decode_int;
+    else if (option == E_LONG || option == E_SLONG)
+        c->decode = cram_beta_decode_long;
+    else if (option == E_BYTE_ARRAY || option == E_BYTE)
+        c->decode = cram_beta_decode_char;
+    else {
+        hts_log_error("BYTE_ARRAYs not supported by this codec");
+        free(c);
+        return NULL;
+    }
+    c->free   = cram_beta_decode_free;
+    c->describe = cram_beta_describe;
+
+    c->u.beta.nbits = -1;
+    c->u.beta.offset = vv->varint_get32(&cp, data + size, NULL);
+    if (cp < data + size) // Ensure test below works
+        c->u.beta.nbits  = vv->varint_get32(&cp, data + size, NULL);
+
+    if (cp - data != size
+        || c->u.beta.nbits < 0 || c->u.beta.nbits > 8 * sizeof(int)) {
+        hts_log_error("Malformed beta header stream");
+        free(c);
+        return NULL;
+    }
+
+    return c;
+}
+
+int cram_beta_encode_store(cram_codec *c, cram_block *b,
+                           char *prefix, int version) {
+    int len = 0, r = 0, n;
+
+    if (prefix) {
+        size_t l = strlen(prefix);
+        BLOCK_APPEND(b, prefix, l);
+        len += l;
+    }
+
+    len += (n = c->vv->varint_put32_blk(b, c->codec)); r |= n;
+    // codec length
+    len += (n = c->vv->varint_put32_blk(b, c->vv->varint_size(c->u.e_beta.offset)
+                                         + c->vv->varint_size(c->u.e_beta.nbits)));
+    r |= n;
+    len += (n = c->vv->varint_put32_blk(b, c->u.e_beta.offset)); r |= n;
+    len += (n = c->vv->varint_put32_blk(b, c->u.e_beta.nbits));  r |= n;
+
+    if (r > 0) return len;
+
+ block_err:
+    return -1;
+}
+
+int cram_beta_encode_long(cram_slice *slice, cram_codec *c,
+                          char *in, int in_size) {
+    int64_t *syms = (int64_t *)in;
+    int i, r = 0;
+
+    for (i = 0; i < in_size; i++)
+        r |= store_bits_MSB(c->out, syms[i] + c->u.e_beta.offset,
+                            c->u.e_beta.nbits);
+
+    return r;
+}
+
+int cram_beta_encode_int(cram_slice *slice, cram_codec *c,
+                         char *in, int in_size) {
+    int *syms = (int *)in;
+    int i, r = 0;
+
+    for (i = 0; i < in_size; i++)
+        r |= store_bits_MSB(c->out, syms[i] + c->u.e_beta.offset,
+                            c->u.e_beta.nbits);
+
+    return r;
+}
+
+int cram_beta_encode_char(cram_slice *slice, cram_codec *c,
+                          char *in, int in_size) {
+    unsigned char *syms = (unsigned char *)in;
+    int i, r = 0;
+
+    for (i = 0; i < in_size; i++)
+        r |= store_bits_MSB(c->out, syms[i] + c->u.e_beta.offset,
+                            c->u.e_beta.nbits);
+
+    return r;
+}
+
+void cram_beta_encode_free(cram_codec *c) {
+    if (c) free(c);
+}
+
+cram_codec *cram_beta_encode_init(cram_stats *st,
+                                  enum cram_encoding codec,
+                                  enum cram_external_type option,
+                                  void *dat,
+                                  int version, varint_vec *vv) {
+    cram_codec *c;
+    hts_pos_t min_val, max_val;
+    int len = 0;
+    int64_t range;
+
+    c = malloc(sizeof(*c));
+    if (!c)
+        return NULL;
+    c->codec  = E_BETA;
+    c->free   = cram_beta_encode_free;
+    if (option == E_INT || option == E_SINT)
+        c->encode = cram_beta_encode_int;
+    else if (option == E_LONG || option == E_SLONG)
+        c->encode = cram_beta_encode_long;
+    else
+        c->encode = cram_beta_encode_char;
+    c->store  = cram_beta_encode_store;
+    c->flush = NULL;
+
+    if (dat) {
+        min_val = ((hts_pos_t *)dat)[0];
+        max_val = ((hts_pos_t *)dat)[1];
+    } else {
+        min_val = INT_MAX;
+        max_val = INT_MIN;
+        int i;
+        for (i = 0; i < MAX_STAT_VAL; i++) {
+            if (!st->freqs[i])
+                continue;
+            if (min_val > i)
+                min_val = i;
+            max_val = i;
+        }
+        if (st->h) {
+            khint_t k;
+
+            for (k = kh_begin(st->h); k != kh_end(st->h); k++) {
+                if (!kh_exist(st->h, k))
+                    continue;
+
+                i = kh_key(st->h, k);
+                if (min_val > i)
+                    min_val = i;
+                if (max_val < i)
+                    max_val = i;
+            }
+        }
+    }
+
+    if (max_val < min_val)
+        goto err;
+
+    range = (int64_t) max_val - min_val;
+    switch (option) {
+    case E_SINT:
+        if (min_val < INT_MIN || range > INT_MAX)
+            goto err;
+        break;
+
+    case E_INT:
+        if (max_val > UINT_MAX || range > UINT_MAX)
+            goto err;
+        break;
+
+    default:
+        break;
+    }
+
+    c->u.e_beta.offset = -min_val;
+    while (range) {
+        len++;
+        range >>= 1;
+    }
+    c->u.e_beta.nbits = len;
+
+    return c;
+
+ err:
+    free(c);
+    return NULL;
+}
+
+/*
+ * ---------------------------------------------------------------------------
+ * XPACK: Packing multiple values into a single byte.  A fast transform that
+ * reduces time taken by entropy encoder and may also improve compression.
+ *
+ * This also has the additional requirement that the data series is not
+ * interleaved with another, permitting efficient encoding and decoding
+ * of all elements enmasse instead of needing to only extract the bits
+ * necessary per item.
+ */
+int cram_xpack_decode_long(cram_slice *slice, cram_codec *c, cram_block *in, char *out, int *out_size) {
+    int64_t *out_i = (int64_t *)out;
+    int i, n = *out_size;
+
+    if (c->u.xpack.nbits) {
+        for (i = 0; i < n; i++)
+            out_i[i] = c->u.xpack.rmap[get_bits_MSB(in, c->u.xpack.nbits)];
+    } else {
+        for (i = 0; i < n; i++)
+            out_i[i] = c->u.xpack.rmap[0];
+    }
+
+    return 0;
+}
+
+int cram_xpack_decode_int(cram_slice *slice, cram_codec *c, cram_block *in, char *out, int *out_size) {
+    int32_t *out_i = (int32_t *)out;
+    int i, n = *out_size;
+
+    if (c->u.xpack.nbits) {
+        if (cram_not_enough_bits(in, c->u.xpack.nbits * n))
+            return -1;
+
+        for (i = 0; i < n; i++)
+            out_i[i] = c->u.xpack.rmap[get_bits_MSB(in, c->u.xpack.nbits)];
+    } else {
+        for (i = 0; i < n; i++)
+            out_i[i] = c->u.xpack.rmap[0];
+    }
+
+    return 0;
+}
+
+static int cram_xpack_decode_expand_char(cram_slice *slice, cram_codec *c) {
+    cram_block *b = slice->block_by_id[512 + c->codec_id];
+    if (b)
+        return 0;
+
+    // get sub-codec data.
+    cram_block *sub_b = c->u.xpack.sub_codec->get_block(slice, c->u.xpack.sub_codec);
+    if (!sub_b)
+        return -1;
+
+    // Allocate local block to expand into
+    b = slice->block_by_id[512 + c->codec_id] = cram_new_block(0, 0);
+    if (!b)
+        return -1;
+    int n = sub_b->uncomp_size * 8/c->u.xpack.nbits;
+    BLOCK_GROW(b, n);
+    b->uncomp_size = n;
+
+    uint8_t p[256];
+    int z;
+    for (z = 0; z < 256; z++)
+        p[z] = c->u.xpack.rmap[z];
+    hts_unpack(sub_b->data, sub_b->uncomp_size, b->data, b->uncomp_size,
+               8 / c->u.xpack.nbits, p);
+
+    return 0;
+
+ block_err:
+    return -1;
+}
+
+int cram_xpack_decode_char(cram_slice *slice, cram_codec *c, cram_block *in, char *out, int *out_size) {
+    // FIXME: we need to ban data-series interleaving in the spec for this to work.
+
+    // Remember this may be called when threaded and multi-slice per container.
+    // Hence one cram_codec instance, multiple slices, multiple blocks.
+    // We therefore have to cache appropriate block info in slice and not codec.
+    //    b = cram_get_block_by_id(slice, c->external.content_id);
+    if (c->u.xpack.nval > 1) {
+        cram_xpack_decode_expand_char(slice, c);
+        cram_block *b = slice->block_by_id[512 + c->codec_id];
+        if (!b)
+            return -1;
+
+        if (out)
+            memcpy(out, b->data + b->byte, *out_size);
+        b->byte += *out_size;
+    } else {
+        memset(out, c->u.xpack.rmap[0], *out_size);
+    }
+
+    return 0;
+}
+
+void cram_xpack_decode_free(cram_codec *c) {
+    if (!c) return;
+
+    if (c->u.xpack.sub_codec)
+        c->u.xpack.sub_codec->free(c->u.xpack.sub_codec);
+
+    //free(slice->block_by_id[512 + c->codec_id]);
+    //slice->block_by_id[512 + c->codec_id] = 0;
+
+    free(c);
+}
+
+int cram_xpack_decode_size(cram_slice *slice, cram_codec *c) {
+    cram_xpack_decode_expand_char(slice, c);
+    return slice->block_by_id[512 + c->codec_id]->uncomp_size;
+}
+
+cram_block *cram_xpack_get_block(cram_slice *slice, cram_codec *c) {
+    cram_xpack_decode_expand_char(slice, c);
+    return slice->block_by_id[512 + c->codec_id];
+}
+
+cram_codec *cram_xpack_decode_init(cram_block_compression_hdr *hdr,
+                                   char *data, int size,
+                                   enum cram_encoding codec,
+                                   enum cram_external_type option,
+                                   int version, varint_vec *vv) {
+    cram_codec *c;
+    char *cp = data;
+    char *endp = data+size;
+
+    if (!(c = calloc(1, sizeof(*c))))
+        return NULL;
+
+    c->codec  = E_XPACK;
+    if (option == E_LONG)
+        c->decode = cram_xpack_decode_long;
+    else if (option == E_INT)
+        c->decode = cram_xpack_decode_int;
+    else if (option == E_BYTE_ARRAY || option == E_BYTE)
+        c->decode = cram_xpack_decode_char;
+    else {
+        fprintf(stderr, "BYTE_ARRAYs not supported by this codec\n");
+        goto malformed;
+    }
+    c->free = cram_xpack_decode_free;
+    c->size = cram_xpack_decode_size;
+    c->get_block = cram_xpack_get_block;
+    c->describe = NULL;
+
+    c->u.xpack.nbits = vv->varint_get32(&cp, endp, NULL);
+    c->u.xpack.nval  = vv->varint_get32(&cp, endp, NULL);
+    if (c->u.xpack.nbits >= 8  || c->u.xpack.nbits < 0 ||
+        c->u.xpack.nval  > 256 || c->u.xpack.nval < 0)
+        goto malformed;
+    int i;
+    for (i = 0; i < c->u.xpack.nval; i++) {
+        uint32_t v = vv->varint_get32(&cp, endp, NULL);
+        if (v >= 256)
+            goto malformed;
+        c->u.xpack.rmap[i] = v; // reverse map: e.g 0-3 to P,A,C,K
+    }
+
+    int encoding = vv->varint_get32(&cp, endp, NULL);
+    int sub_size = vv->varint_get32(&cp, endp, NULL);
+    if (sub_size < 0 || endp - cp < sub_size)
+        goto malformed;
+    c->u.xpack.sub_codec = cram_decoder_init(hdr, encoding, cp, sub_size,
+                                             option, version, vv);
+    if (c->u.xpack.sub_codec == NULL)
+        goto malformed;
+    cp += sub_size;
+
+    if (cp - data != size
+        || c->u.xpack.nbits < 0 || c->u.xpack.nbits > 8 * sizeof(int64_t)) {
+    malformed:
+        fprintf(stderr, "Malformed xpack header stream\n");
+        cram_xpack_decode_free(c);
+        return NULL;
+    }
+
+    return c;
+}
+
+int cram_xpack_encode_flush(cram_codec *c) {
+    // Pack the buffered up data
+    int meta_len;
+    uint64_t out_len;
+    uint8_t out_meta[1024];
+    uint8_t *out = hts_pack(BLOCK_DATA(c->out), BLOCK_SIZE(c->out),
+                            out_meta, &meta_len, &out_len);
+
+    // We now need to pass this through the next layer of transform
+    if (c->u.e_xpack.sub_codec->encode(NULL, // also indicates flush incoming
+                                     c->u.e_xpack.sub_codec,
+                                     (char *)out, out_len))
+        return -1;
+
+    int r = 0;
+    if (c->u.e_xpack.sub_codec->flush)
+        r = c->u.e_xpack.sub_codec->flush(c->u.e_xpack.sub_codec);
+
+    free(out);
+    return r;
+}
+
+int cram_xpack_encode_store(cram_codec *c, cram_block *b,
+                            char *prefix, int version) {
+    int len = 0, r = 0, n;
+
+    if (prefix) {
+        size_t l = strlen(prefix);
+        BLOCK_APPEND(b, prefix, l);
+        len += l;
+    }
+
+    // Store sub-codec
+    cram_codec *tc = c->u.e_xpack.sub_codec;
+    cram_block *tb = cram_new_block(0, 0);
+    if (!tb)
+        return -1;
+    int len2 = tc->store(tc, tb, NULL, version);
+
+    len += (n = c->vv->varint_put32_blk(b, c->codec)); r |= n;
+
+    // codec length
+    int len1 = 0, i;
+    for (i = 0; i < c->u.e_xpack.nval; i++)
+        len1 += (n = c->vv->varint_size(c->u.e_xpack.rmap[i])), r |= n;
+    len += (n = c->vv->varint_put32_blk(b, c->vv->varint_size(c->u.e_xpack.nbits)
+                                        +  c->vv->varint_size(c->u.e_xpack.nval)
+                                        + len1 + len2)); r |= n;
+
+    // The map and sub-codec
+    len += (n = c->vv->varint_put32_blk(b, c->u.e_xpack.nbits)); r |= n;
+    len += (n = c->vv->varint_put32_blk(b, c->u.e_xpack.nval));  r |= n;
+    for (i = 0; i < c->u.e_xpack.nval; i++)
+        len += (n = c->vv->varint_put32_blk(b, c->u.e_xpack.rmap[i])), r |= n;
+
+    BLOCK_APPEND(b, BLOCK_DATA(tb), BLOCK_SIZE(tb));
+
+    cram_free_block(tb);
+
+    return r > 0 ? len + len2 : -1;
+
+ block_err:
+    return -1;
+}
+
+// Same as cram_beta_encode_long
+int cram_xpack_encode_long(cram_slice *slice, cram_codec *c,
+                           char *in, int in_size) {
+    int64_t *syms = (int64_t *)in;
+    int i, r = 0;
+
+    for (i = 0; i < in_size; i++)
+        r |= store_bits_MSB(c->out, c->u.e_xpack.map[syms[i]], c->u.e_xpack.nbits);
+
+    return r;
+}
+
+int cram_xpack_encode_int(cram_slice *slice, cram_codec *c,
+                          char *in, int in_size) {
+    int *syms = (int *)in;
+    int i, r = 0;
+
+    for (i = 0; i < in_size; i++)
+        r |= store_bits_MSB(c->out, c->u.e_xpack.map[syms[i]], c->u.e_xpack.nbits);
+
+    return r;
+}
+
+int cram_xpack_encode_char(cram_slice *slice, cram_codec *c,
+                           char *in, int in_size) {
+    BLOCK_APPEND(c->out, in, in_size);
+    return 0;
+
+ block_err:
+    return -1;
+}
+
+void cram_xpack_encode_free(cram_codec *c) {
+    if (!c) return;
+
+    if (c->u.e_xpack.sub_codec)
+        c->u.e_xpack.sub_codec->free(c->u.e_xpack.sub_codec);
+
+    cram_free_block(c->out);
+
+    free(c);
+}
+
+cram_codec *cram_xpack_encode_init(cram_stats *st,
+                                   enum cram_encoding codec,
+                                   enum cram_external_type option,
+                                   void *dat,
+                                   int version, varint_vec *vv) {
+    cram_codec *c;
+
+    if (!(c = malloc(sizeof(*c))))
+        return NULL;
+
+    c->codec  = E_XPACK;
+    c->free   = cram_xpack_encode_free;
+    if (option == E_LONG)
+        c->encode = cram_xpack_encode_long;
+    else if (option == E_INT)
+        c->encode = cram_xpack_encode_int;
+    else
+        c->encode = cram_xpack_encode_char;
+    c->store  = cram_xpack_encode_store;
+    c->flush  = cram_xpack_encode_flush;
+
+    cram_xpack_encoder *e = (cram_xpack_encoder *)dat;
+    c->u.e_xpack.nbits = e->nbits;
+    c->u.e_xpack.nval = e->nval;
+    c->u.e_xpack.sub_codec = cram_encoder_init(e->sub_encoding, NULL,
+                                               E_BYTE_ARRAY, e->sub_codec_dat,
+                                               version, vv);
+
+    // Initialise fwd and rev maps
+    memcpy(c->u.e_xpack.map, e->map, sizeof(e->map)); // P,A,C,K to 0,1,2,3
+    int i, n;
+    for (i = n = 0; i < 256; i++)
+        if (e->map[i] != -1)
+            c->u.e_xpack.rmap[n++] = i;               // 0,1,2,3 to P,A,C,K
+    if (n != e->nval) {
+        fprintf(stderr, "Incorrectly specified number of map items in PACK\n");
+        return NULL;
+    }
+
+    return c;
+}
+
+/*
+ * ---------------------------------------------------------------------------
+ * XDELTA: subtract successive values, zig-zag to turn +/- to + only,
+ * and then var-int encode the result.
+ *
+ * This also has the additional requirement that the data series is not
+ * interleaved with another, permitting efficient encoding and decoding
+ * of all elements enmasse instead of needing to only extract the bits
+ * necessary per item.
+ */
+
+static uint8_t  zigzag8 (int8_t  x) { return (x << 1) ^ (x >>  7); }
+static uint16_t zigzag16(int16_t x) { return (x << 1) ^ (x >> 15); }
+static uint32_t zigzag32(int32_t x) { return (x << 1) ^ (x >> 31); }
+
+//static int8_t  unzigzag8 (uint8_t  x) { return (x >> 1) ^ -(x & 1); }
+static int16_t unzigzag16(uint16_t x) { return (x >> 1) ^ -(x & 1); }
+static int32_t unzigzag32(uint32_t x) { return (x >> 1) ^ -(x & 1); }
+
+int cram_xdelta_decode_long(cram_slice *slice, cram_codec *c, cram_block *in, char *out, int *out_size) {
+    return -1;
+}
+
+int cram_xdelta_decode_int(cram_slice *slice, cram_codec *c, cram_block *in, char *out, int *out_size) {
+    // Slow value-by-value method for now
+    uint32_t *out32 = (uint32_t *)out;
+    int i;
+    for (i = 0; i < *out_size; i++) {
+        uint32_t v;
+        int one = 1;
+        if (c->u.e_xdelta.sub_codec->decode(slice, c->u.e_xdelta.sub_codec, in,
+                                          (char *)&v, &one) < 0)
+            return -1;
+        uint32_t d = unzigzag32(v);
+        c->u.xdelta.last = out32[i] = d + c->u.xdelta.last;
+    }
+
+    return 0;
+}
+
+static int cram_xdelta_decode_expand_char(cram_slice *slice, cram_codec *c) {
+    return -1;
+}
+
+int cram_xdelta_decode_char(cram_slice *slice, cram_codec *c, cram_block *in, char *out, int *out_size) {
+    return -1;
+}
+
+static inline int16_t le_int2(int16_t i) {
+    int16_t s;
+    i16_to_le(i, (uint8_t *)&s);
+    return s;
+}
+
+int cram_xdelta_decode_block(cram_slice *slice, cram_codec *c, cram_block *in,
+                             char *out_, int *out_size) {
+    cram_block *out = (cram_block *)out_;
+    cram_block *b = c->u.e_xdelta.sub_codec->get_block(slice, c->u.e_xdelta.sub_codec);
+    int i = 0;
+
+    const int w = c->u.xdelta.word_size;
+    uint32_t npad = (w - *out_size%w)%w;
+    uint32_t out_sz = *out_size + npad;
+    c->u.xdelta.last = 0;  // reset for each new array
+
+    for (i = 0; i < out_sz; i += w) {
+        uint16_t v;
+        // Need better interface
+        char *cp = (char *)b->data + b->byte;
+        char *cp_end = (char *)b->data + b->uncomp_size;
+        int err = 0;
+        v = c->vv->varint_get32(&cp, cp_end, &err);
+        if (err)
+            return -1;
+        b->byte = cp - (char *)b->data;
+
+        switch(w) {
+        case 2: {
+            int16_t d = unzigzag16(v), z;
+            c->u.xdelta.last = d + c->u.xdelta.last;
+            z = le_int2(c->u.xdelta.last);
+            BLOCK_APPEND(out, &z, 2-npad);
+            npad = 0;
+            break;
+        }
+        default:
+            fprintf(stderr, "Unsupported word size by XDELTA\n");
+            return -1;
+        }
+    }
+
+    return 0;
+
+ block_err:
+    return -1;
+}
+
+void cram_xdelta_decode_free(cram_codec *c) {
+    if (!c) return;
+
+    if (c->u.xdelta.sub_codec)
+        c->u.xdelta.sub_codec->free(c->u.xdelta.sub_codec);
+
+    free(c);
+}
+
+int cram_xdelta_decode_size(cram_slice *slice, cram_codec *c) {
+    cram_xdelta_decode_expand_char(slice, c);
+    return slice->block_by_id[512 + c->codec_id]->uncomp_size;
+}
+
+cram_block *cram_xdelta_get_block(cram_slice *slice, cram_codec *c) {
+    cram_xdelta_decode_expand_char(slice, c);
+    return slice->block_by_id[512 + c->codec_id];
+}
+
+cram_codec *cram_xdelta_decode_init(cram_block_compression_hdr *hdr,
+                                    char *data, int size,
+                                    enum cram_encoding codec,
+                                    enum cram_external_type option,
+                                    int version, varint_vec *vv) {
+    cram_codec *c;
+    char *cp = data;
+    char *endp = data+size;
+
+    if (!(c = calloc(1, sizeof(*c))))
+        return NULL;
+
+    c->codec  = E_XDELTA;
+    if (option == E_LONG)
+        c->decode = cram_xdelta_decode_long;
+    else if (option == E_INT)
+        c->decode = cram_xdelta_decode_int;
+    else if (option == E_BYTE_ARRAY || option == E_BYTE)
+        c->decode = cram_xdelta_decode_char;
+    else if (option == E_BYTE_ARRAY_BLOCK) {
+        option = E_BYTE_ARRAY;
+        c->decode = cram_xdelta_decode_block;
+    } else {
+        free(c);
+        return NULL;
+    }
+    c->free = cram_xdelta_decode_free;
+    c->size = cram_xdelta_decode_size;
+    c->get_block = cram_xdelta_get_block;
+    c->describe = NULL;
+
+    c->u.xdelta.word_size = vv->varint_get32(&cp, endp, NULL);
+    c->u.xdelta.last = 0;
+
+    int encoding = vv->varint_get32(&cp, endp, NULL);
+    int sub_size = vv->varint_get32(&cp, endp, NULL);
+    if (sub_size < 0 || endp - cp < sub_size)
+        goto malformed;
+    c->u.xdelta.sub_codec = cram_decoder_init(hdr, encoding, cp, sub_size,
+                                              option, version, vv);
+    if (c->u.xdelta.sub_codec == NULL)
+        goto malformed;
+    cp += sub_size;
+
+    if (cp - data != size) {
+    malformed:
+        fprintf(stderr, "Malformed xdelta header stream\n");
+        cram_xdelta_decode_free(c);
+        return NULL;
+    }
+
+    return c;
+}
+
+int cram_xdelta_encode_flush(cram_codec *c) {
+    int r = -1;
+    cram_block *b = cram_new_block(0, 0);
+    if (!b)
+        return -1;
+
+    switch (c->u.e_xdelta.word_size) {
+    case 2: {
+        // Delta + zigzag transform.
+        // Subtracting two 8-bit values has a 9-bit result (-255 to 255).
+        // However think of it as turning a wheel clockwise or anti-clockwise.
+        // If it has 256 gradations then a -ve rotation followed by a +ve
+        // rotation of the same amount reverses it regardless.
+        //
+        // Similarly the zig-zag transformation doesn't invent any extra bits,
+        // so the entire thing can be done in-situ.  This may permit faster
+        // SIMD loops if we break apart the steps.
+
+        // uint16_t last = 0, d;
+        // for (i = 0; i < n; i++) {
+        //     d = io[i] - last;
+        //     last = io[i];
+        //     io[i] = zigzag16(vd);
+        // }
+
+        // --- vs ---
+
+        // for (i = n-1; i >= 1; i--)
+        //     io[i] -= io[i-1];
+        // for (i = 0; i < n; i++)
+        //     io[i] = zigzag16(io[i]);
+
+        // varint: need array variant for speed here.
+        // With zig-zag
+        int i, n = BLOCK_SIZE(c->out)/2;;
+        uint16_t *dat = (uint16_t *)BLOCK_DATA(c->out), last = 0;
+
+        if (n*2 < BLOCK_SIZE(c->out)) {
+            // half word
+            last = *(uint8_t *)dat;
+            c->vv->varint_put32_blk(b, zigzag16(last));
+            dat = (uint16_t *)(((uint8_t *)dat)+1);
+        }
+
+        for (i = 0; i < n; i++) {
+            uint16_t d = dat[i] - last; // possibly unaligned
+            last = dat[i];
+            c->vv->varint_put32_blk(b, zigzag16(d));
+        }
+
+        break;
+    }
+
+    case 4: {
+        int i, n = BLOCK_SIZE(c->out)/4;;
+        uint32_t *dat = (uint32_t *)BLOCK_DATA(c->out), last = 0;
+
+        for (i = 0; i < n; i++) {
+            uint32_t d = dat[i] - last;
+            last = dat[i];
+            c->vv->varint_put32_blk(b, zigzag32(d));
+        }
+
+        break;
+    }
+
+    case 1: {
+        int i, n = BLOCK_SIZE(c->out);;
+        uint8_t *dat = (uint8_t *)BLOCK_DATA(c->out), last = 0;
+
+        for (i = 0; i < n; i++) {
+            uint32_t d = dat[i] - last;
+            last = dat[i];
+            c->vv->varint_put32_blk(b, zigzag8(d));
+        }
+
+        break;
+    }
+
+    default:
+        goto err;
+    }
+
+    if (c->u.e_xdelta.sub_codec->encode(NULL, c->u.e_xdelta.sub_codec,
+                                      (char *)b->data, b->byte))
+        goto err;
+
+    r = 0;
+
+ err:
+    cram_free_block(b);
+    return r;
+
+}
+
+int cram_xdelta_encode_store(cram_codec *c, cram_block *b,
+                            char *prefix, int version) {
+    int len = 0, r = 0, n;
+
+    if (prefix) {
+        size_t l = strlen(prefix);
+        BLOCK_APPEND(b, prefix, l);
+        len += l;
+    }
+
+    // Store sub-codec
+    cram_codec *tc = c->u.e_xdelta.sub_codec;
+    cram_block *tb = cram_new_block(0, 0);
+    if (!tb)
+        return -1;
+    int len2 = tc->store(tc, tb, NULL, version);
+
+    len += (n = c->vv->varint_put32_blk(b, c->codec)); r |= n;
+
+    // codec length
+    len += (n = c->vv->varint_put32_blk(b, c->vv->varint_size(c->u.e_xdelta.word_size)
+                                        + len2)); r |= n;
+
+    // This and sub-codec
+    len += (n = c->vv->varint_put32_blk(b, c->u.e_xdelta.word_size)); r |= n;
+    BLOCK_APPEND(b, BLOCK_DATA(tb), BLOCK_SIZE(tb));
+
+    cram_free_block(tb);
+
+    return r > 0 ? len + len2 : -1;
+
+ block_err:
+    return -1;
+}
+
+// Same as cram_beta_encode_long
+int cram_xdelta_encode_long(cram_slice *slice, cram_codec *c,
+                           char *in, int in_size) {
+    return -1;
+}
+
+int cram_xdelta_encode_int(cram_slice *slice, cram_codec *c,
+                          char *in, int in_size) {
+    return -1;
+}
+
+int cram_xdelta_encode_char(cram_slice *slice, cram_codec *c,
+                            char *in, int in_size) {
+    char *dat = malloc(in_size*5);
+    if (!dat)
+        return -1;
+    char *cp = dat, *cp_end = dat + in_size*5;
+
+    c->u.e_xdelta.last = 0; // reset for each new array
+    if (c->u.e_xdelta.word_size == 2) {
+        int i, part;
+
+        part = in_size%2;
+        if (part) {
+            uint16_t z = in[0];
+            c->u.e_xdelta.last = le_int2(z);
+            cp += c->vv->varint_put32(cp, cp_end, zigzag16(c->u.e_xdelta.last));
+        }
+
+        uint16_t *in16 = (uint16_t *)(in+part);
+        for (i = 0; i < in_size/2; i++) {
+            uint16_t d = le_int2(in16[i]) - c->u.e_xdelta.last;
+            c->u.e_xdelta.last = le_int2(in16[i]);
+            cp += c->vv->varint_put32(cp, cp_end, zigzag16(d));
+        }
+    }
+    if (c->u.e_xdelta.sub_codec->encode(slice, c->u.e_xdelta.sub_codec,
+                                      (char *)dat, cp-dat)) {
+        free(dat);
+        return -1;
+    }
+
+    free(dat);
+    return 0;
+}
+
+void cram_xdelta_encode_free(cram_codec *c) {
+    if (!c) return;
+
+    if (c->u.e_xdelta.sub_codec)
+        c->u.e_xdelta.sub_codec->free(c->u.e_xdelta.sub_codec);
+
+    cram_free_block(c->out);
+
+    free(c);
+}
+
+cram_codec *cram_xdelta_encode_init(cram_stats *st,
+                                    enum cram_encoding codec,
+                                    enum cram_external_type option,
+                                    void *dat,
+                                    int version, varint_vec *vv) {
+    cram_codec *c;
+
+    if (!(c = malloc(sizeof(*c))))
+        return NULL;
+
+    c->codec  = E_XDELTA;
+    c->free   = cram_xdelta_encode_free;
+    if (option == E_LONG)
+        c->encode = cram_xdelta_encode_long;
+    else if (option == E_INT)
+        c->encode = cram_xdelta_encode_int;
+    else
+        c->encode = cram_xdelta_encode_char;
+    c->store  = cram_xdelta_encode_store;
+    c->flush  = cram_xdelta_encode_flush;
+
+    cram_xdelta_encoder *e = (cram_xdelta_encoder *)dat;
+    c->u.e_xdelta.word_size = e->word_size;
+    c->u.e_xdelta.last = 0;
+    c->u.e_xdelta.sub_codec = cram_encoder_init(e->sub_encoding, NULL,
+                                                E_BYTE_ARRAY,
+                                                e->sub_codec_dat,
+                                                version, vv);
+
+    return c;
+}
+
+/*
+ * ---------------------------------------------------------------------------
+ * XRLE
+ *
+ * This also has the additional requirement that the data series is not
+ * interleaved with another, permitting efficient encoding and decoding
+ * of all elements enmasse instead of needing to only extract the bits
+ * necessary per item.
+ */
+int cram_xrle_decode_long(cram_slice *slice, cram_codec *c, cram_block *in, char *out, int *out_size) {
+    // TODO if and when needed
+    return -1;
+}
+
+int cram_xrle_decode_int(cram_slice *slice, cram_codec *c, cram_block *in, char *out, int *out_size) {
+    // TODO if and when needed
+    return -1;
+}
+
+// Expands an XRLE transform and caches result in slice->block_by_id[]
+static int cram_xrle_decode_expand_char(cram_slice *slice, cram_codec *c) {
+    cram_block *b = slice->block_by_id[512 + c->codec_id];
+    if (b)
+        return 0;
+
+    b = slice->block_by_id[512 + c->codec_id] = cram_new_block(0, 0);
+    if (!b)
+        return -1;
+    cram_block *lit_b = c->u.xrle.lit_codec->get_block(slice, c->u.xrle.lit_codec);
+    if (!lit_b)
+        return -1;
+    unsigned char *lit_dat = lit_b->data;
+    unsigned int lit_sz = lit_b->uncomp_size;
+    unsigned int len_sz = c->u.xrle.len_codec->size(slice, c->u.xrle.len_codec);
+
+    cram_block *len_b = c->u.xrle.len_codec->get_block(slice, c->u.xrle.len_codec);
+    if (!len_b)
+        return -1;
+    unsigned char *len_dat = len_b->data;
+
+    uint8_t rle_syms[256];
+    int rle_nsyms = 0;
+    int i;
+    for (i = 0; i < 256; i++) {
+        if (c->u.xrle.rep_score[i] > 0)
+            rle_syms[rle_nsyms++] = i;
+    }
+
+    uint64_t out_sz;
+    int nb = var_get_u64(len_dat, len_dat+len_sz, &out_sz);
+    if (!(b->data = malloc(out_sz)))
+        return -1;
+    hts_rle_decode(lit_dat, lit_sz,
+                   len_dat+nb, len_sz-nb,
+                   rle_syms, rle_nsyms,
+                   b->data, &out_sz);
+    b->uncomp_size = out_sz;
+
+    return 0;
+}
+
+int cram_xrle_decode_size(cram_slice *slice, cram_codec *c) {
+    cram_xrle_decode_expand_char(slice, c);
+    return slice->block_by_id[512 + c->codec_id]->uncomp_size;
+}
+
+cram_block *cram_xrle_get_block(cram_slice *slice, cram_codec *c) {
+    cram_xrle_decode_expand_char(slice, c);
+    return slice->block_by_id[512 + c->codec_id];
+}
+
+int cram_xrle_decode_char(cram_slice *slice, cram_codec *c, cram_block *in, char *out, int *out_size) {
+    int n = *out_size;
+
+    cram_xrle_decode_expand_char(slice, c);
+    cram_block *b = slice->block_by_id[512 + c->codec_id];
+
+    memcpy(out, b->data + b->idx, n);
+    b->idx += n;
+    return 0;
+
+    // Old code when not cached
+    while (n > 0) {
+        if (c->u.xrle.cur_len == 0) {
+            unsigned char lit;
+            int one = 1;
+            if (c->u.xrle.lit_codec->decode(slice, c->u.xrle.lit_codec, in,
+                                          (char *)&lit, &one) < 0)
+                return -1;
+            c->u.xrle.cur_lit = lit;
+
+            if (c->u.xrle.rep_score[lit] > 0) {
+                if (c->u.xrle.len_codec->decode(slice, c->u.xrle.len_codec, in,
+                                              (char *)&c->u.xrle.cur_len, &one) < 0)
+                    return -1;
+            } // else cur_len still zero
+            //else fprintf(stderr, "%d\n", lit);
+
+            c->u.xrle.cur_len++;
+        }
+
+        if (n >= c->u.xrle.cur_len) {
+            memset(out, c->u.xrle.cur_lit, c->u.xrle.cur_len);
+            out += c->u.xrle.cur_len;
+            n -= c->u.xrle.cur_len;
+            c->u.xrle.cur_len = 0;
+        } else {
+            memset(out, c->u.xrle.cur_lit, n);
+            out += n;
+            c->u.xrle.cur_len -= n;
+            n = 0;
+        }
+    }
+
+    return 0;
+}
+
+void cram_xrle_decode_free(cram_codec *c) {
+    if (!c) return;
+
+    if (c->u.xrle.len_codec)
+        c->u.xrle.len_codec->free(c->u.xrle.len_codec);
+
+    if (c->u.xrle.lit_codec)
+        c->u.xrle.lit_codec->free(c->u.xrle.lit_codec);
+
+    free(c);
+}
+
+cram_codec *cram_xrle_decode_init(cram_block_compression_hdr *hdr,
+                                  char *data, int size,
+                                  enum cram_encoding codec,
+                                  enum cram_external_type option,
+                                  int version, varint_vec *vv) {
+    cram_codec *c;
+    char *cp = data;
+    char *endp = data+size;
+    int err = 0;
+
+    if (!(c = calloc(1, sizeof(*c))))
+        return NULL;
+
+    c->codec  = E_XRLE;
+    if (option == E_LONG)
+        c->decode = cram_xrle_decode_long;
+    else if (option == E_INT)
+        c->decode = cram_xrle_decode_int;
+    else if (option == E_BYTE_ARRAY || option == E_BYTE)
+        c->decode = cram_xrle_decode_char;
+    else {
+        fprintf(stderr, "BYTE_ARRAYs not supported by this codec\n");
+        free(c);
+        return NULL;
+    }
+    c->free   = cram_xrle_decode_free;
+    c->size   = cram_xrle_decode_size;
+    c->get_block = cram_xrle_get_block;
+    c->describe = NULL;
+    c->u.xrle.cur_len = 0;
+    c->u.xrle.cur_lit = -1;
+
+    // RLE map
+    int i, j, nrle = vv->varint_get32(&cp, endp, &err);
+    memset(c->u.xrle.rep_score, 0, 256*sizeof(*c->u.xrle.rep_score));
+    for (i = 0; i < nrle && i < 256; i++) {
+        j = vv->varint_get32(&cp, endp, &err);
+        if (j >= 0 && j < 256)
+            c->u.xrle.rep_score[j] = 1;
+    }
+
+    // Length and literal sub encodings
+    c->u.xrle.len_encoding = vv->varint_get32(&cp, endp, &err);
+    int sub_size = vv->varint_get32(&cp, endp, &err);
+    if (sub_size < 0 || endp - cp < sub_size)
+        goto malformed;
+    c->u.xrle.len_codec = cram_decoder_init(hdr, c->u.xrle.len_encoding,
+                                            cp, sub_size, E_INT, version, vv);
+    if (c->u.xrle.len_codec == NULL)
+        goto malformed;
+    cp += sub_size;
+
+    c->u.xrle.lit_encoding = vv->varint_get32(&cp, endp, &err);
+    sub_size = vv->varint_get32(&cp, endp, &err);
+    if (sub_size < 0 || endp - cp < sub_size)
+        goto malformed;
+    c->u.xrle.lit_codec = cram_decoder_init(hdr, c->u.xrle.lit_encoding,
+                                            cp, sub_size, option, version, vv);
+    if (c->u.xrle.lit_codec == NULL)
+        goto malformed;
+    cp += sub_size;
+
+    if (err)
+        goto malformed;
+
+    return c;
+
+ malformed:
+    fprintf(stderr, "Malformed xrle header stream\n");
+    cram_xrle_decode_free(c);
+    return NULL;
+}
+
+int cram_xrle_encode_flush(cram_codec *c) {
+    uint8_t *out_lit, *out_len;
+    uint64_t out_lit_size, out_len_size;
+    uint8_t rle_syms[256];
+    int rle_nsyms = 0, i;
+
+    for (i = 0; i < 256; i++)
+        if (c->u.e_xrle.rep_score[i] > 0)
+            rle_syms[rle_nsyms++] = i;
+
+    if (!c->u.e_xrle.to_flush) {
+        c->u.e_xrle.to_flush = (char *)BLOCK_DATA(c->out);
+        c->u.e_xrle.to_flush_size = BLOCK_SIZE(c->out);
+    }
+
+    out_len = malloc(c->u.e_xrle.to_flush_size+8);
+    if (!out_len)
+        return -1;
+
+    int nb = var_put_u64(out_len, NULL, c->u.e_xrle.to_flush_size);
+
+    out_lit = hts_rle_encode((uint8_t *)c->u.e_xrle.to_flush, c->u.e_xrle.to_flush_size,
+                             out_len+nb, &out_len_size,
+                             rle_syms, &rle_nsyms,
+                             NULL, &out_lit_size);
+    out_len_size += nb;
+
+
+    // TODO: can maybe "gift" the sub codec the data block, to remove
+    // one level of memcpy.
+    if (c->u.e_xrle.len_codec->encode(NULL,
+                                      c->u.e_xrle.len_codec,
+                                      (char *)out_len, out_len_size))
+        return -1;
+
+    if (c->u.e_xrle.lit_codec->encode(NULL,
+                                      c->u.e_xrle.lit_codec,
+                                      (char *)out_lit, out_lit_size))
+        return -1;
+
+    free(out_len);
+    free(out_lit);
+
+    return 0;
+}
+
+int cram_xrle_encode_store(cram_codec *c, cram_block *b,
+                            char *prefix, int version) {
+    int len = 0, r = 0, n;
+    cram_codec *tc;
+    cram_block *b_rle, *b_len, *b_lit;
+
+    if (prefix) {
+        size_t l = strlen(prefix);
+        BLOCK_APPEND(b, prefix, l);
+        len += l;
+    }
+
+    // List of symbols to RLE
+    b_rle = cram_new_block(0, 0);
+    if (!b_rle)
+        return -1;
+    int i, nrle = 0, len1 = 0;
+    for (i = 0; i < 256; i++) {
+        if (c->u.e_xrle.rep_score[i] > 0) {
+            nrle++;
+            len1 += (n = c->vv->varint_put32_blk(b_rle,i)); r |= n;
+        }
+    }
+
+    // Store length and literal sub-codecs to get encoded length
+    tc = c->u.e_xrle.len_codec;
+    b_len = cram_new_block(0, 0);
+    if (!b_len)
+        return -1;
+    int len2 = tc->store(tc, b_len, NULL, version);
+
+    tc = c->u.e_xrle.lit_codec;
+    b_lit = cram_new_block(0, 0);
+    if (!b_lit)
+        return -1;
+    int len3 = tc->store(tc, b_lit, NULL, version);
+
+    len += (n = c->vv->varint_put32_blk(b, c->codec)); r |= n;
+    len += (n = c->vv->varint_put32_blk(b, len1 + len2 + len3
+                                        + c->vv->varint_size(nrle))); r |= n;
+    len += (n = c->vv->varint_put32_blk(b, nrle)); r |= n;
+    BLOCK_APPEND(b, BLOCK_DATA(b_rle), BLOCK_SIZE(b_rle));
+    BLOCK_APPEND(b, BLOCK_DATA(b_len), BLOCK_SIZE(b_len));
+    BLOCK_APPEND(b, BLOCK_DATA(b_lit), BLOCK_SIZE(b_lit));
+
+    cram_free_block(b_rle);
+    cram_free_block(b_len);
+    cram_free_block(b_lit);
+
+    if (r > 0)
+        return len + len1 + len2 + len3;
+
+ block_err:
+    return -1;
+}
+
+int cram_xrle_encode_long(cram_slice *slice, cram_codec *c,
+                           char *in, int in_size) {
+    // TODO if and when needed
+    return -1;
+}
+
+int cram_xrle_encode_int(cram_slice *slice, cram_codec *c,
+                          char *in, int in_size) {
+    // TODO if and when needed
+    return -1;
+}
+
+int cram_xrle_encode_char(cram_slice *slice, cram_codec *c,
+                          char *in, int in_size) {
+    if (c->u.e_xrle.to_flush) {
+        if (!c->out && !(c->out = cram_new_block(0, 0)))
+            return -1;
+        BLOCK_APPEND(c->out, c->u.e_xrle.to_flush, c->u.e_xrle.to_flush_size);
+        c->u.e_xrle.to_flush = NULL;
+        c->u.e_xrle.to_flush_size = 0;
+    }
+
+    if (c->out && BLOCK_SIZE(c->out) > 0) {
+        // Gathering data
+        BLOCK_APPEND(c->out, in, in_size);
+        return 0;
+    }
+
+    // else cache copy of the data we're about to send to flush instead.
+    c->u.e_xrle.to_flush = in;
+    c->u.e_xrle.to_flush_size = in_size;
+    return 0;
+
+ block_err:
+    return -1;
+}
+
+void cram_xrle_encode_free(cram_codec *c) {
+    if (!c) return;
+
+    if (c->u.e_xrle.len_codec)
+        c->u.e_xrle.len_codec->free(c->u.e_xrle.len_codec);
+    if (c->u.e_xrle.lit_codec)
+        c->u.e_xrle.lit_codec->free(c->u.e_xrle.lit_codec);
+
+    cram_free_block(c->out);
+
+    free(c);
+}
+
+cram_codec *cram_xrle_encode_init(cram_stats *st,
+                                  enum cram_encoding codec,
+                                  enum cram_external_type option,
+                                  void *dat,
+                                  int version, varint_vec *vv) {
+    cram_codec *c;
+
+    if (!(c = malloc(sizeof(*c))))
+        return NULL;
+
+    c->codec  = E_XRLE;
+    c->free   = cram_xrle_encode_free;
+    if (option == E_LONG)
+        c->encode = cram_xrle_encode_long;
+    else if (option == E_INT)
+        c->encode = cram_xrle_encode_int;
+    else
+        c->encode = cram_xrle_encode_char;
+    c->store  = cram_xrle_encode_store;
+    c->flush  = cram_xrle_encode_flush;
+
+    cram_xrle_encoder *e = (cram_xrle_encoder *)dat;
+
+    c->u.e_xrle.len_codec = cram_encoder_init(e->len_encoding, NULL,
+                                              E_BYTE, e->len_dat,
+                                              version, vv);
+    c->u.e_xrle.lit_codec = cram_encoder_init(e->lit_encoding, NULL,
+                                              E_BYTE, e->lit_dat,
+                                              version, vv);
+    c->u.e_xrle.cur_lit = -1;
+    c->u.e_xrle.cur_len = -1;
+    c->u.e_xrle.to_flush = NULL;
+    c->u.e_xrle.to_flush_size = 0;
+
+    memcpy(c->u.e_xrle.rep_score, e->rep_score, 256*sizeof(*c->u.e_xrle.rep_score));
+
+    return c;
+}
+
+/*
+ * ---------------------------------------------------------------------------
+ * SUBEXP
+ */
+int cram_subexp_decode(cram_slice *slice, cram_codec *c, cram_block *in, char *out, int *out_size) {
+    int32_t *out_i = (int32_t *)out;
+    int n, count;
+    int k = c->u.subexp.k;
+
+    for (count = 0, n = *out_size; count < n; count++) {
+        int i = 0, tail;
+        int val;
+
+        /* Get number of 1s */
+        //while (get_bit_MSB(in) == 1) i++;
+        i = get_one_bits_MSB(in);
+        if (i < 0 || cram_not_enough_bits(in, i > 0 ? i + k - 1 : k))
+            return -1;
+        /*
+         * Val is
+         * i > 0:  2^(k+i-1) + k+i-1 bits
+         * i = 0:  k bits
+         */
+        if (i) {
+            tail = i + k-1;
+            val = 0;
+            while (tail) {
+                //val = val<<1; val |= get_bit_MSB(in);
+                GET_BIT_MSB(in, val);
+                tail--;
+            }
+            val += 1 << (i + k-1);
+        } else {
+            tail = k;
+            val = 0;
+            while (tail) {
+                //val = val<<1; val |= get_bit_MSB(in);
+                GET_BIT_MSB(in, val);
+                tail--;
+            }
+        }
+
+        out_i[count] = val - c->u.subexp.offset;
+    }
+
+    return 0;
+}
+
+void cram_subexp_decode_free(cram_codec *c) {
+    if (c)
+        free(c);
+}
+
+int cram_subexp_describe(cram_codec *c, kstring_t *ks) {
+    return ksprintf(ks, "SUBEXP(offset=%d,k=%d)",
+                    c->u.subexp.offset,
+                    c->u.subexp.k)
+        < 0 ? -1 : 0;
+}
+
+cram_codec *cram_subexp_decode_init(cram_block_compression_hdr *hdr,
+                                    char *data, int size,
+                                    enum cram_encoding codec,
+                                    enum cram_external_type option,
+                                    int version, varint_vec *vv) {
+    cram_codec *c;
+    char *cp = data;
+
+    if (option != E_INT) {
+        hts_log_error("This codec only supports INT encodings");
+        return NULL;
+    }
+
+    if (!(c = malloc(sizeof(*c))))
+        return NULL;
+
+    c->codec  = E_SUBEXP;
+    c->decode = cram_subexp_decode;
+    c->free   = cram_subexp_decode_free;
+    c->describe = cram_subexp_describe;
+    c->u.subexp.k = -1;
+
+    c->u.subexp.offset = vv->varint_get32(&cp, data + size, NULL);
+    c->u.subexp.k      = vv->varint_get32(&cp, data + size, NULL);
+
+    if (cp - data != size || c->u.subexp.k < 0) {
+        hts_log_error("Malformed subexp header stream");
+        free(c);
+        return NULL;
+    }
+
+    return c;
+}
+
+/*
+ * ---------------------------------------------------------------------------
+ * GAMMA
+ */
+int cram_gamma_decode(cram_slice *slice, cram_codec *c, cram_block *in, char *out, int *out_size) {
+    int32_t *out_i = (int32_t *)out;
+    int i, n;
+
+    for (i = 0, n = *out_size; i < n; i++) {
+        int nz = 0;
+        int val;
+        //while (get_bit_MSB(in) == 0) nz++;
+        nz = get_zero_bits_MSB(in);
+        if (cram_not_enough_bits(in, nz))
+            return -1;
+        val = 1;
+        while (nz > 0) {
+            //val <<= 1; val |= get_bit_MSB(in);
+            GET_BIT_MSB(in, val);
+            nz--;
+        }
+
+        out_i[i] = val - c->u.gamma.offset;
+    }
+
+    return 0;
+}
+
+void cram_gamma_decode_free(cram_codec *c) {
+    if (c)
+        free(c);
+}
+
+int cram_gamma_describe(cram_codec *c, kstring_t *ks) {
+    return ksprintf(ks, "GAMMA(offset=%d)", c->u.subexp.offset)
+        < 0 ? -1 : 0;
+}
+
+cram_codec *cram_gamma_decode_init(cram_block_compression_hdr *hdr,
+                                   char *data, int size,
+                                   enum cram_encoding codec,
+                                   enum cram_external_type option,
+                                   int version, varint_vec *vv) {
+    cram_codec *c = NULL;
+    char *cp = data;
+
+    if (option != E_INT) {
+        hts_log_error("This codec only supports INT encodings");
+        return NULL;
+    }
+
+    if (size < 1)
+        goto malformed;
+
+    if (!(c = malloc(sizeof(*c))))
+        return NULL;
+
+    c->codec  = E_GAMMA;
+    c->decode = cram_gamma_decode;
+    c->free   = cram_gamma_decode_free;
+    c->describe = cram_gamma_describe;
+
+    c->u.gamma.offset = vv->varint_get32(&cp, data+size, NULL);
+
+    if (cp - data != size)
+        goto malformed;
+
+    return c;
+
+ malformed:
+    hts_log_error("Malformed gamma header stream");
+    free(c);
+    return NULL;
+}
+
+/*
+ * ---------------------------------------------------------------------------
+ * HUFFMAN
+ */
+
+static int code_sort(const void *vp1, const void *vp2) {
+    const cram_huffman_code *c1 = (const cram_huffman_code *)vp1;
+    const cram_huffman_code *c2 = (const cram_huffman_code *)vp2;
+
+    if (c1->len != c2->len)
+        return c1->len - c2->len;
+    else
+        return c1->symbol < c2->symbol ? -1 : (c1->symbol > c2->symbol ? 1 : 0);
+}
+
+void cram_huffman_decode_free(cram_codec *c) {
+    if (!c)
+        return;
+
+    if (c->u.huffman.codes)
+        free(c->u.huffman.codes);
+    free(c);
+}
+
+int cram_huffman_decode_null(cram_slice *slice, cram_codec *c,
+                             cram_block *in, char *out, int *out_size) {
+    return -1;
+}
+
+int cram_huffman_decode_char0(cram_slice *slice, cram_codec *c,
+                              cram_block *in, char *out, int *out_size) {
+    int i, n;
+
+    if (!out)
+        return 0;
+
+    /* Special case of 0 length codes */
+    for (i = 0, n = *out_size; i < n; i++) {
+        out[i] = c->u.huffman.codes[0].symbol;
+    }
+    return 0;
+}
+
+int cram_huffman_decode_char(cram_slice *slice, cram_codec *c,
+                             cram_block *in, char *out, int *out_size) {
+    int i, n, ncodes = c->u.huffman.ncodes;
+    const cram_huffman_code * const codes = c->u.huffman.codes;
+
+    for (i = 0, n = *out_size; i < n; i++) {
+        int idx = 0;
+        int val = 0, len = 0, last_len = 0;
+
+        for (;;) {
+            int dlen = codes[idx].len - last_len;
+            if (cram_not_enough_bits(in, dlen))
+                return -1;
+
+            //val <<= dlen;
+            //val  |= get_bits_MSB(in, dlen);
+            //last_len = (len += dlen);
+
+            last_len = (len += dlen);
+            for (; dlen; dlen--) GET_BIT_MSB(in, val);
+
+            idx = val - codes[idx].p;
+            if (idx >= ncodes || idx < 0)
+                return -1;
+
+            if (codes[idx].code == val && codes[idx].len == len) {
+                if (out) out[i] = codes[idx].symbol;
+                break;
+            }
+        }
+    }
+
+    return 0;
+}
+
+int cram_huffman_decode_int0(cram_slice *slice, cram_codec *c,
+                             cram_block *in, char *out, int *out_size) {
+    int32_t *out_i = (int32_t *)out;
+    int i, n;
+    const cram_huffman_code * const codes = c->u.huffman.codes;
+
+    /* Special case of 0 length codes */
+    for (i = 0, n = *out_size; i < n; i++) {
+        out_i[i] = codes[0].symbol;
+    }
+    return 0;
+}
+
+int cram_huffman_decode_int(cram_slice *slice, cram_codec *c,
+                            cram_block *in, char *out, int *out_size) {
+    int32_t *out_i = (int32_t *)out;
+    int i, n, ncodes = c->u.huffman.ncodes;
+    const cram_huffman_code * const codes = c->u.huffman.codes;
+
+    for (i = 0, n = *out_size; i < n; i++) {
+        int idx = 0;
+        int val = 0, len = 0, last_len = 0;
+
+        // Now one bit at a time for remaining checks
+        for (;;) {
+            int dlen = codes[idx].len - last_len;
+            if (cram_not_enough_bits(in, dlen))
+                return -1;
+
+            //val <<= dlen;
+            //val  |= get_bits_MSB(in, dlen);
+            //last_len = (len += dlen);
+
+            last_len = (len += dlen);
+            for (; dlen; dlen--) GET_BIT_MSB(in, val);
+
+            idx = val - codes[idx].p;
+            if (idx >= ncodes || idx < 0)
+                return -1;
+
+            if (codes[idx].code == val && codes[idx].len == len) {
+                out_i[i] = codes[idx].symbol;
+                break;
+            }
+        }
+    }
+
+    return 0;
+}
+
+int cram_huffman_decode_long0(cram_slice *slice, cram_codec *c,
+                              cram_block *in, char *out, int *out_size) {
+    int64_t *out_i = (int64_t *)out;
+    int i, n;
+    const cram_huffman_code * const codes = c->u.huffman.codes;
+
+    /* Special case of 0 length codes */
+    for (i = 0, n = *out_size; i < n; i++) {
+        out_i[i] = codes[0].symbol;
+    }
+    return 0;
+}
+
+int cram_huffman_decode_long(cram_slice *slice, cram_codec *c,
+                             cram_block *in, char *out, int *out_size) {
+    int64_t *out_i = (int64_t *)out;
+    int i, n, ncodes = c->u.huffman.ncodes;
+    const cram_huffman_code * const codes = c->u.huffman.codes;
+
+    for (i = 0, n = *out_size; i < n; i++) {
+        int idx = 0;
+        int val = 0, len = 0, last_len = 0;
+
+        // Now one bit at a time for remaining checks
+        for (;;) {
+            int dlen = codes[idx].len - last_len;
+            if (cram_not_enough_bits(in, dlen))
+                return -1;
+
+            //val <<= dlen;
+            //val  |= get_bits_MSB(in, dlen);
+            //last_len = (len += dlen);
+
+            last_len = (len += dlen);
+            for (; dlen; dlen--) GET_BIT_MSB(in, val);
+
+            idx = val - codes[idx].p;
+            if (idx >= ncodes || idx < 0)
+                return -1;
+
+            if (codes[idx].code == val && codes[idx].len == len) {
+                out_i[i] = codes[idx].symbol;
+                break;
+            }
+        }
+    }
+
+    return 0;
+}
+
+int cram_huffman_describe(cram_codec *c, kstring_t *ks) {
+    int r = 0, n;
+    r |= ksprintf(ks, "HUFFMAN(codes={") < 0;
+    for (n = 0; n < c->u.huffman.ncodes; n++) {
+        r |= ksprintf(ks, "%s%"PRId64, n?",":"",
+                      c->u.huffman.codes[n].symbol);
+    }
+    r |= ksprintf(ks, "},lengths={") < 0;
+    for (n = 0; n < c->u.huffman.ncodes; n++) {
+        r |= ksprintf(ks, "%s%d", n?",":"",
+                      c->u.huffman.codes[n].len);
+    }
+    r |= ksprintf(ks, "})") < 0;
+    return r;
+}
+
+/*
+ * Initialises a huffman decoder from an encoding data stream.
+ */
+cram_codec *cram_huffman_decode_init(cram_block_compression_hdr *hdr,
+                                     char *data, int size,
+                                     enum cram_encoding codec,
+                                     enum cram_external_type option,
+                                     int version, varint_vec *vv) {
+    int32_t ncodes = 0, i, j;
+    char *cp = data, *data_end = &data[size];
+    cram_codec *h;
+    cram_huffman_code *codes = NULL;
+    int32_t val, last_len, max_len = 0;
+    uint32_t max_val; // needs one more bit than val
+    const int max_code_bits = sizeof(val) * 8 - 1;
+    int err = 0;
+
+    if (option == E_BYTE_ARRAY_BLOCK) {
+        hts_log_error("BYTE_ARRAYs not supported by this codec");
+        return NULL;
+    }
+
+    ncodes = vv->varint_get32(&cp, data_end, &err);
+    if (ncodes < 0) {
+        hts_log_error("Invalid number of symbols in huffman stream");
+        return NULL;
+    }
+    if (ncodes >= SIZE_MAX / sizeof(*codes)) {
+        errno = ENOMEM;
+        return NULL;
+    }
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    if (ncodes > FUZZ_ALLOC_LIMIT / sizeof(*codes)) {
+        errno = ENOMEM;
+        return NULL;
+    }
+#endif
+    h = calloc(1, sizeof(*h));
+    if (!h)
+        return NULL;
+
+    h->codec  = E_HUFFMAN;
+    h->free   = cram_huffman_decode_free;
+
+    h->u.huffman.ncodes = ncodes;
+    h->u.huffman.option = option;
+    if (ncodes) {
+        codes = h->u.huffman.codes = malloc(ncodes * sizeof(*codes));
+        if (!codes) {
+            free(h);
+            return NULL;
+        }
+    } else {
+        codes = h->u.huffman.codes = NULL;
+    }
+
+    /* Read symbols and bit-lengths */
+    if (option == E_LONG) {
+        for (i = 0; i < ncodes; i++)
+            codes[i].symbol = vv->varint_get64(&cp, data_end, &err);
+    } else if (option == E_INT || option == E_BYTE) {
+        for (i = 0; i < ncodes; i++)
+            codes[i].symbol = vv->varint_get32(&cp, data_end, &err);
+    } else {
+        goto malformed;
+    }
+
+    if (err)
+        goto malformed;
+
+    i = vv->varint_get32(&cp, data_end, &err);
+    if (i != ncodes)
+        goto malformed;
+
+    if (ncodes == 0) {
+        /* NULL huffman stream.  Ensure it returns an error if
+           anything tries to use it. */
+        h->decode = cram_huffman_decode_null;
+        return h;
+    }
+
+    for (i = 0; i < ncodes; i++) {
+        codes[i].len = vv->varint_get32(&cp, data_end, &err);
+        if (err)
+            break;
+        if (codes[i].len < 0) {
+            hts_log_error("Huffman code length (%d) is negative", codes[i].len);
+            goto malformed;
+        }
+        if (max_len < codes[i].len)
+            max_len = codes[i].len;
+    }
+    if (err || cp - data != size || max_len >= ncodes)
+        goto malformed;
+
+    /* 31 is max. bits available in val */
+    if (max_len > max_code_bits) {
+        hts_log_error("Huffman code length (%d) is greater "
+                      "than maximum supported (%d)", max_len, max_code_bits);
+        goto malformed;
+    }
+
+    /* Sort by bit length and then by symbol value */
+    qsort(codes, ncodes, sizeof(*codes), code_sort);
+
+    /* Assign canonical codes */
+    val = -1, last_len = 0, max_val = 0;
+    for (i = 0; i < ncodes; i++) {
+        val++;
+        if (val > max_val)
+            goto malformed;
+
+        if (codes[i].len > last_len) {
+            val <<= (codes[i].len - last_len);
+            last_len = codes[i].len;
+            max_val = (1U << codes[i].len) - 1;
+        }
+        codes[i].code = val;
+    }
+
+    /*
+     * Compute the next starting point, offset by the i'th value.
+     * For example if codes 10, 11, 12, 13 are 30, 31, 32, 33 then
+     * codes[10..13].p = 30 - 10.
+     */
+    last_len = 0;
+    for (i = j = 0; i < ncodes; i++) {
+        if (codes[i].len > last_len) {
+            j = codes[i].code - i;
+            last_len = codes[i].len;
+        }
+        codes[i].p = j;
+    }
+
+    // puts("==HUFF LEN==");
+    // for (i = 0; i <= last_len+1; i++) {
+    //     printf("len %d=%d prefix %d\n", i, h->u.huffman.lengths[i], h->u.huffman.prefix[i]);
+    // }
+    // puts("===HUFFMAN CODES===");
+    // for (i = 0; i < ncodes; i++) {
+    //     int j;
+    //     printf("%d: %d %d %d ", i, codes[i].symbol, codes[i].len, codes[i].code);
+    //     j = codes[i].len;
+    //     while (j) {
+    //         putchar(codes[i].code & (1 << --j) ? '1' : '0');
+    //     }
+    //     printf(" %d\n", codes[i].code);
+    // }
+
+    if (option == E_BYTE || option == E_BYTE_ARRAY) {
+        if (h->u.huffman.codes[0].len == 0)
+            h->decode = cram_huffman_decode_char0;
+        else
+            h->decode = cram_huffman_decode_char;
+    } else if (option == E_LONG || option == E_SLONG) {
+        if (h->u.huffman.codes[0].len == 0)
+            h->decode = cram_huffman_decode_long0;
+        else
+            h->decode = cram_huffman_decode_long;
+    } else if (option == E_INT || option == E_SINT || option == E_BYTE) {
+        if (h->u.huffman.codes[0].len == 0)
+            h->decode = cram_huffman_decode_int0;
+        else
+            h->decode = cram_huffman_decode_int;
+    } else {
+        return NULL;
+    }
+    h->describe = cram_huffman_describe;
+
+    return (cram_codec *)h;
+
+ malformed:
+    hts_log_error("Malformed huffman header stream");
+    free(codes);
+    free(h);
+    return NULL;
+}
+
+int cram_huffman_encode_char0(cram_slice *slice, cram_codec *c,
+                              char *in, int in_size) {
+    return 0;
+}
+
+int cram_huffman_encode_char(cram_slice *slice, cram_codec *c,
+                             char *in, int in_size) {
+    int i, code, len, r = 0;
+    unsigned char *syms = (unsigned char *)in;
+
+    while (in_size--) {
+        int sym = *syms++;
+        if (sym >= -1 && sym < MAX_HUFF) {
+            i = c->u.e_huffman.val2code[sym+1];
+            assert(c->u.e_huffman.codes[i].symbol == sym);
+            code = c->u.e_huffman.codes[i].code;
+            len  = c->u.e_huffman.codes[i].len;
+        } else {
+            /* Slow - use a lookup table for when sym < MAX_HUFF? */
+            for (i = 0; i < c->u.e_huffman.nvals; i++) {
+                if (c->u.e_huffman.codes[i].symbol == sym)
+                    break;
+            }
+            if (i == c->u.e_huffman.nvals)
+                return -1;
+
+            code = c->u.e_huffman.codes[i].code;
+            len  = c->u.e_huffman.codes[i].len;
+        }
+
+        r |= store_bits_MSB(c->out, code, len);
+    }
+
+    return r;
+}
+
+int cram_huffman_encode_int0(cram_slice *slice, cram_codec *c,
+                             char *in, int in_size) {
+    return 0;
+}
+
+int cram_huffman_encode_int(cram_slice *slice, cram_codec *c,
+                            char *in, int in_size) {
+    int i, code, len, r = 0;
+    int *syms = (int *)in;
+
+    while (in_size--) {
+        int sym = *syms++;
+
+        if (sym >= -1 && sym < MAX_HUFF) {
+            i = c->u.e_huffman.val2code[sym+1];
+            assert(c->u.e_huffman.codes[i].symbol == sym);
+            code = c->u.e_huffman.codes[i].code;
+            len  = c->u.e_huffman.codes[i].len;
+        } else {
+            /* Slow - use a lookup table for when sym < MAX_HUFFMAN_SYM? */
+            for (i = 0; i < c->u.e_huffman.nvals; i++) {
+                if (c->u.e_huffman.codes[i].symbol == sym)
+                    break;
+            }
+            if (i == c->u.e_huffman.nvals)
+                return -1;
+
+            code = c->u.e_huffman.codes[i].code;
+            len  = c->u.e_huffman.codes[i].len;
+        }
+
+        r |= store_bits_MSB(c->out, code, len);
+    }
+
+    return r;
+}
+
+int cram_huffman_encode_long0(cram_slice *slice, cram_codec *c,
+                              char *in, int in_size) {
+    return 0;
+}
+
+int cram_huffman_encode_long(cram_slice *slice, cram_codec *c,
+                             char *in, int in_size) {
+    int i, code, len, r = 0;
+    int64_t *syms = (int64_t *)in;
+
+    while (in_size--) {
+        int sym = *syms++;
+
+        if (sym >= -1 && sym < MAX_HUFF) {
+            i = c->u.e_huffman.val2code[sym+1];
+            assert(c->u.e_huffman.codes[i].symbol == sym);
+            code = c->u.e_huffman.codes[i].code;
+            len  = c->u.e_huffman.codes[i].len;
+        } else {
+            /* Slow - use a lookup table for when sym < MAX_HUFFMAN_SYM? */
+            for (i = 0; i < c->u.e_huffman.nvals; i++) {
+                if (c->u.e_huffman.codes[i].symbol == sym)
+                    break;
+            }
+            if (i == c->u.e_huffman.nvals)
+                return -1;
+
+            code = c->u.e_huffman.codes[i].code;
+            len  = c->u.e_huffman.codes[i].len;
+        }
+
+        r |= store_bits_MSB(c->out, code, len);
+    }
+
+    return r;
+}
+
+void cram_huffman_encode_free(cram_codec *c) {
+    if (!c)
+        return;
+
+    if (c->u.e_huffman.codes)
+        free(c->u.e_huffman.codes);
+    free(c);
+}
+
+/*
+ * Encodes a huffman tree.
+ * Returns number of bytes written.
+ */
+int cram_huffman_encode_store(cram_codec *c, cram_block *b, char *prefix,
+                              int version) {
+    int i, len = 0, r = 0, n;
+    cram_huffman_code *codes = c->u.e_huffman.codes;
+    /*
+     * Up to code length 127 means 2.5e+26 bytes of data required (worst
+     * case huffman tree needs symbols with freqs matching the Fibonacci
+     * series). So guaranteed 1 byte per code.
+     *
+     * Symbols themselves could be 5 bytes (eg -1 is 5 bytes in itf8).
+     *
+     * Therefore 6*ncodes + 5 + 5 + 1 + 5 is max memory
+     */
+    char *tmp = malloc(6*c->u.e_huffman.nvals+16);
+    char *tp = tmp, *tpend = tmp+6*c->u.e_huffman.nvals+16;
+
+    if (!tmp)
+        return -1;
+
+    if (prefix) {
+        size_t l = strlen(prefix);
+        BLOCK_APPEND(b, prefix, l);
+        len += l;
+    }
+
+    tp += c->vv->varint_put32(tp, tpend, c->u.e_huffman.nvals);
+    if (c->u.e_huffman.option == E_LONG) {
+        for (i = 0; i < c->u.e_huffman.nvals; i++) {
+            tp += c->vv->varint_put64(tp, tpend, codes[i].symbol);
+        }
+    } else if (c->u.e_huffman.option == E_SLONG) {
+        for (i = 0; i < c->u.e_huffman.nvals; i++) {
+            tp += c->vv->varint_put64s(tp, tpend, codes[i].symbol);
+        }
+    } else if (c->u.e_huffman.option == E_INT || c->u.e_huffman.option == E_BYTE) {
+        for (i = 0; i < c->u.e_huffman.nvals; i++) {
+            tp += c->vv->varint_put32(tp, tpend, codes[i].symbol);
+        }
+    } else if (c->u.e_huffman.option == E_SINT) {
+        for (i = 0; i < c->u.e_huffman.nvals; i++) {
+            tp += c->vv->varint_put32s(tp, tpend, codes[i].symbol);
+        }
+    } else {
+        return -1;
+    }
+
+    tp += c->vv->varint_put32(tp, tpend, c->u.e_huffman.nvals);
+    for (i = 0; i < c->u.e_huffman.nvals; i++)
+        tp += c->vv->varint_put32(tp, tpend, codes[i].len);
+
+    len += (n = c->vv->varint_put32_blk(b, c->codec)); r |= n;
+    len += (n = c->vv->varint_put32_blk(b, tp-tmp));   r |= n;
+    BLOCK_APPEND(b, tmp, tp-tmp);
+    len += tp-tmp;
+
+    free(tmp);
+
+    if (r > 0)
+        return len;
+
+ block_err:
+    return -1;
+}
+
+cram_codec *cram_huffman_encode_init(cram_stats *st,
+                                     enum cram_encoding codec,
+                                     enum cram_external_type option,
+                                     void *dat,
+                                     int version, varint_vec *vv) {
+    int *vals = NULL, *freqs = NULL, *lens = NULL, code, len;
+    int *new_vals, *new_freqs;
+    int i, max_val = 0, min_val = INT_MAX, k;
+    size_t nvals, vals_alloc = 0;
+    cram_codec *c;
+    cram_huffman_code *codes;
+
+    c = malloc(sizeof(*c));
+    if (!c)
+        return NULL;
+    c->codec = E_HUFFMAN;
+
+    /* Count number of unique symbols */
+    for (nvals = i = 0; i < MAX_STAT_VAL; i++) {
+        if (!st->freqs[i])
+            continue;
+        if (nvals >= vals_alloc) {
+            vals_alloc = vals_alloc ? vals_alloc*2 : 1024;
+            new_vals  = realloc(vals,  vals_alloc * sizeof(int));
+            if (!new_vals) goto nomem;
+            vals = new_vals;
+            new_freqs = realloc(freqs, vals_alloc * sizeof(int));
+            if (!new_freqs) goto nomem;
+            freqs = new_freqs;
+        }
+        vals[nvals] = i;
+        freqs[nvals] = st->freqs[i];
+        assert(st->freqs[i] > 0);
+        if (max_val < i) max_val = i;
+        if (min_val > i) min_val = i;
+        nvals++;
+    }
+    if (st->h) {
+        khint_t k;
+
+        for (k = kh_begin(st->h); k != kh_end(st->h); k++) {
+            if (!kh_exist(st->h, k))
+                continue;
+            if (nvals >= vals_alloc) {
+                vals_alloc = vals_alloc ? vals_alloc*2 : 1024;
+                new_vals  = realloc(vals,  vals_alloc * sizeof(int));
+                if (!new_vals) goto nomem;
+                vals = new_vals;
+                new_freqs = realloc(freqs, vals_alloc * sizeof(int));
+                if (!new_freqs) goto nomem;
+                freqs = new_freqs;
+            }
+            vals[nvals]= kh_key(st->h, k);
+            freqs[nvals] = kh_val(st->h, k);
+            assert(freqs[nvals] > 0);
+            if (max_val < i) max_val = i;
+            if (min_val > i) min_val = i;
+            nvals++;
+        }
+    }
+
+    assert(nvals > 0);
+
+    new_freqs = realloc(freqs, 2*nvals*sizeof(*freqs));
+    if (!new_freqs) goto nomem;
+    freqs = new_freqs;
+    lens = calloc(2*nvals, sizeof(*lens));
+    if (!lens) goto nomem;
+
+    /* Inefficient, use pointers to form chain so we can insert and maintain
+     * a sorted list? This is currently O(nvals^2) complexity.
+     */
+    for (;;) {
+        int low1 = INT_MAX, low2 = INT_MAX;
+        int ind1 = 0, ind2 = 0;
+        for (i = 0; i < nvals; i++) {
+            if (freqs[i] < 0)
+                continue;
+            if (low1 > freqs[i])
+                low2 = low1, ind2 = ind1, low1 = freqs[i], ind1 = i;
+            else if (low2 > freqs[i])
+                low2 = freqs[i], ind2 = i;
+        }
+        if (low2 == INT_MAX)
+            break;
+
+        freqs[nvals] = low1 + low2;
+        lens[ind1] = nvals;
+        lens[ind2] = nvals;
+        freqs[ind1] *= -1;
+        freqs[ind2] *= -1;
+        nvals++;
+    }
+    nvals = nvals/2+1;
+
+    /* Assign lengths */
+    for (i = 0; i < nvals; i++) {
+        int code_len = 0;
+        for (k = lens[i]; k; k = lens[k])
+            code_len++;
+        lens[i] = code_len;
+        freqs[i] *= -1;
+        //fprintf(stderr, "%d / %d => %d\n", vals[i], freqs[i], lens[i]);
+    }
+
+
+    /* Sort, need in a struct */
+    if (!(codes = malloc(nvals * sizeof(*codes))))
+        goto nomem;
+    for (i = 0; i < nvals; i++) {
+        codes[i].symbol = vals[i];
+        codes[i].len = lens[i];
+    }
+    qsort(codes, nvals, sizeof(*codes), code_sort);
+
+    /*
+     * Generate canonical codes from lengths.
+     * Sort by length.
+     * Start with 0.
+     * Every new code of same length is +1.
+     * Every new code of new length is +1 then <<1 per extra length.
+     *
+     * /\
+     * a/\
+     * /\/\
+     * bcd/\
+     *    ef
+     *
+     * a 1  0
+     * b 3  4 (0+1)<<2
+     * c 3  5
+     * d 3  6
+     * e 4  14  (6+1)<<1
+     * f 5  15
+     */
+    code = 0; len = codes[0].len;
+    for (i = 0; i < nvals; i++) {
+        while (len != codes[i].len) {
+            code<<=1;
+            len++;
+        }
+        codes[i].code = code++;
+
+        if (codes[i].symbol >= -1 && codes[i].symbol < MAX_HUFF)
+            c->u.e_huffman.val2code[codes[i].symbol+1] = i;
+
+        //fprintf(stderr, "sym %d, code %d, len %d\n",
+        //      codes[i].symbol, codes[i].code, codes[i].len);
+    }
+
+    free(lens);
+    free(vals);
+    free(freqs);
+
+    c->u.e_huffman.codes = codes;
+    c->u.e_huffman.nvals = nvals;
+    c->u.e_huffman.option = option;
+
+    c->free = cram_huffman_encode_free;
+    if (option == E_BYTE || option == E_BYTE_ARRAY) {
+        if (c->u.e_huffman.codes[0].len == 0)
+            c->encode = cram_huffman_encode_char0;
+        else
+            c->encode = cram_huffman_encode_char;
+    } else if (option == E_INT || option == E_SINT) {
+        if (c->u.e_huffman.codes[0].len == 0)
+            c->encode = cram_huffman_encode_int0;
+        else
+            c->encode = cram_huffman_encode_int;
+    } else if (option == E_LONG || option == E_SLONG) {
+        if (c->u.e_huffman.codes[0].len == 0)
+            c->encode = cram_huffman_encode_long0;
+        else
+            c->encode = cram_huffman_encode_long;
+    } else {
+        return NULL;
+    }
+    c->store = cram_huffman_encode_store;
+    c->flush = NULL;
+
+    return c;
+
+ nomem:
+    hts_log_error("Out of memory");
+    free(vals);
+    free(freqs);
+    free(lens);
+    free(c);
+    return NULL;
+}
+
+/*
+ * ---------------------------------------------------------------------------
+ * BYTE_ARRAY_LEN
+ */
+int cram_byte_array_len_decode(cram_slice *slice, cram_codec *c,
+                               cram_block *in, char *out,
+                               int *out_size) {
+    /* Fetch length */
+    int32_t len = 0, one = 1;
+    int r;
+
+    r = c->u.byte_array_len.len_codec->decode(slice, c->u.byte_array_len.len_codec,
+                                              in, (char *)&len, &one);
+    //printf("ByteArray Len=%d\n", len);
+
+    if (!r && c->u.byte_array_len.val_codec && len >= 0) {
+        r = c->u.byte_array_len.val_codec->decode(slice,
+                                                  c->u.byte_array_len.val_codec,
+                                                  in, out, &len);
+    } else {
+        return -1;
+    }
+
+    *out_size = len;
+
+    return r;
+}
+
+void cram_byte_array_len_decode_free(cram_codec *c) {
+    if (!c) return;
+
+    if (c->u.byte_array_len.len_codec)
+        c->u.byte_array_len.len_codec->free(c->u.byte_array_len.len_codec);
+
+    if (c->u.byte_array_len.val_codec)
+        c->u.byte_array_len.val_codec->free(c->u.byte_array_len.val_codec);
+
+    free(c);
+}
+
+int cram_byte_array_len_describe(cram_codec *c, kstring_t *ks) {
+    int r = 0;
+    r |= ksprintf(ks, "BYTE_ARRAY_LEN(len_codec={") < 0;
+    cram_byte_array_len_decoder *l = &c->u.byte_array_len;
+    r |=  l->len_codec->describe
+        ? l->len_codec->describe(l->len_codec, ks)
+        : (ksprintf(ks, "?")<0);
+    r |= ksprintf(ks, "},val_codec={") < 0;
+    r |=  l->val_codec->describe
+        ? l->val_codec->describe(l->val_codec, ks)
+        : (ksprintf(ks, "?")<0);
+    r |= ksprintf(ks, "}") < 0;
+
+    return r;
+}
+
+cram_codec *cram_byte_array_len_decode_init(cram_block_compression_hdr *hdr,
+                                            char *data, int size,
+                                            enum cram_encoding codec,
+                                            enum cram_external_type option,
+                                            int version, varint_vec *vv) {
+    cram_codec *c;
+    char *cp   = data;
+    char *endp = data + size;
+
+    if (!(c = malloc(sizeof(*c))))
+        return NULL;
+
+    c->codec  = E_BYTE_ARRAY_LEN;
+    c->decode = cram_byte_array_len_decode;
+    c->free   = cram_byte_array_len_decode_free;
+    c->describe = cram_byte_array_len_describe;
+    c->u.byte_array_len.len_codec = NULL;
+    c->u.byte_array_len.val_codec = NULL;
+
+    int encoding = vv->varint_get32(&cp, endp, NULL);
+    int sub_size = vv->varint_get32(&cp, endp, NULL);
+    if (sub_size < 0 || endp - cp < sub_size)
+        goto malformed;
+    c->u.byte_array_len.len_codec = cram_decoder_init(hdr, encoding, cp, sub_size,
+                                                      E_INT, version, vv);
+    if (c->u.byte_array_len.len_codec == NULL)
+        goto no_codec;
+    cp += sub_size;
+
+    encoding = vv->varint_get32(&cp, endp, NULL);
+    sub_size = vv->varint_get32(&cp, endp, NULL);
+    if (sub_size < 0 || endp - cp < sub_size)
+        goto malformed;
+    c->u.byte_array_len.val_codec = cram_decoder_init(hdr, encoding, cp, sub_size,
+                                                      option, version, vv);
+    if (c->u.byte_array_len.val_codec == NULL)
+        goto no_codec;
+    cp += sub_size;
+
+    if (cp - data != size)
+        goto malformed;
+
+    return c;
+
+ malformed:
+    hts_log_error("Malformed byte_array_len header stream");
+ no_codec:
+    cram_byte_array_len_decode_free(c);
+    return NULL;
+}
+
+int cram_byte_array_len_encode(cram_slice *slice, cram_codec *c,
+                               char *in, int in_size) {
+    int32_t i32 = in_size;
+    int r = 0;
+
+    r |= c->u.e_byte_array_len.len_codec->encode(slice,
+                                                 c->u.e_byte_array_len.len_codec,
+                                                 (char *)&i32, 1);
+    r |= c->u.e_byte_array_len.val_codec->encode(slice,
+                                                 c->u.e_byte_array_len.val_codec,
+                                                 in, in_size);
+    return r;
+}
+
+void cram_byte_array_len_encode_free(cram_codec *c) {
+    if (!c)
+        return;
+
+    if (c->u.e_byte_array_len.len_codec)
+        c->u.e_byte_array_len.len_codec->free(c->u.e_byte_array_len.len_codec);
+
+    if (c->u.e_byte_array_len.val_codec)
+        c->u.e_byte_array_len.val_codec->free(c->u.e_byte_array_len.val_codec);
+
+    free(c);
+}
+
+int cram_byte_array_len_encode_store(cram_codec *c, cram_block *b,
+                                     char *prefix, int version) {
+    int len = 0, len2, len3, r = 0, n;
+    cram_codec *tc;
+    cram_block *b_len = NULL, *b_val = NULL;
+
+    if (prefix) {
+        size_t l = strlen(prefix);
+        BLOCK_APPEND(b, prefix, l);
+        len += l;
+    }
+
+    tc = c->u.e_byte_array_len.len_codec;
+    b_len = cram_new_block(0, 0);
+    if (!b_len) goto block_err;
+    len2 = tc->store(tc, b_len, NULL, version);
+    if (len2 < 0) goto block_err;
+
+    tc = c->u.e_byte_array_len.val_codec;
+    b_val = cram_new_block(0, 0);
+    if (!b_val) goto block_err;
+    len3 = tc->store(tc, b_val, NULL, version);
+    if (len3 < 0) goto block_err;
+
+    len += (n = c->vv->varint_put32_blk(b, c->codec));  r |= n;
+    len += (n = c->vv->varint_put32_blk(b, len2+len3)); r |= n;
+    BLOCK_APPEND(b, BLOCK_DATA(b_len), BLOCK_SIZE(b_len));
+    BLOCK_APPEND(b, BLOCK_DATA(b_val), BLOCK_SIZE(b_val));
+
+    cram_free_block(b_len);
+    cram_free_block(b_val);
+
+    if (r > 0)
+        return len + len2 + len3;
+
+ block_err:
+    if (b_len) cram_free_block(b_len);
+    if (b_val) cram_free_block(b_val);
+    return -1;
+}
+
+cram_codec *cram_byte_array_len_encode_init(cram_stats *st,
+                                            enum cram_encoding codec,
+                                            enum cram_external_type option,
+                                            void *dat,
+                                            int version, varint_vec *vv) {
+    cram_codec *c;
+    cram_byte_array_len_encoder *e = (cram_byte_array_len_encoder *)dat;
+
+    c = malloc(sizeof(*c));
+    if (!c)
+        return NULL;
+    c->codec = E_BYTE_ARRAY_LEN;
+    c->free = cram_byte_array_len_encode_free;
+    c->encode = cram_byte_array_len_encode;
+    c->store = cram_byte_array_len_encode_store;
+    c->flush = NULL;
+
+    c->u.e_byte_array_len.len_codec = cram_encoder_init(e->len_encoding,
+                                                        st, E_INT,
+                                                        e->len_dat,
+                                                        version, vv);
+    c->u.e_byte_array_len.val_codec = cram_encoder_init(e->val_encoding,
+                                                        NULL, E_BYTE_ARRAY,
+                                                        e->val_dat,
+                                                        version, vv);
+
+    if (!c->u.e_byte_array_len.len_codec ||
+        !c->u.e_byte_array_len.val_codec) {
+        cram_byte_array_len_encode_free(c);
+        return NULL;
+    }
+
+    return c;
+}
+
+/*
+ * ---------------------------------------------------------------------------
+ * BYTE_ARRAY_STOP
+ */
+static int cram_byte_array_stop_decode_char(cram_slice *slice, cram_codec *c,
+                                            cram_block *in, char *out,
+                                            int *out_size) {
+    char *cp, ch;
+    cram_block *b = NULL;
+
+    b = cram_get_block_by_id(slice, c->u.byte_array_stop.content_id);
+    if (!b)
+        return *out_size?-1:0;
+
+    if (b->idx >= b->uncomp_size)
+        return -1;
+
+    cp = (char *)b->data + b->idx;
+    if (out) {
+        while ((ch = *cp) != (char)c->u.byte_array_stop.stop) {
+            if (cp - (char *)b->data >= b->uncomp_size)
+                return -1;
+            *out++ = ch;
+            cp++;
+        }
+    } else {
+        // Consume input, but produce no output
+        while ((ch = *cp) != (char)c->u.byte_array_stop.stop) {
+            if (cp - (char *)b->data >= b->uncomp_size)
+                return -1;
+            cp++;
+        }
+    }
+
+    *out_size = cp - (char *)(b->data + b->idx);
+    b->idx = cp - (char *)b->data + 1;
+
+    return 0;
+}
+
+int cram_byte_array_stop_decode_block(cram_slice *slice, cram_codec *c,
+                                      cram_block *in, char *out_,
+                                      int *out_size) {
+    cram_block *b;
+    cram_block *out = (cram_block *)out_;
+    unsigned char *cp, *cp_end;
+    unsigned char stop;
+
+    b = cram_get_block_by_id(slice, c->u.byte_array_stop.content_id);
+    if (!b)
+        return *out_size?-1:0;
+
+    if (b->idx >= b->uncomp_size)
+        return -1;
+    cp = b->data + b->idx;
+    cp_end = b->data + b->uncomp_size;
+
+    stop = c->u.byte_array_stop.stop;
+    if (cp_end - cp < out->alloc - out->byte) {
+        unsigned char *out_cp = BLOCK_END(out);
+        while (cp != cp_end && *cp != stop)
+            *out_cp++ = *cp++;
+        BLOCK_SIZE(out) = out_cp - BLOCK_DATA(out);
+    } else {
+        unsigned char *cp_start;
+        for (cp_start = cp; cp != cp_end && *cp != stop; cp++)
+            ;
+        BLOCK_APPEND(out, cp_start, cp - cp_start);
+        BLOCK_GROW(out, cp - cp_start);
+    }
+
+    *out_size = cp - (b->data + b->idx);
+    b->idx = cp - b->data + 1;
+
+    return 0;
+
+ block_err:
+    return -1;
+}
+
+void cram_byte_array_stop_decode_free(cram_codec *c) {
+    if (!c) return;
+
+    free(c);
+}
+
+int cram_byte_array_stop_describe(cram_codec *c, kstring_t *ks) {
+    return ksprintf(ks, "BYTE_ARRAY_STOP(stop=%d,id=%d)",
+                    c->u.byte_array_stop.stop,
+                    c->u.byte_array_stop.content_id)
+        < 0 ? -1 : 0;
+}
+
+cram_codec *cram_byte_array_stop_decode_init(cram_block_compression_hdr *hdr,
+                                             char *data, int size,
+                                             enum cram_encoding codec,
+                                             enum cram_external_type option,
+                                             int version, varint_vec *vv) {
+    cram_codec *c = NULL;
+    unsigned char *cp = (unsigned char *)data;
+    int err = 0;
+
+    if (size < (CRAM_MAJOR_VERS(version) == 1 ? 5 : 2))
+        goto malformed;
+
+    if (!(c = malloc(sizeof(*c))))
+        return NULL;
+
+    c->codec  = E_BYTE_ARRAY_STOP;
+    switch (option) {
+    case E_BYTE_ARRAY_BLOCK:
+        c->decode = cram_byte_array_stop_decode_block;
+        break;
+    case E_BYTE_ARRAY:
+        c->decode = cram_byte_array_stop_decode_char;
+        break;
+    default:
+        hts_log_error("The byte_array_stop codec only supports BYTE_ARRAYs");
+        free(c);
+        return NULL;
+    }
+    c->free   = cram_byte_array_stop_decode_free;
+    c->describe = cram_byte_array_stop_describe;
+
+    c->u.byte_array_stop.stop = *cp++;
+    if (CRAM_MAJOR_VERS(version) == 1) {
+        c->u.byte_array_stop.content_id = cp[0] + (cp[1]<<8) + (cp[2]<<16)
+            + ((unsigned int) cp[3]<<24);
+        cp += 4;
+    } else {
+        c->u.byte_array_stop.content_id = vv->varint_get32((char **)&cp, data+size, &err);
+    }
+
+    if ((char *)cp - data != size || err)
+        goto malformed;
+
+    return c;
+
+ malformed:
+    hts_log_error("Malformed byte_array_stop header stream");
+    free(c);
+    return NULL;
+}
+
+int cram_byte_array_stop_encode(cram_slice *slice, cram_codec *c,
+                                char *in, int in_size) {
+    BLOCK_APPEND(c->out, in, in_size);
+    BLOCK_APPEND_CHAR(c->out, c->u.e_byte_array_stop.stop);
+    return 0;
+
+ block_err:
+    return -1;
+}
+
+void cram_byte_array_stop_encode_free(cram_codec *c) {
+    if (!c)
+        return;
+    free(c);
+}
+
+int cram_byte_array_stop_encode_store(cram_codec *c, cram_block *b,
+                                      char *prefix, int version) {
+    int len = 0;
+    char buf[20], *cp = buf;
+
+    if (prefix) {
+        size_t l = strlen(prefix);
+        BLOCK_APPEND(b, prefix, l);
+        len += l;
+    }
+
+    cp += c->vv->varint_put32(cp, buf+20, c->codec);
+
+    if (CRAM_MAJOR_VERS(version) == 1) {
+        cp += c->vv->varint_put32(cp, buf+20, 5);
+        *cp++ = c->u.e_byte_array_stop.stop;
+        *cp++ = (c->u.e_byte_array_stop.content_id >>  0) & 0xff;
+        *cp++ = (c->u.e_byte_array_stop.content_id >>  8) & 0xff;
+        *cp++ = (c->u.e_byte_array_stop.content_id >> 16) & 0xff;
+        *cp++ = (c->u.e_byte_array_stop.content_id >> 24) & 0xff;
+    } else {
+        cp += c->vv->varint_put32(cp, buf+20, 1 +
+                                  c->vv->varint_size(c->u.e_byte_array_stop.content_id));
+        *cp++ = c->u.e_byte_array_stop.stop;
+        cp += c->vv->varint_put32(cp, buf+20, c->u.e_byte_array_stop.content_id);
+    }
+
+    BLOCK_APPEND(b, buf, cp-buf);
+    len += cp-buf;
+
+    return len;
+
+ block_err:
+    return -1;
+}
+
+cram_codec *cram_byte_array_stop_encode_init(cram_stats *st,
+                                             enum cram_encoding codec,
+                                             enum cram_external_type option,
+                                             void *dat,
+                                             int version, varint_vec *vv) {
+    cram_codec *c;
+
+    c = malloc(sizeof(*c));
+    if (!c)
+        return NULL;
+    c->codec = E_BYTE_ARRAY_STOP;
+    c->free = cram_byte_array_stop_encode_free;
+    c->encode = cram_byte_array_stop_encode;
+    c->store = cram_byte_array_stop_encode_store;
+    c->flush = NULL;
+
+    c->u.e_byte_array_stop.stop = ((int *)dat)[0];
+    c->u.e_byte_array_stop.content_id = ((int *)dat)[1];
+
+    return c;
+}
+
+/*
+ * ---------------------------------------------------------------------------
+ */
+
+const char *cram_encoding2str(enum cram_encoding t) {
+    switch (t) {
+    case E_NULL:            return "NULL";
+    case E_EXTERNAL:        return "EXTERNAL";
+    case E_GOLOMB:          return "GOLOMB";
+    case E_HUFFMAN:         return "HUFFMAN";
+    case E_BYTE_ARRAY_LEN:  return "BYTE_ARRAY_LEN";
+    case E_BYTE_ARRAY_STOP: return "BYTE_ARRAY_STOP";
+    case E_BETA:            return "BETA";
+    case E_SUBEXP:          return "SUBEXP";
+    case E_GOLOMB_RICE:     return "GOLOMB_RICE";
+    case E_GAMMA:           return "GAMMA";
+
+    case E_VARINT_UNSIGNED: return "VARINT_UNSIGNED";
+    case E_VARINT_SIGNED:   return "VARINT_SIGNED";
+    case E_CONST_BYTE:      return "CONST_BYTE";
+    case E_CONST_INT:       return "CONST_INT";
+
+    case E_NUM_CODECS:
+    default:                return "?";
+    }
+}
+
+static cram_codec *(*decode_init[])(cram_block_compression_hdr *hdr,
+                                    char *data,
+                                    int size,
+                                    enum cram_encoding codec,
+                                    enum cram_external_type option,
+                                    int version, varint_vec *vv) = {
+    // CRAM 3.0 valid codecs
+    NULL, // null codec
+    cram_external_decode_init,
+    NULL, // golomb
+    cram_huffman_decode_init,
+    cram_byte_array_len_decode_init,
+    cram_byte_array_stop_decode_init,
+    cram_beta_decode_init,
+    cram_subexp_decode_init,
+    NULL, // golomb rice
+    cram_gamma_decode_init,
+
+    // Gap between CRAM 3 and CRAM 4; 9 to 39 inclusive
+    NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
+
+    NULL,                      // was xbyte
+    cram_varint_decode_init,   // varint unsigned
+    cram_varint_decode_init,   // varint signed
+    cram_const_decode_init,    // const byte
+    cram_const_decode_init,    // const int
+
+    // Gap to CRAM 4 transfomrations; 45 to 49 inclusive
+    NULL, NULL, NULL, NULL, NULL,
+
+    NULL, // xhuffman
+    cram_xpack_decode_init,
+    cram_xrle_decode_init,
+    cram_xdelta_decode_init,
+};
+
+cram_codec *cram_decoder_init(cram_block_compression_hdr *hdr,
+                              enum cram_encoding codec,
+                              char *data, int size,
+                              enum cram_external_type option,
+                              int version, varint_vec *vv) {
+    if (codec >= E_NULL && codec < E_NUM_CODECS && decode_init[codec]) {
+        cram_codec *r = decode_init[codec](hdr, data, size, codec,
+                                           option, version, vv);
+        if (r) {
+            r->vv = vv;
+            r->codec_id = hdr->ncodecs++;
+        }
+        return r;
+    } else {
+        hts_log_error("Unimplemented codec of type %s", cram_encoding2str(codec));
+        return NULL;
+    }
+}
+
+static cram_codec *(*encode_init[])(cram_stats *stx,
+                                    enum cram_encoding codec,
+                                    enum cram_external_type option,
+                                    void *opt,
+                                    int version, varint_vec *vv) = {
+    // CRAM 3.0 valid codecs
+    NULL, // null codec
+    cram_external_encode_init, // int/bytes in cram 3, byte only in cram 4
+    NULL, // golomb
+    cram_huffman_encode_init,
+    cram_byte_array_len_encode_init,
+    cram_byte_array_stop_encode_init,
+    cram_beta_encode_init,
+    NULL, // subexponential (we support decode only)
+    NULL, // golomb rice
+    NULL, // gamma (we support decode only)
+
+    // Gap between CRAM 3 and CRAM 4; 9 to 39 inclusive
+    NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
+
+    NULL, // was xbyte
+    cram_varint_encode_init, // varint unsigned
+    cram_varint_encode_init, // varint signed
+    cram_const_encode_init,  // const byte
+    cram_const_encode_init,  // const int
+
+    // Gap to CRAM 4 transfomrations; 45 to 49 inclusive
+    NULL, NULL, NULL, NULL, NULL,
+
+    NULL, // xhuffman
+    cram_xpack_encode_init,
+    cram_xrle_encode_init,
+    cram_xdelta_encode_init,
+};
+
+cram_codec *cram_encoder_init(enum cram_encoding codec,
+                              cram_stats *st,
+                              enum cram_external_type option,
+                              void *dat,
+                              int version, varint_vec *vv) {
+    if (st && !st->nvals)
+        return NULL;
+
+    // cram_stats_encoding assumes integer data, but if option
+    // is E_BYTE then tweak the requested encoding.  This ought
+    // to be fixed in cram_stats_encoding instead.
+    if (option == E_BYTE || option == E_BYTE_ARRAY ||
+       option == E_BYTE_ARRAY_BLOCK) {
+       if (codec == E_VARINT_SIGNED || codec == E_VARINT_UNSIGNED)
+           codec = E_EXTERNAL;
+       else if (codec == E_CONST_INT)
+           codec = E_CONST_BYTE;
+    }
+
+    if (encode_init[codec]) {
+        cram_codec *r;
+        if ((r = encode_init[codec](st, codec, option, dat, version, vv)))
+            r->out = NULL;
+        if (!r) {
+            hts_log_error("Unable to initialise codec of type %s", cram_encoding2str(codec));
+            return NULL;
+        }
+        r->vv = vv;
+        return r;
+    } else {
+        hts_log_error("Unimplemented codec of type %s", cram_encoding2str(codec));
+        abort();
+    }
+}
+
+/*
+ * Returns the content_id used by this codec, also in id2 if byte_array_len.
+ * Returns -1 for the CORE block and -2 for unneeded.
+ * id2 is only filled out for BYTE_ARRAY_LEN which uses 2 codecs.
+ */
+int cram_codec_to_id(cram_codec *c, int *id2) {
+    int bnum1, bnum2 = -2;
+
+    switch (c->codec) {
+    case E_CONST_INT:
+    case E_CONST_BYTE:
+        bnum1 = -2; // no blocks used
+        break;
+
+    case E_HUFFMAN:
+        bnum1 = c->u.huffman.ncodes == 1 ? -2 : -1;
+        break;
+
+    case E_GOLOMB:
+    case E_BETA:
+    case E_SUBEXP:
+    case E_GOLOMB_RICE:
+    case E_GAMMA:
+        // CORE block
+        bnum1 = -1;
+        break;
+
+    case E_EXTERNAL:
+    case E_VARINT_UNSIGNED:
+    case E_VARINT_SIGNED:
+        bnum1 = c->u.external.content_id;
+        break;
+
+    case E_BYTE_ARRAY_LEN:
+        bnum1 = cram_codec_to_id(c->u.byte_array_len.len_codec, NULL);
+        bnum2 = cram_codec_to_id(c->u.byte_array_len.val_codec, NULL);
+        break;
+
+    case E_BYTE_ARRAY_STOP:
+        bnum1 = c->u.byte_array_stop.content_id;
+        break;
+
+    case E_NULL:
+        bnum1 = -2;
+        break;
+
+    default:
+        hts_log_error("Unknown codec type %d", c->codec);
+        bnum1 = -1;
+    }
+
+    if (id2)
+        *id2 = bnum2;
+    return bnum1;
+}
+
+
+/*
+ * cram_codec structures are specialised for decoding or encoding.
+ * Unfortunately this makes turning a decoder into an encoder (such as
+ * when transcoding files) problematic.
+ *
+ * This function converts a cram decoder codec into an encoder version
+ * in-place (ie it modifiers the codec itself).
+ *
+ * Returns 0 on success;
+ *        -1 on failure.
+ */
+int cram_codec_decoder2encoder(cram_fd *fd, cram_codec *c) {
+    int j;
+
+    switch (c->codec) {
+    case E_CONST_INT:
+    case E_CONST_BYTE:
+        // shares struct with decode
+        c->store = cram_const_encode_store;
+        break;
+
+    case E_EXTERNAL:
+        // shares struct with decode
+        c->free = cram_external_encode_free;
+        c->store = cram_external_encode_store;
+        if (c->decode == cram_external_decode_int)
+            c->encode = cram_external_encode_int;
+        else if (c->decode == cram_external_decode_long)
+            c->encode = cram_external_encode_long;
+        else if (c->decode == cram_external_decode_char)
+            c->encode = cram_external_encode_char;
+        else if (c->decode == cram_external_decode_block)
+            c->encode = cram_external_encode_char;
+        else
+            return -1;
+        break;
+
+    case E_VARINT_SIGNED:
+    case E_VARINT_UNSIGNED:
+        // shares struct with decode
+        c->free = cram_varint_encode_free;
+        c->store = cram_varint_encode_store;
+        if (c->decode == cram_varint_decode_int)
+            c->encode = cram_varint_encode_int;
+        else if (c->decode == cram_varint_decode_sint)
+            c->encode = cram_varint_encode_sint;
+        else if (c->decode == cram_varint_decode_long)
+            c->encode = cram_varint_encode_long;
+        else if (c->decode == cram_varint_decode_slong)
+            c->encode = cram_varint_encode_slong;
+        else
+            return -1;
+        break;
+
+    case E_HUFFMAN: {
+        // New structure, so switch.
+        // FIXME: we huffman and e_huffman structs amended, we could
+        // unify this.
+        cram_codec *t = malloc(sizeof(*t));
+        if (!t) return -1;
+        t->vv     = c->vv;
+        t->codec = E_HUFFMAN;
+        t->free = cram_huffman_encode_free;
+        t->store = cram_huffman_encode_store;
+        t->u.e_huffman.codes = c->u.huffman.codes;
+        t->u.e_huffman.nvals = c->u.huffman.ncodes;
+        t->u.e_huffman.option = c->u.huffman.option;
+        for (j = 0; j < t->u.e_huffman.nvals; j++) {
+            int32_t sym = t->u.e_huffman.codes[j].symbol;
+            if (sym >= -1 && sym < MAX_HUFF)
+                t->u.e_huffman.val2code[sym+1] = j;
+        }
+
+        if (c->decode == cram_huffman_decode_char0)
+            t->encode = cram_huffman_encode_char0;
+        else if (c->decode == cram_huffman_decode_char)
+            t->encode = cram_huffman_encode_char;
+        else if (c->decode == cram_huffman_decode_int0)
+            t->encode = cram_huffman_encode_int0;
+        else if (c->decode == cram_huffman_decode_int)
+            t->encode = cram_huffman_encode_int;
+        else if (c->decode == cram_huffman_decode_long0)
+            t->encode = cram_huffman_encode_long0;
+        else if (c->decode == cram_huffman_decode_long)
+            t->encode = cram_huffman_encode_long;
+        else {
+            free(t);
+            return -1;
+        }
+        *c = *t;
+        free(t);
+        break;
+    }
+
+    case E_BETA:
+        // shares struct with decode
+        c->free = cram_beta_encode_free;
+        c->store = cram_beta_encode_store;
+        if (c->decode == cram_beta_decode_int)
+            c->encode = cram_beta_encode_int;
+        else if (c->decode == cram_beta_decode_long)
+            c->encode = cram_beta_encode_long;
+        else if (c->decode == cram_beta_decode_char)
+            c->encode = cram_beta_encode_char;
+        else
+            return -1;
+        break;
+
+    case E_XPACK: {
+        // shares struct with decode
+        cram_codec t = *c;
+        t.free = cram_xpack_encode_free;
+        t.store = cram_xpack_encode_store;
+        if (t.decode == cram_xpack_decode_long)
+            t.encode = cram_xpack_encode_long;
+        else if (t.decode == cram_xpack_decode_int)
+            t.encode = cram_xpack_encode_int;
+        else if (t.decode == cram_xpack_decode_char)
+            t.encode = cram_xpack_encode_char;
+        else
+            return -1;
+        t.u.e_xpack.sub_codec = t.u.xpack.sub_codec;
+        if (cram_codec_decoder2encoder(fd, t.u.e_xpack.sub_codec) == -1)
+            return -1;
+        *c = t;
+        break;
+    }
+
+    case E_BYTE_ARRAY_LEN: {
+        cram_codec *t = malloc(sizeof(*t));
+        if (!t) return -1;
+        t->vv     = c->vv;
+        t->codec  = E_BYTE_ARRAY_LEN;
+        t->free   = cram_byte_array_len_encode_free;
+        t->store  = cram_byte_array_len_encode_store;
+        t->encode = cram_byte_array_len_encode;
+        t->u.e_byte_array_len.len_codec = c->u.byte_array_len.len_codec;
+        t->u.e_byte_array_len.val_codec = c->u.byte_array_len.val_codec;
+        if (cram_codec_decoder2encoder(fd, t->u.e_byte_array_len.len_codec) == -1 ||
+            cram_codec_decoder2encoder(fd, t->u.e_byte_array_len.val_codec) == -1) {
+            t->free(t);
+            return -1;
+        }
+
+        // {len,val}_{encoding,dat} are undefined, but unused.
+        // Leaving them unset here means we can test that assertion.
+        *c = *t;
+        free(t);
+        break;
+    }
+
+    case E_BYTE_ARRAY_STOP:
+        // shares struct with decode
+        c->free   = cram_byte_array_stop_encode_free;
+        c->store  = cram_byte_array_stop_encode_store;
+        c->encode = cram_byte_array_stop_encode;
+        break;
+
+    default:
+        return -1;
+    }
+
+    return 0;
+}
+
+int cram_codec_describe(cram_codec *c, kstring_t *ks) {
+    if (c && c->describe)
+        return c->describe(c, ks);
+    else
+        return ksprintf(ks, "?");
+}
diff --git a/ext/htslib/cram/cram_codecs.h b/ext/htslib/cram/cram_codecs.h
new file mode 100644
index 0000000..d93d995
--- /dev/null
+++ b/ext/htslib/cram/cram_codecs.h
@@ -0,0 +1,264 @@
+/*
+Copyright (c) 2012-2015, 2018, 2020, 2023 Genome Research Ltd.
+Author: James Bonfield <jkb@sanger.ac.uk>
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+Institute nor the names of its contributors may be used to endorse or promote
+products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#ifndef CRAM_CODECS_H
+#define CRAM_CODECS_H
+
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct cram_codec;
+
+/*
+ * Slow but simple huffman decoder to start with.
+ * Read a bit at a time, keeping track of {length, value}
+ * eg. 1 1 0 1 => {1,1},  {2,3}, {3,6}, {4,13}
+ *
+ * Keep track of this through the huffman code table.
+ * For fast scanning we have an index of where the first code of length X
+ * appears.
+ */
+typedef struct {
+    int64_t symbol;
+    int32_t p; // next code start value, minus index to codes[]
+    int32_t code;
+    int32_t len;
+} cram_huffman_code;
+
+typedef struct {
+    int ncodes;
+    cram_huffman_code *codes;
+    int option;
+} cram_huffman_decoder;
+
+#define MAX_HUFF 128
+typedef struct {
+    cram_huffman_code *codes;
+    int nvals;
+    int val2code[MAX_HUFF+1]; // value to code lookup for small values
+    int option;
+} cram_huffman_encoder;
+
+typedef struct {
+    int32_t offset;
+    int32_t nbits;
+} cram_beta_decoder;
+
+// A PACK transform, packing multiple values into a single byte
+typedef struct {
+    int32_t nbits;
+    enum cram_encoding sub_encoding;
+    void *sub_codec_dat;
+    struct cram_codec *sub_codec;
+    int nval;  // number of items in maps
+    uint32_t rmap[256]; // 0,1,2,3 -> P,A,C,K
+    int map[256];       // P,A,C,K -> 0,1,2,3 // NB: max input is uint8_tb? Or use hash?
+} cram_xpack_decoder;
+typedef cram_xpack_decoder cram_xpack_encoder;
+
+// Transforms symbols X,Y,Z to bytes 0,1,2.
+typedef struct {
+    enum cram_encoding len_encoding;
+    enum cram_encoding lit_encoding;
+    void *len_dat;
+    void *lit_dat;
+    struct cram_codec *len_codec;
+    struct cram_codec *lit_codec;
+    int cur_len;
+    int cur_lit;
+    int rep_score[256];
+    char *to_flush;
+    size_t to_flush_size;
+} cram_xrle_decoder;
+typedef cram_xrle_decoder cram_xrle_encoder;
+
+// DELTA + zigzag + varint encoding
+typedef struct {
+    // FIXME: define endian here too.  Require little endian?
+    int64_t last;
+    uint8_t word_size; // 1, 2, 4, 8
+    //uint8_t sign;      // true if input data is already signed
+    enum cram_encoding sub_encoding;
+    void *sub_codec_dat;
+    struct cram_codec *sub_codec;
+} cram_xdelta_decoder;
+typedef cram_xdelta_decoder cram_xdelta_encoder;
+
+typedef struct {
+    int32_t offset;
+} cram_gamma_decoder;
+
+typedef struct {
+    int32_t offset;
+    int32_t k;
+} cram_subexp_decoder;
+
+typedef struct {
+    int32_t content_id;
+    enum cram_external_type type;
+} cram_external_decoder;
+
+typedef struct {
+    int32_t content_id;
+    int64_t offset;
+    enum cram_external_type type;
+} cram_varint_decoder;
+
+typedef struct {
+    struct cram_codec *len_codec;
+    struct cram_codec *val_codec;
+} cram_byte_array_len_decoder;
+
+typedef struct {
+    unsigned char stop;
+    int32_t content_id;
+} cram_byte_array_stop_decoder;
+
+typedef struct {
+    enum cram_encoding len_encoding;
+    enum cram_encoding val_encoding;
+    void *len_dat;
+    void *val_dat;
+    struct cram_codec *len_codec;
+    struct cram_codec *val_codec;
+} cram_byte_array_len_encoder;
+
+typedef struct {
+    int64_t val;
+} cram_const_codec;
+
+/*
+ * A generic codec structure.
+ */
+struct cram_codec {
+    enum cram_encoding codec;
+    cram_block *out;
+    varint_vec *vv;
+    int codec_id;
+    void (*free)(struct cram_codec *codec);
+    int (*decode)(cram_slice *slice, struct cram_codec *codec,
+                  cram_block *in, char *out, int *out_size);
+    int (*encode)(cram_slice *slice, struct cram_codec *codec,
+                  char *in, int in_size);
+    int (*store)(struct cram_codec *codec, cram_block *b, char *prefix,
+                 int version);
+    int (*size)(cram_slice *slice, struct cram_codec *codec);
+    int (*flush)(struct cram_codec *codec);
+    cram_block *(*get_block)(cram_slice *slice, struct cram_codec *codec);
+    int (*describe)(struct cram_codec *codec, kstring_t *ks);
+
+    union {
+        cram_huffman_decoder         huffman;
+        cram_external_decoder        external;
+        cram_beta_decoder            beta;
+        cram_gamma_decoder           gamma;
+        cram_subexp_decoder          subexp;
+        cram_byte_array_len_decoder  byte_array_len;
+        cram_byte_array_stop_decoder byte_array_stop;
+        cram_xpack_decoder           xpack;
+        cram_xrle_decoder            xrle;
+        cram_xdelta_decoder          xdelta;
+        cram_const_codec             xconst;
+        cram_varint_decoder          varint;
+
+        cram_huffman_encoder         e_huffman;
+        cram_external_decoder        e_external;
+        cram_byte_array_stop_decoder e_byte_array_stop;
+        cram_byte_array_len_encoder  e_byte_array_len;
+        cram_beta_decoder            e_beta;
+        cram_xpack_decoder           e_xpack;
+        cram_xrle_decoder            e_xrle;
+        cram_xdelta_decoder          e_xdelta;
+        cram_const_codec             e_xconst;
+        cram_varint_decoder          e_varint;
+    } u;
+};
+
+const char *cram_encoding2str(enum cram_encoding t);
+
+cram_codec *cram_decoder_init(cram_block_compression_hdr *hdr,
+                              enum cram_encoding codec, char *data, int size,
+                              enum cram_external_type option,
+                              int version, varint_vec *vv);
+cram_codec *cram_encoder_init(enum cram_encoding codec, cram_stats *st,
+                              enum cram_external_type option, void *dat,
+                              int version, varint_vec *vv);
+
+//int cram_decode(void *codes, char *in, int in_size, char *out, int *out_size);
+//void cram_decoder_free(void *codes);
+
+//#define GET_BIT_MSB(b,v) (void)(v<<=1, v|=(b->data[b->byte] >> b->bit)&1, (--b->bit == -1) && (b->bit = 7, b->byte++))
+
+#define GET_BIT_MSB(b,v) (void)(v<<=1, v|=(b->data[b->byte] >> b->bit)&1, b->byte += (--b->bit<0), b->bit&=7)
+
+/*
+ * Check that enough bits are left in a block to satisy a bit-based decoder.
+ * Return  0 if there are enough
+ *         1 if not.
+ */
+
+static inline int cram_not_enough_bits(cram_block *blk, int nbits) {
+    if (nbits < 0 ||
+        (blk->byte >= blk->uncomp_size && nbits > 0) ||
+        (blk->uncomp_size - blk->byte <= INT32_MAX / 8 + 1 &&
+         (blk->uncomp_size - blk->byte) * 8 + blk->bit - 7 < nbits)) {
+        return 1;
+    }
+    return 0;
+}
+
+/*
+ * Returns the content_id used by this codec, also in id2 if byte_array_len.
+ * Returns -1 for the CORE block and -2 for unneeded.
+ * id2 is only filled out for BYTE_ARRAY_LEN which uses 2 codecs.
+ */
+int cram_codec_to_id(cram_codec *c, int *id2);
+
+/*
+ * cram_codec structures are specialised for decoding or encoding.
+ * Unfortunately this makes turning a decoder into an encoder (such as
+ * when transcoding files) problematic.
+ *
+ * This function converts a cram decoder codec into an encoder version
+ * in-place (ie it modifiers the codec itself).
+ *
+ * Returns 0 on success;
+ *        -1 on failure.
+ */
+int cram_codec_decoder2encoder(cram_fd *fd, cram_codec *c);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* CRAM_CODECS_H */
diff --git a/ext/htslib/cram/cram_decode.c b/ext/htslib/cram/cram_decode.c
new file mode 100644
index 0000000..2b2ad60
--- /dev/null
+++ b/ext/htslib/cram/cram_decode.c
@@ -0,0 +1,3587 @@
+/*
+Copyright (c) 2012-2020, 2022-2024 Genome Research Ltd.
+Author: James Bonfield <jkb@sanger.ac.uk>
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+Institute nor the names of its contributors may be used to endorse or promote
+products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+/*
+ * - In-memory decoding of CRAM data structures.
+ * - Iterator for reading CRAM record by record.
+ */
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+
+#include <stdio.h>
+#include <errno.h>
+#include <assert.h>
+#include <stdlib.h>
+#include <string.h>
+#include <zlib.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <math.h>
+#include <stdint.h>
+#include <inttypes.h>
+
+#include "cram.h"
+#include "os.h"
+#include "../htslib/hts.h"
+
+//Whether CIGAR has just M or uses = and X to indicate match and mismatch
+//#define USE_X
+
+/* ----------------------------------------------------------------------
+ * CRAM compression headers
+ */
+
+/*
+ * Decodes the Tag Dictionary record in the preservation map
+ * Updates the cram compression header.
+ *
+ * Returns number of bytes decoded on success
+ *        -1 on failure
+ */
+int cram_decode_TD(cram_fd *fd, char *cp, const char *endp,
+                   cram_block_compression_hdr *h) {
+    char *op = cp;
+    unsigned char *dat;
+    cram_block *b;
+    int32_t blk_size = 0;
+    int nTL, i, sz, err = 0;
+
+    if (!(b = cram_new_block(0, 0)))
+        return -1;
+
+    if (h->TD_blk || h->TL) {
+        hts_log_warning("More than one TD block found in compression header");
+        cram_free_block(h->TD_blk);
+        free(h->TL);
+        h->TD_blk = NULL;
+        h->TL = NULL;
+    }
+
+    /* Decode */
+    blk_size = fd->vv.varint_get32(&cp, endp, &err);
+    if (!blk_size) {
+        h->nTL = 0;
+        cram_free_block(b);
+        return cp - op;
+    }
+
+    if (err || blk_size < 0 || endp - cp < blk_size) {
+        cram_free_block(b);
+        return -1;
+    }
+
+    BLOCK_APPEND(b, cp, blk_size);
+    cp += blk_size;
+    sz = cp - op;
+    // Force nul termination if missing
+    if (BLOCK_DATA(b)[BLOCK_SIZE(b)-1])
+        BLOCK_APPEND_CHAR(b, '\0');
+
+    /* Set up TL lookup table */
+    dat = BLOCK_DATA(b);
+
+    // Count
+    for (nTL = i = 0; i < BLOCK_SIZE(b); i++) {
+        nTL++;
+        while (dat[i])
+            i++;
+    }
+
+    // Copy
+    if (!(h->TL = calloc(nTL, sizeof(*h->TL)))) {
+        cram_free_block(b);
+        return -1;
+    }
+    for (nTL = i = 0; i < BLOCK_SIZE(b); i++) {
+        h->TL[nTL++] = &dat[i];
+        while (dat[i])
+            i++;
+    }
+    h->TD_blk = b;
+    h->nTL = nTL;
+
+    return sz;
+
+ block_err:
+    cram_free_block(b);
+    return -1;
+}
+
+/*
+ * Decodes a CRAM block compression header.
+ * Returns header ptr on success
+ *         NULL on failure
+ */
+cram_block_compression_hdr *cram_decode_compression_header(cram_fd *fd,
+                                                           cram_block *b) {
+    char *cp, *endp, *cp_copy;
+    cram_block_compression_hdr *hdr = calloc(1, sizeof(*hdr));
+    int i, err = 0;
+    int32_t map_size = 0, map_count = 0;
+
+    if (!hdr)
+        return NULL;
+
+    if (b->method != RAW) {
+        if (cram_uncompress_block(b)) {
+            free(hdr);
+            return NULL;
+        }
+    }
+
+    cp = (char *)b->data;
+    endp = cp + b->uncomp_size;
+
+    if (CRAM_MAJOR_VERS(fd->version) == 1) {
+        hdr->ref_seq_id = fd->vv.varint_get32(&cp, endp, &err);
+        if (CRAM_MAJOR_VERS(fd->version) >= 4) {
+            hdr->ref_seq_start = fd->vv.varint_get64(&cp, endp, &err);
+            hdr->ref_seq_span  = fd->vv.varint_get64(&cp, endp, &err);
+        } else {
+            hdr->ref_seq_start = fd->vv.varint_get32(&cp, endp, &err);
+            hdr->ref_seq_span  = fd->vv.varint_get32(&cp, endp, &err);
+        }
+        hdr->num_records   = fd->vv.varint_get32(&cp, endp, &err);
+        hdr->num_landmarks = fd->vv.varint_get32(&cp, endp, &err);
+        if (hdr->num_landmarks < 0 ||
+            hdr->num_landmarks >= SIZE_MAX / sizeof(int32_t) ||
+            endp - cp < hdr->num_landmarks) {
+            free(hdr);
+            return NULL;
+        }
+        if (!(hdr->landmark = malloc(hdr->num_landmarks * sizeof(int32_t)))) {
+            free(hdr);
+            return NULL;
+        }
+        for (i = 0; i < hdr->num_landmarks; i++)
+            hdr->landmark[i] = fd->vv.varint_get32(&cp, endp, &err);;
+    }
+
+    hdr->preservation_map = kh_init(map);
+
+    memset(hdr->rec_encoding_map, 0,
+           CRAM_MAP_HASH * sizeof(hdr->rec_encoding_map[0]));
+    memset(hdr->tag_encoding_map, 0,
+           CRAM_MAP_HASH * sizeof(hdr->tag_encoding_map[0]));
+
+    if (!hdr->preservation_map) {
+        cram_free_compression_header(hdr);
+        return NULL;
+    }
+
+    /* Initialise defaults for preservation map */
+    hdr->read_names_included = 0;
+    hdr->AP_delta = 1;
+    hdr->qs_seq_orient = 1;
+    memcpy(hdr->substitution_matrix, "CGTNAGTNACTNACGNACGT", 20);
+
+    /* Preservation map */
+    map_size  = fd->vv.varint_get32(&cp, endp, &err); cp_copy = cp;
+    map_count = fd->vv.varint_get32(&cp, endp, &err);
+    for (i = 0; i < map_count; i++) {
+        pmap_t hd;
+        khint_t k;
+        int r;
+
+        if (endp - cp < 3) {
+            cram_free_compression_header(hdr);
+            return NULL;
+        }
+        cp += 2;
+        switch(CRAM_KEY(cp[-2],cp[-1])) {
+        case CRAM_KEY('M','I'): // was mapped QS included in V1.0
+        case CRAM_KEY('U','I'): // was unmapped QS included in V1.0
+        case CRAM_KEY('P','I'): // was unmapped placed in V1.0
+            hd.i = *cp++;
+            break;
+
+        case CRAM_KEY('R','N'):
+            hd.i = *cp++;
+            k = kh_put(map, hdr->preservation_map, "RN", &r);
+            if (-1 == r) {
+                cram_free_compression_header(hdr);
+                return NULL;
+            }
+
+            kh_val(hdr->preservation_map, k) = hd;
+            hdr->read_names_included = hd.i;
+            break;
+
+        case CRAM_KEY('A','P'):
+            hd.i = *cp++;
+            k = kh_put(map, hdr->preservation_map, "AP", &r);
+            if (-1 == r) {
+                cram_free_compression_header(hdr);
+                return NULL;
+            }
+
+            kh_val(hdr->preservation_map, k) = hd;
+            hdr->AP_delta = hd.i;
+            break;
+
+        case CRAM_KEY('R','R'):
+            hd.i = *cp++;
+            k = kh_put(map, hdr->preservation_map, "RR", &r);
+            if (-1 == r) {
+                cram_free_compression_header(hdr);
+                return NULL;
+            }
+
+            kh_val(hdr->preservation_map, k) = hd;
+            hdr->no_ref = !hd.i;
+            break;
+
+        case CRAM_KEY('Q','O'):
+            hd.i = *cp++;
+            k = kh_put(map, hdr->preservation_map, "QO", &r);
+            if (-1 == r) {
+                cram_free_compression_header(hdr);
+                return NULL;
+            }
+
+            kh_val(hdr->preservation_map, k) = hd;
+            hdr->qs_seq_orient = hd.i;
+            break;
+
+        case CRAM_KEY('S','M'):
+            if (endp - cp < 5) {
+                cram_free_compression_header(hdr);
+                return NULL;
+            }
+            hdr->substitution_matrix[0][(cp[0]>>6)&3] = 'C';
+            hdr->substitution_matrix[0][(cp[0]>>4)&3] = 'G';
+            hdr->substitution_matrix[0][(cp[0]>>2)&3] = 'T';
+            hdr->substitution_matrix[0][(cp[0]>>0)&3] = 'N';
+
+            hdr->substitution_matrix[1][(cp[1]>>6)&3] = 'A';
+            hdr->substitution_matrix[1][(cp[1]>>4)&3] = 'G';
+            hdr->substitution_matrix[1][(cp[1]>>2)&3] = 'T';
+            hdr->substitution_matrix[1][(cp[1]>>0)&3] = 'N';
+
+            hdr->substitution_matrix[2][(cp[2]>>6)&3] = 'A';
+            hdr->substitution_matrix[2][(cp[2]>>4)&3] = 'C';
+            hdr->substitution_matrix[2][(cp[2]>>2)&3] = 'T';
+            hdr->substitution_matrix[2][(cp[2]>>0)&3] = 'N';
+
+            hdr->substitution_matrix[3][(cp[3]>>6)&3] = 'A';
+            hdr->substitution_matrix[3][(cp[3]>>4)&3] = 'C';
+            hdr->substitution_matrix[3][(cp[3]>>2)&3] = 'G';
+            hdr->substitution_matrix[3][(cp[3]>>0)&3] = 'N';
+
+            hdr->substitution_matrix[4][(cp[4]>>6)&3] = 'A';
+            hdr->substitution_matrix[4][(cp[4]>>4)&3] = 'C';
+            hdr->substitution_matrix[4][(cp[4]>>2)&3] = 'G';
+            hdr->substitution_matrix[4][(cp[4]>>0)&3] = 'T';
+
+            hd.p = cp;
+            cp += 5;
+
+            k = kh_put(map, hdr->preservation_map, "SM", &r);
+            if (-1 == r) {
+                cram_free_compression_header(hdr);
+                return NULL;
+            }
+            kh_val(hdr->preservation_map, k) = hd;
+            break;
+
+        case CRAM_KEY('T','D'): {
+            int sz = cram_decode_TD(fd, cp, endp, hdr); // tag dictionary
+            if (sz < 0) {
+                cram_free_compression_header(hdr);
+                return NULL;
+            }
+
+            hd.p = cp;
+            cp += sz;
+
+            k = kh_put(map, hdr->preservation_map, "TD", &r);
+            if (-1 == r) {
+                cram_free_compression_header(hdr);
+                return NULL;
+            }
+            kh_val(hdr->preservation_map, k) = hd;
+            break;
+        }
+
+        default:
+            hts_log_warning("Unrecognised preservation map key %c%c", cp[-2], cp[-1]);
+            // guess byte;
+            cp++;
+            break;
+        }
+    }
+    if (cp - cp_copy != map_size) {
+        cram_free_compression_header(hdr);
+        return NULL;
+    }
+
+    /* Record encoding map */
+    map_size  = fd->vv.varint_get32(&cp, endp, &err); cp_copy = cp;
+    map_count = fd->vv.varint_get32(&cp, endp, &err);
+    int is_v4 = CRAM_MAJOR_VERS(fd->version) >= 4 ? 1 : 0;
+    for (i = 0; i < map_count; i++) {
+        char *key = cp;
+        int32_t encoding = E_NULL;
+        int32_t size = 0;
+        ptrdiff_t offset;
+        cram_map *m;
+        enum cram_DS_ID ds_id;
+        enum cram_external_type type;
+
+        if (endp - cp < 4) {
+            cram_free_compression_header(hdr);
+            return NULL;
+        }
+
+        cp += 2;
+        encoding = fd->vv.varint_get32(&cp, endp, &err);
+        size     = fd->vv.varint_get32(&cp, endp, &err);
+
+        offset = cp - (char *)b->data;
+
+        if (encoding == E_NULL)
+            continue;
+
+        if (size < 0 || endp - cp < size) {
+            cram_free_compression_header(hdr);
+            return NULL;
+        }
+
+        //printf("%s codes for %.2s\n", cram_encoding2str(encoding), key);
+
+        /*
+         * For CRAM1.0 CF and BF are Byte and not Int.
+         * Practically speaking it makes no difference unless we have a
+         * 1.0 format file that stores these in EXTERNAL as only then
+         * does Byte vs Int matter.
+         *
+         * Neither this C code nor Java reference implementations did this,
+         * so we gloss over it and treat them as int.
+         */
+        ds_id = DS_CORE;
+        if (key[0] == 'B' && key[1] == 'F') {
+            ds_id = DS_BF; type = E_INT;
+        } else if (key[0] == 'C' && key[1] == 'F') {
+            ds_id = DS_CF; type = E_INT;
+        } else if (key[0] == 'R' && key[1] == 'I') {
+            ds_id = DS_RI; type = E_INT;
+        } else if (key[0] == 'R' && key[1] == 'L') {
+            ds_id = DS_RL; type = E_INT;
+        } else if (key[0] == 'A' && key[1] == 'P') {
+            ds_id = DS_AP;
+            type = is_v4 ? E_SLONG : E_INT;
+        } else if (key[0] == 'R' && key[1] == 'G') {
+            ds_id = DS_RG;
+            type = E_INT;
+        } else if (key[0] == 'M' && key[1] == 'F') {
+            ds_id = DS_MF; type = E_INT;
+        } else if (key[0] == 'N' && key[1] == 'S') {
+            ds_id = DS_NS; type = E_INT;
+        } else if (key[0] == 'N' && key[1] == 'P') {
+            ds_id = DS_NP;
+            type = is_v4 ? E_LONG : E_INT;
+        } else if (key[0] == 'T' && key[1] == 'S') {
+            ds_id = DS_TS;
+            type = is_v4 ? E_SLONG : E_INT;
+        } else if (key[0] == 'N' && key[1] == 'F') {
+            ds_id = DS_NF; type = E_INT;
+        } else if (key[0] == 'T' && key[1] == 'C') {
+            ds_id = DS_TC; type = E_BYTE;
+        } else if (key[0] == 'T' && key[1] == 'N') {
+            ds_id = DS_TN; type = E_INT;
+        } else if (key[0] == 'F' && key[1] == 'N') {
+            ds_id = DS_FN; type = E_INT;
+        } else if (key[0] == 'F' && key[1] == 'C') {
+            ds_id = DS_FC; type = E_BYTE;
+        } else if (key[0] == 'F' && key[1] == 'P') {
+            ds_id = DS_FP; type = E_INT;
+        } else if (key[0] == 'B' && key[1] == 'S') {
+            ds_id = DS_BS; type = E_BYTE;
+        } else if (key[0] == 'I' && key[1] == 'N') {
+            ds_id = DS_IN; type = E_BYTE_ARRAY;
+        } else if (key[0] == 'S' && key[1] == 'C') {
+            ds_id = DS_SC; type = E_BYTE_ARRAY;
+        } else if (key[0] == 'D' && key[1] == 'L') {
+            ds_id = DS_DL; type = E_INT;
+        } else if (key[0] == 'B' && key[1] == 'A') {
+            ds_id = DS_BA; type = E_BYTE;
+        } else if (key[0] == 'B' && key[1] == 'B') {
+            ds_id = DS_BB; type = E_BYTE_ARRAY;
+        } else if (key[0] == 'R' && key[1] == 'S') {
+            ds_id = DS_RS; type = E_INT;
+        } else if (key[0] == 'P' && key[1] == 'D') {
+            ds_id = DS_PD; type = E_INT;
+        } else if (key[0] == 'H' && key[1] == 'C') {
+            ds_id = DS_HC; type = E_INT;
+        } else if (key[0] == 'M' && key[1] == 'Q') {
+            ds_id = DS_MQ; type = E_INT;
+        } else if (key[0] == 'R' && key[1] == 'N') {
+            ds_id = DS_RN; type = E_BYTE_ARRAY_BLOCK;
+        } else if (key[0] == 'Q' && key[1] == 'S') {
+            ds_id = DS_QS; type = E_BYTE;
+        } else if (key[0] == 'Q' && key[1] == 'Q') {
+            ds_id = DS_QQ; type = E_BYTE_ARRAY;
+        } else if (key[0] == 'T' && key[1] == 'L') {
+            ds_id = DS_TL; type = E_INT;
+        } else if (key[0] == 'T' && key[1] == 'M') {
+        } else if (key[0] == 'T' && key[1] == 'V') {
+        } else {
+            hts_log_warning("Unrecognised key: %.2s", key);
+        }
+
+        if (ds_id != DS_CORE) {
+            if (hdr->codecs[ds_id] != NULL) {
+                hts_log_warning("Codec for key %.2s defined more than once",
+                                key);
+                hdr->codecs[ds_id]->free(hdr->codecs[ds_id]);
+            }
+            hdr->codecs[ds_id] = cram_decoder_init(hdr, encoding, cp, size,
+                                                   type, fd->version, &fd->vv);
+            if (!hdr->codecs[ds_id]) {
+                cram_free_compression_header(hdr);
+                return NULL;
+            }
+        }
+
+        cp += size;
+
+        // Fill out cram_map purely for cram_dump to dump out.
+        m = malloc(sizeof(*m));
+        if (!m) {
+            cram_free_compression_header(hdr);
+            return NULL;
+        }
+        m->key = CRAM_KEY(key[0], key[1]);
+        m->encoding = encoding;
+        m->size     = size;
+        m->offset   = offset;
+        m->codec = NULL;
+
+        m->next = hdr->rec_encoding_map[CRAM_MAP(key[0], key[1])];
+        hdr->rec_encoding_map[CRAM_MAP(key[0], key[1])] = m;
+    }
+    if (cp - cp_copy != map_size) {
+        cram_free_compression_header(hdr);
+        return NULL;
+    }
+
+    /* Tag encoding map */
+    map_size  = fd->vv.varint_get32(&cp, endp, &err); cp_copy = cp;
+    map_count = fd->vv.varint_get32(&cp, endp, &err);
+    for (i = 0; i < map_count; i++) {
+        int32_t encoding = E_NULL;
+        int32_t size = 0;
+        cram_map *m = malloc(sizeof(*m)); // FIXME: use pooled_alloc
+        uint8_t key[3];
+
+        if (!m || endp - cp < 6) {
+            free(m);
+            cram_free_compression_header(hdr);
+            return NULL;
+        }
+
+        m->key = fd->vv.varint_get32(&cp, endp, &err);
+        key[0] = m->key>>16;
+        key[1] = m->key>>8;
+        key[2] = m->key;
+        encoding = fd->vv.varint_get32(&cp, endp, &err);
+        size     = fd->vv.varint_get32(&cp, endp, &err);
+
+        m->encoding = encoding;
+        m->size     = size;
+        m->offset   = cp - (char *)b->data;
+        if (size < 0 || endp - cp < size ||
+            !(m->codec = cram_decoder_init(hdr, encoding, cp, size,
+                                           E_BYTE_ARRAY_BLOCK, fd->version, &fd->vv))) {
+            cram_free_compression_header(hdr);
+            free(m);
+            return NULL;
+        }
+
+        cp += size;
+
+        m->next = hdr->tag_encoding_map[CRAM_MAP(key[0],key[1])];
+        hdr->tag_encoding_map[CRAM_MAP(key[0],key[1])] = m;
+    }
+    if (err || cp - cp_copy != map_size) {
+        cram_free_compression_header(hdr);
+        return NULL;
+    }
+
+    return hdr;
+}
+
+/*
+ * Note we also need to scan through the record encoding map to
+ * see which data series share the same block, either external or
+ * CORE. For example if we need the BF data series but MQ and CF
+ * are also encoded in the same block then we need to add those in
+ * as a dependency in order to correctly decode BF.
+ *
+ * Returns 0 on success
+ *        -1 on failure
+ */
+int cram_dependent_data_series(cram_fd *fd,
+                               cram_block_compression_hdr *hdr,
+                               cram_slice *s) {
+    int *block_used;
+    int core_used = 0;
+    int i;
+    static int i_to_id[] = {
+        DS_BF, DS_AP, DS_FP, DS_RL, DS_DL, DS_NF, DS_BA, DS_QS,
+        DS_FC, DS_FN, DS_BS, DS_IN, DS_RG, DS_MQ, DS_TL, DS_RN,
+        DS_NS, DS_NP, DS_TS, DS_MF, DS_CF, DS_RI, DS_RS, DS_PD,
+        DS_HC, DS_SC, DS_BB, DS_QQ,
+    };
+    uint32_t orig_ds;
+
+    /*
+     * Set the data_series bit field based on fd->required_fields
+     * contents.
+     */
+    if (fd->required_fields && fd->required_fields != INT_MAX) {
+        s->data_series = 0;
+
+        if (fd->required_fields & SAM_QNAME)
+            s->data_series |= CRAM_RN;
+
+        if (fd->required_fields & SAM_FLAG)
+            s->data_series |= CRAM_BF;
+
+        if (fd->required_fields & SAM_RNAME)
+            s->data_series |= CRAM_RI | CRAM_BF;
+
+        if (fd->required_fields & SAM_POS)
+            s->data_series |= CRAM_AP | CRAM_BF;
+
+        if (fd->required_fields & SAM_MAPQ)
+            s->data_series |= CRAM_MQ;
+
+        if (fd->required_fields & SAM_CIGAR)
+            s->data_series |= CRAM_CIGAR;
+
+        if (fd->required_fields & SAM_RNEXT)
+            s->data_series |= CRAM_CF | CRAM_NF | CRAM_RI | CRAM_NS |CRAM_BF;
+
+        if (fd->required_fields & SAM_PNEXT)
+            s->data_series |= CRAM_CF | CRAM_NF | CRAM_AP | CRAM_NP | CRAM_BF;
+
+        if (fd->required_fields & SAM_TLEN)
+            s->data_series |= CRAM_CF | CRAM_NF | CRAM_AP | CRAM_TS |
+                CRAM_BF | CRAM_MF | CRAM_RI | CRAM_CIGAR;
+
+        if (fd->required_fields & SAM_SEQ)
+            s->data_series |= CRAM_SEQ;
+
+        if (!(fd->required_fields & SAM_AUX))
+            // No easy way to get MD/NM without other tags at present
+            s->decode_md = 0;
+
+        if (fd->required_fields & SAM_QUAL)
+            s->data_series |= CRAM_QUAL;
+
+        if (fd->required_fields & SAM_AUX)
+            s->data_series |= CRAM_RG | CRAM_TL | CRAM_aux;
+
+        if (fd->required_fields & SAM_RGAUX)
+            s->data_series |= CRAM_RG | CRAM_BF;
+
+        // Always uncompress CORE block
+        if (cram_uncompress_block(s->block[0]))
+            return -1;
+    } else {
+        s->data_series = CRAM_ALL;
+
+        for (i = 0; i < s->hdr->num_blocks; i++) {
+            if (cram_uncompress_block(s->block[i]))
+                return -1;
+        }
+
+        return 0;
+    }
+
+    block_used = calloc(s->hdr->num_blocks+1, sizeof(int));
+    if (!block_used)
+        return -1;
+
+    do {
+        /*
+         * Also set data_series based on code prerequisites. Eg if we need
+         * CRAM_QS then we also need to know CRAM_RL so we know how long it
+         * is, or if we need FC/FP then we also need FN (number of features).
+         *
+         * It's not reciprocal though. We may be needing to decode FN
+         * but have no need to decode FC, FP and cigar ops.
+         */
+        if (s->data_series & CRAM_RS)    s->data_series |= CRAM_FC|CRAM_FP;
+        if (s->data_series & CRAM_PD)    s->data_series |= CRAM_FC|CRAM_FP;
+        if (s->data_series & CRAM_HC)    s->data_series |= CRAM_FC|CRAM_FP;
+        if (s->data_series & CRAM_QS)    s->data_series |= CRAM_FC|CRAM_FP;
+        if (s->data_series & CRAM_IN)    s->data_series |= CRAM_FC|CRAM_FP;
+        if (s->data_series & CRAM_SC)    s->data_series |= CRAM_FC|CRAM_FP;
+        if (s->data_series & CRAM_BS)    s->data_series |= CRAM_FC|CRAM_FP;
+        if (s->data_series & CRAM_DL)    s->data_series |= CRAM_FC|CRAM_FP;
+        if (s->data_series & CRAM_BA)    s->data_series |= CRAM_FC|CRAM_FP;
+        if (s->data_series & CRAM_BB)    s->data_series |= CRAM_FC|CRAM_FP;
+        if (s->data_series & CRAM_QQ)    s->data_series |= CRAM_FC|CRAM_FP;
+
+        // cram_decode_seq() needs seq[] array
+        if (s->data_series & (CRAM_SEQ|CRAM_CIGAR)) s->data_series |= CRAM_RL;
+
+        if (s->data_series & CRAM_FP)    s->data_series |= CRAM_FC;
+        if (s->data_series & CRAM_FC)    s->data_series |= CRAM_FN;
+        if (s->data_series & CRAM_aux)   s->data_series |= CRAM_TL;
+        if (s->data_series & CRAM_MF)    s->data_series |= CRAM_CF;
+        if (s->data_series & CRAM_MQ)    s->data_series |= CRAM_BF;
+        if (s->data_series & CRAM_BS)    s->data_series |= CRAM_RI;
+        if (s->data_series & (CRAM_MF |CRAM_NS |CRAM_NP |CRAM_TS |CRAM_NF))
+            s->data_series |= CRAM_CF;
+        if (!hdr->read_names_included && s->data_series & CRAM_RN)
+            s->data_series |= CRAM_CF | CRAM_NF;
+        if (s->data_series & (CRAM_BA | CRAM_QS | CRAM_BB | CRAM_QQ))
+            s->data_series |= CRAM_BF | CRAM_CF | CRAM_RL;
+        if (s->data_series & CRAM_FN) {
+            // The CRAM_FN loop checks for reference length boundaries,
+            // which needs a working seq_pos.  Some fields are fixed size
+            // irrespective of if we decode (BS), but others need to know
+            // the size of the string fetched back (SC, IN, BB).
+            s->data_series |= CRAM_SC | CRAM_IN | CRAM_BB;
+        }
+
+        orig_ds = s->data_series;
+
+        // Find which blocks are in use.
+        for (i = 0; i < sizeof(i_to_id)/sizeof(*i_to_id); i++) {
+            int bnum1, bnum2, j;
+            cram_codec *c = hdr->codecs[i_to_id[i]];
+
+            if (!(s->data_series & (1<<i)))
+                continue;
+
+            if (!c)
+                continue;
+
+            bnum1 = cram_codec_to_id(c, &bnum2);
+
+            for (;;) {
+                switch (bnum1) {
+                case -2:
+                    break;
+
+                case -1:
+                    core_used = 1;
+                    break;
+
+                default:
+                    for (j = 0; j < s->hdr->num_blocks; j++) {
+                        if (s->block[j]->content_type == EXTERNAL &&
+                            s->block[j]->content_id == bnum1) {
+                            block_used[j] = 1;
+                            if (cram_uncompress_block(s->block[j])) {
+                                free(block_used);
+                                return -1;
+                            }
+                        }
+                    }
+                    break;
+                }
+
+                if (bnum2 == -2 || bnum1 == bnum2)
+                    break;
+
+                bnum1 = bnum2; // 2nd pass
+            }
+        }
+
+        // Tags too
+        if ((fd->required_fields & SAM_AUX) ||
+            (s->data_series & CRAM_aux)) {
+            for (i = 0; i < CRAM_MAP_HASH; i++) {
+                int bnum1, bnum2, j;
+                cram_map *m = hdr->tag_encoding_map[i];
+
+                while (m) {
+                    cram_codec *c = m->codec;
+                    if (!c) {
+                        m = m->next;
+                        continue;
+                    }
+
+                    bnum1 = cram_codec_to_id(c, &bnum2);
+
+                    for (;;) {
+                        switch (bnum1) {
+                        case -2:
+                            break;
+
+                        case -1:
+                            core_used = 1;
+                            break;
+
+                        default:
+                            for (j = 0; j < s->hdr->num_blocks; j++) {
+                                if (s->block[j]->content_type == EXTERNAL &&
+                                    s->block[j]->content_id == bnum1) {
+                                    block_used[j] = 1;
+                                    if (cram_uncompress_block(s->block[j])) {
+                                        free(block_used);
+                                        return -1;
+                                    }
+                                }
+                            }
+                            break;
+                        }
+
+                        if (bnum2 == -2 || bnum1 == bnum2)
+                            break;
+
+                        bnum1 = bnum2; // 2nd pass
+                    }
+
+                    m = m->next;
+                }
+            }
+        }
+
+        // We now know which blocks are in used, so repeat and find
+        // which other data series need to be added.
+        for (i = 0; i < sizeof(i_to_id)/sizeof(*i_to_id); i++) {
+            int bnum1, bnum2, j;
+            cram_codec *c = hdr->codecs[i_to_id[i]];
+
+            if (!c)
+                continue;
+
+            bnum1 = cram_codec_to_id(c, &bnum2);
+
+            for (;;) {
+                switch (bnum1) {
+                case -2:
+                    break;
+
+                case -1:
+                    if (core_used) {
+                        //printf(" + data series %08x:\n", 1<<i);
+                        s->data_series |= 1<<i;
+                    }
+                    break;
+
+                default:
+                    for (j = 0; j < s->hdr->num_blocks; j++) {
+                        if (s->block[j]->content_type == EXTERNAL &&
+                            s->block[j]->content_id == bnum1) {
+                            if (block_used[j]) {
+                                //printf(" + data series %08x:\n", 1<<i);
+                                s->data_series |= 1<<i;
+                            }
+                        }
+                    }
+                    break;
+                }
+
+                if (bnum2 == -2 || bnum1 == bnum2)
+                    break;
+
+                bnum1 = bnum2; // 2nd pass
+            }
+        }
+
+        // Tags too
+        for (i = 0; i < CRAM_MAP_HASH; i++) {
+            int bnum1, bnum2, j;
+            cram_map *m = hdr->tag_encoding_map[i];
+
+            while (m) {
+                cram_codec *c = m->codec;
+                if (!c) {
+                    m = m->next;
+                    continue;
+                }
+
+                bnum1 = cram_codec_to_id(c, &bnum2);
+
+                for (;;) {
+                    switch (bnum1) {
+                    case -2:
+                        break;
+
+                    case -1:
+                        //printf(" + data series %08x:\n", CRAM_aux);
+                        s->data_series |= CRAM_aux;
+                        break;
+
+                    default:
+                        for (j = 0; j < s->hdr->num_blocks; j++) {
+                            if (s->block[j]->content_type == EXTERNAL &&
+                                s->block[j]->content_id == bnum1) {
+                                if (block_used[j]) {
+                                    //printf(" + data series %08x:\n",
+                                    //       CRAM_aux);
+                                    s->data_series |= CRAM_aux;
+                                }
+                            }
+                        }
+                        break;
+                    }
+
+                    if (bnum2 == -2 || bnum1 == bnum2)
+                        break;
+
+                    bnum1 = bnum2; // 2nd pass
+                }
+
+                m = m->next;
+            }
+        }
+    } while (orig_ds != s->data_series);
+
+    free(block_used);
+    return 0;
+}
+
+/*
+ * Checks whether an external block is used solely by a single data series.
+ * Returns the codec type if so (EXTERNAL, BYTE_ARRAY_LEN, BYTE_ARRAY_STOP)
+ *         or 0 if not (E_NULL).
+ */
+static int cram_ds_unique(cram_block_compression_hdr *hdr, cram_codec *c,
+                          int id) {
+    int i, n_id = 0;
+    enum cram_encoding e_type = 0;
+
+    for (i = 0; i < DS_END; i++) {
+        cram_codec *c;
+        int bnum1, bnum2, old_n_id;
+
+        if (!(c = hdr->codecs[i]))
+            continue;
+
+        bnum1 = cram_codec_to_id(c, &bnum2);
+
+        old_n_id = n_id;
+        if (bnum1 == id) {
+            n_id++;
+            e_type = c->codec;
+        }
+        if (bnum2 == id) {
+            n_id++;
+            e_type = c->codec;
+        }
+
+        if (n_id == old_n_id+2)
+            n_id--; // len/val in same place counts once only.
+    }
+
+    return n_id == 1 ? e_type : 0;
+}
+
+/*
+ * Attempts to estimate the size of some blocks so we can preallocate them
+ * before decoding.  Although decoding will automatically grow the blocks,
+ * it is typically more efficient to preallocate.
+ */
+void cram_decode_estimate_sizes(cram_block_compression_hdr *hdr, cram_slice *s,
+                                int *qual_size, int *name_size,
+                                int *q_id) {
+    int bnum1, bnum2;
+    cram_codec *cd;
+
+    *qual_size = 0;
+    *name_size = 0;
+
+    /* Qual */
+    cd = hdr->codecs[DS_QS];
+    if (cd == NULL) return;
+    bnum1 = cram_codec_to_id(cd, &bnum2);
+    if (bnum1 < 0 && bnum2 >= 0) bnum1 = bnum2;
+    if (cram_ds_unique(hdr, cd, bnum1)) {
+        cram_block *b = cram_get_block_by_id(s, bnum1);
+        if (b) *qual_size = b->uncomp_size;
+        if (q_id && cd->codec == E_EXTERNAL)
+            *q_id = bnum1;
+    }
+
+    /* Name */
+    cd = hdr->codecs[DS_RN];
+    if (cd == NULL) return;
+    bnum1 = cram_codec_to_id(cd, &bnum2);
+    if (bnum1 < 0 && bnum2 >= 0) bnum1 = bnum2;
+    if (cram_ds_unique(hdr, cd, bnum1)) {
+        cram_block *b = cram_get_block_by_id(s, bnum1);
+        if (b) *name_size = b->uncomp_size;
+    }
+}
+
+
+/* ----------------------------------------------------------------------
+ * CRAM slices
+ */
+
+/*
+ * Decodes a CRAM (un)mapped slice header block.
+ * Returns slice header ptr on success
+ *         NULL on failure
+ */
+cram_block_slice_hdr *cram_decode_slice_header(cram_fd *fd, cram_block *b) {
+    cram_block_slice_hdr *hdr;
+    unsigned char *cp;
+    unsigned char *cp_end;
+    int i, err = 0;
+
+    if (b->method != RAW) {
+        /* Spec. says slice header should be RAW, but we can future-proof
+           by trying to decode it if it isn't. */
+        if (cram_uncompress_block(b) < 0)
+            return NULL;
+    }
+    cp =  (unsigned char *)BLOCK_DATA(b);
+    cp_end = cp + b->uncomp_size;
+
+    if (b->content_type != MAPPED_SLICE &&
+        b->content_type != UNMAPPED_SLICE)
+        return NULL;
+
+    if (!(hdr  = calloc(1, sizeof(*hdr))))
+        return NULL;
+
+    hdr->content_type = b->content_type;
+
+    if (b->content_type == MAPPED_SLICE) {
+        hdr->ref_seq_id = fd->vv.varint_get32s((char **)&cp, (char *)cp_end, &err);
+        if (CRAM_MAJOR_VERS(fd->version) >= 4) {
+            hdr->ref_seq_start = fd->vv.varint_get64((char **)&cp, (char *)cp_end, &err);
+            hdr->ref_seq_span  = fd->vv.varint_get64((char **)&cp, (char *)cp_end, &err);
+        } else {
+            hdr->ref_seq_start = fd->vv.varint_get32((char **)&cp, (char *)cp_end, &err);
+            hdr->ref_seq_span  = fd->vv.varint_get32((char **)&cp, (char *)cp_end, &err);
+        }
+        if (hdr->ref_seq_start < 0 || hdr->ref_seq_span < 0) {
+            free(hdr);
+            hts_log_error("Negative values not permitted for header "
+                          "sequence start or span fields");
+            return NULL;
+        }
+    }
+    hdr->num_records = fd->vv.varint_get32((char **)&cp, (char *) cp_end, &err);
+    hdr->record_counter = 0;
+    if (CRAM_MAJOR_VERS(fd->version) == 2) {
+        hdr->record_counter = fd->vv.varint_get32((char **)&cp, (char *)cp_end, &err);
+    } else if (CRAM_MAJOR_VERS(fd->version) >= 3) {
+        hdr->record_counter = fd->vv.varint_get64((char **)&cp, (char *)cp_end, &err);
+    }
+    hdr->num_blocks      = fd->vv.varint_get32((char **)&cp, (char *)cp_end, &err);
+    hdr->num_content_ids = fd->vv.varint_get32((char **)&cp, (char *)cp_end, &err);
+    if (hdr->num_content_ids < 1 ||
+        hdr->num_content_ids >= 10000) {
+        // Slice must have at least one data block, and there is no need
+        // for more than 2 per possible aux-tag plus ancillary.
+        free(hdr);
+        return NULL;
+    }
+    hdr->block_content_ids = malloc(hdr->num_content_ids * sizeof(int32_t));
+    if (!hdr->block_content_ids) {
+        free(hdr);
+        return NULL;
+    }
+
+    for (i = 0; i < hdr->num_content_ids; i++)
+        hdr->block_content_ids[i] = fd->vv.varint_get32((char **)&cp,
+                                                         (char *)cp_end,
+                                                         &err);
+    if (err) {
+        free(hdr->block_content_ids);
+        free(hdr);
+        return NULL;
+    }
+
+    if (b->content_type == MAPPED_SLICE)
+        hdr->ref_base_id = fd->vv.varint_get32((char **)&cp, (char *) cp_end, &err);
+
+    if (CRAM_MAJOR_VERS(fd->version) != 1) {
+        if (cp_end - cp < 16) {
+            free(hdr->block_content_ids);
+            free(hdr);
+            return NULL;
+        }
+        memcpy(hdr->md5, cp, 16);
+    } else {
+        memset(hdr->md5, 0, 16);
+    }
+
+    if (!err)
+        return hdr;
+
+    free(hdr->block_content_ids);
+    free(hdr);
+    return NULL;
+}
+
+
+#if 0
+/* Returns the number of bits set in val; it the highest bit used */
+static int nbits(int v) {
+    static const int MultiplyDeBruijnBitPosition[32] = {
+        1, 10, 2, 11, 14, 22, 3, 30, 12, 15, 17, 19, 23, 26, 4, 31,
+        9, 13, 21, 29, 16, 18, 25, 8, 20, 28, 24, 7, 27, 6, 5, 32
+    };
+
+    v |= v >> 1; // first up to set all bits 1 after the first 1 */
+    v |= v >> 2;
+    v |= v >> 4;
+    v |= v >> 8;
+    v |= v >> 16;
+
+    // DeBruijn magic to find top bit
+    return MultiplyDeBruijnBitPosition[(uint32_t)(v * 0x07C4ACDDU) >> 27];
+}
+#endif
+
+#if 0
+static int sort_freqs(const void *vp1, const void *vp2) {
+    const int i1 = *(const int *)vp1;
+    const int i2 = *(const int *)vp2;
+    return i1-i2;
+}
+#endif
+
+/* ----------------------------------------------------------------------
+ * Primary CRAM sequence decoder
+ */
+
+static inline int add_md_char(cram_slice *s, int decode_md, char c, int32_t *md_dist) {
+    if (decode_md) {
+        BLOCK_APPEND_UINT(s->aux_blk, *md_dist);
+        BLOCK_APPEND_CHAR(s->aux_blk, c);
+        *md_dist = 0;
+    }
+    return 0;
+
+ block_err:
+    return -1;
+}
+
+/*
+ * Internal part of cram_decode_slice().
+ * Generates the sequence, quality and cigar components.
+ */
+static int cram_decode_seq(cram_fd *fd, cram_container *c, cram_slice *s,
+                           cram_block *blk, cram_record *cr, sam_hdr_t *sh,
+                           int cf, char *seq, char *qual,
+                           int has_MD, int has_NM) {
+    int prev_pos = 0, f, r = 0, out_sz = 1;
+    int seq_pos = 1;
+    int cig_len = 0;
+    int64_t ref_pos = cr->apos;
+    int32_t fn, i32;
+    enum cigar_op cig_op = BAM_CMATCH;
+    uint32_t *cigar = s->cigar;
+    uint32_t ncigar = s->ncigar;
+    uint32_t cigar_alloc = s->cigar_alloc;
+    uint32_t nm = 0;
+    int32_t md_dist = 0;
+    int orig_aux = 0;
+    // CRAM <  4.0 decode_md is off/on
+    // CRAM >= 4.0 decode_md is auto/on (auto=on if MD* present, off otherwise)
+    int do_md = CRAM_MAJOR_VERS(fd->version) >= 4
+        ? (s->decode_md > 0)
+        : (s->decode_md != 0);
+    int decode_md = s->ref && cr->ref_id >= 0 && ((do_md && !has_MD) || has_MD < 0);
+    int decode_nm = s->ref && cr->ref_id >= 0 && ((do_md && !has_NM) || has_NM < 0);
+    uint32_t ds = s->data_series;
+    sam_hrecs_t *bfd = sh->hrecs;
+
+    cram_codec **codecs = c->comp_hdr->codecs;
+
+    if ((ds & CRAM_QS) && !(cf & CRAM_FLAG_PRESERVE_QUAL_SCORES)) {
+        memset(qual, 255, cr->len);
+    }
+
+    if (cr->cram_flags & CRAM_FLAG_NO_SEQ)
+        decode_md = decode_nm = 0;
+
+    if (decode_md) {
+        orig_aux = BLOCK_SIZE(s->aux_blk);
+        if (has_MD == 0)
+            BLOCK_APPEND(s->aux_blk, "MDZ", 3);
+    }
+
+    if (ds & CRAM_FN) {
+        if (!codecs[DS_FN]) return -1;
+        r |= codecs[DS_FN]->decode(s,codecs[DS_FN],
+                                   blk, (char *)&fn, &out_sz);
+        if (r) return r;
+    } else {
+        fn = 0;
+    }
+
+    ref_pos--; // count from 0
+    cr->cigar = ncigar;
+
+    if (!(ds & (CRAM_FC | CRAM_FP)))
+        goto skip_cigar;
+
+    if (fn) {
+        if ((ds & CRAM_FC) && !codecs[DS_FC])
+            return -1;
+        if ((ds & CRAM_FP) && !codecs[DS_FP])
+            return -1;
+    }
+
+    for (f = 0; f < fn; f++) {
+        int32_t pos = 0;
+        char op;
+
+        if (ncigar+2 >= cigar_alloc) {
+            cigar_alloc = cigar_alloc ? cigar_alloc*2 : 1024;
+            if (!(cigar = realloc(s->cigar, cigar_alloc * sizeof(*cigar))))
+                return -1;
+            s->cigar = cigar;
+        }
+
+        if (ds & CRAM_FC) {
+            r |= codecs[DS_FC]->decode(s,
+                                       codecs[DS_FC],
+                                       blk,
+                                       &op,  &out_sz);
+            if (r) return r;
+        }
+
+        if (!(ds & CRAM_FP))
+            continue;
+
+        r |= codecs[DS_FP]->decode(s,
+                                   codecs[DS_FP],
+                                   blk,
+                                   (char *)&pos, &out_sz);
+        if (r) return r;
+        pos += prev_pos;
+
+        if (pos <= 0) {
+            hts_log_error("Feature position %d before start of read", pos);
+            return -1;
+        }
+
+        if (pos > seq_pos) {
+            if (pos > cr->len+1)
+                return -1;
+
+            if (s->ref && cr->ref_id >= 0) {
+                if (ref_pos + pos - seq_pos > bfd->ref[cr->ref_id].len) {
+                    static int whinged = 0;
+                    int rlen;
+                    if (!whinged)
+                        hts_log_warning("Ref pos outside of ref sequence boundary");
+                    whinged = 1;
+                    rlen = bfd->ref[cr->ref_id].len - ref_pos;
+                    // May miss MD/NM cases where both seq/ref are N, but this is a
+                    // malformed cram file anyway.
+                    if (rlen > 0) {
+                        if (ref_pos + rlen > s->ref_end)
+                            goto beyond_slice;
+
+                        memcpy(&seq[seq_pos-1],
+                               &s->ref[ref_pos - s->ref_start +1], rlen);
+                        if ((pos - seq_pos) - rlen > 0)
+                            memset(&seq[seq_pos-1+rlen], 'N',
+                                   (pos - seq_pos) - rlen);
+                    } else {
+                        memset(&seq[seq_pos-1], 'N', cr->len - seq_pos + 1);
+                    }
+                    if (md_dist >= 0)
+                        md_dist += pos - seq_pos;
+                } else {
+                    // 'N' in both ref and seq is also mismatch for NM/MD
+                    if (ref_pos + pos-seq_pos > s->ref_end)
+                        goto beyond_slice;
+
+                    const char *refp = s->ref + ref_pos - s->ref_start + 1;
+                    const int frag_len = pos - seq_pos;
+                    int do_cpy = 1;
+                    if (decode_md || decode_nm) {
+                        char *N = memchr(refp, 'N', frag_len);
+                        if (N) {
+                            int i;
+                            for (i = 0; i < frag_len; i++) {
+                                char base = refp[i];
+                                if (base == 'N') {
+                                    if (add_md_char(s, decode_md,
+                                                    'N', &md_dist) < 0)
+                                        return -1;
+                                    nm++;
+                                } else {
+                                    md_dist++;
+                                }
+                                seq[seq_pos-1+i] = base;
+                            }
+                            do_cpy = 0;
+                        } else {
+                            md_dist += frag_len;
+                        }
+                    }
+                    if (do_cpy)
+                        memcpy(&seq[seq_pos-1], refp, frag_len);
+                }
+            }
+#ifdef USE_X
+            if (cig_len && cig_op != BAM_CBASE_MATCH) {
+                cigar[ncigar++] = (cig_len<<4) + cig_op;
+                cig_len = 0;
+            }
+            cig_op = BAM_CBASE_MATCH;
+#else
+            if (cig_len && cig_op != BAM_CMATCH) {
+                cigar[ncigar++] = (cig_len<<4) + cig_op;
+                cig_len = 0;
+            }
+            cig_op = BAM_CMATCH;
+#endif
+            cig_len += pos - seq_pos;
+            ref_pos += pos - seq_pos;
+            seq_pos = pos;
+        }
+
+        prev_pos = pos;
+
+        if (!(ds & CRAM_FC))
+            goto skip_cigar;
+
+        switch(op) {
+        case 'S': { // soft clip: IN
+            int32_t out_sz2 = 1;
+            int have_sc = 0;
+
+            if (cig_len) {
+                cigar[ncigar++] = (cig_len<<4) + cig_op;
+                cig_len = 0;
+            }
+            switch (CRAM_MAJOR_VERS(fd->version)) {
+            case 1:
+                if (ds & CRAM_IN) {
+                    r |= codecs[DS_IN]
+                        ? codecs[DS_IN]->decode(s, codecs[DS_IN],
+                                                blk,
+                                                cr->len ? &seq[pos-1] : NULL,
+                                                &out_sz2)
+                        : (seq[pos-1] = 'N', out_sz2 = 1, 0);
+                    have_sc = 1;
+                }
+                break;
+            case 2:
+            default:
+                if (ds & CRAM_SC) {
+                    r |= codecs[DS_SC]
+                        ? codecs[DS_SC]->decode(s, codecs[DS_SC],
+                                                blk,
+                                                cr->len ? &seq[pos-1] : NULL,
+                                                &out_sz2)
+                        : (seq[pos-1] = 'N', out_sz2 = 1, 0);
+                    have_sc = 1;
+                }
+                break;
+
+                //default:
+                //    r |= codecs[DS_BB]
+                //        ? codecs[DS_BB]->decode(s, codecs[DS_BB],
+                //                                blk, &seq[pos-1], &out_sz2)
+                //        : (seq[pos-1] = 'N', out_sz2 = 1, 0);
+            }
+            if (have_sc) {
+                if (r) return r;
+                cigar[ncigar++] = (out_sz2<<4) + BAM_CSOFT_CLIP;
+                cig_op = BAM_CSOFT_CLIP;
+                seq_pos += out_sz2;
+            }
+            break;
+        }
+
+        case 'X': { // Substitution; BS
+            unsigned char base;
+#ifdef USE_X
+            if (cig_len && cig_op != BAM_CBASE_MISMATCH) {
+                cigar[ncigar++] = (cig_len<<4) + cig_op;
+                cig_len = 0;
+            }
+            if (ds & CRAM_BS) {
+                if (!codecs[DS_BS]) return -1;
+                r |= codecs[DS_BS]->decode(s, codecs[DS_BS], blk,
+                                           (char *)&base, &out_sz);
+                if (pos-1 < cr->len)
+                    seq[pos-1] = 'N'; // FIXME look up BS=base value
+            }
+            cig_op = BAM_CBASE_MISMATCH;
+#else
+            int ref_base;
+            if (cig_len && cig_op != BAM_CMATCH) {
+                cigar[ncigar++] = (cig_len<<4) + cig_op;
+                cig_len = 0;
+            }
+            if (ds & CRAM_BS) {
+                if (!codecs[DS_BS]) return -1;
+                r |= codecs[DS_BS]->decode(s, codecs[DS_BS], blk,
+                                           (char *)&base, &out_sz);
+                if (r) return -1;
+                if (cr->ref_id < 0 || ref_pos >= bfd->ref[cr->ref_id].len || !s->ref) {
+                    if (pos-1 < cr->len)
+                        seq[pos-1] = c->comp_hdr->
+                            substitution_matrix[fd->L1['N']][base];
+                    if (decode_md || decode_nm) {
+                        if (md_dist >= 0 && decode_md)
+                            BLOCK_APPEND_UINT(s->aux_blk, md_dist);
+                        md_dist = -1;
+                        nm--;
+                    }
+                } else {
+                    unsigned char ref_call = ref_pos < s->ref_end
+                        ? (uc)s->ref[ref_pos - s->ref_start +1]
+                        : 'N';
+                    ref_base = fd->L1[ref_call];
+                    if (pos-1 < cr->len)
+                        seq[pos-1] = c->comp_hdr->
+                            substitution_matrix[ref_base][base];
+                    if (add_md_char(s, decode_md, ref_call, &md_dist) < 0)
+                        return -1;
+                }
+            }
+            cig_op = BAM_CMATCH;
+#endif
+            nm++;
+            cig_len++;
+            seq_pos++;
+            ref_pos++;
+            break;
+        }
+
+        case 'D': { // Deletion; DL
+            if (cig_len && cig_op != BAM_CDEL) {
+                cigar[ncigar++] = (cig_len<<4) + cig_op;
+                cig_len = 0;
+            }
+            if (ds & CRAM_DL) {
+                if (!codecs[DS_DL]) return -1;
+                r |= codecs[DS_DL]->decode(s, codecs[DS_DL], blk,
+                                           (char *)&i32, &out_sz);
+                if (r) return r;
+                if (decode_md || decode_nm) {
+                    if (ref_pos + i32 > s->ref_end)
+                        goto beyond_slice;
+                    if (md_dist >= 0 && decode_md)
+                        BLOCK_APPEND_UINT(s->aux_blk, md_dist);
+                    if (ref_pos + i32 <= bfd->ref[cr->ref_id].len) {
+                        if (decode_md) {
+                            BLOCK_APPEND_CHAR(s->aux_blk, '^');
+                            BLOCK_APPEND(s->aux_blk,
+                                         &s->ref[ref_pos - s->ref_start +1],
+                                         i32);
+                            md_dist = 0;
+                        }
+                        nm += i32;
+                    } else {
+                        uint32_t dlen;
+                        if (bfd->ref[cr->ref_id].len >= ref_pos) {
+                            if (decode_md) {
+                                BLOCK_APPEND_CHAR(s->aux_blk, '^');
+                                BLOCK_APPEND(s->aux_blk,
+                                             &s->ref[ref_pos - s->ref_start+1],
+                                             bfd->ref[cr->ref_id].len-ref_pos);
+                                BLOCK_APPEND_UINT(s->aux_blk, 0);
+                            }
+                            dlen = i32 - (bfd->ref[cr->ref_id].len - ref_pos);
+                            nm += i32 - dlen;
+                        } else {
+                            dlen = i32;
+                        }
+
+                        md_dist = -1;
+                    }
+                }
+                cig_op = BAM_CDEL;
+                cig_len += i32;
+                ref_pos += i32;
+                //printf("  %d: DL = %d (ret %d)\n", f, i32, r);
+            }
+            break;
+        }
+
+        case 'I': { // Insertion (several bases); IN
+            int32_t out_sz2 = 1;
+
+            if (cig_len && cig_op != BAM_CINS) {
+                cigar[ncigar++] = (cig_len<<4) + cig_op;
+                cig_len = 0;
+            }
+
+            if (ds & CRAM_IN) {
+                if (!codecs[DS_IN]) return -1;
+                r |= codecs[DS_IN]->decode(s, codecs[DS_IN], blk,
+                                           cr->len ? &seq[pos-1] : NULL,
+                                           &out_sz2);
+                if (r) return r;
+                cig_op = BAM_CINS;
+                cig_len += out_sz2;
+                seq_pos += out_sz2;
+                nm      += out_sz2;
+                //printf("  %d: IN(I) = %.*s (ret %d, out_sz %d)\n", f, out_sz2, dat, r, out_sz2);
+            }
+            break;
+        }
+
+        case 'i': { // Insertion (single base); BA
+            if (cig_len && cig_op != BAM_CINS) {
+                cigar[ncigar++] = (cig_len<<4) + cig_op;
+                cig_len = 0;
+            }
+            if (ds & CRAM_BA) {
+                if (!codecs[DS_BA]) return -1;
+                r |= codecs[DS_BA]->decode(s, codecs[DS_BA], blk,
+                                           cr->len ? &seq[pos-1] : NULL,
+                                           &out_sz);
+                if (r) return r;
+            }
+            cig_op = BAM_CINS;
+            cig_len++;
+            seq_pos++;
+            nm++;
+            break;
+        }
+
+        case 'b': { // Several bases
+            int32_t len = 1;
+
+            if (cig_len && cig_op != BAM_CMATCH) {
+                cigar[ncigar++] = (cig_len<<4) + cig_op;
+                cig_len = 0;
+            }
+
+            if (ds & CRAM_BB) {
+                if (!codecs[DS_BB]) return -1;
+                r |= codecs[DS_BB]->decode(s, codecs[DS_BB], blk,
+                                           cr->len ? &seq[pos-1] : NULL,
+                                           &len);
+                if (r) return r;
+
+                if (decode_md || decode_nm) {
+                    int x;
+                    if (md_dist >= 0 && decode_md)
+                        BLOCK_APPEND_UINT(s->aux_blk, md_dist);
+
+                    for (x = 0; x < len; x++) {
+                        if (x && decode_md)
+                            BLOCK_APPEND_UINT(s->aux_blk, 0);
+                        if (ref_pos+x >= bfd->ref[cr->ref_id].len || !s->ref) {
+                            md_dist = -1;
+                            break;
+                        } else {
+                            if (decode_md) {
+                                if (ref_pos + x > s->ref_end)
+                                    goto beyond_slice;
+                                char r = s->ref[ref_pos+x-s->ref_start +1];
+                                BLOCK_APPEND_CHAR(s->aux_blk, r);
+                            }
+                        }
+                    }
+
+                    nm += x;
+                    md_dist = 0;
+                }
+            }
+
+            cig_op = BAM_CMATCH;
+
+            cig_len+=len;
+            seq_pos+=len;
+            ref_pos+=len;
+            //prev_pos+=len;
+            break;
+        }
+
+        case 'q': { // Several quality values
+            int32_t len = 1;
+
+            if (cig_len && cig_op != BAM_CMATCH) {
+                cigar[ncigar++] = (cig_len<<4) + cig_op;
+                cig_len = 0;
+            }
+
+            if (ds & CRAM_QQ) {
+                if (!codecs[DS_QQ]) return -1;
+                if ((ds & CRAM_QS) && !(cf & CRAM_FLAG_PRESERVE_QUAL_SCORES)
+                    && (unsigned char)*qual == 255)
+                    memset(qual, 30, cr->len); // ?
+                r |= codecs[DS_QQ]->decode(s, codecs[DS_QQ], blk,
+                                           (char *)&qual[pos-1], &len);
+                if (r) return r;
+            }
+
+            cig_op = BAM_CMATCH;
+
+            //prev_pos+=len;
+            break;
+        }
+
+        case 'B': { // Read base; BA, QS
+#ifdef USE_X
+            if (cig_len && cig_op != BAM_CBASE_MISMATCH) {
+                cigar[ncigar++] = (cig_len<<4) + cig_op;
+                cig_len = 0;
+            }
+#else
+            if (cig_len && cig_op != BAM_CMATCH) {
+                cigar[ncigar++] = (cig_len<<4) + cig_op;
+                cig_len = 0;
+            }
+#endif
+            if (ds & CRAM_BA) {
+                if (!codecs[DS_BA]) return -1;
+                r |= codecs[DS_BA]->decode(s, codecs[DS_BA], blk,
+                                           cr->len ? &seq[pos-1] : NULL,
+                                           &out_sz);
+
+                if (decode_md || decode_nm) {
+                    if (md_dist >= 0 && decode_md)
+                        BLOCK_APPEND_UINT(s->aux_blk, md_dist);
+                    if (ref_pos >= bfd->ref[cr->ref_id].len || !s->ref) {
+                        md_dist = -1;
+                    } else {
+                        if (decode_md) {
+                            if (ref_pos > s->ref_end)
+                                goto beyond_slice;
+                            BLOCK_APPEND_CHAR(s->aux_blk,
+                                              s->ref[ref_pos-s->ref_start +1]);
+                        }
+                        nm++;
+                        md_dist = 0;
+                    }
+                }
+            }
+            if (ds & CRAM_QS) {
+                if (!codecs[DS_QS]) return -1;
+                if (!(cf & CRAM_FLAG_PRESERVE_QUAL_SCORES)
+                    && (unsigned char)*qual == 255)
+                    memset(qual, 30, cr->len); // ASCII ?.  Same as htsjdk
+                r |= codecs[DS_QS]->decode(s, codecs[DS_QS], blk,
+                                           (char *)&qual[pos-1], &out_sz);
+            }
+#ifdef USE_X
+            cig_op = BAM_CBASE_MISMATCH;
+#else
+            cig_op = BAM_CMATCH;
+#endif
+            cig_len++;
+            seq_pos++;
+            ref_pos++;
+            //printf("  %d: BA/QS(B) = %c/%d (ret %d)\n", f, i32, qc, r);
+            break;
+        }
+
+        case 'Q': { // Quality score; QS
+            if (ds & CRAM_QS) {
+                if (!codecs[DS_QS]) return -1;
+                if (!(cf & CRAM_FLAG_PRESERVE_QUAL_SCORES) &&
+                    (unsigned char)*qual == 255)
+                    memset(qual, 30, cr->len); // ?
+                r |= codecs[DS_QS]->decode(s, codecs[DS_QS], blk,
+                                           (char *)&qual[pos-1], &out_sz);
+                //printf("  %d: QS = %d (ret %d)\n", f, qc, r);
+            }
+            break;
+        }
+
+        case 'H': { // hard clip; HC
+            if (cig_len && cig_op != BAM_CHARD_CLIP) {
+                cigar[ncigar++] = (cig_len<<4) + cig_op;
+                cig_len = 0;
+            }
+            if (ds & CRAM_HC) {
+                if (!codecs[DS_HC]) return -1;
+                r |= codecs[DS_HC]->decode(s, codecs[DS_HC], blk,
+                                           (char *)&i32, &out_sz);
+                if (r) return r;
+                cig_op = BAM_CHARD_CLIP;
+                cig_len += i32;
+            }
+            break;
+        }
+
+        case 'P': { // padding; PD
+            if (cig_len && cig_op != BAM_CPAD) {
+                cigar[ncigar++] = (cig_len<<4) + cig_op;
+                cig_len = 0;
+            }
+            if (ds & CRAM_PD) {
+                if (!codecs[DS_PD]) return -1;
+                r |= codecs[DS_PD]->decode(s, codecs[DS_PD], blk,
+                                           (char *)&i32, &out_sz);
+                if (r) return r;
+                cig_op = BAM_CPAD;
+                cig_len += i32;
+            }
+            break;
+        }
+
+        case 'N': { // Ref skip; RS
+            if (cig_len && cig_op != BAM_CREF_SKIP) {
+                cigar[ncigar++] = (cig_len<<4) + cig_op;
+                cig_len = 0;
+            }
+            if (ds & CRAM_RS) {
+                if (!codecs[DS_RS]) return -1;
+                r |= codecs[DS_RS]->decode(s, codecs[DS_RS], blk,
+                                           (char *)&i32, &out_sz);
+                if (r) return r;
+                cig_op = BAM_CREF_SKIP;
+                cig_len += i32;
+                ref_pos += i32;
+            }
+            break;
+        }
+
+        default:
+            hts_log_error("Unknown feature code '%c'", op);
+            return -1;
+        }
+    }
+
+    if (!(ds & CRAM_FC))
+        goto skip_cigar;
+
+    /* An implicit match op for any unaccounted for bases */
+    if ((ds & CRAM_FN) && cr->len >= seq_pos) {
+        if (s->ref && cr->ref_id >= 0) {
+            if (ref_pos + cr->len - seq_pos + 1 > bfd->ref[cr->ref_id].len) {
+                static int whinged = 0;
+                int rlen;
+                if (!whinged)
+                    hts_log_warning("Ref pos outside of ref sequence boundary");
+                whinged = 1;
+                rlen = bfd->ref[cr->ref_id].len - ref_pos;
+                // May miss MD/NM cases where both seq/ref are N, but this is a
+                // malformed cram file anyway.
+                if (rlen > 0) {
+                    if (seq_pos-1 + rlen < cr->len)
+                        memcpy(&seq[seq_pos-1],
+                               &s->ref[ref_pos - s->ref_start +1], rlen);
+                    if ((cr->len - seq_pos + 1) - rlen > 0)
+                        memset(&seq[seq_pos-1+rlen], 'N',
+                               (cr->len - seq_pos + 1) - rlen);
+                } else {
+                    if (cr->len - seq_pos + 1 > 0)
+                        memset(&seq[seq_pos-1], 'N', cr->len - seq_pos + 1);
+                }
+                if (md_dist >= 0)
+                    md_dist += cr->len - seq_pos + 1;
+            } else {
+                if (cr->len - seq_pos + 1 > 0) {
+                    if (ref_pos + cr->len-seq_pos +1 > s->ref_end)
+                        goto beyond_slice;
+                    int remainder = cr->len - (seq_pos-1);
+                    int j = ref_pos - s->ref_start + 1;
+                    if (decode_md || decode_nm) {
+                        int i;
+                        char *N = memchr(&s->ref[j], 'N', remainder);
+                        if (!N) {
+                            // short cut the common case
+                            md_dist += cr->len - (seq_pos-1);
+                        } else {
+                            char *refp = &s->ref[j-(seq_pos-1)];
+                            md_dist += N-&s->ref[j];
+                            int i_start = seq_pos-1 + (N - &s->ref[j]);
+                            for (i = i_start; i < cr->len; i++) {
+                                char base = refp[i];
+                                if (base == 'N') {
+                                    if (add_md_char(s, decode_md, 'N',
+                                                    &md_dist) < 0)
+                                        return -1;
+                                    nm++;
+                                } else {
+                                    md_dist++;
+                                }
+                            }
+                        }
+                    }
+                    memcpy(&seq[seq_pos-1], &s->ref[j], remainder);
+                }
+                ref_pos += cr->len - seq_pos + 1;
+            }
+        } else if (cr->ref_id >= 0) {
+            // So alignment end can be computed even when not decoding sequence
+            ref_pos += cr->len - seq_pos + 1;
+        }
+
+        if (ncigar+1 >= cigar_alloc) {
+            cigar_alloc = cigar_alloc ? cigar_alloc*2 : 1024;
+            if (!(cigar = realloc(s->cigar, cigar_alloc * sizeof(*cigar))))
+                return -1;
+            s->cigar = cigar;
+        }
+#ifdef USE_X
+        if (cig_len && cig_op != BAM_CBASE_MATCH) {
+            cigar[ncigar++] = (cig_len<<4) + cig_op;
+            cig_len = 0;
+        }
+        cig_op = BAM_CBASE_MATCH;
+#else
+        if (cig_len && cig_op != BAM_CMATCH) {
+            cigar[ncigar++] = (cig_len<<4) + cig_op;
+            cig_len = 0;
+        }
+        cig_op = BAM_CMATCH;
+#endif
+        cig_len += cr->len - seq_pos+1;
+    }
+
+ skip_cigar:
+
+    if ((ds & CRAM_FN) && decode_md) {
+        if (md_dist >= 0)
+            BLOCK_APPEND_UINT(s->aux_blk, md_dist);
+    }
+
+    if (cig_len) {
+        if (ncigar >= cigar_alloc) {
+            cigar_alloc = cigar_alloc ? cigar_alloc*2 : 1024;
+            if (!(cigar = realloc(s->cigar, cigar_alloc * sizeof(*cigar))))
+                return -1;
+            s->cigar = cigar;
+        }
+
+        cigar[ncigar++] = (cig_len<<4) + cig_op;
+    }
+
+    cr->ncigar = ncigar - cr->cigar;
+    cr->aend = ref_pos > cr->apos ? ref_pos : cr->apos;
+
+    //printf("2: %.*s %d .. %d\n", cr->name_len, DSTRING_STR(name_ds) + cr->name, cr->apos, ref_pos);
+
+    if (ds & CRAM_MQ) {
+        if (!codecs[DS_MQ]) return -1;
+        r |= codecs[DS_MQ]->decode(s, codecs[DS_MQ], blk,
+                                   (char *)&cr->mqual, &out_sz);
+    } else {
+        cr->mqual = 40;
+    }
+
+    if ((ds & CRAM_QS) && (cf & CRAM_FLAG_PRESERVE_QUAL_SCORES)) {
+        int32_t out_sz2 = cr->len;
+
+        if (!codecs[DS_QS]) return -1;
+        r |= codecs[DS_QS]->decode(s, codecs[DS_QS], blk,
+                                   qual, &out_sz2);
+    }
+
+    s->cigar = cigar;
+    s->cigar_alloc = cigar_alloc;
+    s->ncigar = ncigar;
+
+    if (cr->cram_flags & CRAM_FLAG_NO_SEQ)
+        cr->len = 0;
+
+    if (decode_md) {
+        BLOCK_APPEND_CHAR(s->aux_blk, '\0'); // null terminate MD:Z:
+        size_t sz = BLOCK_SIZE(s->aux_blk) - orig_aux;
+        if (has_MD < 0) {
+            // has_MD < 0; already have MDZ allocated in aux at -has_MD,
+            // but wrote MD to end of aux (at orig_aux).
+            // We need some memmoves to shuffle it around.
+            char tmp_MD_[1024], *tmp_MD = tmp_MD_;
+            unsigned char *orig_aux_p = BLOCK_DATA(s->aux_blk) + orig_aux;
+            if (sz > 1024) {
+                tmp_MD = malloc(sz);
+                if (!tmp_MD)
+                    return -1;
+            }
+            memcpy(tmp_MD, orig_aux_p, sz);
+            memmove(&BLOCK_DATA(s->aux_blk)[-has_MD] + sz,
+                    &BLOCK_DATA(s->aux_blk)[-has_MD],
+                    orig_aux_p - &BLOCK_DATA(s->aux_blk)[-has_MD]);
+            memcpy(&BLOCK_DATA(s->aux_blk)[-has_MD], tmp_MD, sz);
+            if (tmp_MD != tmp_MD_)
+                free(tmp_MD);
+
+            if (-has_NM > -has_MD)
+                // we inserted before NM, so move it up a bit
+                has_NM -= sz;
+        }
+        // else has_MD == 0 and we've already appended MD to the end.
+
+        cr->aux_size += sz;
+    }
+
+    if (decode_nm) {
+        if (has_NM == 0) {
+            char buf[7];
+            size_t buf_size;
+            buf[0] = 'N'; buf[1] = 'M';
+            if (nm <= UINT8_MAX) {
+                buf_size = 4;
+                buf[2] = 'C';
+                buf[3] = (nm>> 0) & 0xff;
+            } else if (nm <= UINT16_MAX) {
+                buf_size = 5;
+                buf[2] = 'S';
+                buf[3] = (nm>> 0) & 0xff;
+                buf[4] = (nm>> 8) & 0xff;
+            } else {
+                buf_size = 7;
+                buf[2] = 'I';
+                buf[3] = (nm>> 0) & 0xff;
+                buf[4] = (nm>> 8) & 0xff;
+                buf[5] = (nm>>16) & 0xff;
+                buf[6] = (nm>>24) & 0xff;
+            }
+            BLOCK_APPEND(s->aux_blk, buf, buf_size);
+            cr->aux_size += buf_size;
+        } else {
+            // Preallocated space for NM at -has_NM into aux block
+            unsigned char *buf = BLOCK_DATA(s->aux_blk) + -has_NM;
+            buf[0] = (nm>> 0) & 0xff;
+            buf[1] = (nm>> 8) & 0xff;
+            buf[2] = (nm>>16) & 0xff;
+            buf[3] = (nm>>24) & 0xff;
+        }
+    }
+
+    return r;
+
+ beyond_slice:
+    // Cramtools can create CRAMs that have sequence features outside the
+    // stated range of the container & slice reference extents (start + span).
+    // We have to check for these in many places, but for brevity have the
+    // error reporting in only one.
+    hts_log_error("CRAM CIGAR extends beyond slice reference extents");
+    return -1;
+
+ block_err:
+    return -1;
+}
+
+/*
+ * Quick and simple hash lookup for cram_map arrays
+ */
+static cram_map *map_find(cram_map **map, unsigned char *key, int id) {
+    cram_map *m;
+
+    m = map[CRAM_MAP(key[0],key[1])];
+    while (m && m->key != id)
+        m= m->next;
+
+    return m;
+}
+
+//#define map_find(M,K,I) M[CRAM_MAP(K[0],K[1])];while (m && m->key != I);m= m->next
+
+
+static int cram_decode_aux_1_0(cram_container *c, cram_slice *s,
+                               cram_block *blk, cram_record *cr) {
+    int i, r = 0, out_sz = 1;
+    unsigned char ntags;
+
+    if (!c->comp_hdr->codecs[DS_TC]) return -1;
+    r |= c->comp_hdr->codecs[DS_TC]->decode(s, c->comp_hdr->codecs[DS_TC], blk,
+                                            (char *)&ntags, &out_sz);
+    cr->ntags = ntags;
+
+    //printf("TC=%d\n", cr->ntags);
+    cr->aux_size = 0;
+    cr->aux = BLOCK_SIZE(s->aux_blk);
+
+    for (i = 0; i < cr->ntags; i++) {
+        int32_t id, out_sz = 1;
+        unsigned char tag_data[3];
+        cram_map *m;
+
+        //printf("Tag %d/%d\n", i+1, cr->ntags);
+        if (!c->comp_hdr->codecs[DS_TN]) return -1;
+        r |= c->comp_hdr->codecs[DS_TN]->decode(s, c->comp_hdr->codecs[DS_TN],
+                                                blk, (char *)&id, &out_sz);
+        if (out_sz == 3) {
+            // Tag name stored as 3 chars instead of an int?
+            memcpy(tag_data, &id, 3);
+        } else {
+            tag_data[0] = (id>>16) & 0xff;
+            tag_data[1] = (id>>8)  & 0xff;
+            tag_data[2] = id       & 0xff;
+        }
+
+        m = map_find(c->comp_hdr->tag_encoding_map, tag_data, id);
+        if (!m)
+            return -1;
+        BLOCK_APPEND(s->aux_blk, (char *)tag_data, 3);
+
+        if (!m->codec) return -1;
+        r |= m->codec->decode(s, m->codec, blk, (char *)s->aux_blk, &out_sz);
+
+        cr->aux_size += out_sz + 3;
+    }
+
+    return r;
+
+ block_err:
+    return -1;
+}
+
+// has_MD and has_NM are filled out with 0 for none present,
+// 1 for present and verbatim, and -pos for present as placeholder
+// (MD*, NM*) to be generated and filled out at offset +pos.
+static int cram_decode_aux(cram_fd *fd,
+                           cram_container *c, cram_slice *s,
+                           cram_block *blk, cram_record *cr,
+                           int *has_MD, int *has_NM) {
+    int i, r = 0, out_sz = 1;
+    int32_t TL = 0;
+    unsigned char *TN;
+    uint32_t ds = s->data_series;
+
+    if (!(ds & (CRAM_TL|CRAM_aux))) {
+        cr->aux = 0;
+        cr->aux_size = 0;
+        return 0;
+    }
+
+    if (!c->comp_hdr->codecs[DS_TL]) return -1;
+    r |= c->comp_hdr->codecs[DS_TL]->decode(s, c->comp_hdr->codecs[DS_TL], blk,
+                                            (char *)&TL, &out_sz);
+    if (r || TL < 0 || TL >= c->comp_hdr->nTL)
+        return -1;
+
+    TN = c->comp_hdr->TL[TL];
+    cr->ntags = strlen((char *)TN)/3; // optimise to remove strlen
+
+    //printf("TC=%d\n", cr->ntags);
+    cr->aux_size = 0;
+    cr->aux = BLOCK_SIZE(s->aux_blk);
+
+    if (!(ds & CRAM_aux))
+        return 0;
+
+    for (i = 0; i < cr->ntags; i++) {
+        int32_t id, out_sz = 1;
+        unsigned char tag_data[7];
+        cram_map *m;
+
+        if (TN[0] == 'M' && TN[1] == 'D' && has_MD)
+            *has_MD = (BLOCK_SIZE(s->aux_blk)+3) * (TN[2] == '*' ? -1 : 1);
+        if (TN[0] == 'N' && TN[1] == 'M' && has_NM)
+            *has_NM = (BLOCK_SIZE(s->aux_blk)+3) * (TN[2] == '*' ? -1 : 1);;
+
+        //printf("Tag %d/%d\n", i+1, cr->ntags);
+        tag_data[0] = TN[0];
+        tag_data[1] = TN[1];
+        tag_data[2] = TN[2];
+        id = (tag_data[0]<<16) | (tag_data[1]<<8) | tag_data[2];
+
+        if (CRAM_MAJOR_VERS(fd->version) >= 4 && TN[2] == '*') {
+            // Place holder, fill out contents later.
+            int tag_data_size;
+            if (TN[0] == 'N' && TN[1] == 'M') {
+                // Use a fixed size, so we can allocate room for it now.
+                memcpy(&tag_data[2], "I\0\0\0\0", 5);
+                tag_data_size = 7;
+            } else if (TN[0] == 'R' && TN[1] == 'G') {
+                // RG is variable size, but known already.  Insert now
+                TN += 3;
+                // Equiv to fd->header->hrecs->rg[cr->rg], but this is the
+                // new header API equivalent.
+                const char *rg = sam_hdr_line_name(fd->header, "RG", cr->rg);
+                if (!rg)
+                    continue;
+
+                size_t rg_len = strlen(rg);
+                tag_data[2] = 'Z';
+                BLOCK_APPEND(s->aux_blk, (char *)tag_data, 3);
+                BLOCK_APPEND(s->aux_blk, rg, rg_len);
+                BLOCK_APPEND_CHAR(s->aux_blk, '\0');
+                cr->aux_size += 3 + rg_len + 1;
+                cr->rg = -1; // prevents auto-add later
+                continue;
+            } else {
+                // Unknown size.  We'll insert MD into stream later.
+                tag_data[2] = 'Z';
+                tag_data_size = 3;
+            }
+            BLOCK_APPEND(s->aux_blk, (char *)tag_data, tag_data_size);
+            cr->aux_size += tag_data_size;
+            TN += 3;
+        } else {
+            TN += 3;
+            m = map_find(c->comp_hdr->tag_encoding_map, tag_data, id);
+            if (!m)
+                return -1;
+
+            BLOCK_APPEND(s->aux_blk, (char *)tag_data, 3);
+
+            if (!m->codec) return -1;
+            r |= m->codec->decode(s, m->codec, blk, (char *)s->aux_blk, &out_sz);
+            if (r) break;
+            cr->aux_size += out_sz + 3;
+
+            // cF CRAM flags.
+            if (TN[-3]=='c' && TN[-2]=='F' && TN[-1]=='C' && out_sz == 1) {
+                // Remove cF tag
+                uint8_t cF = BLOCK_END(s->aux_blk)[-1];
+                BLOCK_SIZE(s->aux_blk) -= out_sz+3;
+                cr->aux_size -= out_sz+3;
+
+                // bit 1 => don't auto-decode MD.
+                // Pretend MD is present verbatim, so we don't auto-generate
+                if ((cF & 1) && has_MD && *has_MD == 0)
+                    *has_MD = 1;
+
+                // bit 1 => don't auto-decode NM
+                if ((cF & 2) && has_NM && *has_NM == 0)
+                    *has_NM = 1;
+            }
+        }
+
+        // We could go to 2^32 fine, but we shouldn't be hitting this anyway,
+        // and it's protecting against memory hogs too.
+        if (BLOCK_SIZE(s->aux_blk) > (1u<<31)) {
+            hts_log_error("CRAM->BAM aux block size overflow");
+            goto block_err;
+        }
+    }
+
+    return r;
+
+ block_err:
+    return -1;
+}
+
+/* Resolve mate pair cross-references between recs within this slice */
+static int cram_decode_slice_xref(cram_slice *s, int required_fields) {
+    int rec;
+
+    if (!(required_fields & (SAM_RNEXT | SAM_PNEXT | SAM_TLEN))) {
+        for (rec = 0; rec < s->hdr->num_records; rec++) {
+            cram_record *cr = &s->crecs[rec];
+
+            cr->tlen = 0;
+            cr->mate_pos = 0;
+            cr->mate_ref_id = -1;
+        }
+
+        return 0;
+    }
+
+    for (rec = 0; rec < s->hdr->num_records; rec++) {
+        cram_record *cr = &s->crecs[rec];
+
+        if (cr->mate_line >= 0) {
+            if (cr->mate_line < s->hdr->num_records) {
+                /*
+                 * On the first read, loop through computing lengths.
+                 * It's not perfect as we have one slice per reference so we
+                 * cannot detect when TLEN should be zero due to seqs that
+                 * map to multiple references.
+                 *
+                 * We also cannot set tlen correct when it spans a slice for
+                 * other reasons. This may make tlen too small. Should we
+                 * fix this by forcing TLEN to be stored verbatim in such cases?
+                 *
+                 * Or do we just admit defeat and output 0 for tlen? It's the
+                 * safe option...
+                 */
+                if (cr->tlen == INT64_MIN) {
+                    int id1 = rec, id2 = rec;
+                    int64_t aleft = cr->apos, aright = cr->aend;
+                    int64_t tlen;
+                    int ref = cr->ref_id;
+
+                    // number of segments starting at the same point.
+                    int left_cnt = 0;
+
+                    do {
+                        if (aleft > s->crecs[id2].apos)
+                            aleft = s->crecs[id2].apos, left_cnt = 1;
+                        else if (aleft == s->crecs[id2].apos)
+                            left_cnt++;
+                        if (aright < s->crecs[id2].aend)
+                            aright = s->crecs[id2].aend;
+                        if (s->crecs[id2].mate_line == -1) {
+                            s->crecs[id2].mate_line = rec;
+                            break;
+                        }
+                        if (s->crecs[id2].mate_line <= id2 ||
+                            s->crecs[id2].mate_line >= s->hdr->num_records)
+                            return -1;
+                        id2 = s->crecs[id2].mate_line;
+
+                        if (s->crecs[id2].ref_id != ref)
+                            ref = -1;
+                    } while (id2 != id1);
+
+                    if (ref != -1) {
+                        tlen = aright - aleft + 1;
+                        id1 = id2 = rec;
+
+                        /*
+                         * When we have two seqs with identical start and
+                         * end coordinates, set +/- tlen based on 1st/last
+                         * bit flags instead, as a tie breaker.
+                         */
+                        if (s->crecs[id2].apos == aleft) {
+                            if (left_cnt == 1 ||
+                                (s->crecs[id2].flags & BAM_FREAD1))
+                                s->crecs[id2].tlen = tlen;
+                            else
+                                s->crecs[id2].tlen = -tlen;
+                        } else {
+                            s->crecs[id2].tlen = -tlen;
+                        }
+
+                        id2 = s->crecs[id2].mate_line;
+                        while (id2 != id1) {
+                            if (s->crecs[id2].apos == aleft) {
+                                if (left_cnt == 1 ||
+                                    (s->crecs[id2].flags & BAM_FREAD1))
+                                    s->crecs[id2].tlen = tlen;
+                                else
+                                    s->crecs[id2].tlen = -tlen;
+                            } else {
+                                s->crecs[id2].tlen = -tlen;
+                            }
+                            id2 = s->crecs[id2].mate_line;
+                        }
+                    } else {
+                        id1 = id2 = rec;
+
+                        s->crecs[id2].tlen = 0;
+                        id2 = s->crecs[id2].mate_line;
+                        while (id2 != id1) {
+                            s->crecs[id2].tlen = 0;
+                            id2 = s->crecs[id2].mate_line;
+                        }
+                    }
+                }
+
+                cr->mate_pos = s->crecs[cr->mate_line].apos;
+                cr->mate_ref_id = s->crecs[cr->mate_line].ref_id;
+
+                // paired
+                cr->flags |= BAM_FPAIRED;
+
+                // set mate unmapped if needed
+                if (s->crecs[cr->mate_line].flags & BAM_FUNMAP) {
+                    cr->flags |= BAM_FMUNMAP;
+                    cr->tlen = 0;
+                }
+                if (cr->flags & BAM_FUNMAP) {
+                    cr->tlen = 0;
+                }
+
+                // set mate reversed if needed
+                if (s->crecs[cr->mate_line].flags & BAM_FREVERSE)
+                    cr->flags |= BAM_FMREVERSE;
+            } else {
+                hts_log_error("Mate line out of bounds: %d vs [0, %d]",
+                              cr->mate_line, s->hdr->num_records-1);
+            }
+
+            /* FIXME: construct read names here too if needed */
+        } else {
+            if (cr->mate_flags & CRAM_M_REVERSE) {
+                cr->flags |= BAM_FPAIRED | BAM_FMREVERSE;
+            }
+            if (cr->mate_flags & CRAM_M_UNMAP) {
+                cr->flags |= BAM_FMUNMAP;
+                //cr->mate_ref_id = -1;
+            }
+            if (!(cr->flags & BAM_FPAIRED))
+                cr->mate_ref_id = -1;
+        }
+
+        if (cr->tlen == INT64_MIN)
+            cr->tlen = 0; // Just incase
+    }
+
+    for (rec = 0; rec < s->hdr->num_records; rec++) {
+        cram_record *cr = &s->crecs[rec];
+        if (cr->explicit_tlen != INT64_MIN)
+            cr->tlen = cr->explicit_tlen;
+    }
+
+    return 0;
+}
+
+static char *md5_print(unsigned char *md5, char *out) {
+    int i;
+    for (i = 0; i < 16; i++) {
+        out[i*2+0] = "0123456789abcdef"[md5[i]>>4];
+        out[i*2+1] = "0123456789abcdef"[md5[i]&15];
+    }
+    out[32] = 0;
+
+    return out;
+}
+
+/*
+ * Utility function to decode tlen (ISIZE), as it's called
+ * in multiple places.
+ *
+ * Returns codec return value (0 on success).
+ */
+static int cram_decode_tlen(cram_fd *fd, cram_container *c, cram_slice *s,
+                            cram_block *blk, int64_t *tlen) {
+    int out_sz = 1, r = 0;
+
+    if (!c->comp_hdr->codecs[DS_TS]) return -1;
+    if (CRAM_MAJOR_VERS(fd->version) < 4) {
+        int32_t i32;
+        r |= c->comp_hdr->codecs[DS_TS]
+            ->decode(s, c->comp_hdr->codecs[DS_TS], blk,
+                     (char *)&i32, &out_sz);
+        *tlen = i32;
+    } else {
+        r |= c->comp_hdr->codecs[DS_TS]
+            ->decode(s, c->comp_hdr->codecs[DS_TS], blk,
+                     (char *)tlen, &out_sz);
+    }
+    return r;
+}
+
+/*
+ * Decode an entire slice from container blocks. Fills out s->crecs[] array.
+ * Returns 0 on success
+ *        -1 on failure
+ */
+int cram_decode_slice(cram_fd *fd, cram_container *c, cram_slice *s,
+                      sam_hdr_t *sh) {
+    cram_block *blk = s->block[0];
+    int32_t bf, ref_id;
+    unsigned char cf;
+    int out_sz, r = 0;
+    int rec;
+    char *seq = NULL, *qual = NULL;
+    int unknown_rg = -1;
+    int embed_ref;
+    char **refs = NULL;
+    uint32_t ds;
+    sam_hrecs_t *bfd = sh->hrecs;
+
+    if (cram_dependent_data_series(fd, c->comp_hdr, s) != 0)
+        return -1;
+
+    ds = s->data_series;
+
+    blk->bit = 7; // MSB first
+
+    // Study the blocks and estimate approx sizes to preallocate.
+    // This looks to speed up decoding by around 8-9%.
+    // We can always shrink back down at the end if we overestimated.
+    // However it's likely that this also saves memory as own growth
+    // factor (*=1.5) is never applied.
+    {
+        int qsize, nsize, q_id;
+        cram_decode_estimate_sizes(c->comp_hdr, s, &qsize, &nsize, &q_id);
+        //fprintf(stderr, "qsize=%d nsize=%d\n", qsize, nsize);
+
+        if (qsize && (ds & CRAM_RL)) BLOCK_RESIZE_EXACT(s->seqs_blk, qsize+1);
+        if (qsize && (ds & CRAM_RL)) BLOCK_RESIZE_EXACT(s->qual_blk, qsize+1);
+        if (nsize && (ds & CRAM_NS)) BLOCK_RESIZE_EXACT(s->name_blk, nsize+1);
+
+        // To do - consider using q_id here to usurp the quality block and
+        // avoid a memcpy during decode.
+        // Specifically when quality is an external block uniquely used by
+        // DS_QS only, then we can set s->qual_blk directly to this
+        // block and save the codec->decode() calls. (Approx 3% cpu saving)
+    }
+
+    /* Look for unknown RG, added as last by Java CRAM? */
+    if (bfd->nrg > 0 &&
+        bfd->rg[bfd->nrg-1].name != NULL &&
+        !strcmp(bfd->rg[bfd->nrg-1].name, "UNKNOWN"))
+        unknown_rg = bfd->nrg-1;
+
+    if (blk->content_type != CORE)
+        return -1;
+
+    if (s->crecs)
+        free(s->crecs);
+    if (!(s->crecs = malloc(s->hdr->num_records * sizeof(*s->crecs))))
+        return -1;
+
+    ref_id = s->hdr->ref_seq_id;
+    if (CRAM_MAJOR_VERS(fd->version) < 4)
+       embed_ref = s->hdr->ref_base_id >= 0 ? 1 : 0;
+    else
+       embed_ref = s->hdr->ref_base_id > 0 ? 1 : 0;
+
+    if (ref_id >= 0) {
+        if (embed_ref) {
+            cram_block *b;
+            if (s->hdr->ref_base_id < 0) {
+                hts_log_error("No reference specified and no embedded reference is available"
+                              " at #%d:%"PRId64"-%"PRId64, ref_id, s->hdr->ref_seq_start,
+                              s->hdr->ref_seq_start + s->hdr->ref_seq_span-1);
+                return -1;
+            }
+            b = cram_get_block_by_id(s, s->hdr->ref_base_id);
+            if (!b)
+                return -1;
+            if (cram_uncompress_block(b) != 0)
+                return -1;
+            s->ref = (char *)BLOCK_DATA(b);
+            s->ref_start = s->hdr->ref_seq_start;
+            s->ref_end   = s->hdr->ref_seq_start + s->hdr->ref_seq_span-1;
+            if (s->hdr->ref_seq_span > b->uncomp_size) {
+                hts_log_error("Embedded reference is too small at #%d:%"PRIhts_pos"-%"PRIhts_pos,
+                              ref_id, s->ref_start, s->ref_end);
+                return -1;
+            }
+        } else if (!c->comp_hdr->no_ref) {
+            //// Avoid Java cramtools bug by loading entire reference seq
+            //s->ref = cram_get_ref(fd, s->hdr->ref_seq_id, 1, 0);
+            //s->ref_start = 1;
+
+            if (fd->required_fields & SAM_SEQ) {
+                s->ref =
+                cram_get_ref(fd, s->hdr->ref_seq_id,
+                             s->hdr->ref_seq_start,
+                             s->hdr->ref_seq_start + s->hdr->ref_seq_span -1);
+            }
+            s->ref_start = s->hdr->ref_seq_start;
+            s->ref_end   = s->hdr->ref_seq_start + s->hdr->ref_seq_span-1;
+
+            /* Sanity check */
+            if (s->ref_start < 0) {
+                hts_log_warning("Slice starts before base 1"
+                                " at #%d:%"PRId64"-%"PRId64, ref_id, s->hdr->ref_seq_start,
+                                s->hdr->ref_seq_start + s->hdr->ref_seq_span-1);
+                s->ref_start = 0;
+            }
+            pthread_mutex_lock(&fd->ref_lock);
+            pthread_mutex_lock(&fd->refs->lock);
+            if ((fd->required_fields & SAM_SEQ) &&
+                ref_id < fd->refs->nref && fd->refs->ref_id &&
+                s->ref_end > fd->refs->ref_id[ref_id]->length) {
+                s->ref_end = fd->refs->ref_id[ref_id]->length;
+            }
+            pthread_mutex_unlock(&fd->refs->lock);
+            pthread_mutex_unlock(&fd->ref_lock);
+        }
+    }
+
+    if ((fd->required_fields & SAM_SEQ) &&
+        s->ref == NULL && s->hdr->ref_seq_id >= 0 && !c->comp_hdr->no_ref) {
+        hts_log_error("Unable to fetch reference #%d:%"PRId64"-%"PRId64"\n",
+                      ref_id, s->hdr->ref_seq_start,
+                      s->hdr->ref_seq_start + s->hdr->ref_seq_span-1);
+        return -1;
+    }
+
+    if (CRAM_MAJOR_VERS(fd->version) != 1
+        && (fd->required_fields & SAM_SEQ)
+        && s->hdr->ref_seq_id >= 0
+        && !fd->ignore_md5
+        && memcmp(s->hdr->md5, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 16)) {
+        hts_md5_context *md5;
+        unsigned char digest[16];
+
+        if (s->ref && s->hdr->ref_seq_id >= 0) {
+            int start, len;
+
+            if (s->hdr->ref_seq_start >= s->ref_start) {
+                start = s->hdr->ref_seq_start - s->ref_start;
+            } else {
+                hts_log_warning("Slice starts before base 1 at #%d:%"PRIhts_pos"-%"PRIhts_pos,
+                                ref_id, s->ref_start, s->ref_end);
+                start = 0;
+            }
+
+            if (s->hdr->ref_seq_span <= s->ref_end - s->ref_start + 1) {
+                len = s->hdr->ref_seq_span;
+            } else {
+                hts_log_warning("Slice ends beyond reference end at #%d:%"PRIhts_pos"-%"PRIhts_pos,
+                                ref_id, s->ref_start, s->ref_end);
+                len = s->ref_end - s->ref_start + 1;
+            }
+
+            if (!(md5 = hts_md5_init()))
+                return -1;
+            if (start + len > s->ref_end - s->ref_start + 1)
+                len = s->ref_end - s->ref_start + 1 - start;
+            if (len >= 0)
+                hts_md5_update(md5, s->ref + start, len);
+            hts_md5_final(digest, md5);
+            hts_md5_destroy(md5);
+        } else if (!s->ref && s->hdr->ref_base_id >= 0) {
+            cram_block *b = cram_get_block_by_id(s, s->hdr->ref_base_id);
+            if (b) {
+                if (!(md5 = hts_md5_init()))
+                    return -1;
+                hts_md5_update(md5, b->data, b->uncomp_size);
+                hts_md5_final(digest, md5);
+                hts_md5_destroy(md5);
+            }
+        }
+
+        if (!c->comp_hdr->no_ref &&
+            ((!s->ref && s->hdr->ref_base_id < 0)
+             || memcmp(digest, s->hdr->md5, 16) != 0)) {
+            char M[33];
+            const char *rname = sam_hdr_tid2name(sh, ref_id);
+            if (!rname) rname="?"; // cannot happen normally
+            hts_log_error("MD5 checksum reference mismatch at %s:%"PRIhts_pos"-%"PRIhts_pos,
+                          rname, s->ref_start, s->ref_end);
+            hts_log_error("CRAM  : %s", md5_print(s->hdr->md5, M));
+            hts_log_error("Ref   : %s", md5_print(digest, M));
+            kstring_t ks = KS_INITIALIZE;
+            if (sam_hdr_find_tag_id(sh, "SQ", "SN", rname, "M5", &ks) == 0)
+                hts_log_error("@SQ M5: %s", ks.s);
+            hts_log_error("Please check the reference given is correct");
+            ks_free(&ks);
+            return -1;
+        }
+    }
+
+    if (ref_id == -2) {
+        pthread_mutex_lock(&fd->ref_lock);
+        pthread_mutex_lock(&fd->refs->lock);
+        refs = calloc(fd->refs->nref, sizeof(char *));
+        pthread_mutex_unlock(&fd->refs->lock);
+        pthread_mutex_unlock(&fd->ref_lock);
+        if (!refs)
+            return -1;
+    }
+
+    int last_ref_id = -9; // Arbitrary -ve marker for not-yet-set
+    for (rec = 0; rec < s->hdr->num_records; rec++) {
+        cram_record *cr = &s->crecs[rec];
+        int has_MD, has_NM;
+
+        //fprintf(stderr, "Decode seq %d, %d/%d\n", rec, blk->byte, blk->bit);
+
+        cr->s = s;
+
+        out_sz = 1; /* decode 1 item */
+        if (ds & CRAM_BF) {
+            if (!c->comp_hdr->codecs[DS_BF]) goto block_err;
+            r |= c->comp_hdr->codecs[DS_BF]
+                            ->decode(s, c->comp_hdr->codecs[DS_BF], blk,
+                                     (char *)&bf, &out_sz);
+            if (r || bf < 0 ||
+                bf >= sizeof(fd->bam_flag_swap)/sizeof(*fd->bam_flag_swap))
+                goto block_err;
+            bf = fd->bam_flag_swap[bf];
+            cr->flags = bf;
+        } else {
+            cr->flags = bf = 0x4; // unmapped
+        }
+
+        if (ds & CRAM_CF) {
+            if (CRAM_MAJOR_VERS(fd->version) == 1) {
+                /* CF is byte in 1.0, int32 in 2.0 */
+                if (!c->comp_hdr->codecs[DS_CF]) goto block_err;
+                r |= c->comp_hdr->codecs[DS_CF]
+                                ->decode(s, c->comp_hdr->codecs[DS_CF], blk,
+                                         (char *)&cf, &out_sz);
+                if (r) goto block_err;
+                cr->cram_flags = cf;
+            } else {
+                if (!c->comp_hdr->codecs[DS_CF]) goto block_err;
+                r |= c->comp_hdr->codecs[DS_CF]
+                                ->decode(s, c->comp_hdr->codecs[DS_CF], blk,
+                                         (char *)&cr->cram_flags, &out_sz);
+                if (r) goto block_err;
+                cf = cr->cram_flags;
+            }
+        } else {
+            cf = cr->cram_flags = 0;
+        }
+
+        if (CRAM_MAJOR_VERS(fd->version) != 1 && ref_id == -2) {
+            if (ds & CRAM_RI) {
+                if (!c->comp_hdr->codecs[DS_RI]) goto block_err;
+                r |= c->comp_hdr->codecs[DS_RI]
+                                ->decode(s, c->comp_hdr->codecs[DS_RI], blk,
+                                         (char *)&cr->ref_id, &out_sz);
+                if (r) goto block_err;
+                if ((fd->required_fields & (SAM_SEQ|SAM_TLEN))
+                    && cr->ref_id >= 0
+                    && cr->ref_id != last_ref_id) {
+                    if (!c->comp_hdr->no_ref) {
+                        // Range(fd):  seq >= 0, unmapped -1, unspecified   -2
+                        // Slice(s):   seq >= 0, unmapped -1, multiple refs -2
+                        // Record(cr): seq >= 0, unmapped -1
+                        pthread_mutex_lock(&fd->range_lock);
+                        int need_ref = (fd->range.refid == -2 || cr->ref_id == fd->range.refid);
+                        pthread_mutex_unlock(&fd->range_lock);
+                        if  (need_ref) {
+                            if (!refs[cr->ref_id])
+                                refs[cr->ref_id] = cram_get_ref(fd, cr->ref_id, 1, 0);
+                            if (!(s->ref = refs[cr->ref_id]))
+                                goto block_err;
+                        } else {
+                            // For multi-ref containers, we don't need to fetch all
+                            // refs if we're only querying one.
+                            s->ref = NULL;
+                        }
+
+                        pthread_mutex_lock(&fd->range_lock);
+                        int discard_last_ref = (last_ref_id >= 0 &&
+                                                refs[last_ref_id] &&
+                                                (fd->range.refid == -2 ||
+                                                 last_ref_id == fd->range.refid));
+                        pthread_mutex_unlock(&fd->range_lock);
+                        if (discard_last_ref) {
+                            pthread_mutex_lock(&fd->ref_lock);
+                            discard_last_ref = !fd->unsorted;
+                            pthread_mutex_unlock(&fd->ref_lock);
+                        }
+                        if (discard_last_ref) {
+                            cram_ref_decr(fd->refs, last_ref_id);
+                            refs[last_ref_id] = NULL;
+                        }
+                    }
+                    s->ref_start = 1;
+                    pthread_mutex_lock(&fd->ref_lock);
+                    pthread_mutex_lock(&fd->refs->lock);
+                    s->ref_end = fd->refs->ref_id[cr->ref_id]->length;
+                    pthread_mutex_unlock(&fd->refs->lock);
+                    pthread_mutex_unlock(&fd->ref_lock);
+
+                    last_ref_id = cr->ref_id;
+                }
+            } else {
+                cr->ref_id = -1;
+            }
+        } else {
+            cr->ref_id = ref_id; // Forced constant in CRAM 1.0
+        }
+        if (cr->ref_id < -1 || cr->ref_id >= bfd->nref) {
+            hts_log_error("Requested unknown reference ID %d", cr->ref_id);
+            goto block_err;
+        }
+
+        if (ds & CRAM_RL) {
+            if (!c->comp_hdr->codecs[DS_RL]) goto block_err;
+            r |= c->comp_hdr->codecs[DS_RL]
+                            ->decode(s, c->comp_hdr->codecs[DS_RL], blk,
+                                     (char *)&cr->len, &out_sz);
+            if (r) goto block_err;
+            if (cr->len < 0) {
+                hts_log_error("Read has negative length");
+                goto block_err;
+            }
+        }
+
+        if (ds & CRAM_AP) {
+            if (!c->comp_hdr->codecs[DS_AP]) goto block_err;
+            if (CRAM_MAJOR_VERS(fd->version) >= 4) {
+                r |= c->comp_hdr->codecs[DS_AP]
+                                ->decode(s, c->comp_hdr->codecs[DS_AP], blk,
+                                         (char *)&cr->apos, &out_sz);
+            } else  {
+                int32_t i32;
+                r |= c->comp_hdr->codecs[DS_AP]
+                                ->decode(s, c->comp_hdr->codecs[DS_AP], blk,
+                                         (char *)&i32, &out_sz);
+                cr->apos = i32;
+            }
+            if (r) goto block_err;;
+            if (c->comp_hdr->AP_delta) {
+                if (cr->apos < 0 && c->unsorted == 0) {
+                    // cache locally in c->unsorted so we don't have an
+                    // excessive number of locks
+                    pthread_mutex_lock(&fd->ref_lock);
+                    c->unsorted = fd->unsorted = 1;
+                    pthread_mutex_unlock(&fd->ref_lock);
+                }
+                cr->apos += s->last_apos;
+            }
+            s->last_apos=  cr->apos;
+        } else {
+            cr->apos = c->ref_seq_start;
+        }
+
+        if (ds & CRAM_RG) {
+            if (!c->comp_hdr->codecs[DS_RG]) goto block_err;
+            r |= c->comp_hdr->codecs[DS_RG]
+                           ->decode(s, c->comp_hdr->codecs[DS_RG], blk,
+                                    (char *)&cr->rg, &out_sz);
+            if (r) goto block_err;
+            if (cr->rg == unknown_rg)
+                cr->rg = -1;
+        } else {
+            cr->rg = -1;
+        }
+
+        cr->name_len = 0;
+
+        if (c->comp_hdr->read_names_included) {
+            int32_t out_sz2 = 1;
+
+            // Read directly into name cram_block
+            cr->name = BLOCK_SIZE(s->name_blk);
+            if (ds & CRAM_RN) {
+                if (!c->comp_hdr->codecs[DS_RN]) goto block_err;
+                r |= c->comp_hdr->codecs[DS_RN]
+                                ->decode(s, c->comp_hdr->codecs[DS_RN], blk,
+                                         (char *)s->name_blk, &out_sz2);
+                if (r) goto block_err;
+                cr->name_len = out_sz2;
+            }
+        }
+
+        cr->mate_pos = 0;
+        cr->mate_line = -1;
+        cr->mate_ref_id = -1;
+        cr->explicit_tlen = INT64_MIN;
+        if ((ds & CRAM_CF) && (cf & CRAM_FLAG_DETACHED)) {
+            if (ds & CRAM_MF) {
+                if (CRAM_MAJOR_VERS(fd->version) == 1) {
+                    /* MF is byte in 1.0, int32 in 2.0 */
+                    unsigned char mf;
+                    if (!c->comp_hdr->codecs[DS_MF]) goto block_err;
+                    r |= c->comp_hdr->codecs[DS_MF]
+                                    ->decode(s, c->comp_hdr->codecs[DS_MF],
+                                             blk, (char *)&mf, &out_sz);
+                    if (r) goto block_err;
+                    cr->mate_flags = mf;
+                } else {
+                    if (!c->comp_hdr->codecs[DS_MF]) goto block_err;
+                    r |= c->comp_hdr->codecs[DS_MF]
+                                    ->decode(s, c->comp_hdr->codecs[DS_MF],
+                                             blk,
+                                             (char *)&cr->mate_flags,
+                                             &out_sz);
+                    if (r) goto block_err;
+                }
+            } else {
+                cr->mate_flags = 0;
+            }
+
+            if (!c->comp_hdr->read_names_included) {
+                int32_t out_sz2 = 1;
+
+                // Read directly into name cram_block
+                cr->name = BLOCK_SIZE(s->name_blk);
+                if (ds & CRAM_RN) {
+                    if (!c->comp_hdr->codecs[DS_RN]) goto block_err;
+                    r |= c->comp_hdr->codecs[DS_RN]
+                                    ->decode(s, c->comp_hdr->codecs[DS_RN],
+                                             blk, (char *)s->name_blk,
+                                             &out_sz2);
+                    if (r) goto block_err;
+                    cr->name_len = out_sz2;
+                }
+            }
+
+            if (ds & CRAM_NS) {
+                if (!c->comp_hdr->codecs[DS_NS]) goto block_err;
+                r |= c->comp_hdr->codecs[DS_NS]
+                                ->decode(s, c->comp_hdr->codecs[DS_NS], blk,
+                                         (char *)&cr->mate_ref_id, &out_sz);
+                if (r) goto block_err;
+            }
+
+            // Skip as mate_ref of "*" is legit. It doesn't mean unmapped, just unknown.
+            // if (cr->mate_ref_id == -1 && cr->flags & 0x01) {
+            //     /* Paired, but unmapped */
+            //     cr->flags |= BAM_FMUNMAP;
+            // }
+
+            if (ds & CRAM_NP) {
+                if (!c->comp_hdr->codecs[DS_NP]) goto block_err;;
+                if (CRAM_MAJOR_VERS(fd->version) < 4) {
+                    int32_t i32;
+                    r |= c->comp_hdr->codecs[DS_NP]
+                                    ->decode(s, c->comp_hdr->codecs[DS_NP], blk,
+                                             (char *)&i32, &out_sz);
+                    cr->mate_pos = i32;
+                } else {
+                    r |= c->comp_hdr->codecs[DS_NP]
+                        ->decode(s, c->comp_hdr->codecs[DS_NP], blk,
+                                 (char *)&cr->mate_pos, &out_sz);
+                }
+                if (r) goto block_err;
+            }
+
+            if (ds & CRAM_TS) {
+                if (!c->comp_hdr->codecs[DS_TS]) goto block_err;
+                r = cram_decode_tlen(fd, c, s, blk, &cr->tlen);
+                if (r) goto block_err;
+            } else {
+                cr->tlen = INT64_MIN;
+            }
+        } else if ((ds & CRAM_CF) && (cf & CRAM_FLAG_MATE_DOWNSTREAM)) {
+            // else not detached
+            if (ds & CRAM_NF) {
+                if (!c->comp_hdr->codecs[DS_NF]) goto block_err;
+                r |= c->comp_hdr->codecs[DS_NF]
+                                ->decode(s, c->comp_hdr->codecs[DS_NF], blk,
+                                         (char *)&cr->mate_line, &out_sz);
+                if (r) goto block_err;
+                cr->mate_line += rec + 1;
+
+                //cr->name_len = sprintf(name, "%d", name_id++);
+                //cr->name = DSTRING_LEN(name_ds);
+                //dstring_nappend(name_ds, name, cr->name_len);
+
+                cr->mate_ref_id = -1;
+                cr->tlen = INT64_MIN;
+                cr->mate_pos = 0;
+            } else  {
+                cr->mate_flags = 0;
+                cr->tlen = INT64_MIN;
+            }
+            if ((ds & CRAM_CF) && (cf & CRAM_FLAG_EXPLICIT_TLEN)) {
+                if (ds & CRAM_TS) {
+                    r = cram_decode_tlen(fd, c, s, blk, &cr->explicit_tlen);
+                    if (r) return r;
+                } else {
+                    cr->mate_flags = 0;
+                    cr->tlen = INT64_MIN;
+                }
+            }
+        } else if ((ds & CRAM_CF) && (cf & CRAM_FLAG_EXPLICIT_TLEN)) {
+            if (ds & CRAM_TS) {
+                r = cram_decode_tlen(fd, c, s, blk, &cr->explicit_tlen);
+                if (r) return r;
+            } else {
+                cr->mate_flags = 0;
+                cr->tlen = INT64_MIN;
+            }
+        } else {
+            cr->mate_flags = 0;
+            cr->tlen = INT64_MIN;
+        }
+        /*
+        else if (!name[0]) {
+            //name[0] = '?'; name[1] = 0;
+            //cr->name_len = 1;
+            //cr->name=  DSTRING_LEN(s->name_ds);
+            //dstring_nappend(s->name_ds, "?", 1);
+
+            cr->mate_ref_id = -1;
+            cr->tlen = 0;
+            cr->mate_pos = 0;
+        }
+        */
+
+        /* Auxiliary tags */
+        has_MD = has_NM = 0;
+        if (CRAM_MAJOR_VERS(fd->version) == 1)
+            r |= cram_decode_aux_1_0(c, s, blk, cr);
+        else
+            r |= cram_decode_aux(fd, c, s, blk, cr, &has_MD, &has_NM);
+        if (r) goto block_err;
+
+        /* Fake up dynamic string growth and appending */
+        if (ds & CRAM_RL) {
+            cr->seq = BLOCK_SIZE(s->seqs_blk);
+            BLOCK_GROW(s->seqs_blk, cr->len);
+            seq = (char *)BLOCK_END(s->seqs_blk);
+            BLOCK_SIZE(s->seqs_blk) += cr->len;
+
+            if (!seq)
+                goto block_err;
+
+            cr->qual = BLOCK_SIZE(s->qual_blk);
+            BLOCK_GROW(s->qual_blk, cr->len);
+            qual = (char *)BLOCK_END(s->qual_blk);
+            BLOCK_SIZE(s->qual_blk) += cr->len;
+
+            if (!s->ref)
+                memset(seq, '=', cr->len);
+        }
+
+        if (!(bf & BAM_FUNMAP)) {
+            if ((ds & CRAM_AP) && cr->apos <= 0) {
+                hts_log_error("Read has alignment position %"PRId64
+                              " but no unmapped flag",
+                              cr->apos);
+                goto block_err;
+            }
+            /* Decode sequence and generate CIGAR */
+            if (ds & (CRAM_SEQ | CRAM_MQ)) {
+                r |= cram_decode_seq(fd, c, s, blk, cr, sh, cf, seq, qual,
+                                     has_MD, has_NM);
+                if (r) goto block_err;
+            } else {
+                cr->cigar = 0;
+                cr->ncigar = 0;
+                cr->aend = cr->apos;
+                cr->mqual = 0;
+            }
+        } else {
+            int out_sz2 = cr->len;
+
+            //puts("Unmapped");
+            cr->cigar = 0;
+            cr->ncigar = 0;
+            cr->aend = cr->apos;
+            cr->mqual = 0;
+
+            if (ds & CRAM_BA && cr->len) {
+                if (!c->comp_hdr->codecs[DS_BA]) goto block_err;
+                r |= c->comp_hdr->codecs[DS_BA]
+                                ->decode(s, c->comp_hdr->codecs[DS_BA], blk,
+                                         (char *)seq, &out_sz2);
+                if (r) goto block_err;
+            }
+
+            if ((ds & CRAM_CF) && (cf & CRAM_FLAG_PRESERVE_QUAL_SCORES)) {
+                out_sz2 = cr->len;
+                if (ds & CRAM_QS && cr->len >= 0) {
+                    if (!c->comp_hdr->codecs[DS_QS]) goto block_err;
+                    r |= c->comp_hdr->codecs[DS_QS]
+                                    ->decode(s, c->comp_hdr->codecs[DS_QS],
+                                             blk, qual, &out_sz2);
+                    if (r) goto block_err;
+                }
+            } else {
+                if (ds & CRAM_RL)
+                    memset(qual, 255, cr->len);
+            }
+        }
+
+        if (!c->comp_hdr->qs_seq_orient && (ds & CRAM_QS) && (cr->flags & BAM_FREVERSE)) {
+            int i, j;
+            for (i = 0, j = cr->len-1; i < j; i++, j--) {
+                unsigned char c;
+                c = qual[i];
+                qual[i] = qual[j];
+                qual[j] = c;
+            }
+        }
+    }
+
+    pthread_mutex_lock(&fd->ref_lock);
+    if (refs) {
+        int i;
+        for (i = 0; i < fd->refs->nref; i++) {
+            if (refs[i])
+                cram_ref_decr(fd->refs, i);
+        }
+        free(refs);
+        refs = NULL;
+    } else if (ref_id >= 0 && s->ref != fd->ref_free && !embed_ref) {
+        cram_ref_decr(fd->refs, ref_id);
+    }
+    pthread_mutex_unlock(&fd->ref_lock);
+
+    /* Resolve mate pair cross-references between recs within this slice */
+    r |= cram_decode_slice_xref(s, fd->required_fields);
+
+    // Free the original blocks as we no longer need these.
+    {
+        int i;
+        for (i = 0; i < s->hdr->num_blocks; i++) {
+            cram_block *b = s->block[i];
+            cram_free_block(b);
+            s->block[i] = NULL;
+        }
+    }
+
+    // Also see initial BLOCK_RESIZE_EXACT at top of function.
+    // As we grow blocks we overallocate by up to 50%. So shrink
+    // back to their final sizes here.
+    //
+    //fprintf(stderr, "%d %d // %d %d // %d %d // %d %d\n",
+    //      (int)s->seqs_blk->byte, (int)s->seqs_blk->alloc,
+    //      (int)s->qual_blk->byte, (int)s->qual_blk->alloc,
+    //      (int)s->name_blk->byte, (int)s->name_blk->alloc,
+    //      (int)s->aux_blk->byte,  (int)s->aux_blk->alloc);
+    BLOCK_RESIZE_EXACT(s->seqs_blk, BLOCK_SIZE(s->seqs_blk)+1);
+    BLOCK_RESIZE_EXACT(s->qual_blk, BLOCK_SIZE(s->qual_blk)+1);
+    BLOCK_RESIZE_EXACT(s->name_blk, BLOCK_SIZE(s->name_blk)+1);
+    BLOCK_RESIZE_EXACT(s->aux_blk,  BLOCK_SIZE(s->aux_blk)+1);
+
+    return r;
+
+ block_err:
+    if (refs) {
+        int i;
+        pthread_mutex_lock(&fd->ref_lock);
+        for (i = 0; i < fd->refs->nref; i++) {
+            if (refs[i])
+                cram_ref_decr(fd->refs, i);
+        }
+        free(refs);
+        pthread_mutex_unlock(&fd->ref_lock);
+    }
+
+    return -1;
+}
+
+typedef struct {
+    cram_fd *fd;
+    cram_container *c;
+    cram_slice *s;
+    sam_hdr_t *h;
+    int exit_code;
+} cram_decode_job;
+
+void *cram_decode_slice_thread(void *arg) {
+    cram_decode_job *j = (cram_decode_job *)arg;
+
+    j->exit_code = cram_decode_slice(j->fd, j->c, j->s, j->h);
+
+    return j;
+}
+
+/*
+ * Spawn a multi-threaded version of cram_decode_slice().
+ */
+int cram_decode_slice_mt(cram_fd *fd, cram_container *c, cram_slice *s,
+                         sam_hdr_t *bfd) {
+    cram_decode_job *j;
+    int nonblock;
+
+    if (!fd->pool)
+        return cram_decode_slice(fd, c, s, bfd);
+
+    if (!(j = malloc(sizeof(*j))))
+        return -1;
+
+    j->fd = fd;
+    j->c  = c;
+    j->s  = s;
+    j->h  = bfd;
+
+    nonblock = hts_tpool_process_sz(fd->rqueue) ? 1 : 0;
+
+    int saved_errno = errno;
+    errno = 0;
+    if (-1 == hts_tpool_dispatch2(fd->pool, fd->rqueue, cram_decode_slice_thread,
+                                  j, nonblock)) {
+        /* Would block */
+        if (errno != EAGAIN)
+            return -1;
+        fd->job_pending = j;
+    } else {
+        fd->job_pending = NULL;
+    }
+    errno = saved_errno;
+
+    // flush too
+    return 0;
+}
+
+
+/* ----------------------------------------------------------------------
+ * CRAM sequence iterators.
+ */
+
+/*
+ * Converts a cram in-memory record into a bam in-memory record. We
+ * pass a pointer to a bam_seq_t pointer along with the a pointer to
+ * the allocated size. These can initially be pointers to NULL and zero.
+ *
+ * This function will reallocate the bam buffer as required and update
+ * (*bam)->alloc accordingly, allowing it to be used within a loop
+ * efficiently without needing to allocate new bam objects over and
+ * over again.
+ *
+ * Returns the used size of the bam record on success
+ *         -1 on failure.
+ */
+int cram_to_bam(sam_hdr_t *sh, cram_fd *fd, cram_slice *s,
+                cram_record *cr, int rec, bam_seq_t **bam) {
+    int ret, rg_len;
+    char name_a[1024], *name;
+    int name_len;
+    char *aux;
+    char *seq, *qual;
+    sam_hrecs_t *bfd = sh->hrecs;
+
+    /* Assign names if not explicitly set */
+    if (fd->required_fields & SAM_QNAME) {
+        if (cr->name_len) {
+            name = (char *)BLOCK_DATA(s->name_blk) + cr->name;
+            name_len = cr->name_len;
+        } else {
+            name = name_a;
+            if (cr->mate_line >= 0 && cr->mate_line < s->max_rec &&
+                s->crecs[cr->mate_line].name_len > 0) {
+                // Copy our mate if non-zero.
+                memcpy(name_a, BLOCK_DATA(s->name_blk)+s->crecs[cr->mate_line].name,
+                       s->crecs[cr->mate_line].name_len);
+                name = name_a + s->crecs[cr->mate_line].name_len;
+            } else {
+                // Otherwise generate a name based on prefix
+                name_len = strlen(fd->prefix);
+                memcpy(name, fd->prefix, name_len);
+                name += name_len;
+                *name++ = ':';
+                if (cr->mate_line >= 0 && cr->mate_line < rec) {
+                    name = (char *)append_uint64((unsigned char *)name,
+                                                 s->hdr->record_counter +
+                                                 cr->mate_line + 1);
+                } else {
+                    name = (char *)append_uint64((unsigned char *)name,
+                                                 s->hdr->record_counter +
+                                                 rec + 1);
+                }
+            }
+            name_len = name - name_a;
+            name = name_a;
+        }
+    } else {
+        name = "?";
+        name_len = 1;
+    }
+
+    /* Generate BAM record */
+    if (cr->rg < -1 || cr->rg >= bfd->nrg)
+        return -1;
+    rg_len = (cr->rg != -1) ? bfd->rg[cr->rg].name_len + 4 : 0;
+
+    if (fd->required_fields & (SAM_SEQ | SAM_QUAL)) {
+        if (!BLOCK_DATA(s->seqs_blk))
+            return -1;
+        seq = (char *)BLOCK_DATA(s->seqs_blk) + cr->seq;
+    } else {
+        seq = "*";
+        cr->len = 0;
+    }
+
+    if (fd->required_fields & SAM_QUAL) {
+        if (!BLOCK_DATA(s->qual_blk))
+            return -1;
+        qual = (char *)BLOCK_DATA(s->qual_blk) + cr->qual;
+    } else {
+        qual = NULL;
+    }
+
+    ret = bam_set1(*bam,
+                   name_len, name,
+                   cr->flags, cr->ref_id, cr->apos - 1, cr->mqual,
+                   cr->ncigar, &s->cigar[cr->cigar],
+                   cr->mate_ref_id, cr->mate_pos - 1, cr->tlen,
+                   cr->len, seq, qual,
+                   cr->aux_size + rg_len);
+    if (ret < 0) {
+        return ret;
+    }
+
+    aux = (char *)bam_aux(*bam);
+
+    /* Auxiliary strings */
+    if (cr->aux_size != 0) {
+        memcpy(aux, BLOCK_DATA(s->aux_blk) + cr->aux, cr->aux_size);
+        aux += cr->aux_size;
+        (*bam)->l_data += cr->aux_size;
+    }
+
+    /* RG:Z: */
+    if (rg_len > 0) {
+        *aux++ = 'R'; *aux++ = 'G'; *aux++ = 'Z';
+        int len = bfd->rg[cr->rg].name_len;
+        memcpy(aux, bfd->rg[cr->rg].name, len);
+        aux += len;
+        *aux++ = 0;
+        (*bam)->l_data += rg_len;
+    }
+
+    return (*bam)->l_data;
+}
+
+/*
+ * Here be dragons! The multi-threading code in this is crufty beyond belief.
+ */
+
+/*
+ * Load first container.
+ * Called when fd->ctr is NULL>
+ *
+ * Returns container on success
+ *        NULL on failure.
+ */
+static cram_container *cram_first_slice(cram_fd *fd) {
+    cram_container *c;
+
+    do {
+        if (fd->ctr)
+            cram_free_container(fd->ctr);
+
+        if (!(c = fd->ctr = cram_read_container(fd)))
+            return NULL;
+        c->curr_slice_mt = c->curr_slice;
+    } while (c->length == 0);
+
+    /*
+     * The first container may be a result of a sub-range query.
+     * In which case it may still not be the optimal starting point
+     * due to skipped containers/slices in the index.
+     */
+    // No need for locks here as we're in the main thread.
+    if (fd->range.refid != -2) {
+        while (c->ref_seq_id != -2 &&
+               (c->ref_seq_id < fd->range.refid ||
+                (fd->range.refid >= 0 && c->ref_seq_id == fd->range.refid
+                 && c->ref_seq_start + c->ref_seq_span-1 < fd->range.start))) {
+            if (0 != cram_seek(fd, c->length, SEEK_CUR))
+                return NULL;
+            cram_free_container(fd->ctr);
+            do {
+                if (!(c = fd->ctr = cram_read_container(fd)))
+                    return NULL;
+            } while (c->length == 0);
+        }
+
+        if (c->ref_seq_id != -2 && c->ref_seq_id != fd->range.refid) {
+            fd->eof = 1;
+            return NULL;
+        }
+    }
+
+    if (!(c->comp_hdr_block = cram_read_block(fd)))
+        return NULL;
+    if (c->comp_hdr_block->content_type != COMPRESSION_HEADER)
+        return NULL;
+
+    c->comp_hdr = cram_decode_compression_header(fd, c->comp_hdr_block);
+    if (!c->comp_hdr)
+        return NULL;
+    if (!c->comp_hdr->AP_delta &&
+        sam_hrecs_sort_order(fd->header->hrecs) != ORDER_COORD) {
+        pthread_mutex_lock(&fd->ref_lock);
+        fd->unsorted = 1;
+        pthread_mutex_unlock(&fd->ref_lock);
+    }
+
+    return c;
+}
+
+cram_slice *cram_next_slice(cram_fd *fd, cram_container **cp) {
+    cram_container *c_curr;  // container being consumed via cram_get_seq()
+    cram_slice *s_curr = NULL;
+
+    // Populate the first container if unknown.
+    if (!(c_curr = fd->ctr)) {
+        if (!(c_curr = cram_first_slice(fd)))
+            return NULL;
+    }
+
+    // Discard previous slice
+    if ((s_curr = c_curr->slice)) {
+        c_curr->slice = NULL;
+        cram_free_slice(s_curr);
+        s_curr = NULL;
+    }
+
+    // If we've consumed all slices in this container, also discard
+    // the container too.
+    if (c_curr->curr_slice == c_curr->max_slice) {
+        if (fd->ctr == c_curr)
+            fd->ctr = NULL;
+        if (fd->ctr_mt == c_curr)
+            fd->ctr_mt = NULL;
+        cram_free_container(c_curr);
+        c_curr = NULL;
+    }
+
+    if (!fd->ctr_mt)
+        fd->ctr_mt = c_curr;
+
+    // Fetch the next slice (and the container if necessary).
+    //
+    // If single threaded this loop bails out as soon as it finds
+    // a slice in range.  In this case c_next and c_curr end up being
+    // the same thing.
+    //
+    // If multi-threaded, we loop until we have filled out
+    // thread pool input queue.  Here c_next and c_curr *may* differ, as
+    // can fd->ctr and fd->ctr_mt.
+    for (;;) {
+        cram_container *c_next = fd->ctr_mt;
+        cram_slice *s_next = NULL;
+
+        // Next slice; either from the last job we failed to push
+        // to the input queue or via more I/O.
+        if (fd->job_pending) {
+            cram_decode_job *j = (cram_decode_job *)fd->job_pending;
+            c_next = j->c;
+            s_next = j->s;
+            free(fd->job_pending);
+            fd->job_pending = NULL;
+        } else if (!fd->ooc) {
+        empty_container:
+            if (!c_next || c_next->curr_slice_mt == c_next->max_slice) {
+                // new container
+                for(;;) {
+                    if (!(c_next = cram_read_container(fd))) {
+                        if (fd->pool) {
+                            fd->ooc = 1;
+                            break;
+                        }
+
+                        return NULL;
+                    }
+                    c_next->curr_slice_mt = c_next->curr_slice;
+
+                    if (c_next->length != 0)
+                        break;
+
+                    cram_free_container(c_next);
+                }
+                if (fd->ooc)
+                    break;
+
+                /* Skip containers not yet spanning our range */
+                if (fd->range.refid != -2 && c_next->ref_seq_id != -2) {
+                    // ref_id beyond end of range; bail out
+                    if (c_next->ref_seq_id != fd->range.refid) {
+                        cram_free_container(c_next);
+                        fd->ctr_mt = NULL;
+                        fd->ooc = 1;
+                        break;
+                    }
+
+                    // position beyond end of range; bail out
+                    if (fd->range.refid != -1 &&
+                        c_next->ref_seq_start > fd->range.end) {
+                        cram_free_container(c_next);
+                        fd->ctr_mt = NULL;
+                        fd->ooc = 1;
+                        break;
+                    }
+
+                    // before start of range; skip to next container
+                    if (fd->range.refid != -1 &&
+                        c_next->ref_seq_start + c_next->ref_seq_span-1 <
+                        fd->range.start) {
+                        c_next->curr_slice_mt = c_next->max_slice;
+                        cram_seek(fd, c_next->length, SEEK_CUR);
+                        cram_free_container(c_next);
+                        c_next = NULL;
+                        continue;
+                    }
+                }
+
+                // Container is valid range, so remember it for restarting
+                // this function.
+                fd->ctr_mt = c_next;
+
+                if (!(c_next->comp_hdr_block = cram_read_block(fd)))
+                    return NULL;
+                if (c_next->comp_hdr_block->content_type != COMPRESSION_HEADER)
+                    return NULL;
+
+                c_next->comp_hdr =
+                    cram_decode_compression_header(fd, c_next->comp_hdr_block);
+                if (!c_next->comp_hdr)
+                    return NULL;
+
+                if (!c_next->comp_hdr->AP_delta &&
+                    sam_hrecs_sort_order(fd->header->hrecs) != ORDER_COORD) {
+                    pthread_mutex_lock(&fd->ref_lock);
+                    fd->unsorted = 1;
+                    pthread_mutex_unlock(&fd->ref_lock);
+                }
+            }
+
+            if (c_next->num_records == 0) {
+                if (fd->ctr == c_next)
+                    fd->ctr = NULL;
+                if (c_curr == c_next)
+                    c_curr = NULL;
+                if (fd->ctr_mt == c_next)
+                    fd->ctr_mt = NULL;
+                cram_free_container(c_next);
+                c_next = NULL;
+                goto empty_container;
+            }
+
+            if (!(s_next = c_next->slice = cram_read_slice(fd)))
+                return NULL;
+
+            s_next->slice_num = ++c_next->curr_slice_mt;
+            s_next->curr_rec = 0;
+            s_next->max_rec = s_next->hdr->num_records;
+
+            s_next->last_apos = s_next->hdr->ref_seq_start;
+
+            // We know the container overlaps our range, but with multi-slice
+            // containers we may have slices that do not.  Skip these also.
+            if (fd->range.refid != -2 && s_next->hdr->ref_seq_id != -2) {
+                // ref_id beyond end of range; bail out
+                if (s_next->hdr->ref_seq_id != fd->range.refid) {
+                    fd->ooc = 1;
+                    cram_free_slice(s_next);
+                    c_next->slice = s_next = NULL;
+                    break;
+                }
+
+                // position beyond end of range; bail out
+                if (fd->range.refid != -1 &&
+                    s_next->hdr->ref_seq_start > fd->range.end) {
+                    fd->ooc = 1;
+                    cram_free_slice(s_next);
+                    c_next->slice = s_next = NULL;
+                    break;
+                }
+
+                // before start of range; skip to next slice
+                if (fd->range.refid != -1 &&
+                    s_next->hdr->ref_seq_start + s_next->hdr->ref_seq_span-1 <
+                    fd->range.start) {
+                    cram_free_slice(s_next);
+                    c_next->slice = s_next = NULL;
+                    continue;
+                }
+            }
+        } // end: if (!fd->ooc)
+
+        if (!c_next || !s_next)
+            break;
+
+        // Decode the slice, either right now (non-threaded) or by pushing
+        // it to the a decode queue (threaded).
+        if (cram_decode_slice_mt(fd, c_next, s_next, fd->header) != 0) {
+            hts_log_error("Failure to decode slice");
+            cram_free_slice(s_next);
+            c_next->slice = NULL;
+            return NULL;
+        }
+
+        // No thread pool, so don't loop again
+        if (!fd->pool) {
+            c_curr = c_next;
+            s_curr = s_next;
+            break;
+        }
+
+        // With thread pool, but we have a job pending so our decode queue
+        // is full.
+        if (fd->job_pending)
+            break;
+
+        // Otherwise we're threaded with room in the decode input queue, so
+        // keep reading slices for decode.
+        // Push it a bit far, to qsize in queue rather than pending arrival,
+        // as cram tends to be a bit bursty in decode timings.
+        if (hts_tpool_process_len(fd->rqueue) >
+            hts_tpool_process_qsize(fd->rqueue))
+            break;
+    } // end of for(;;)
+
+
+    // When not threaded we've already have c_curr and s_curr.
+    // Otherwise we need get them by pulling off the decode output queue.
+    if (fd->pool) {
+        hts_tpool_result *res;
+        cram_decode_job *j;
+
+        if (fd->ooc && hts_tpool_process_empty(fd->rqueue)) {
+            fd->eof = 1;
+            return NULL;
+        }
+
+        res = hts_tpool_next_result_wait(fd->rqueue);
+
+        if (!res || !hts_tpool_result_data(res)) {
+            hts_log_error("Call to hts_tpool_next_result failed");
+            return NULL;
+        }
+
+        j = (cram_decode_job *)hts_tpool_result_data(res);
+        c_curr = j->c;
+        s_curr = j->s;
+
+        if (j->exit_code != 0) {
+            hts_log_error("Slice decode failure");
+            fd->eof = 0;
+            hts_tpool_delete_result(res, 1);
+            return NULL;
+        }
+
+        hts_tpool_delete_result(res, 1);
+    }
+
+    *cp = c_curr;
+
+    // Update current slice being processed (as opposed to current
+    // slice in the multi-threaded reahead.
+    fd->ctr = c_curr;
+    if (c_curr) {
+        c_curr->slice = s_curr;
+        if (s_curr)
+            c_curr->curr_slice = s_curr->slice_num;
+    }
+    if (s_curr)
+        s_curr->curr_rec = 0;
+    else
+        fd->eof = 1;
+
+    return s_curr;
+}
+
+/*
+ * Read the next cram record and return it.
+ * Note that to decode cram_record the caller will need to look up some data
+ * in the current slice, pointed to by fd->ctr->slice. This is valid until
+ * the next call to cram_get_seq (which may invalidate it).
+ *
+ * Returns record pointer on success (do not free)
+ *        NULL on failure
+ */
+cram_record *cram_get_seq(cram_fd *fd) {
+    cram_container *c;
+    cram_slice *s;
+
+    for (;;) {
+        c = fd->ctr;
+        if (c && c->slice && c->slice->curr_rec < c->slice->max_rec) {
+            s = c->slice;
+        } else {
+            if (!(s = cram_next_slice(fd, &c)))
+                return NULL;
+            continue; /* In case slice contains no records */
+        }
+
+        // No need to lock here as get_seq is running in the main thread,
+        // which is also the same one that does the range modifications.
+        if (fd->range.refid != -2) {
+            if (fd->range.refid == -1 && s->crecs[s->curr_rec].ref_id != -1) {
+                // Special case when looking for unmapped blocks at end.
+                // If these are mixed in with mapped data (c->ref_id == -2)
+                // then we need skip until we find the unmapped data, if at all
+                s->curr_rec++;
+                continue;
+            }
+            if (s->crecs[s->curr_rec].ref_id < fd->range.refid &&
+                s->crecs[s->curr_rec].ref_id != -1) {
+                // Looking for a mapped read, but not there yet.  Special case
+                // as -1 (unmapped) shouldn't be considered < refid.
+                s->curr_rec++;
+                continue;
+            }
+
+            if (s->crecs[s->curr_rec].ref_id != fd->range.refid) {
+                fd->eof = 1;
+                cram_free_slice(s);
+                c->slice = NULL;
+                return NULL;
+            }
+
+            if (fd->range.refid != -1 && s->crecs[s->curr_rec].apos > fd->range.end) {
+                fd->eof = 1;
+                cram_free_slice(s);
+                c->slice = NULL;
+                return NULL;
+            }
+
+            if (fd->range.refid != -1 && s->crecs[s->curr_rec].aend < fd->range.start) {
+                s->curr_rec++;
+                continue;
+            }
+        }
+
+        break;
+    }
+
+    fd->ctr = c;
+    c->slice = s;
+    return &s->crecs[s->curr_rec++];
+}
+
+/*
+ * Read the next cram record and convert it to a bam_seq_t struct.
+ *
+ * Returns >= 0 success (number of bytes written to *bam)
+ *        -1 on EOF or failure (check fd->err)
+ */
+int cram_get_bam_seq(cram_fd *fd, bam_seq_t **bam) {
+    cram_record *cr;
+    cram_container *c;
+    cram_slice *s;
+
+    if (!(cr = cram_get_seq(fd)))
+        return -1;
+
+    c = fd->ctr;
+    s = c->slice;
+
+    return cram_to_bam(fd->header, fd, s, cr, s->curr_rec-1, bam);
+}
+
+/*
+ * Drains and frees the decode read-queue for a multi-threaded reader.
+ */
+void cram_drain_rqueue(cram_fd *fd) {
+    cram_container *lc = NULL;
+
+    if (!fd->pool || !fd->rqueue)
+        return;
+
+    // drain queue of any in-flight decode jobs
+    while (!hts_tpool_process_empty(fd->rqueue)) {
+        hts_tpool_result *r = hts_tpool_next_result_wait(fd->rqueue);
+        if (!r)
+            break;
+        cram_decode_job *j = (cram_decode_job *)hts_tpool_result_data(r);
+        if (j->c->slice == j->s)
+            j->c->slice = NULL;
+        if (j->c != lc) {
+            if (lc) {
+                if (fd->ctr == lc)
+                    fd->ctr = NULL;
+                if (fd->ctr_mt == lc)
+                    fd->ctr_mt = NULL;
+                cram_free_container(lc);
+            }
+            lc = j->c;
+        }
+        cram_free_slice(j->s);
+        hts_tpool_delete_result(r, 1);
+    }
+
+    // Also tidy up any pending decode job that we didn't submit to the workers
+    // due to the input queue being full.
+    if (fd->job_pending) {
+        cram_decode_job *j = (cram_decode_job *)fd->job_pending;
+        if (j->c->slice == j->s)
+            j->c->slice = NULL;
+        if (j->c != lc) {
+            if (lc) {
+                if (fd->ctr == lc)
+                    fd->ctr = NULL;
+                if (fd->ctr_mt == lc)
+                    fd->ctr_mt = NULL;
+                cram_free_container(lc);
+            }
+            lc = j->c;
+        }
+        cram_free_slice(j->s);
+        free(j);
+        fd->job_pending = NULL;
+    }
+
+    if (lc) {
+        if (fd->ctr == lc)
+            fd->ctr = NULL;
+        if (fd->ctr_mt == lc)
+            fd->ctr_mt = NULL;
+        cram_free_container(lc);
+    }
+}
diff --git a/ext/htslib/cram/cram_decode.h b/ext/htslib/cram/cram_decode.h
new file mode 100644
index 0000000..16d87a0
--- /dev/null
+++ b/ext/htslib/cram/cram_decode.h
@@ -0,0 +1,142 @@
+/*
+Copyright (c) 2012-2013, 2018, 2024 Genome Research Ltd.
+Author: James Bonfield <jkb@sanger.ac.uk>
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+Institute nor the names of its contributors may be used to endorse or promote
+products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+/*! \file
+ * Include cram.h instead.
+ *
+ * This is an internal part of the CRAM system and is automatically included
+ * when you #include cram.h.
+ *
+ * Implements the decoding portion of CRAM I/O. Also see
+ * cram_codecs.[ch] for the actual encoding functions themselves.
+ */
+
+#ifndef CRAM_DECODE_H
+#define CRAM_DECODE_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* ----------------------------------------------------------------------
+ * CRAM sequence iterators.
+ */
+
+/*! Read the next cram record and return it as a cram_record.
+ *
+ * Note that to decode cram_record the caller will need to look up some data
+ * in the current slice, pointed to by fd->ctr->slice. This is valid until
+ * the next call to cram_get_seq (which may invalidate it).
+ *
+ * @return
+ * Returns record pointer on success (do not free);
+ *        NULL on failure
+ */
+cram_record *cram_get_seq(cram_fd *fd);
+
+/*! Read the next cram record and convert it to a bam_seq_t struct.
+ *
+ * @return
+ * Returns 0 on success;
+ *        -1 on EOF or failure (check fd->err)
+ */
+int cram_get_bam_seq(cram_fd *fd, bam_seq_t **bam);
+
+
+/* ----------------------------------------------------------------------
+ * Internal functions
+ */
+
+/*! INTERNAL:
+ * Decodes a CRAM block compression header.
+ *
+ * @return
+ * Returns header ptr on success;
+ *         NULL on failure
+ */
+cram_block_compression_hdr *cram_decode_compression_header(cram_fd *fd,
+                                                           cram_block *b);
+
+/*! INTERNAL:
+ * Decodes a CRAM (un)mapped slice header block.
+ *
+ * @return
+ * Returns slice header ptr on success;
+ *         NULL on failure
+ */
+cram_block_slice_hdr *cram_decode_slice_header(cram_fd *fd, cram_block *b);
+
+
+/*! INTERNAL:
+ * Loads and decodes the next slice worth of data.
+ *
+ * @return
+ * Returns cram slice pointer on success;
+ *         NULL on failure
+ */
+cram_slice *cram_next_slice(cram_fd *fd, cram_container **cp);
+
+/*! INTERNAL:
+ * Decode an entire slice from container blocks. Fills out s->crecs[] array.
+ *
+ * @return
+ * Returns 0 on success;
+ *        -1 on failure
+ */
+int cram_decode_slice(cram_fd *fd, cram_container *c, cram_slice *s,
+                      sam_hdr_t *hdr);
+
+
+/*! INTERNAL:
+ * Converts a cram in-memory record into a bam in-memory record. We
+ * pass a pointer to a bam_seq_t pointer along with the a pointer to
+ * the allocated size. These can initially be pointers to NULL and zero.
+ *
+ * This function will reallocate the bam buffer as required and update
+ * (*bam)->alloc accordingly, allowing it to be used within a loop
+ * efficiently without needing to allocate new bam objects over and
+ * over again.
+ *
+ * Returns the used size of the bam record on success
+ *         -1 on failure.
+ */
+int cram_to_bam(sam_hdr_t *sh, cram_fd *fd, cram_slice *s,
+                cram_record *cr, int rec, bam_seq_t **bam);
+
+/*
+ * Drains and frees the decode read-queue for a multi-threaded reader.
+ */
+void cram_drain_rqueue(cram_fd *fd);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/ext/htslib/cram/cram_encode.c b/ext/htslib/cram/cram_encode.c
new file mode 100644
index 0000000..5d22db5
--- /dev/null
+++ b/ext/htslib/cram/cram_encode.c
@@ -0,0 +1,4180 @@
+/*
+Copyright (c) 2012-2020, 2022-2024 Genome Research Ltd.
+Author: James Bonfield <jkb@sanger.ac.uk>
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+Institute nor the names of its contributors may be used to endorse or promote
+products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+
+#include <stdio.h>
+#include <errno.h>
+#include <assert.h>
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+#include <zlib.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <math.h>
+#include <inttypes.h>
+#include <ctype.h>
+
+#include "cram.h"
+#include "os.h"
+#include "../sam_internal.h" // for nibble2base
+#include "../htslib/hts.h"
+#include "../htslib/hts_endian.h"
+#include "../textutils_internal.h"
+
+KHASH_MAP_INIT_STR(m_s2u64, uint64_t)
+
+#define Z_CRAM_STRAT Z_FILTERED
+//#define Z_CRAM_STRAT Z_RLE
+//#define Z_CRAM_STRAT Z_HUFFMAN_ONLY
+//#define Z_CRAM_STRAT Z_DEFAULT_STRATEGY
+
+static int process_one_read(cram_fd *fd, cram_container *c,
+                            cram_slice *s, cram_record *cr,
+                            bam_seq_t *b, int rnum, kstring_t *MD,
+                            int embed_ref, int no_ref);
+
+/*
+ * Returns index of val into key.
+ * Basically strchr(key, val)-key;
+ */
+static int sub_idx(char *key, char val) {
+    int i;
+
+    for (i = 0; i < 4 && *key++ != val; i++);
+    return i;
+}
+
+/*
+ * Encodes a compression header block into a generic cram_block structure.
+ *
+ * Returns cram_block ptr on success
+ *         NULL on failure
+ */
+cram_block *cram_encode_compression_header(cram_fd *fd, cram_container *c,
+                                           cram_block_compression_hdr *h,
+                                           int embed_ref) {
+    cram_block *cb  = cram_new_block(COMPRESSION_HEADER, 0);
+    cram_block *map = cram_new_block(COMPRESSION_HEADER, 0);
+    int i, mc, r = 0;
+
+    int no_ref = c->no_ref;
+
+    if (!cb || !map)
+        return NULL;
+
+    /*
+     * This is a concatenation of several blocks of data:
+     * header + landmarks, preservation map, read encoding map, and the tag
+     * encoding map.
+     * All 4 are variable sized and we need to know how large these are
+     * before creating the compression header itself as this starts with
+     * the total size (stored as a variable length string).
+     */
+
+    // Duplicated from container itself, and removed in 1.1
+    if (CRAM_MAJOR_VERS(fd->version) == 1) {
+        r |= itf8_put_blk(cb, h->ref_seq_id);
+        r |= itf8_put_blk(cb, h->ref_seq_start);
+        r |= itf8_put_blk(cb, h->ref_seq_span);
+        r |= itf8_put_blk(cb, h->num_records);
+        r |= itf8_put_blk(cb, h->num_landmarks);
+        for (i = 0; i < h->num_landmarks; i++) {
+            r |= itf8_put_blk(cb, h->landmark[i]);
+        }
+    }
+
+    if (h->preservation_map) {
+        kh_destroy(map, h->preservation_map);
+        h->preservation_map = NULL;
+    }
+
+    /* Create in-memory preservation map */
+    /* FIXME: should create this when we create the container */
+    if (c->num_records > 0) {
+        khint_t k;
+        int r;
+
+        if (!(h->preservation_map = kh_init(map)))
+            return NULL;
+
+        k = kh_put(map, h->preservation_map, "RN", &r);
+        if (-1 == r) return NULL;
+        kh_val(h->preservation_map, k).i = !fd->lossy_read_names;
+
+        if (CRAM_MAJOR_VERS(fd->version) == 1) {
+            k = kh_put(map, h->preservation_map, "PI", &r);
+            if (-1 == r) return NULL;
+            kh_val(h->preservation_map, k).i = 0;
+
+            k = kh_put(map, h->preservation_map, "UI", &r);
+            if (-1 == r) return NULL;
+            kh_val(h->preservation_map, k).i = 1;
+
+            k = kh_put(map, h->preservation_map, "MI", &r);
+            if (-1 == r) return NULL;
+            kh_val(h->preservation_map, k).i = 1;
+
+        } else {
+            // Technically SM was in 1.0, but wasn't in Java impl.
+            k = kh_put(map, h->preservation_map, "SM", &r);
+            if (-1 == r) return NULL;
+            kh_val(h->preservation_map, k).i = 0;
+
+            k = kh_put(map, h->preservation_map, "TD", &r);
+            if (-1 == r) return NULL;
+            kh_val(h->preservation_map, k).i = 0;
+
+            k = kh_put(map, h->preservation_map, "AP", &r);
+            if (-1 == r) return NULL;
+            kh_val(h->preservation_map, k).i = h->AP_delta;
+
+            if (CRAM_MAJOR_VERS(fd->version) >= 4) {
+                k = kh_put(map, h->preservation_map, "QO", &r);
+                if (-1 == r) return NULL;
+                kh_val(h->preservation_map, k).i = h->qs_seq_orient;
+            }
+
+            if (no_ref || embed_ref>0) {
+                // Reference Required == No
+                k = kh_put(map, h->preservation_map, "RR", &r);
+                if (-1 == r) return NULL;
+                kh_val(h->preservation_map, k).i = 0;
+            }
+        }
+    }
+
+    /* Encode preservation map; could collapse this and above into one */
+    mc = 0;
+    BLOCK_SIZE(map) = 0;
+    if (h->preservation_map) {
+        khint_t k;
+
+        for (k = kh_begin(h->preservation_map);
+             k != kh_end(h->preservation_map);
+             k++) {
+            const char *key;
+            khash_t(map) *pmap = h->preservation_map;
+
+
+            if (!kh_exist(pmap, k))
+                continue;
+
+            key = kh_key(pmap, k);
+            BLOCK_APPEND(map, key, 2);
+
+            switch(CRAM_KEY(key[0], key[1])) {
+            case CRAM_KEY('M','I'):
+            case CRAM_KEY('U','I'):
+            case CRAM_KEY('P','I'):
+            case CRAM_KEY('A','P'):
+            case CRAM_KEY('R','N'):
+            case CRAM_KEY('R','R'):
+            case CRAM_KEY('Q','O'):
+                BLOCK_APPEND_CHAR(map, kh_val(pmap, k).i);
+                break;
+
+            case CRAM_KEY('S','M'): {
+                char smat[5], *mp = smat;
+                // Output format is for order ACGTN (minus ref base)
+                // to store the code value 0-3 for each symbol.
+                //
+                // Note this is different to storing the symbols in order
+                // that the codes occur from 0-3, which is what we used to
+                // do.  (It didn't matter as we always had a fixed table in
+                // the order.)
+                *mp++ =
+                    (sub_idx(h->substitution_matrix[0], 'C') << 6) |
+                    (sub_idx(h->substitution_matrix[0], 'G') << 4) |
+                    (sub_idx(h->substitution_matrix[0], 'T') << 2) |
+                    (sub_idx(h->substitution_matrix[0], 'N') << 0);
+                *mp++ =
+                    (sub_idx(h->substitution_matrix[1], 'A') << 6) |
+                    (sub_idx(h->substitution_matrix[1], 'G') << 4) |
+                    (sub_idx(h->substitution_matrix[1], 'T') << 2) |
+                    (sub_idx(h->substitution_matrix[1], 'N') << 0);
+                *mp++ =
+                    (sub_idx(h->substitution_matrix[2], 'A') << 6) |
+                    (sub_idx(h->substitution_matrix[2], 'C') << 4) |
+                    (sub_idx(h->substitution_matrix[2], 'T') << 2) |
+                    (sub_idx(h->substitution_matrix[2], 'N') << 0);
+                *mp++ =
+                    (sub_idx(h->substitution_matrix[3], 'A') << 6) |
+                    (sub_idx(h->substitution_matrix[3], 'C') << 4) |
+                    (sub_idx(h->substitution_matrix[3], 'G') << 2) |
+                    (sub_idx(h->substitution_matrix[3], 'N') << 0);
+                *mp++ =
+                    (sub_idx(h->substitution_matrix[4], 'A') << 6) |
+                    (sub_idx(h->substitution_matrix[4], 'C') << 4) |
+                    (sub_idx(h->substitution_matrix[4], 'G') << 2) |
+                    (sub_idx(h->substitution_matrix[4], 'T') << 0);
+                BLOCK_APPEND(map, smat, 5);
+                break;
+            }
+
+            case CRAM_KEY('T','D'): {
+                r |= (fd->vv.varint_put32_blk(map, BLOCK_SIZE(h->TD_blk)) <= 0);
+                BLOCK_APPEND(map,
+                             BLOCK_DATA(h->TD_blk),
+                             BLOCK_SIZE(h->TD_blk));
+                break;
+            }
+
+            default:
+                hts_log_warning("Unknown preservation key '%.2s'", key);
+                break;
+            }
+
+            mc++;
+        }
+    }
+    r |= (fd->vv.varint_put32_blk(cb, BLOCK_SIZE(map) + fd->vv.varint_size(mc)) <= 0);
+    r |= (fd->vv.varint_put32_blk(cb, mc) <= 0);
+    BLOCK_APPEND(cb, BLOCK_DATA(map), BLOCK_SIZE(map));
+
+    /* rec encoding map */
+    mc = 0;
+    BLOCK_SIZE(map) = 0;
+    if (h->codecs[DS_BF]) {
+        if (-1 == h->codecs[DS_BF]->store(h->codecs[DS_BF], map, "BF",
+                                          fd->version))
+            return NULL;
+        mc++;
+    }
+    if (h->codecs[DS_CF]) {
+        if (-1 == h->codecs[DS_CF]->store(h->codecs[DS_CF], map, "CF",
+                                          fd->version))
+            return NULL;
+        mc++;
+    }
+    if (h->codecs[DS_RL]) {
+        if (-1 == h->codecs[DS_RL]->store(h->codecs[DS_RL], map, "RL",
+                                          fd->version))
+            return NULL;
+        mc++;
+    }
+    if (h->codecs[DS_AP]) {
+        if (-1 == h->codecs[DS_AP]->store(h->codecs[DS_AP], map, "AP",
+                                          fd->version))
+            return NULL;
+        mc++;
+    }
+    if (h->codecs[DS_RG]) {
+        if (-1 == h->codecs[DS_RG]->store(h->codecs[DS_RG], map, "RG",
+                                          fd->version))
+            return NULL;
+        mc++;
+    }
+    if (h->codecs[DS_MF]) {
+        if (-1 == h->codecs[DS_MF]->store(h->codecs[DS_MF], map, "MF",
+                                          fd->version))
+            return NULL;
+        mc++;
+    }
+    if (h->codecs[DS_NS]) {
+        if (-1 == h->codecs[DS_NS]->store(h->codecs[DS_NS], map, "NS",
+                                          fd->version))
+            return NULL;
+        mc++;
+    }
+    if (h->codecs[DS_NP]) {
+        if (-1 == h->codecs[DS_NP]->store(h->codecs[DS_NP], map, "NP",
+                                          fd->version))
+            return NULL;
+        mc++;
+    }
+    if (h->codecs[DS_TS]) {
+        if (-1 == h->codecs[DS_TS]->store(h->codecs[DS_TS], map, "TS",
+                                          fd->version))
+            return NULL;
+        mc++;
+    }
+    if (h->codecs[DS_NF]) {
+        if (-1 == h->codecs[DS_NF]->store(h->codecs[DS_NF], map, "NF",
+                                          fd->version))
+            return NULL;
+        mc++;
+    }
+    if (h->codecs[DS_TC]) {
+        if (-1 == h->codecs[DS_TC]->store(h->codecs[DS_TC], map, "TC",
+                                          fd->version))
+            return NULL;
+        mc++;
+    }
+    if (h->codecs[DS_TN]) {
+        if (-1 == h->codecs[DS_TN]->store(h->codecs[DS_TN], map, "TN",
+                                          fd->version))
+            return NULL;
+        mc++;
+    }
+    if (h->codecs[DS_TL]) {
+        if (-1 == h->codecs[DS_TL]->store(h->codecs[DS_TL], map, "TL",
+                                          fd->version))
+            return NULL;
+        mc++;
+    }
+    if (h->codecs[DS_FN]) {
+        if (-1 == h->codecs[DS_FN]->store(h->codecs[DS_FN], map, "FN",
+                                          fd->version))
+            return NULL;
+        mc++;
+    }
+    if (h->codecs[DS_FC]) {
+        if (-1 == h->codecs[DS_FC]->store(h->codecs[DS_FC], map, "FC",
+                                          fd->version))
+            return NULL;
+        mc++;
+    }
+    if (h->codecs[DS_FP]) {
+        if (-1 == h->codecs[DS_FP]->store(h->codecs[DS_FP], map, "FP",
+                                          fd->version))
+            return NULL;
+        mc++;
+    }
+    if (h->codecs[DS_BS]) {
+        if (-1 == h->codecs[DS_BS]->store(h->codecs[DS_BS], map, "BS",
+                                          fd->version))
+            return NULL;
+        mc++;
+    }
+    if (h->codecs[DS_IN]) {
+        if (-1 == h->codecs[DS_IN]->store(h->codecs[DS_IN], map, "IN",
+                                          fd->version))
+            return NULL;
+        mc++;
+    }
+    if (h->codecs[DS_DL]) {
+        if (-1 == h->codecs[DS_DL]->store(h->codecs[DS_DL], map, "DL",
+                                          fd->version))
+            return NULL;
+        mc++;
+    }
+    if (h->codecs[DS_BA]) {
+        if (-1 == h->codecs[DS_BA]->store(h->codecs[DS_BA], map, "BA",
+                                          fd->version))
+            return NULL;
+        mc++;
+    }
+    if (h->codecs[DS_BB]) {
+        if (-1 == h->codecs[DS_BB]->store(h->codecs[DS_BB], map, "BB",
+                                          fd->version))
+            return NULL;
+        mc++;
+    }
+    if (h->codecs[DS_MQ]) {
+        if (-1 == h->codecs[DS_MQ]->store(h->codecs[DS_MQ], map, "MQ",
+                                          fd->version))
+            return NULL;
+        mc++;
+    }
+    if (h->codecs[DS_RN]) {
+        if (-1 == h->codecs[DS_RN]->store(h->codecs[DS_RN], map, "RN",
+                                          fd->version))
+            return NULL;
+        mc++;
+    }
+    if (h->codecs[DS_QS]) {
+        if (-1 == h->codecs[DS_QS]->store(h->codecs[DS_QS], map, "QS",
+                                          fd->version))
+            return NULL;
+        mc++;
+    }
+    if (h->codecs[DS_QQ]) {
+        if (-1 == h->codecs[DS_QQ]->store(h->codecs[DS_QQ], map, "QQ",
+                                          fd->version))
+            return NULL;
+        mc++;
+    }
+    if (h->codecs[DS_RI]) {
+        if (-1 == h->codecs[DS_RI]->store(h->codecs[DS_RI], map, "RI",
+                                          fd->version))
+            return NULL;
+        mc++;
+    }
+    if (CRAM_MAJOR_VERS(fd->version) != 1) {
+        if (h->codecs[DS_SC]) {
+            if (-1 == h->codecs[DS_SC]->store(h->codecs[DS_SC], map, "SC",
+                                              fd->version))
+                return NULL;
+            mc++;
+        }
+        if (h->codecs[DS_RS]) {
+            if (-1 == h->codecs[DS_RS]->store(h->codecs[DS_RS], map, "RS",
+                                              fd->version))
+                return NULL;
+            mc++;
+        }
+        if (h->codecs[DS_PD]) {
+            if (-1 == h->codecs[DS_PD]->store(h->codecs[DS_PD], map, "PD",
+                                              fd->version))
+                return NULL;
+            mc++;
+        }
+        if (h->codecs[DS_HC]) {
+            if (-1 == h->codecs[DS_HC]->store(h->codecs[DS_HC], map, "HC",
+                                              fd->version))
+                return NULL;
+            mc++;
+        }
+    }
+    if (h->codecs[DS_TM]) {
+        if (-1 == h->codecs[DS_TM]->store(h->codecs[DS_TM], map, "TM",
+                                          fd->version))
+            return NULL;
+        mc++;
+    }
+    if (h->codecs[DS_TV]) {
+        if (-1 == h->codecs[DS_TV]->store(h->codecs[DS_TV], map, "TV",
+                                          fd->version))
+            return NULL;
+        mc++;
+    }
+    r |= (fd->vv.varint_put32_blk(cb, BLOCK_SIZE(map) + fd->vv.varint_size(mc)) <= 0);
+    r |= (fd->vv.varint_put32_blk(cb, mc) <= 0);
+    BLOCK_APPEND(cb, BLOCK_DATA(map), BLOCK_SIZE(map));
+
+    /* tag encoding map */
+    mc = 0;
+    BLOCK_SIZE(map) = 0;
+    if (c->tags_used) {
+        khint_t k;
+
+        for (k = kh_begin(c->tags_used); k != kh_end(c->tags_used); k++) {
+            int key;
+            if (!kh_exist(c->tags_used, k))
+                continue;
+
+            key = kh_key(c->tags_used, k);
+            cram_codec *cd = kh_val(c->tags_used, k)->codec;
+
+            r |= (fd->vv.varint_put32_blk(map, key) <= 0);
+            if (-1 == cd->store(cd, map, NULL, fd->version))
+                return NULL;
+
+            mc++;
+        }
+    }
+
+    r |= (fd->vv.varint_put32_blk(cb, BLOCK_SIZE(map) + fd->vv.varint_size(mc)) <= 0);
+    r |= (fd->vv.varint_put32_blk(cb, mc) <= 0);
+    BLOCK_APPEND(cb, BLOCK_DATA(map), BLOCK_SIZE(map));
+
+    hts_log_info("Wrote compression block header in %d bytes", (int)BLOCK_SIZE(cb));
+
+    BLOCK_UPLEN(cb);
+
+    cram_free_block(map);
+
+    if (r >= 0)
+        return cb;
+
+ block_err:
+    return NULL;
+}
+
+
+/*
+ * Encodes a slice compression header.
+ *
+ * Returns cram_block on success
+ *         NULL on failure
+ */
+cram_block *cram_encode_slice_header(cram_fd *fd, cram_slice *s) {
+    char *buf;
+    char *cp;
+    cram_block *b = cram_new_block(MAPPED_SLICE, 0);
+    int j;
+
+    if (!b)
+        return NULL;
+
+    cp = buf = malloc(22+16+5*(8+s->hdr->num_blocks));
+    if (NULL == buf) {
+        cram_free_block(b);
+        return NULL;
+    }
+
+    cp += fd->vv.varint_put32s(cp, NULL, s->hdr->ref_seq_id);
+    if (CRAM_MAJOR_VERS(fd->version) >= 4) {
+        cp += fd->vv.varint_put64(cp, NULL, s->hdr->ref_seq_start);
+        cp += fd->vv.varint_put64(cp, NULL, s->hdr->ref_seq_span);
+    } else {
+        if (s->hdr->ref_seq_start < 0 || s->hdr->ref_seq_start > INT_MAX) {
+            hts_log_error("Reference position too large for CRAM 3");
+            cram_free_block(b);
+            free(buf);
+            return NULL;
+        }
+        cp += fd->vv.varint_put32(cp, NULL, s->hdr->ref_seq_start);
+        cp += fd->vv.varint_put32(cp, NULL, s->hdr->ref_seq_span);
+    }
+    cp += fd->vv.varint_put32(cp, NULL, s->hdr->num_records);
+    if (CRAM_MAJOR_VERS(fd->version) == 2)
+        cp += fd->vv.varint_put32(cp, NULL, s->hdr->record_counter);
+    else if (CRAM_MAJOR_VERS(fd->version) >= 3)
+        cp += fd->vv.varint_put64(cp, NULL, s->hdr->record_counter);
+    cp += fd->vv.varint_put32(cp, NULL, s->hdr->num_blocks);
+    cp += fd->vv.varint_put32(cp, NULL, s->hdr->num_content_ids);
+    for (j = 0; j < s->hdr->num_content_ids; j++) {
+        cp += fd->vv.varint_put32(cp, NULL, s->hdr->block_content_ids[j]);
+    }
+    if (s->hdr->content_type == MAPPED_SLICE)
+        cp += fd->vv.varint_put32(cp, NULL, s->hdr->ref_base_id);
+
+    if (CRAM_MAJOR_VERS(fd->version) != 1) {
+        memcpy(cp, s->hdr->md5, 16); cp += 16;
+    }
+
+    assert(cp-buf <= 22+16+5*(8+s->hdr->num_blocks));
+
+    b->data = (unsigned char *)buf;
+    b->comp_size = b->uncomp_size = cp-buf;
+
+    return b;
+}
+
+
+/*
+ * Encodes a single read.
+ *
+ * Returns 0 on success
+ *        -1 on failure
+ */
+static int cram_encode_slice_read(cram_fd *fd,
+                                  cram_container *c,
+                                  cram_block_compression_hdr *h,
+                                  cram_slice *s,
+                                  cram_record *cr,
+                                  int64_t *last_pos) {
+    int r = 0;
+    int32_t i32;
+    int64_t i64;
+    unsigned char uc;
+
+    //fprintf(stderr, "Encode seq %d, %d/%d FN=%d, %s\n", rec, core->byte, core->bit, cr->nfeature, s->name_ds->str + cr->name);
+
+    //printf("BF=0x%x\n", cr->flags);
+    //      bf = cram_flag_swap[cr->flags];
+    i32 = fd->cram_flag_swap[cr->flags & 0xfff];
+    r |= h->codecs[DS_BF]->encode(s, h->codecs[DS_BF], (char *)&i32, 1);
+
+    i32 = cr->cram_flags & CRAM_FLAG_MASK;
+    r |= h->codecs[DS_CF]->encode(s, h->codecs[DS_CF], (char *)&i32, 1);
+
+    if (CRAM_MAJOR_VERS(fd->version) != 1 && s->hdr->ref_seq_id == -2)
+        r |= h->codecs[DS_RI]->encode(s, h->codecs[DS_RI], (char *)&cr->ref_id, 1);
+
+    r |= h->codecs[DS_RL]->encode(s, h->codecs[DS_RL], (char *)&cr->len, 1);
+
+    if (c->pos_sorted) {
+        if (CRAM_MAJOR_VERS(fd->version) >= 4) {
+            i64 = cr->apos - *last_pos;
+            r |= h->codecs[DS_AP]->encode(s, h->codecs[DS_AP], (char *)&i64, 1);
+        } else {
+            i32 = cr->apos - *last_pos;
+            r |= h->codecs[DS_AP]->encode(s, h->codecs[DS_AP], (char *)&i32, 1);
+        }
+        *last_pos = cr->apos;
+    } else {
+        if (CRAM_MAJOR_VERS(fd->version) >= 4) {
+            i64 = cr->apos;
+            r |= h->codecs[DS_AP]->encode(s, h->codecs[DS_AP], (char *)&i64, 1);
+        } else {
+            i32 = cr->apos;
+            r |= h->codecs[DS_AP]->encode(s, h->codecs[DS_AP], (char *)&i32, 1);
+        }
+    }
+
+    r |= h->codecs[DS_RG]->encode(s, h->codecs[DS_RG], (char *)&cr->rg, 1);
+
+    if (cr->cram_flags & CRAM_FLAG_DETACHED) {
+        i32 = cr->mate_flags;
+        r |= h->codecs[DS_MF]->encode(s, h->codecs[DS_MF], (char *)&i32, 1);
+
+        r |= h->codecs[DS_NS]->encode(s, h->codecs[DS_NS],
+                                      (char *)&cr->mate_ref_id, 1);
+
+        if (CRAM_MAJOR_VERS(fd->version) >= 4) {
+            r |= h->codecs[DS_NP]->encode(s, h->codecs[DS_NP],
+                                          (char *)&cr->mate_pos, 1);
+            r |= h->codecs[DS_TS]->encode(s, h->codecs[DS_TS],
+                                          (char *)&cr->tlen, 1);
+        } else {
+            i32 = cr->mate_pos;
+            r |= h->codecs[DS_NP]->encode(s, h->codecs[DS_NP],
+                                          (char *)&i32, 1);
+            i32 = cr->tlen;
+            r |= h->codecs[DS_TS]->encode(s, h->codecs[DS_TS],
+                                          (char *)&i32, 1);
+        }
+    } else {
+        if (cr->cram_flags & CRAM_FLAG_MATE_DOWNSTREAM) {
+            r |= h->codecs[DS_NF]->encode(s, h->codecs[DS_NF],
+                                          (char *)&cr->mate_line, 1);
+        }
+        if (cr->cram_flags & CRAM_FLAG_EXPLICIT_TLEN) {
+            if (CRAM_MAJOR_VERS(fd->version) >= 4) {
+                r |= h->codecs[DS_TS]->encode(s, h->codecs[DS_TS],
+                                              (char *)&cr->tlen, 1);
+            }
+        }
+    }
+
+    /* Aux tags */
+    if (CRAM_MAJOR_VERS(fd->version) == 1) {
+        int j;
+        uc = cr->ntags;
+        r |= h->codecs[DS_TC]->encode(s, h->codecs[DS_TC], (char *)&uc, 1);
+
+        for (j = 0; j < cr->ntags; j++) {
+            uint32_t i32 = s->TN[cr->TN_idx + j]; // id
+            r |= h->codecs[DS_TN]->encode(s, h->codecs[DS_TN], (char *)&i32, 1);
+        }
+    } else {
+        r |= h->codecs[DS_TL]->encode(s, h->codecs[DS_TL], (char *)&cr->TL, 1);
+    }
+
+    // qual
+    // QS codec : Already stored in block[2].
+
+    // features (diffs)
+    if (!(cr->flags & BAM_FUNMAP)) {
+        int prev_pos = 0, j;
+
+        r |= h->codecs[DS_FN]->encode(s, h->codecs[DS_FN],
+                                      (char *)&cr->nfeature, 1);
+        for (j = 0; j < cr->nfeature; j++) {
+            cram_feature *f = &s->features[cr->feature + j];
+
+            uc = f->X.code;
+            r |= h->codecs[DS_FC]->encode(s, h->codecs[DS_FC], (char *)&uc, 1);
+            i32 = f->X.pos - prev_pos;
+            r |= h->codecs[DS_FP]->encode(s, h->codecs[DS_FP], (char *)&i32, 1);
+            prev_pos = f->X.pos;
+
+            switch(f->X.code) {
+                //char *seq;
+
+            case 'X':
+                //fprintf(stderr, "    FC=%c FP=%d base=%d\n", f->X.code, i32, f->X.base);
+
+                uc = f->X.base;
+                r |= h->codecs[DS_BS]->encode(s, h->codecs[DS_BS],
+                                              (char *)&uc, 1);
+                break;
+            case 'S':
+                // Already done
+                //r |= h->codecs[DS_SC]->encode(s, h->codecs[DS_SC],
+                //                              BLOCK_DATA(s->soft_blk) + f->S.seq_idx,
+                //                              f->S.len);
+
+                //if (CRAM_MAJOR_VERS(fd->version) >= 3) {
+                //    r |= h->codecs[DS_BB]->encode(s, h->codecs[DS_BB],
+                //                                  BLOCK_DATA(s->seqs_blk) + f->S.seq_idx,
+                //                                  f->S.len);
+                //}
+                break;
+            case 'I':
+                //seq = DSTRING_STR(s->seqs_ds) + f->S.seq_idx;
+                //r |= h->codecs[DS_IN]->encode(s, h->codecs[DS_IN],
+                //                           seq, f->S.len);
+                //if (CRAM_MAJOR_VERS(fd->version) >= 3) {
+                //    r |= h->codecs[DS_BB]->encode(s, h->codecs[DS_BB],
+                //                                  BLOCK_DATA(s->seqs_blk) + f->I.seq_idx,
+                //                                  f->I.len);
+                //}
+                break;
+            case 'i':
+                uc = f->i.base;
+                r |= h->codecs[DS_BA]->encode(s, h->codecs[DS_BA],
+                                              (char *)&uc, 1);
+                //seq = DSTRING_STR(s->seqs_ds) + f->S.seq_idx;
+                //r |= h->codecs[DS_IN]->encode(s, h->codecs[DS_IN],
+                //                           seq, 1);
+                break;
+            case 'D':
+                i32 = f->D.len;
+                r |= h->codecs[DS_DL]->encode(s, h->codecs[DS_DL],
+                                              (char *)&i32, 1);
+                break;
+
+            case 'B':
+                //                  // Used when we try to store a non ACGTN base or an N
+                //                  // that aligns against a non ACGTN reference
+
+                uc  = f->B.base;
+                r |= h->codecs[DS_BA]->encode(s, h->codecs[DS_BA],
+                                              (char *)&uc, 1);
+
+                //                  Already added
+                //                  uc  = f->B.qual;
+                //                  r |= h->codecs[DS_QS]->encode(s, h->codecs[DS_QS],
+                //                                           (char *)&uc, 1);
+                break;
+
+            case 'b':
+                // string of bases
+                r |= h->codecs[DS_BB]->encode(s, h->codecs[DS_BB],
+                                              (char *)BLOCK_DATA(s->seqs_blk)
+                                                      + f->b.seq_idx,
+                                              f->b.len);
+                break;
+
+            case 'Q':
+                //                  Already added
+                //                  uc  = f->B.qual;
+                //                  r |= h->codecs[DS_QS]->encode(s, h->codecs[DS_QS],
+                //                                           (char *)&uc, 1);
+                break;
+
+            case 'N':
+                i32 = f->N.len;
+                r |= h->codecs[DS_RS]->encode(s, h->codecs[DS_RS],
+                                              (char *)&i32, 1);
+                break;
+
+            case 'P':
+                i32 = f->P.len;
+                r |= h->codecs[DS_PD]->encode(s, h->codecs[DS_PD],
+                                              (char *)&i32, 1);
+                break;
+
+            case 'H':
+                i32 = f->H.len;
+                r |= h->codecs[DS_HC]->encode(s, h->codecs[DS_HC],
+                                              (char *)&i32, 1);
+                break;
+
+
+            default:
+                hts_log_error("Unhandled feature code %c", f->X.code);
+                return -1;
+            }
+        }
+
+        r |= h->codecs[DS_MQ]->encode(s, h->codecs[DS_MQ],
+                                      (char *)&cr->mqual, 1);
+    } else {
+        char *seq = (char *)BLOCK_DATA(s->seqs_blk) + cr->seq;
+        if (cr->len)
+            r |= h->codecs[DS_BA]->encode(s, h->codecs[DS_BA], seq, cr->len);
+    }
+
+    return r ? -1 : 0;
+}
+
+
+/*
+ * Applies various compression methods to specific blocks, depending on
+ * known observations of how data series compress.
+ *
+ * Returns 0 on success
+ *        -1 on failure
+ */
+static int cram_compress_slice(cram_fd *fd, cram_container *c, cram_slice *s) {
+    int level = fd->level, i;
+    int method = 1<<GZIP | 1<<GZIP_RLE, methodF = method;
+    int v31_or_above = (fd->version >= (3<<8)+1);
+
+    /* Compress the CORE Block too, with minimal zlib level */
+    if (level > 5 && s->block[0]->uncomp_size > 500)
+        cram_compress_block2(fd, s, s->block[0], NULL, 1<<GZIP, 1);
+
+    if (fd->use_bz2)
+        method |= 1<<BZIP2;
+
+    int method_rans   = (1<<RANS0) | (1<<RANS1);
+    int method_ranspr = method_rans;
+
+    if (fd->use_rans) {
+        method_ranspr = (1<<RANS_PR0)   | (1<<RANS_PR1);
+        if (level > 1)
+            method_ranspr |=
+                  (1<<RANS_PR64)  | (1<<RANS_PR9)
+                | (1<<RANS_PR128) | (1<<RANS_PR193);
+        if (level > 5)
+            method_ranspr |= (1<<RANS_PR129) | (1<<RANS_PR192);
+    }
+
+    if (fd->use_rans) {
+        methodF |= v31_or_above ? method_ranspr : method_rans;
+        method  |= v31_or_above ? method_ranspr : method_rans;
+    }
+
+    int method_arith   = 0;
+    if (fd->use_arith) {
+        method_arith = (1<<ARITH_PR0)   | (1<<ARITH_PR1);
+        if (level > 1)
+            method_arith |=
+                  (1<<ARITH_PR64)  | (1<<ARITH_PR9)
+                | (1<<ARITH_PR128) | (1<<ARITH_PR129)
+                | (1<<ARITH_PR192) | (1u<<ARITH_PR193);
+    }
+    if (fd->use_arith && v31_or_above) {
+        methodF |= method_arith;
+        method  |= method_arith;
+    }
+
+    if (fd->use_lzma)
+        method |= (1<<LZMA);
+
+    /* Faster method for data series we only need entropy encoding on */
+    methodF = method & ~(1<<GZIP | 1<<BZIP2 | 1<<LZMA);
+    if (level >= 5) {
+        method |= 1<<GZIP_1;
+        methodF = method;
+    }
+    if (level == 1) {
+        method &= ~(1<<GZIP);
+        method |=   1<<GZIP_1;
+        methodF = method;
+    }
+
+    int qmethod  = method;
+    int qmethodF = method;
+    if (v31_or_above && fd->use_fqz) {
+        qmethod  |= 1<<FQZ;
+        qmethodF |= 1<<FQZ;
+        if (fd->level > 4) {
+            qmethod  |= 1<<FQZ_b;
+            qmethodF |= 1<<FQZ_b;
+        }
+        if (fd->level > 6) {
+            qmethod  |= (1<<FQZ_c) | (1<<FQZ_d);
+            qmethodF |= (1<<FQZ_c) | (1<<FQZ_d);
+        }
+    }
+
+    pthread_mutex_lock(&fd->metrics_lock);
+    for (i = 0; i < DS_END; i++)
+        if (c->stats[i] && c->stats[i]->nvals > 16)
+            fd->m[i]->unpackable = 1;
+    pthread_mutex_unlock(&fd->metrics_lock);
+
+    /* Specific compression methods for certain block types */
+    if (cram_compress_block2(fd, s, s->block[DS_IN], fd->m[DS_IN], //IN (seq)
+                             method, level))
+        return -1;
+
+    if (fd->level == 0) {
+        /* Do nothing */
+    } else if (fd->level == 1) {
+        if (cram_compress_block2(fd, s, s->block[DS_QS], fd->m[DS_QS],
+                                 qmethodF, 1))
+            return -1;
+        for (i = DS_aux; i <= DS_aux_oz; i++) {
+            if (s->block[i])
+                if (cram_compress_block2(fd, s, s->block[i], fd->m[i],
+                                         method, 1))
+                    return -1;
+        }
+    } else if (fd->level < 3) {
+        if (cram_compress_block2(fd, s, s->block[DS_QS], fd->m[DS_QS],
+                                 qmethod, 1))
+            return -1;
+        if (cram_compress_block2(fd, s, s->block[DS_BA], fd->m[DS_BA],
+                                 method, 1))
+            return -1;
+        if (s->block[DS_BB])
+            if (cram_compress_block2(fd, s, s->block[DS_BB], fd->m[DS_BB],
+                                     method, 1))
+                return -1;
+        for (i = DS_aux; i <= DS_aux_oz; i++) {
+            if (s->block[i])
+                if (cram_compress_block2(fd, s, s->block[i], fd->m[i],
+                                         method, level))
+                    return -1;
+        }
+    } else {
+        if (cram_compress_block2(fd, s, s->block[DS_QS], fd->m[DS_QS],
+                                 qmethod, level))
+            return -1;
+        if (cram_compress_block2(fd, s, s->block[DS_BA], fd->m[DS_BA],
+                                 method, level))
+            return -1;
+        if (s->block[DS_BB])
+            if (cram_compress_block2(fd, s, s->block[DS_BB], fd->m[DS_BB],
+                                     method, level))
+                return -1;
+        for (i = DS_aux; i <= DS_aux_oz; i++) {
+            if (s->block[i])
+                if (cram_compress_block2(fd, s, s->block[i], fd->m[i],
+                                         method, level))
+                    return -1;
+        }
+    }
+
+    // NAME: best is generally xz, bzip2, zlib then rans1
+    int method_rn = method & ~(method_rans | method_ranspr | 1<<GZIP_RLE);
+    if (fd->version >= (3<<8)+1 && fd->use_tok)
+        method_rn |= fd->use_arith ? (1<<TOKA) : (1<<TOK3);
+    if (cram_compress_block2(fd, s, s->block[DS_RN], fd->m[DS_RN],
+                             method_rn, level))
+        return -1;
+
+    // NS shows strong local correlation as rearrangements are localised
+    if (s->block[DS_NS] && s->block[DS_NS] != s->block[0])
+        if (cram_compress_block2(fd, s, s->block[DS_NS], fd->m[DS_NS],
+                                 method, level))
+            return -1;
+
+
+    /*
+     * Compress any auxiliary tags with their own per-tag metrics
+     */
+    {
+        int i;
+        for (i = DS_END /*num_blk - naux_blk*/; i < s->hdr->num_blocks; i++) {
+            if (!s->block[i] || s->block[i] == s->block[0])
+                continue;
+
+            if (s->block[i]->method != RAW)
+                continue;
+
+            if (cram_compress_block2(fd, s, s->block[i], s->block[i]->m,
+                                     method, level))
+                return -1;
+        }
+    }
+
+    /*
+     * Minimal compression of any block still uncompressed, bar CORE
+     */
+    {
+        int i;
+        for (i = 1; i < s->hdr->num_blocks && i < DS_END; i++) {
+            if (!s->block[i] || s->block[i] == s->block[0])
+                continue;
+
+            if (s->block[i]->method != RAW)
+                continue;
+
+            if (cram_compress_block2(fd, s, s->block[i], fd->m[i],
+                                    methodF, level))
+                return -1;
+        }
+    }
+
+    return 0;
+}
+
+/*
+ * Allocates a block associated with the cram codec associated with
+ * data series ds_id or the internal codec_id (depending on codec
+ * type).
+ *
+ * The ds_ids are what end up written to disk as an external block.
+ * The c_ids are internal and used when daisy-chaining transforms
+ * such as MAP and RLE.  These blocks are also allocated, but
+ * are ephemeral in nature.  (The codecs themselves cannot allocate
+ * these as the same codec pointer may be operating on multiple slices
+ * if we're using a multi-slice container.)
+ *
+ * Returns 0 on success
+ *        -1 on failure
+ */
+static int cram_allocate_block(cram_codec *codec, cram_slice *s, int ds_id) {
+    if (!codec)
+        return 0;
+
+    switch(codec->codec) {
+    // Codecs which are hard-coded to use the CORE block
+    case E_GOLOMB:
+    case E_HUFFMAN:
+    case E_BETA:
+    case E_SUBEXP:
+    case E_GOLOMB_RICE:
+    case E_GAMMA:
+        codec->out = s->block[0];
+        break;
+
+    // Codecs which don't use external blocks
+    case E_CONST_BYTE:
+    case E_CONST_INT:
+       codec->out = NULL;
+       break;
+
+    // Codecs that emit directly to external blocks
+    case E_EXTERNAL:
+    case E_VARINT_UNSIGNED:
+    case E_VARINT_SIGNED:
+        if (!(s->block[ds_id] = cram_new_block(EXTERNAL, ds_id)))
+            return -1;
+        codec->u.external.content_id = ds_id;
+        codec->out = s->block[ds_id];
+        break;
+
+    case E_BYTE_ARRAY_STOP: // Why no sub-codec?
+        if (!(s->block[ds_id] = cram_new_block(EXTERNAL, ds_id)))
+            return -1;
+        codec->u.byte_array_stop.content_id = ds_id;
+        codec->out = s->block[ds_id];
+        break;
+
+
+    // Codecs that contain sub-codecs which may in turn emit to external blocks
+    case E_BYTE_ARRAY_LEN: {
+        cram_codec *bal = codec->u.e_byte_array_len.len_codec;
+        if (cram_allocate_block(bal, s, bal->u.external.content_id))
+            return -1;
+        bal = codec->u.e_byte_array_len.val_codec;
+        if (cram_allocate_block(bal, s, bal->u.external.content_id))
+            return -1;
+
+        break;
+    }
+
+    case E_XRLE:
+        if (cram_allocate_block(codec->u.e_xrle.len_codec, s, ds_id))
+                                //ds_id == DS_QS ? DS_QS_len : ds_id))
+            return -1;
+        if (cram_allocate_block(codec->u.e_xrle.lit_codec, s, ds_id))
+            return -1;
+
+        break;
+
+    case E_XPACK:
+        if (cram_allocate_block(codec->u.e_xpack.sub_codec, s, ds_id))
+            return -1;
+        codec->out = cram_new_block(0, 0); // ephemeral
+        if (!codec->out)
+            return -1;
+
+        break;
+
+    case E_XDELTA:
+        if (cram_allocate_block(codec->u.e_xdelta.sub_codec, s, ds_id))
+            return -1;
+        codec->out = cram_new_block(0, 0); // ephemeral
+        if (!codec->out)
+            return -1;
+
+        break;
+
+    default:
+        break;
+    }
+
+    return 0;
+}
+
+/*
+ * Encodes a single slice from a container
+ *
+ * Returns 0 on success
+ *        -1 on failure
+ */
+static int cram_encode_slice(cram_fd *fd, cram_container *c,
+                             cram_block_compression_hdr *h, cram_slice *s,
+                             int embed_ref) {
+    int rec, r = 0;
+    int64_t last_pos;
+    enum cram_DS_ID id;
+
+    /*
+     * Slice external blocks:
+     * ID 0 => base calls (insertions, soft-clip)
+     * ID 1 => qualities
+     * ID 2 => names
+     * ID 3 => TS (insert size), NP (next frag)
+     * ID 4 => tag values
+     * ID 6 => tag IDs (TN), if CRAM_V1.0
+     * ID 7 => TD tag dictionary, if !CRAM_V1.0
+     */
+
+    /* Create cram slice header */
+    s->hdr->ref_base_id = embed_ref>0 && s->hdr->ref_seq_span > 0
+        ? DS_ref
+        : (CRAM_MAJOR_VERS(fd->version) >= 4 ? 0 : -1);
+    s->hdr->record_counter = c->num_records + c->record_counter;
+    c->num_records += s->hdr->num_records;
+
+    int ntags = c->tags_used ? c->tags_used->n_occupied : 0;
+    s->block = calloc(DS_END + ntags*2, sizeof(s->block[0]));
+    s->hdr->block_content_ids = malloc(DS_END * sizeof(int32_t));
+    if (!s->block || !s->hdr->block_content_ids)
+        return -1;
+
+    // Create first fixed blocks, always external.
+    // CORE
+    if (!(s->block[0] = cram_new_block(CORE, 0)))
+        return -1;
+
+    // TN block for CRAM v1
+    if (CRAM_MAJOR_VERS(fd->version) == 1) {
+        if (h->codecs[DS_TN]->codec == E_EXTERNAL) {
+            if (!(s->block[DS_TN] = cram_new_block(EXTERNAL,DS_TN))) return -1;
+            h->codecs[DS_TN]->u.external.content_id = DS_TN;
+        } else {
+            s->block[DS_TN] = s->block[0];
+        }
+    }
+
+    // Embedded reference
+    if (embed_ref>0) {
+        if (!(s->block[DS_ref] = cram_new_block(EXTERNAL, DS_ref)))
+            return -1;
+        s->ref_id = DS_ref; // needed?
+        BLOCK_APPEND(s->block[DS_ref],
+                     c->ref + s->hdr->ref_seq_start - c->ref_start,
+                     s->hdr->ref_seq_span);
+    }
+
+    /*
+     * All the data-series blocks if appropriate.
+     */
+    for (id = DS_QS; id < DS_TN; id++) {
+        if (cram_allocate_block(h->codecs[id], s, id) < 0)
+            return -1;
+    }
+
+    /*
+     * Add in the external tag blocks too.
+     */
+    if (c->tags_used) {
+        int n;
+        s->hdr->num_blocks = DS_END;
+        for (n = 0; n < s->naux_block; n++) {
+            s->block[s->hdr->num_blocks++] = s->aux_block[n];
+            s->aux_block[n] = NULL;
+        }
+    }
+
+    /* Encode reads */
+    last_pos = s->hdr->ref_seq_start;
+    for (rec = 0; rec < s->hdr->num_records; rec++) {
+        cram_record *cr = &s->crecs[rec];
+        if (cram_encode_slice_read(fd, c, h, s, cr, &last_pos) == -1)
+            return -1;
+    }
+
+    s->block[0]->uncomp_size = s->block[0]->byte + (s->block[0]->bit < 7);
+    s->block[0]->comp_size = s->block[0]->uncomp_size;
+
+    // Make sure the fixed blocks point to the correct sources
+    if (s->block[DS_IN]) cram_free_block(s->block[DS_IN]);
+    s->block[DS_IN] = s->base_blk; s->base_blk = NULL;
+    if (s->block[DS_QS]) cram_free_block(s->block[DS_QS]);
+    s->block[DS_QS] = s->qual_blk; s->qual_blk = NULL;
+    if (s->block[DS_RN]) cram_free_block(s->block[DS_RN]);
+    s->block[DS_RN] = s->name_blk; s->name_blk = NULL;
+    if (s->block[DS_SC]) cram_free_block(s->block[DS_SC]);
+    s->block[DS_SC] = s->soft_blk; s->soft_blk = NULL;
+
+    // Finalise any data transforms.
+    for (id = DS_QS; id < DS_TN; id++) {
+       if (h->codecs[id] && h->codecs[id]->flush)
+           h->codecs[id]->flush(h->codecs[id]);
+    }
+
+    // Ensure block sizes are up to date.
+    for (id = 1; id < s->hdr->num_blocks; id++) {
+        if (!s->block[id] || s->block[id] == s->block[0])
+            continue;
+
+        if (s->block[id]->uncomp_size == 0)
+            BLOCK_UPLEN(s->block[id]);
+    }
+
+    // Compress it all
+    if (cram_compress_slice(fd, c, s) == -1)
+        return -1;
+
+    // Collapse empty blocks and create hdr_block
+    {
+        int i, j;
+
+        s->hdr->block_content_ids = realloc(s->hdr->block_content_ids,
+                                            s->hdr->num_blocks * sizeof(int32_t));
+        if (!s->hdr->block_content_ids)
+            return -1;
+
+        for (i = j = 1; i < s->hdr->num_blocks; i++) {
+            if (!s->block[i] || s->block[i] == s->block[0])
+                continue;
+            if (s->block[i]->uncomp_size == 0) {
+                cram_free_block(s->block[i]);
+                s->block[i] = NULL;
+                continue;
+            }
+            s->block[j] = s->block[i];
+            s->hdr->block_content_ids[j-1] = s->block[i]->content_id;
+            j++;
+        }
+        s->hdr->num_content_ids = j-1;
+        s->hdr->num_blocks = j;
+
+        if (!(s->hdr_block = cram_encode_slice_header(fd, s)))
+            return -1;
+    }
+
+    return r ? -1 : 0;
+
+ block_err:
+    return -1;
+}
+
+static inline const char *bam_data_end(bam1_t *b) {
+    return (const char *)b->data + b->l_data;
+}
+
+/*
+ * A bounds checking version of bam_aux2i.
+ */
+static inline int bam_aux2i_end(const uint8_t *aux, const uint8_t *aux_end) {
+    int type = *aux++;
+    switch (type) {
+        case 'c':
+            if (aux_end - aux < 1) {
+                errno = EINVAL;
+                return 0;
+            }
+            return *(int8_t *)aux;
+        case 'C':
+            if (aux_end - aux < 1) {
+                errno = EINVAL;
+                return 0;
+            }
+            return *aux;
+        case 's':
+            if (aux_end - aux < 2) {
+                errno = EINVAL;
+                return 0;
+            }
+            return le_to_i16(aux);
+        case 'S':
+            if (aux_end - aux < 2) {
+                errno = EINVAL;
+                return 0;
+            }
+            return le_to_u16(aux);
+        case 'i':
+            if (aux_end - aux < 4) {
+                errno = EINVAL;
+                return 0;
+            }
+            return le_to_i32(aux);
+        case 'I':
+            if (aux_end - aux < 4) {
+                errno = EINVAL;
+                return 0;
+            }
+            return le_to_u32(aux);
+        default:
+            errno = EINVAL;
+    }
+    return 0;
+}
+
+/*
+ * Returns the number of expected read names for this record.
+ */
+static int expected_template_count(bam_seq_t *b) {
+    int expected = bam_flag(b) & BAM_FPAIRED ? 2 : 1;
+
+    uint8_t *TC = (uint8_t *)bam_aux_get(b, "TC");
+    if (TC) {
+        int n = bam_aux2i_end(TC, (uint8_t *)bam_data_end(b));
+        if (expected < n)
+            expected = n;
+    }
+
+    if (!TC && bam_aux_get(b, "SA")) {
+        // We could count the semicolons, but we'd have to do this for
+        // read1, read2 and read(not-1-or-2) combining the results
+        // together.  This is a cheap and safe alternative for now.
+        expected = INT_MAX;
+    }
+
+    return expected;
+}
+
+/*
+ * Lossily reject read names.
+ *
+ * The rule here is that if all reads for this template reside in the
+ * same slice then we can lose the name.  Otherwise we keep them as we
+ * do not know when (or if) the other reads will turn up.
+ *
+ * Note there may be only 1 read (non-paired library) or more than 2
+ * reads (paired library with supplementary reads), or other weird
+ * setups.  We need to know how many are expected.  Ways to guess:
+ *
+ * - Flags (0x1 - has > 1 read)
+ * - TC aux field (not mandatory)
+ * - SA tags (count semicolons, NB per fragment so sum - hard)
+ * - RNEXT/PNEXT uniqueness count. (not implemented, tricky)
+ *
+ * Returns 0 on success
+ *        -1 on failure
+ */
+static int lossy_read_names(cram_fd *fd, cram_container *c, cram_slice *s,
+                            int bam_start) {
+    int r1, r2, ret = -1;
+
+    // Initialise cram_flags
+    for (r2 = 0; r2 < s->hdr->num_records; r2++)
+        s->crecs[r2].cram_flags = 0;
+
+    if (!fd->lossy_read_names)
+        return 0;
+
+    khash_t(m_s2u64) *names = kh_init(m_s2u64);
+    if (!names)
+        goto fail;
+
+    // 1: Iterate through names to count frequency
+    for (r1 = bam_start, r2 = 0; r2 < s->hdr->num_records; r1++, r2++) {
+        //cram_record *cr = &s->crecs[r2];
+        bam_seq_t *b = c->bams[r1];
+        khint_t k;
+        int n;
+        uint64_t e;
+        union {
+            uint64_t i64;
+            struct {
+                int32_t e,c; // expected & observed counts.
+            } counts;
+        } u;
+
+        e = expected_template_count(b);
+        u.counts.e = e; u.counts.c = 1;
+
+        k = kh_put(m_s2u64, names, bam_name(b), &n);
+        if (n == -1)
+            goto fail;
+
+        if (n == 0) {
+            // not a new name
+            u.i64 = kh_val(names, k);
+            if (u.counts.e != e) {
+                // different expectation or already hit the max
+                //fprintf(stderr, "Err computing no. %s recs\n", bam_name(b));
+                kh_val(names, k) = 0;
+            } else {
+                u.counts.c++;
+                if (u.counts.e == u.counts.c) {
+                    // Reached expected count.
+                    kh_val(names, k) = -1;
+                } else {
+                    kh_val(names, k) = u.i64;
+                }
+            }
+        } else {
+            // new name
+            kh_val(names, k) = u.i64;
+        }
+    }
+
+    // 2: Remove names if all present (hd.i == -1)
+    for (r1 = bam_start, r2 = 0; r2 < s->hdr->num_records; r1++, r2++) {
+        cram_record *cr = &s->crecs[r2];
+        bam_seq_t *b = c->bams[r1];
+        khint_t k;
+
+        k = kh_get(m_s2u64, names, bam_name(b));
+
+        if (k == kh_end(names))
+            goto fail;
+
+        if (kh_val(names, k) == -1)
+            cr->cram_flags = CRAM_FLAG_DISCARD_NAME;
+    }
+
+    ret = 0;
+ fail: // ret==-1
+
+    if (names)
+        kh_destroy(m_s2u64, names);
+
+    return ret;
+}
+
+/*
+ * Adds the reading names.  We do this here as a separate pass rather
+ * than per record in the process_one_read calls as that function can
+ * go back and change the CRAM_FLAG_DETACHED status of a previously
+ * processed read if it subsequently determines the TLEN field is
+ * incorrect.  Given DETACHED reads always try to decode read names,
+ * we need to know their status before generating the read-name block.
+ *
+ * Output is an update s->name_blk, and cr->name / cr->name_len
+ * fields.
+ */
+static int add_read_names(cram_fd *fd, cram_container *c, cram_slice *s,
+                          int bam_start) {
+    int r1, r2;
+    int keep_names = !fd->lossy_read_names;
+
+    for (r1 = bam_start, r2 = 0;
+         r1 < c->curr_c_rec && r2 < s->hdr->num_records;
+         r1++, r2++) {
+        cram_record *cr = &s->crecs[r2];
+        bam_seq_t *b = c->bams[r1];
+
+        cr->name        = BLOCK_SIZE(s->name_blk);
+        if ((cr->cram_flags & CRAM_FLAG_DETACHED) || keep_names) {
+            if (CRAM_MAJOR_VERS(fd->version) >= 4
+                && (cr->cram_flags & CRAM_FLAG_MATE_DOWNSTREAM)
+                && cr->mate_line) {
+                // Dedup read names in V4
+                BLOCK_APPEND(s->name_blk, "\0", 1);
+                cr->name_len    = 1;
+            } else {
+                BLOCK_APPEND(s->name_blk, bam_name(b), bam_name_len(b));
+                cr->name_len    = bam_name_len(b);
+            }
+        } else {
+            // Can only discard duplicate names if not detached
+            cr->name_len = 0;
+        }
+
+        if (cram_stats_add(c->stats[DS_RN], cr->name_len) < 0)
+            goto block_err;
+    }
+
+    return 0;
+
+ block_err:
+    return -1;
+}
+
+// CRAM version >= 3.1
+#define CRAM_ge31(v) ((v) >= 0x301)
+
+// Returns the next cigar op code: one of the BAM_C* codes,
+// or -1 if no more are present.
+static inline
+int next_cigar_op(uint32_t *cigar, uint32_t ncigar, int *skip, int *spos,
+                  uint32_t *cig_ind, uint32_t *cig_op, uint32_t *cig_len) {
+    for(;;) {
+        while (*cig_len == 0) {
+            if (*cig_ind < ncigar) {
+                *cig_op  = cigar[*cig_ind] & BAM_CIGAR_MASK;
+                *cig_len = cigar[*cig_ind] >> BAM_CIGAR_SHIFT;
+                (*cig_ind)++;
+            } else {
+                return -1;
+            }
+        }
+
+        if (skip[*cig_op]) {
+            *spos += (bam_cigar_type(*cig_op)&1) * *cig_len;
+            *cig_len = 0;
+            continue;
+        }
+
+        (*cig_len)--;
+        break;
+    }
+
+    return *cig_op;
+}
+
+// Ensure ref and hist are large enough.
+static inline int extend_ref(char **ref, uint32_t (**hist)[5], hts_pos_t pos,
+                             hts_pos_t ref_start, hts_pos_t *ref_end) {
+    if (pos < ref_start)
+        return -1;
+    if (pos < *ref_end)
+        return 0;
+
+    // realloc
+    if (pos - ref_start > UINT_MAX)
+        return -2; // protect overflow in new_end calculation
+
+    hts_pos_t old_end = *ref_end ? *ref_end : ref_start;
+    hts_pos_t new_end = ref_start + 1000 + (pos-ref_start)*1.5;
+
+    // Refuse to work on excessively large blocks.
+    // We'll just switch to referenceless encoding, which is probably better
+    // here as this must be very sparse data anyway.
+    if (new_end - ref_start > UINT_MAX/sizeof(**hist)/2)
+        return -2;
+
+    char *tmp = realloc(*ref, new_end-ref_start+1);
+    if (!tmp)
+        return -1;
+    *ref = tmp;
+
+    uint32_t (*tmp5)[5] = realloc(**hist,
+                                  (new_end - ref_start)*sizeof(**hist));
+    if (!tmp5)
+        return -1;
+    *hist = tmp5;
+    *ref_end = new_end;
+
+    // initialise
+    old_end -= ref_start;
+    new_end -= ref_start;
+    memset(&(*ref)[old_end],  0,  new_end-old_end);
+    memset(&(*hist)[old_end], 0, (new_end-old_end)*sizeof(**hist));
+
+    return 0;
+}
+
+// Walk through MD + seq to generate ref
+// Returns 1 on success, <0 on failure
+static int cram_add_to_ref_MD(bam1_t *b, char **ref, uint32_t (**hist)[5],
+                              hts_pos_t ref_start, hts_pos_t *ref_end,
+                              const uint8_t *MD) {
+    uint8_t *seq = bam_get_seq(b);
+    uint32_t *cigar = bam_get_cigar(b);
+    uint32_t ncigar = b->core.n_cigar;
+    uint32_t cig_op = 0, cig_len = 0, cig_ind = 0;
+
+    int iseq = 0, next_op;
+    hts_pos_t iref = b->core.pos - ref_start;
+
+    // Skip INS, REF_SKIP, *CLIP, PAD. and BACK.
+    static int cig_skip[16] = {0,1,0,1,1,1,1,0,0,1,1,1,1,1,1,1};
+    while (iseq < b->core.l_qseq && *MD) {
+        if (isdigit(*MD)) {
+            // match
+            int overflow = 0;
+            int len = hts_str2uint((char *)MD, (char **)&MD, 31, &overflow);
+            if (overflow ||
+                extend_ref(ref, hist, iref+ref_start + len,
+                           ref_start, ref_end) < 0)
+                return -1;
+            while (iseq < b->core.l_qseq && len) {
+                // rewrite to have internal loops?
+                if ((next_op = next_cigar_op(cigar, ncigar, cig_skip,
+                                             &iseq, &cig_ind, &cig_op,
+                                             &cig_len)) < 0)
+                    return -1;
+
+                if (next_op != BAM_CMATCH &&
+                    next_op != BAM_CEQUAL) {
+                    hts_log_info("MD:Z and CIGAR are incompatible for "
+                                 "record %s", bam_get_qname(b));
+                    return -1;
+                }
+
+                // Short-cut loop over same cigar op for efficiency
+                cig_len++;
+                do {
+                    cig_len--;
+                    (*ref)[iref++] = seq_nt16_str[bam_seqi(seq, iseq)];
+                    iseq++;
+                    len--;
+                } while (cig_len && iseq < b->core.l_qseq && len);
+            }
+            if (len > 0)
+                return -1; // MD is longer than seq
+        } else if (*MD == '^') {
+            // deletion
+            MD++;
+            while (isalpha(*MD)) {
+                if (extend_ref(ref, hist, iref+ref_start, ref_start,
+                               ref_end) < 0)
+                    return -1;
+                if ((next_op = next_cigar_op(cigar, ncigar, cig_skip,
+                                             &iseq, &cig_ind, &cig_op,
+                                             &cig_len)) < 0)
+                    return -1;
+
+                if (next_op != BAM_CDEL) {
+                    hts_log_info("MD:Z and CIGAR are incompatible");
+                    return -1;
+                }
+
+                (*ref)[iref++] = *MD++ & ~0x20;
+            }
+        } else {
+            // substitution
+            if (extend_ref(ref, hist, iref+ref_start, ref_start, ref_end) < 0)
+                return -1;
+            if ((next_op = next_cigar_op(cigar, ncigar, cig_skip,
+                                         &iseq, &cig_ind, &cig_op,
+                                         &cig_len)) < 0)
+                return -1;
+
+            if (next_op != BAM_CMATCH && next_op != BAM_CDIFF) {
+                hts_log_info("MD:Z and CIGAR are incompatible");
+                return -1;
+            }
+
+            (*ref)[iref++] = *MD++ & ~0x20;
+            iseq++;
+        }
+    }
+
+    return 1;
+}
+
+// Append a sequence to a ref/consensus structure.
+// We maintain both an absolute refefence (ACGTN where MD:Z is
+// present) and a 5-way frequency array for when no MD:Z is known.
+// We then subsequently convert the 5-way frequencies to a consensus
+// ref in a second pass.
+//
+// Returns >=0 on success,
+//         -1 on failure (eg inconsistent data)
+static int cram_add_to_ref(bam1_t *b, char **ref, uint32_t (**hist)[5],
+                           hts_pos_t ref_start, hts_pos_t *ref_end) {
+    const uint8_t *MD = bam_aux_get(b, "MD");
+    int ret = 0;
+    if (MD && *MD == 'Z') {
+        // We can use MD to directly compute the reference
+        int ret = cram_add_to_ref_MD(b, ref, hist, ref_start, ref_end, MD+1);
+
+        if (ret > 0)
+            return ret;
+    }
+
+    // Otherwise we just use SEQ+CIGAR and build a consensus which we later
+    // turn into a fake reference
+    uint32_t *cigar = bam_get_cigar(b);
+    uint32_t ncigar = b->core.n_cigar;
+    uint32_t i, j;
+    hts_pos_t iseq = 0, iref = b->core.pos - ref_start;
+    uint8_t *seq = bam_get_seq(b);
+    for (i = 0; i < ncigar; i++) {
+        switch (bam_cigar_op(cigar[i])) {
+        case BAM_CSOFT_CLIP:
+        case BAM_CINS:
+            iseq += bam_cigar_oplen(cigar[i]);
+            break;
+
+        case BAM_CMATCH:
+        case BAM_CEQUAL:
+        case BAM_CDIFF: {
+            int len = bam_cigar_oplen(cigar[i]);
+            // Maps an nt16 (A=1 C=2 G=4 T=8 bits) to 0123 plus N=4
+            static uint8_t L16[16] = {4,0,1,4, 2,4,4,4, 3,4,4,4, 4,4,4,4};
+
+            if (extend_ref(ref, hist, iref+ref_start + len,
+                           ref_start, ref_end) < 0)
+                return -1;
+            if (iseq + len <= b->core.l_qseq) {
+                // Nullify failed MD:Z if appropriate
+                if (ret < 0)
+                    memset(&(*ref)[iref], 0, len);
+
+                for (j = 0; j < len; j++, iref++, iseq++)
+                    (*hist)[iref][L16[bam_seqi(seq, iseq)]]++;
+            } else {
+                // Probably a 2ndary read with seq "*"
+                iseq += len;
+                iref += len;
+            }
+            break;
+        }
+
+        case BAM_CDEL:
+        case BAM_CREF_SKIP:
+            iref += bam_cigar_oplen(cigar[i]);
+        }
+    }
+
+    return 1;
+}
+
+// Automatically generates the reference and stashed it in c->ref, also
+// setting c->ref_start and c->ref_end.
+//
+// If we have MD:Z tags then we use them to directly infer the reference,
+// along with SEQ + CIGAR.  Otherwise we use SEQ/CIGAR only to build up
+// a consensus and then assume the reference as the majority rule.
+//
+// In this latter scenario we need to be wary of auto-generating MD and NM
+// during decode, but that's handled elsewhere via an additional aux tag.
+//
+// Returns 0 on success,
+//        -1 on failure
+static int cram_generate_reference(cram_container *c, cram_slice *s, int r1) {
+    // TODO: if we can find an external reference then use it, even if the
+    // user told us to do embed_ref=2.
+    char *ref = NULL;
+    uint32_t (*hist)[5] = NULL;
+    hts_pos_t ref_start = c->bams[r1]->core.pos, ref_end = 0;
+    if (ref_start < 0)
+        return -1; // cannot build consensus from unmapped data
+
+    // initial allocation
+    if (extend_ref(&ref, &hist,
+                   c->bams[r1 + s->hdr->num_records-1]->core.pos +
+                   c->bams[r1 + s->hdr->num_records-1]->core.l_qseq,
+                   ref_start, &ref_end) < 0)
+        return -1;
+
+    // Add each bam file to the reference/consensus arrays
+    int r2;
+    hts_pos_t last_pos = -1;
+    for (r2 = 0; r1 < c->curr_c_rec && r2 < s->hdr->num_records; r1++, r2++) {
+        if (c->bams[r1]->core.pos < last_pos) {
+            hts_log_error("Cannot build reference with unsorted data");
+            goto err;
+        }
+        last_pos = c->bams[r1]->core.pos;
+        if (cram_add_to_ref(c->bams[r1], &ref, &hist, ref_start, &ref_end) < 0)
+            goto err;
+    }
+
+    // Compute the consensus
+    hts_pos_t i;
+    for (i = 0; i < ref_end-ref_start; i++) {
+        if (!ref[i]) {
+            int max_v = 0, max_j = 4, j;
+            for (j = 0; j < 4; j++)
+                // don't call N (j==4) unless no coverage
+                if (max_v < hist[i][j])
+                    max_v = hist[i][j], max_j = j;
+            ref[i] = "ACGTN"[max_j];
+        }
+    }
+    free(hist);
+
+    // Put the reference in place so it appears to be an external
+    // ref file.
+    c->ref       = ref;
+    c->ref_start = ref_start+1;
+    c->ref_end   = ref_end+1;
+    c->ref_free  = 1;
+
+    return 0;
+
+ err:
+    free(ref);
+    free(hist);
+    return -1;
+}
+
+// Check if the SQ M5 tag matches the reference we've loaded.
+static int validate_md5(cram_fd *fd, int ref_id) {
+    if (fd->ignore_md5 || ref_id < 0 || ref_id >= fd->refs->nref)
+        return 0;
+
+    // Have we already checked this ref?
+    if (fd->refs->ref_id[ref_id]->validated_md5)
+        return 0;
+
+    // Check if we have the MD5 known.
+    // We should, but maybe we're using embedded references?
+    sam_hrecs_t *hrecs = fd->header->hrecs;
+    sam_hrec_type_t *ty = sam_hrecs_find_type_id(hrecs, "SQ", "SN",
+                                                 hrecs->ref[ref_id].name);
+    if (!ty)
+        return 0;
+
+    sam_hrec_tag_t *m5tag = sam_hrecs_find_key(ty, "M5", NULL);
+    if (!m5tag)
+        return 0;
+
+    // It's known, so compute md5 on the loaded reference sequence.
+    char *ref = fd->refs->ref_id[ref_id]->seq;
+    int64_t len = fd->refs->ref_id[ref_id]->length;
+    hts_md5_context *md5;
+    char unsigned buf[16];
+    char buf2[33];
+
+    if (!(md5 = hts_md5_init()))
+        return -1;
+    hts_md5_update(md5, ref, len);
+    hts_md5_final(buf, md5);
+    hts_md5_destroy(md5);
+    hts_md5_hex(buf2, buf);
+
+    // Compare it to header @SQ M5 tag
+    if (strcmp(m5tag->str+3, buf2)) {
+        hts_log_error("SQ header M5 tag discrepancy for reference '%s'",
+                      hrecs->ref[ref_id].name);
+        hts_log_error("Please use the correct reference, or "
+                      "consider using embed_ref=2");
+        return -1;
+    }
+    fd->refs->ref_id[ref_id]->validated_md5 = 1;
+
+    return 0;
+}
+
+/*
+ * Encodes all slices in a container into blocks.
+ * Returns 0 on success
+ *        -1 on failure
+ */
+int cram_encode_container(cram_fd *fd, cram_container *c) {
+    int i, j, slice_offset;
+    cram_block_compression_hdr *h = c->comp_hdr;
+    cram_block *c_hdr;
+    int multi_ref = 0;
+    int r1, r2, sn, nref, embed_ref, no_ref;
+    spare_bams *spares;
+
+    if (!c->bams)
+        goto err;
+
+    if (CRAM_MAJOR_VERS(fd->version) == 1)
+        goto err;
+
+//#define goto_err {fprintf(stderr, "ERR at %s:%d\n", __FILE__, __LINE__);goto err;}
+#define goto_err goto err
+
+    // Don't try embed ref if we repeatedly fail
+    pthread_mutex_lock(&fd->ref_lock);
+    int failed_embed = (fd->no_ref_counter >= 5); // maximum 5 tries
+    if (!failed_embed && c->embed_ref == -2) {
+        hts_log_warning("Retrying embed_ref=2 mode for #%d/5", fd->no_ref_counter);
+        fd->no_ref = c->no_ref = 0;
+        fd->embed_ref = c->embed_ref = 2;
+    } else if (failed_embed && c->embed_ref == -2) {
+        // We've tried several times, so this time give up for good
+        hts_log_warning("Keeping non-ref mode from now on");
+        fd->embed_ref = c->embed_ref = 0;
+    }
+    pthread_mutex_unlock(&fd->ref_lock);
+
+ restart:
+    /* Cache references up-front if we have unsorted access patterns */
+    pthread_mutex_lock(&fd->ref_lock);
+    nref = fd->refs->nref;
+    pthread_mutex_unlock(&fd->ref_lock);
+    embed_ref = c->embed_ref;
+    no_ref = c->no_ref;
+
+    /* To create M5 strings */
+    /* Fetch reference sequence */
+    if (!no_ref) {
+        if (!c->bams || !c->curr_c_rec || !c->bams[0])
+            goto_err;
+        bam_seq_t *b = c->bams[0];
+
+        if (embed_ref <= 1) {
+            char *ref = cram_get_ref(fd, bam_ref(b), 1, 0);
+            if (!ref && bam_ref(b) >= 0) {
+                if (!c->pos_sorted) {
+                    // TODO: maybe also check fd->no_ref?
+                    hts_log_warning("Failed to load reference #%d",
+                                    bam_ref(b));
+                    hts_log_warning("Switching to non-ref mode");
+
+                    pthread_mutex_lock(&fd->ref_lock);
+                    c->embed_ref = fd->embed_ref = 0;
+                    c->no_ref = fd->no_ref = 1;
+                    pthread_mutex_unlock(&fd->ref_lock);
+                    goto restart;
+                }
+
+                if (c->multi_seq || embed_ref == 0) {
+                    hts_log_error("Failed to load reference #%d", bam_ref(b));
+                    return -1;
+                }
+                hts_log_warning("Failed to load reference #%d", bam_ref(b));
+                hts_log_warning("Enabling embed_ref=2 mode to auto-generate"
+                                " reference");
+                if (embed_ref <= 0)
+                    hts_log_warning("NOTE: the CRAM file will be bigger than"
+                                    " using an external reference");
+                pthread_mutex_lock(&fd->ref_lock);
+                embed_ref = c->embed_ref = fd->embed_ref = 2;
+                pthread_mutex_unlock(&fd->ref_lock);
+                goto auto_ref;
+            } else if (ref) {
+                if (validate_md5(fd, c->ref_seq_id) < 0)
+                    goto_err;
+            }
+            if ((c->ref_id = bam_ref(b)) >= 0) {
+                c->ref_seq_id = c->ref_id;
+                c->ref       = fd->refs->ref_id[c->ref_seq_id]->seq;
+                c->ref_start = 1;
+                c->ref_end   = fd->refs->ref_id[c->ref_seq_id]->length;
+            }
+        } else {
+        auto_ref:
+            // Auto-embed ref.
+            // This starts as 'N' and is amended on-the-fly as we go
+            // based on MD:Z tags.
+            if ((c->ref_id = bam_ref(b)) >= 0) {
+                c->ref = NULL;
+                // c->ref_free is boolean; whether to free c->ref.  In this
+                // case c->ref will be our auto-embedded sequence instead of
+                // a "global" portion of reference from fd->refs.
+                // Do not confuse with fd->ref_free which is a pointer to a
+                // reference string to free.
+                c->ref_free = 1;
+            }
+        }
+        c->ref_seq_id = c->ref_id;
+    } else {
+        c->ref_id = bam_ref(c->bams[0]);
+        cram_ref_incr(fd->refs, c->ref_id);
+        c->ref_seq_id = c->ref_id;
+    }
+
+    if (!no_ref && c->refs_used) {
+        for (i = 0; i < nref; i++) {
+            if (c->refs_used[i]) {
+                if (cram_get_ref(fd, i, 1, 0)) {
+                    if (validate_md5(fd, i) < 0)
+                        goto_err;
+                } else {
+                    hts_log_warning("Failed to find reference, "
+                                    "switching to non-ref mode");
+                    no_ref = c->no_ref = 1;
+                }
+            }
+        }
+    }
+
+    /* Turn bams into cram_records and gather basic stats */
+    for (r1 = sn = 0; r1 < c->curr_c_rec; sn++) {
+        cram_slice *s = c->slices[sn];
+        int64_t first_base = INT64_MAX, last_base = INT64_MIN;
+
+        int r1_start = r1;
+
+        assert(sn < c->curr_slice);
+
+        // Discover which read names *may* be safely removed.
+        // Ie which ones have all their records in this slice.
+        if (lossy_read_names(fd, c, s, r1_start) != 0)
+            return -1;
+
+        // Tracking of MD tags so we can spot when the auto-generated values
+        // will differ from the current stored ones.  The kstring here is
+        // simply to avoid excessive malloc and free calls.  All initialisation
+        // is done within process_one_read().
+        kstring_t MD = {0};
+
+        // Embed consensus / MD-generated ref
+        if (embed_ref == 2) {
+            if (cram_generate_reference(c, s, r1) < 0) {
+                // Should this be a permanent thing via fd->no_ref?
+                // Doing so means we cannot easily switch back again should
+                // things fix themselves later on.  This is likely not a
+                // concern though as failure to generate a reference implies
+                // unsorted data which is rarely recovered from.
+
+                // Only if sn == 0.  We're hosed if we're on the 2nd slice and
+                // the first worked, as no-ref is a container global param.
+                if (sn > 0) {
+                    hts_log_error("Failed to build reference, "
+                                  "switching to non-ref mode");
+                    return -1;
+                } else {
+                    hts_log_warning("Failed to build reference, "
+                                    "switching to non-ref mode");
+                }
+                pthread_mutex_lock(&fd->ref_lock);
+                c->embed_ref = fd->embed_ref = -2; // was previously embed_ref
+                c->no_ref = fd->no_ref = 1;
+                fd->no_ref_counter++; // more likely to keep permanent action
+                pthread_mutex_unlock(&fd->ref_lock);
+                failed_embed = 1;
+                goto restart;
+            } else {
+                pthread_mutex_lock(&fd->ref_lock);
+                fd->no_ref_counter -= (fd->no_ref_counter > 0);
+                pthread_mutex_unlock(&fd->ref_lock);
+            }
+        }
+
+        // Iterate through records creating the cram blocks for some
+        // fields and just gathering stats for others.
+        for (r2 = 0; r1 < c->curr_c_rec && r2 < s->hdr->num_records; r1++, r2++) {
+            cram_record *cr = &s->crecs[r2];
+            bam_seq_t *b = c->bams[r1];
+
+            /* If multi-ref we need to cope with changing reference per seq */
+            if (c->multi_seq && !no_ref) {
+                if (bam_ref(b) != c->ref_seq_id && bam_ref(b) >= 0) {
+                    if (c->ref_seq_id >= 0)
+                        cram_ref_decr(fd->refs, c->ref_seq_id);
+
+                    if (!cram_get_ref(fd, bam_ref(b), 1, 0)) {
+                        hts_log_error("Failed to load reference #%d", bam_ref(b));
+                        free(MD.s);
+                        return -1;
+                    }
+                    if (validate_md5(fd, bam_ref(b)) < 0)
+                        return -1;
+
+                    c->ref_seq_id = bam_ref(b); // overwritten later by -2
+                    if (!fd->refs->ref_id[c->ref_seq_id]->seq)
+                        return -1;
+                    c->ref       = fd->refs->ref_id[c->ref_seq_id]->seq;
+                    c->ref_start = 1;
+                    c->ref_end   = fd->refs->ref_id[c->ref_seq_id]->length;
+                }
+            }
+
+            if (process_one_read(fd, c, s, cr, b, r2, &MD, embed_ref,
+                                 no_ref) != 0) {
+                free(MD.s);
+                return -1;
+            }
+
+            if (first_base > cr->apos)
+                first_base = cr->apos;
+
+            if (last_base < cr->aend)
+                last_base = cr->aend;
+        }
+
+        free(MD.s);
+
+        // Process_one_read doesn't add read names as it can change
+        // its mind during the loop on the CRAM_FLAG_DETACHED setting
+        // of earlier records (if it detects the auto-generation of
+        // TLEN is incorrect).  This affects which read-names can be
+        // lossily compressed, so we do these in another pass.
+        if (add_read_names(fd, c, s, r1_start) < 0)
+            return -1;
+
+        if (c->multi_seq) {
+            s->hdr->ref_seq_id    = -2;
+            s->hdr->ref_seq_start = 0;
+            s->hdr->ref_seq_span  = 0;
+        } else if (c->ref_id == -1 && CRAM_ge31(fd->version)) {
+            // Spec states span=0, but it broke our range queries.
+            // See commit message for this and prior.
+            s->hdr->ref_seq_id    = -1;
+            s->hdr->ref_seq_start = 0;
+            s->hdr->ref_seq_span  = 0;
+        } else {
+            s->hdr->ref_seq_id    = c->ref_id;
+            s->hdr->ref_seq_start = first_base;
+            s->hdr->ref_seq_span  = MAX(0, last_base - first_base + 1);
+        }
+        s->hdr->num_records = r2;
+
+        // Processed a slice, now stash the aux blocks so the next
+        // slice can start aggregating them from the start again.
+        if (c->tags_used->n_occupied) {
+            int ntags = c->tags_used->n_occupied;
+            s->aux_block = calloc(ntags*2, sizeof(*s->aux_block));
+            if (!s->aux_block)
+                return -1;
+
+            khint_t k;
+
+            s->naux_block = 0;
+            for (k = kh_begin(c->tags_used); k != kh_end(c->tags_used); k++) {
+                if (!kh_exist(c->tags_used, k))
+                    continue;
+
+                cram_tag_map *tm = kh_val(c->tags_used, k);
+                if (!tm) goto_err;
+                if (!tm->blk) continue;
+                s->aux_block[s->naux_block++] = tm->blk;
+                tm->blk = NULL;
+                if (!tm->blk2) continue;
+                s->aux_block[s->naux_block++] = tm->blk2;
+                tm->blk2 = NULL;
+            }
+            assert(s->naux_block <= 2*c->tags_used->n_occupied);
+        }
+    }
+
+    if (c->multi_seq && !no_ref) {
+        if (c->ref_seq_id >= 0)
+            cram_ref_decr(fd->refs, c->ref_seq_id);
+    }
+
+    /* Link our bams[] array onto the spare bam list for reuse */
+    spares = malloc(sizeof(*spares));
+    if (!spares) goto_err;
+    pthread_mutex_lock(&fd->bam_list_lock);
+    spares->bams = c->bams;
+    spares->next = fd->bl;
+    fd->bl = spares;
+    pthread_mutex_unlock(&fd->bam_list_lock);
+    c->bams = NULL;
+
+    /* Detect if a multi-seq container */
+    cram_stats_encoding(fd, c->stats[DS_RI]);
+    multi_ref = c->stats[DS_RI]->nvals > 1;
+    pthread_mutex_lock(&fd->metrics_lock);
+    fd->last_RI_count = c->stats[DS_RI]->nvals;
+    pthread_mutex_unlock(&fd->metrics_lock);
+
+
+    if (multi_ref) {
+        hts_log_info("Multi-ref container");
+        c->ref_seq_id = -2;
+        c->ref_seq_start = 0;
+        c->ref_seq_span = 0;
+    }
+
+
+    /* Compute MD5s */
+    no_ref = c->no_ref;
+    int is_v4 = CRAM_MAJOR_VERS(fd->version) >= 4 ? 1 : 0;
+
+    for (i = 0; i < c->curr_slice; i++) {
+        cram_slice *s = c->slices[i];
+
+        if (CRAM_MAJOR_VERS(fd->version) != 1) {
+            if (s->hdr->ref_seq_id >= 0 && c->multi_seq == 0 && !no_ref) {
+                hts_md5_context *md5 = hts_md5_init();
+                if (!md5)
+                    return -1;
+                hts_md5_update(md5,
+                               c->ref + s->hdr->ref_seq_start - c->ref_start,
+                               s->hdr->ref_seq_span);
+                hts_md5_final(s->hdr->md5, md5);
+                hts_md5_destroy(md5);
+            } else {
+                memset(s->hdr->md5, 0, 16);
+            }
+        }
+    }
+
+    c->num_records = 0;
+    c->num_blocks = 1; // cram_block_compression_hdr
+    c->length = 0;
+
+    //fprintf(stderr, "=== BF ===\n");
+    h->codecs[DS_BF] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_BF]),
+                                         c->stats[DS_BF], E_INT, NULL,
+                                         fd->version, &fd->vv);
+    if (c->stats[DS_BF]->nvals && !h->codecs[DS_BF]) goto_err;
+
+    //fprintf(stderr, "=== CF ===\n");
+    h->codecs[DS_CF] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_CF]),
+                                         c->stats[DS_CF], E_INT, NULL,
+                                         fd->version, &fd->vv);
+    if (c->stats[DS_CF]->nvals && !h->codecs[DS_CF]) goto_err;
+
+    //fprintf(stderr, "=== RN ===\n");
+    //h->codecs[DS_RN] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_RN]),
+    //                                     c->stats[DS_RN], E_BYTE_ARRAY, NULL,
+    //                                     fd->version);
+
+    //fprintf(stderr, "=== AP ===\n");
+    if (c->pos_sorted || CRAM_MAJOR_VERS(fd->version) >= 4) {
+        if (c->pos_sorted)
+            h->codecs[DS_AP] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_AP]),
+                                                 c->stats[DS_AP],
+                                                 is_v4 ? E_LONG : E_INT,
+                                                 NULL, fd->version, &fd->vv);
+        else
+            // Unsorted data has no stats, but hard-code VARINT_SIGNED / EXT.
+            h->codecs[DS_AP] = cram_encoder_init(is_v4 ? E_VARINT_SIGNED
+                                                       : E_EXTERNAL,
+                                                 NULL,
+                                                 is_v4 ? E_LONG : E_INT,
+                                                 NULL, fd->version, &fd->vv);
+    } else {
+        // Removed BETA in v4.0.
+        // Should we consider dropping use of it for 3.0 too?
+        hts_pos_t p[2] = {0, c->max_apos};
+        h->codecs[DS_AP] = cram_encoder_init(E_BETA, NULL,
+                                             is_v4 ? E_LONG : E_INT,
+                                             p, fd->version, &fd->vv);
+//      cram_xdelta_encoder e;
+//      e.word_size = is_v4 ? 8 : 4;
+//      e.sub_encoding = E_EXTERNAL;
+//      e.sub_codec_dat = (void *)DS_AP;
+//
+//      h->codecs[DS_AP] = cram_encoder_init(E_XDELTA, NULL,
+//                                           is_v4 ? E_LONG : E_INT,
+//                                           &e, fd->version, &fd->vv);
+    }
+    if (!h->codecs[DS_AP]) goto_err;
+
+    //fprintf(stderr, "=== RG ===\n");
+    h->codecs[DS_RG] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_RG]),
+                                         c->stats[DS_RG],
+                                         E_INT,
+                                         NULL,
+                                         fd->version, &fd->vv);
+    if (c->stats[DS_RG]->nvals && !h->codecs[DS_RG]) goto_err;
+
+    //fprintf(stderr, "=== MQ ===\n");
+    h->codecs[DS_MQ] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_MQ]),
+                                         c->stats[DS_MQ], E_INT, NULL,
+                                         fd->version, &fd->vv);
+    if (c->stats[DS_MQ]->nvals && !h->codecs[DS_MQ]) goto_err;
+
+    //fprintf(stderr, "=== NS ===\n");
+    h->codecs[DS_NS] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_NS]),
+                                         c->stats[DS_NS], E_INT, NULL,
+                                         fd->version, &fd->vv);
+    if (c->stats[DS_NS]->nvals && !h->codecs[DS_NS]) goto_err;
+
+    //fprintf(stderr, "=== MF ===\n");
+    h->codecs[DS_MF] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_MF]),
+                                         c->stats[DS_MF], E_INT, NULL,
+                                         fd->version, &fd->vv);
+    if (c->stats[DS_MF]->nvals && !h->codecs[DS_MF]) goto_err;
+
+    //fprintf(stderr, "=== TS ===\n");
+    h->codecs[DS_TS] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_TS]),
+                                         c->stats[DS_TS],
+                                         is_v4 ? E_LONG : E_INT,
+                                         NULL, fd->version, &fd->vv);
+    if (c->stats[DS_TS]->nvals && !h->codecs[DS_TS]) goto_err;
+
+    //fprintf(stderr, "=== NP ===\n");
+    h->codecs[DS_NP] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_NP]),
+                                         c->stats[DS_NP],
+                                         is_v4 ? E_LONG : E_INT,
+                                         NULL, fd->version, &fd->vv);
+    if (c->stats[DS_NP]->nvals && !h->codecs[DS_NP]) goto_err;
+
+    //fprintf(stderr, "=== NF ===\n");
+    h->codecs[DS_NF] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_NF]),
+                                         c->stats[DS_NF], E_INT, NULL,
+                                         fd->version, &fd->vv);
+    if (c->stats[DS_NF]->nvals && !h->codecs[DS_NF]) goto_err;
+
+    //fprintf(stderr, "=== RL ===\n");
+    h->codecs[DS_RL] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_RL]),
+                                         c->stats[DS_RL], E_INT, NULL,
+                                         fd->version, &fd->vv);
+    if (c->stats[DS_RL]->nvals && !h->codecs[DS_RL]) goto_err;
+
+    //fprintf(stderr, "=== FN ===\n");
+    h->codecs[DS_FN] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_FN]),
+                                         c->stats[DS_FN], E_INT, NULL,
+                                         fd->version, &fd->vv);
+    if (c->stats[DS_FN]->nvals && !h->codecs[DS_FN]) goto_err;
+
+    //fprintf(stderr, "=== FC ===\n");
+    h->codecs[DS_FC] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_FC]),
+                                         c->stats[DS_FC], E_BYTE, NULL,
+                                         fd->version, &fd->vv);
+    if (c->stats[DS_FC]->nvals && !h->codecs[DS_FC]) goto_err;
+
+    //fprintf(stderr, "=== FP ===\n");
+    h->codecs[DS_FP] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_FP]),
+                                         c->stats[DS_FP], E_INT, NULL,
+                                         fd->version, &fd->vv);
+    if (c->stats[DS_FP]->nvals && !h->codecs[DS_FP]) goto_err;
+
+    //fprintf(stderr, "=== DL ===\n");
+    h->codecs[DS_DL] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_DL]),
+                                         c->stats[DS_DL], E_INT, NULL,
+                                         fd->version, &fd->vv);
+    if (c->stats[DS_DL]->nvals && !h->codecs[DS_DL]) goto_err;
+
+    //fprintf(stderr, "=== BA ===\n");
+    h->codecs[DS_BA] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_BA]),
+                                         c->stats[DS_BA], E_BYTE, NULL,
+                                         fd->version, &fd->vv);
+    if (c->stats[DS_BA]->nvals && !h->codecs[DS_BA]) goto_err;
+
+    if (CRAM_MAJOR_VERS(fd->version) >= 3) {
+        cram_byte_array_len_encoder e;
+
+        e.len_encoding = CRAM_MAJOR_VERS(fd->version) >= 4
+            ? E_VARINT_UNSIGNED
+            : E_EXTERNAL;
+        e.len_dat = (void *)DS_BB_len;
+        //e.len_dat = (void *)DS_BB;
+
+        e.val_encoding = E_EXTERNAL;
+        e.val_dat = (void *)DS_BB;
+
+        h->codecs[DS_BB] = cram_encoder_init(E_BYTE_ARRAY_LEN, NULL,
+                                             E_BYTE_ARRAY, (void *)&e,
+                                             fd->version, &fd->vv);
+        if (!h->codecs[DS_BB]) goto_err;
+    } else {
+        h->codecs[DS_BB] = NULL;
+    }
+
+    //fprintf(stderr, "=== BS ===\n");
+    h->codecs[DS_BS] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_BS]),
+                                         c->stats[DS_BS], E_BYTE, NULL,
+                                         fd->version, &fd->vv);
+    if (c->stats[DS_BS]->nvals && !h->codecs[DS_BS]) goto_err;
+
+    if (CRAM_MAJOR_VERS(fd->version) == 1) {
+        h->codecs[DS_TL] = NULL;
+        h->codecs[DS_RI] = NULL;
+        h->codecs[DS_RS] = NULL;
+        h->codecs[DS_PD] = NULL;
+        h->codecs[DS_HC] = NULL;
+        h->codecs[DS_SC] = NULL;
+
+        //fprintf(stderr, "=== TC ===\n");
+        h->codecs[DS_TC] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_TC]),
+                                             c->stats[DS_TC], E_BYTE, NULL,
+                                             fd->version, &fd->vv);
+        if (c->stats[DS_TC]->nvals && !h->codecs[DS_TC]) goto_err;
+
+        //fprintf(stderr, "=== TN ===\n");
+        h->codecs[DS_TN] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_TN]),
+                                             c->stats[DS_TN], E_INT, NULL,
+                                             fd->version, &fd->vv);
+        if (c->stats[DS_TN]->nvals && !h->codecs[DS_TN]) goto_err;
+    } else {
+        h->codecs[DS_TC] = NULL;
+        h->codecs[DS_TN] = NULL;
+
+        //fprintf(stderr, "=== TL ===\n");
+        h->codecs[DS_TL] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_TL]),
+                                             c->stats[DS_TL], E_INT, NULL,
+                                             fd->version, &fd->vv);
+        if (c->stats[DS_TL]->nvals && !h->codecs[DS_TL]) goto_err;
+
+
+        //fprintf(stderr, "=== RI ===\n");
+        h->codecs[DS_RI] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_RI]),
+                                             c->stats[DS_RI], E_INT, NULL,
+                                             fd->version, &fd->vv);
+        if (c->stats[DS_RI]->nvals && !h->codecs[DS_RI]) goto_err;
+
+        //fprintf(stderr, "=== RS ===\n");
+        h->codecs[DS_RS] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_RS]),
+                                             c->stats[DS_RS], E_INT, NULL,
+                                             fd->version, &fd->vv);
+        if (c->stats[DS_RS]->nvals && !h->codecs[DS_RS]) goto_err;
+
+        //fprintf(stderr, "=== PD ===\n");
+        h->codecs[DS_PD] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_PD]),
+                                             c->stats[DS_PD], E_INT, NULL,
+                                             fd->version, &fd->vv);
+        if (c->stats[DS_PD]->nvals && !h->codecs[DS_PD]) goto_err;
+
+        //fprintf(stderr, "=== HC ===\n");
+        h->codecs[DS_HC] = cram_encoder_init(cram_stats_encoding(fd, c->stats[DS_HC]),
+                                             c->stats[DS_HC], E_INT, NULL,
+                                             fd->version, &fd->vv);
+        if (c->stats[DS_HC]->nvals && !h->codecs[DS_HC]) goto_err;
+
+        //fprintf(stderr, "=== SC ===\n");
+        if (1) {
+            int i2[2] = {0, DS_SC};
+
+            h->codecs[DS_SC] = cram_encoder_init(E_BYTE_ARRAY_STOP, NULL,
+                                                 E_BYTE_ARRAY, (void *)i2,
+                                                 fd->version, &fd->vv);
+        } else {
+            // Appears to be no practical benefit to using this method,
+            // but it may work better if we start mixing SC, IN and BB
+            // elements into the same external block.
+            cram_byte_array_len_encoder e;
+
+            e.len_encoding = CRAM_MAJOR_VERS(fd->version) >= 4
+                ? E_VARINT_UNSIGNED
+                : E_EXTERNAL;
+            e.len_dat = (void *)DS_SC_len;
+
+            e.val_encoding = E_EXTERNAL;
+            e.val_dat = (void *)DS_SC;
+
+            h->codecs[DS_SC] = cram_encoder_init(E_BYTE_ARRAY_LEN, NULL,
+                                                 E_BYTE_ARRAY, (void *)&e,
+                                                 fd->version, &fd->vv);
+        }
+        if (!h->codecs[DS_SC]) goto_err;
+    }
+
+    //fprintf(stderr, "=== IN ===\n");
+    {
+        int i2[2] = {0, DS_IN};
+        h->codecs[DS_IN] = cram_encoder_init(E_BYTE_ARRAY_STOP, NULL,
+                                             E_BYTE_ARRAY, (void *)i2,
+                                             fd->version, &fd->vv);
+        if (!h->codecs[DS_IN]) goto_err;
+    }
+
+    h->codecs[DS_QS] = cram_encoder_init(E_EXTERNAL, NULL, E_BYTE,
+                                         (void *)DS_QS,
+                                         fd->version, &fd->vv);
+    if (!h->codecs[DS_QS]) goto_err;
+    {
+        int i2[2] = {0, DS_RN};
+        h->codecs[DS_RN] = cram_encoder_init(E_BYTE_ARRAY_STOP, NULL,
+                                             E_BYTE_ARRAY, (void *)i2,
+                                             fd->version, &fd->vv);
+        if (!h->codecs[DS_RN]) goto_err;
+    }
+
+
+    /* Encode slices */
+    for (i = 0; i < c->curr_slice; i++) {
+        hts_log_info("Encode slice %d", i);
+
+        int local_embed_ref =
+            embed_ref>0 && c->slices[i]->hdr->ref_seq_id != -1 ? 1 : 0;
+        if (cram_encode_slice(fd, c, h, c->slices[i], local_embed_ref) != 0)
+            return -1;
+    }
+
+    /* Create compression header */
+    {
+        h->ref_seq_id    = c->ref_seq_id;
+        h->ref_seq_start = c->ref_seq_start;
+        h->ref_seq_span  = c->ref_seq_span;
+        h->num_records   = c->num_records;
+        h->qs_seq_orient = c->qs_seq_orient;
+        // slight misnomer - sorted or treat as-if sorted (ap_delta force to 1)
+        h->AP_delta      = c->pos_sorted;
+        memcpy(h->substitution_matrix, CRAM_SUBST_MATRIX, 20);
+
+        if (!(c_hdr = cram_encode_compression_header(fd, c, h, embed_ref)))
+            return -1;
+    }
+
+    /* Compute landmarks */
+    /* Fill out slice landmarks */
+    c->num_landmarks = c->curr_slice;
+    c->landmark = malloc(c->num_landmarks * sizeof(*c->landmark));
+    if (!c->landmark)
+        return -1;
+
+    /*
+     * Slice offset starts after the first block, so we need to simulate
+     * writing it to work out the correct offset
+     */
+    {
+        slice_offset = c_hdr->method == RAW
+            ? c_hdr->uncomp_size
+            : c_hdr->comp_size;
+        slice_offset += 2 + 4*(CRAM_MAJOR_VERS(fd->version) >= 3) +
+            fd->vv.varint_size(c_hdr->content_id) +
+            fd->vv.varint_size(c_hdr->comp_size) +
+            fd->vv.varint_size(c_hdr->uncomp_size);
+    }
+
+    c->ref_seq_id    = c->slices[0]->hdr->ref_seq_id;
+    if (c->ref_seq_id == -1 && CRAM_ge31(fd->version)) {
+        // Spec states span=0, but it broke our range queries.
+        // See commit message for this and prior.
+        c->ref_seq_start = 0;
+        c->ref_seq_span  = 0;
+    } else {
+        c->ref_seq_start = c->slices[0]->hdr->ref_seq_start;
+        c->ref_seq_span  = c->slices[0]->hdr->ref_seq_span;
+    }
+    for (i = 0; i < c->curr_slice; i++) {
+        cram_slice *s = c->slices[i];
+
+        c->num_blocks += s->hdr->num_blocks + 1; // slice header
+        c->landmark[i] = slice_offset;
+
+        if (s->hdr->ref_seq_start + s->hdr->ref_seq_span >
+            c->ref_seq_start + c->ref_seq_span) {
+            c->ref_seq_span = s->hdr->ref_seq_start + s->hdr->ref_seq_span
+                - c->ref_seq_start;
+        }
+
+        slice_offset += s->hdr_block->method == RAW
+            ? s->hdr_block->uncomp_size
+            : s->hdr_block->comp_size;
+
+        slice_offset += 2 + 4*(CRAM_MAJOR_VERS(fd->version) >= 3) +
+            fd->vv.varint_size(s->hdr_block->content_id) +
+            fd->vv.varint_size(s->hdr_block->comp_size) +
+            fd->vv.varint_size(s->hdr_block->uncomp_size);
+
+        for (j = 0; j < s->hdr->num_blocks; j++) {
+            slice_offset += 2 + 4*(CRAM_MAJOR_VERS(fd->version) >= 3) +
+                fd->vv.varint_size(s->block[j]->content_id) +
+                fd->vv.varint_size(s->block[j]->comp_size) +
+                fd->vv.varint_size(s->block[j]->uncomp_size);
+
+            slice_offset += s->block[j]->method == RAW
+                ? s->block[j]->uncomp_size
+                : s->block[j]->comp_size;
+        }
+    }
+    c->length += slice_offset; // just past the final slice
+
+    c->comp_hdr_block = c_hdr;
+
+    if (c->ref_seq_id >= 0) {
+        if (c->ref_free) {
+            free(c->ref);
+            c->ref = NULL;
+        } else {
+            cram_ref_decr(fd->refs, c->ref_seq_id);
+        }
+    }
+
+    /* Cache references up-front if we have unsorted access patterns */
+    if (!no_ref && c->refs_used) {
+        for (i = 0; i < fd->refs->nref; i++) {
+            if (c->refs_used[i])
+                cram_ref_decr(fd->refs, i);
+        }
+    }
+
+    return 0;
+
+ err:
+    return -1;
+}
+
+
+/*
+ * Adds a feature code to a read within a slice. For purposes of minimising
+ * memory allocations and fragmentation we have one array of features for all
+ * reads within the slice. We return the index into this array for this new
+ * feature.
+ *
+ * Returns feature index on success
+ *         -1 on failure.
+ */
+static int cram_add_feature(cram_container *c, cram_slice *s,
+                            cram_record *r, cram_feature *f) {
+    if (s->nfeatures >= s->afeatures) {
+        s->afeatures = s->afeatures ? s->afeatures*2 : 1024;
+        s->features = realloc(s->features, s->afeatures*sizeof(*s->features));
+        if (!s->features)
+            return -1;
+    }
+
+    if (!r->nfeature++) {
+        r->feature = s->nfeatures;
+        if (cram_stats_add(c->stats[DS_FP], f->X.pos) < 0)
+            return -1;
+    } else {
+        if (cram_stats_add(c->stats[DS_FP],
+                           f->X.pos - s->features[r->feature + r->nfeature-2].X.pos) < 0)
+            return -1;
+
+    }
+    if (cram_stats_add(c->stats[DS_FC], f->X.code) < 0)
+        return -1;
+
+    s->features[s->nfeatures++] = *f;
+
+    return 0;
+}
+
+static int cram_add_substitution(cram_fd *fd, cram_container *c,
+                                 cram_slice *s, cram_record *r,
+                                 int pos, char base, char qual, char ref) {
+    cram_feature f;
+
+    // seq=ACGTN vs ref=ACGT or seq=ACGT vs ref=ACGTN
+    if (fd->L2[(uc)base]<4 || (fd->L2[(uc)base]<5 && fd->L2[(uc)ref]<4)) {
+        f.X.pos = pos+1;
+        f.X.code = 'X';
+        f.X.base = fd->cram_sub_matrix[ref&0x1f][base&0x1f];
+        if (cram_stats_add(c->stats[DS_BS], f.X.base) < 0)
+            return -1;
+    } else {
+        f.B.pos = pos+1;
+        f.B.code = 'B';
+        f.B.base = base;
+        f.B.qual = qual;
+        if (cram_stats_add(c->stats[DS_BA], f.B.base) < 0) return -1;
+        if (cram_stats_add(c->stats[DS_QS], f.B.qual) < 0) return -1;
+        BLOCK_APPEND_CHAR(s->qual_blk, qual);
+    }
+    return cram_add_feature(c, s, r, &f);
+
+ block_err:
+    return -1;
+}
+
+static int cram_add_bases(cram_fd *fd, cram_container *c,
+                          cram_slice *s, cram_record *r,
+                          int pos, int len, char *base) {
+    cram_feature f;
+
+    f.b.pos = pos+1;
+    f.b.code = 'b';
+    f.b.seq_idx = base - (char *)BLOCK_DATA(s->seqs_blk);
+    f.b.len = len;
+
+    return cram_add_feature(c, s, r, &f);
+}
+
+static int cram_add_base(cram_fd *fd, cram_container *c,
+                         cram_slice *s, cram_record *r,
+                         int pos, char base, char qual) {
+    cram_feature f;
+    f.B.pos = pos+1;
+    f.B.code = 'B';
+    f.B.base = base;
+    f.B.qual = qual;
+    if (cram_stats_add(c->stats[DS_BA], base) < 0) return -1;
+    if (cram_stats_add(c->stats[DS_QS], qual) < 0) return -1;
+    BLOCK_APPEND_CHAR(s->qual_blk, qual);
+    return cram_add_feature(c, s, r, &f);
+
+ block_err:
+    return -1;
+}
+
+static int cram_add_quality(cram_fd *fd, cram_container *c,
+                            cram_slice *s, cram_record *r,
+                            int pos, char qual) {
+    cram_feature f;
+    f.Q.pos = pos+1;
+    f.Q.code = 'Q';
+    f.Q.qual = qual;
+    if (cram_stats_add(c->stats[DS_QS], qual) < 0) return -1;
+    BLOCK_APPEND_CHAR(s->qual_blk, qual);
+    return cram_add_feature(c, s, r, &f);
+
+ block_err:
+    return -1;
+}
+
+static int cram_add_deletion(cram_container *c, cram_slice *s, cram_record *r,
+                             int pos, int len, char *base) {
+    cram_feature f;
+    f.D.pos = pos+1;
+    f.D.code = 'D';
+    f.D.len = len;
+    if (cram_stats_add(c->stats[DS_DL], len) < 0) return -1;
+    return cram_add_feature(c, s, r, &f);
+}
+
+static int cram_add_softclip(cram_container *c, cram_slice *s, cram_record *r,
+                             int pos, int len, char *base, int version) {
+    cram_feature f;
+    f.S.pos = pos+1;
+    f.S.code = 'S';
+    f.S.len = len;
+    switch (CRAM_MAJOR_VERS(version)) {
+    case 1:
+        f.S.seq_idx = BLOCK_SIZE(s->base_blk);
+        BLOCK_APPEND(s->base_blk, base, len);
+        BLOCK_APPEND_CHAR(s->base_blk, '\0');
+        break;
+
+    case 2:
+    default:
+        f.S.seq_idx = BLOCK_SIZE(s->soft_blk);
+        if (base) {
+            BLOCK_APPEND(s->soft_blk, base, len);
+        } else {
+            int i;
+            for (i = 0; i < len; i++)
+                BLOCK_APPEND_CHAR(s->soft_blk, 'N');
+        }
+        BLOCK_APPEND_CHAR(s->soft_blk, '\0');
+        break;
+
+        //default:
+        //    // v3.0 onwards uses BB data-series
+        //    f.S.seq_idx = BLOCK_SIZE(s->soft_blk);
+    }
+    return cram_add_feature(c, s, r, &f);
+
+ block_err:
+    return -1;
+}
+
+static int cram_add_hardclip(cram_container *c, cram_slice *s, cram_record *r,
+                             int pos, int len, char *base) {
+    cram_feature f;
+    f.S.pos = pos+1;
+    f.S.code = 'H';
+    f.S.len = len;
+    if (cram_stats_add(c->stats[DS_HC], len) < 0) return -1;
+    return cram_add_feature(c, s, r, &f);
+}
+
+static int cram_add_skip(cram_container *c, cram_slice *s, cram_record *r,
+                         int pos, int len, char *base) {
+    cram_feature f;
+    f.S.pos = pos+1;
+    f.S.code = 'N';
+    f.S.len = len;
+    if (cram_stats_add(c->stats[DS_RS], len) < 0) return -1;
+    return cram_add_feature(c, s, r, &f);
+}
+
+static int cram_add_pad(cram_container *c, cram_slice *s, cram_record *r,
+                        int pos, int len, char *base) {
+    cram_feature f;
+    f.S.pos = pos+1;
+    f.S.code = 'P';
+    f.S.len = len;
+    if (cram_stats_add(c->stats[DS_PD], len) < 0) return -1;
+    return cram_add_feature(c, s, r, &f);
+}
+
+static int cram_add_insertion(cram_container *c, cram_slice *s, cram_record *r,
+                              int pos, int len, char *base) {
+    cram_feature f;
+    f.I.pos = pos+1;
+    if (len == 1) {
+        char b = base ? *base : 'N';
+        f.i.code = 'i';
+        f.i.base = b;
+        if (cram_stats_add(c->stats[DS_BA], b) < 0) return -1;
+    } else {
+        f.I.code = 'I';
+        f.I.len = len;
+        f.S.seq_idx = BLOCK_SIZE(s->base_blk);
+        if (base) {
+            BLOCK_APPEND(s->base_blk, base, len);
+        } else {
+            int i;
+            for (i = 0; i < len; i++)
+                BLOCK_APPEND_CHAR(s->base_blk, 'N');
+        }
+        BLOCK_APPEND_CHAR(s->base_blk, '\0');
+    }
+    return cram_add_feature(c, s, r, &f);
+
+ block_err:
+    return -1;
+}
+
+/*
+ * Encodes auxiliary data. Largely duplicated from above, but done so to
+ * keep it simple and avoid a myriad of version ifs.
+ *
+ * Returns the RG header line pointed to by the BAM aux fields on success,
+ *         NULL on failure or no rg present, also sets "*err" to non-zero
+ */
+static sam_hrec_rg_t *cram_encode_aux(cram_fd *fd, bam_seq_t *b,
+                                      cram_container *c,
+                                      cram_slice *s, cram_record *cr,
+                                      int verbatim_NM, int verbatim_MD,
+                                      int NM, kstring_t *MD, int cf_tag,
+                                      int no_ref, int *err) {
+    char *aux, *orig;
+    sam_hrec_rg_t *brg = NULL;
+    int aux_size = bam_get_l_aux(b);
+    const char *aux_end = bam_data_end(b);
+    cram_block *td_b = c->comp_hdr->TD_blk;
+    int TD_blk_size = BLOCK_SIZE(td_b), new;
+    char *key;
+    khint_t k;
+
+    if (err) *err = 1;
+
+    orig = aux = (char *)bam_aux(b);
+
+
+    // cF:i  => Extra CRAM bit flags.
+    // 1:  Don't auto-decode MD (may be invalid)
+    // 2:  Don't auto-decode NM (may be invalid)
+    if (cf_tag && CRAM_MAJOR_VERS(fd->version) < 4) {
+        // Temporary copy of aux so we can ammend it.
+        aux = malloc(aux_size+4);
+        if (!aux)
+            return NULL;
+
+        memcpy(aux, orig, aux_size);
+        aux[aux_size++] = 'c';
+        aux[aux_size++] = 'F';
+        aux[aux_size++] = 'C';
+        aux[aux_size++] = cf_tag;
+        orig = aux;
+        aux_end = aux + aux_size;
+    }
+
+    // Copy aux keys to td_b and aux values to slice aux blocks
+    while (aux_end - aux >= 1 && aux[0] != 0) {
+        int r;
+
+        // Room for code + type + at least 1 byte of data
+        if (aux - orig >= aux_size - 3)
+            goto err;
+
+        // RG:Z
+        if (aux[0] == 'R' && aux[1] == 'G' && aux[2] == 'Z') {
+            char *rg = &aux[3];
+            aux = rg;
+            while (aux < aux_end && *aux++);
+            if (aux == aux_end && aux[-1] != '\0') {
+                hts_log_error("Unterminated RG:Z tag for read \"%s\"",
+                              bam_get_qname(b));
+                goto err;
+            }
+            brg = sam_hrecs_find_rg(fd->header->hrecs, rg);
+            if (brg) {
+                if (CRAM_MAJOR_VERS(fd->version) >= 4)
+                    BLOCK_APPEND(td_b, "RG*", 3);
+                continue;
+            } else {
+                // RG:Z tag will be stored verbatim
+                hts_log_warning("Missing @RG header for RG \"%s\"", rg);
+                aux = rg - 3;
+            }
+        }
+
+        // MD:Z
+        if (aux[0] == 'M' && aux[1] == 'D' && aux[2] == 'Z') {
+            if (cr->len && !no_ref && !(cr->flags & BAM_FUNMAP) && !verbatim_MD) {
+                if (MD && MD->s && strncasecmp(MD->s, aux+3, orig + aux_size - (aux+3)) == 0) {
+                    while (aux < aux_end && *aux++);
+                    if (aux == aux_end && aux[-1] != '\0') {
+                        hts_log_error("Unterminated MD:Z tag for read \"%s\"",
+                                      bam_get_qname(b));
+                        goto err;
+                    }
+                    if (CRAM_MAJOR_VERS(fd->version) >= 4)
+                        BLOCK_APPEND(td_b, "MD*", 3);
+                    continue;
+                }
+            }
+        }
+
+        // NM:i
+        if (aux[0] == 'N' && aux[1] == 'M') {
+            if (cr->len && !no_ref && !(cr->flags & BAM_FUNMAP) && !verbatim_NM) {
+                int NM_ = bam_aux2i_end((uint8_t *)aux+2, (uint8_t *)aux_end);
+                if (NM_ == NM) {
+                    switch(aux[2]) {
+                    case 'A': case 'C': case 'c': aux+=4; break;
+                    case 'S': case 's':           aux+=5; break;
+                    case 'I': case 'i': case 'f': aux+=7; break;
+                    default:
+                        hts_log_error("Unhandled type code for NM tag");
+                        goto err;
+                    }
+                    if (CRAM_MAJOR_VERS(fd->version) >= 4)
+                        BLOCK_APPEND(td_b, "NM*", 3);
+                    continue;
+                }
+            }
+        }
+
+        BLOCK_APPEND(td_b, aux, 3);
+
+        // Container level tags_used, for TD series
+        // Maps integer key ('X0i') to cram_tag_map struct.
+        int key = (((unsigned char *) aux)[0]<<16 |
+                   ((unsigned char *) aux)[1]<<8  |
+                   ((unsigned char *) aux)[2]);
+        k = kh_put(m_tagmap, c->tags_used, key, &r);
+        if (-1 == r)
+            goto err;
+        else if (r != 0)
+            kh_val(c->tags_used, k) = NULL;
+
+        if (r == 1) {
+            khint_t k_global;
+
+            // Global tags_used for cram_metrics support
+            pthread_mutex_lock(&fd->metrics_lock);
+            k_global = kh_put(m_metrics, fd->tags_used, key, &r);
+            if (-1 == r) {
+                pthread_mutex_unlock(&fd->metrics_lock);
+                goto err;
+            }
+            if (r >= 1) {
+                kh_val(fd->tags_used, k_global) = cram_new_metrics();
+                if (!kh_val(fd->tags_used, k_global)) {
+                    kh_del(m_metrics, fd->tags_used, k_global);
+                    pthread_mutex_unlock(&fd->metrics_lock);
+                    goto err;
+                }
+            }
+
+            pthread_mutex_unlock(&fd->metrics_lock);
+
+            int i2[2] = {'\t',key};
+            size_t sk = key;
+            cram_tag_map *m = calloc(1, sizeof(*m));
+            if (!m)
+                goto_err;
+            kh_val(c->tags_used, k) = m;
+
+            cram_codec *c;
+
+            // Use a block content id based on the tag id.
+            // Codec type depends on tag data type.
+            switch(aux[2]) {
+            case 'Z': case 'H':
+                // string as byte_array_stop
+                c = cram_encoder_init(E_BYTE_ARRAY_STOP, NULL,
+                                      E_BYTE_ARRAY, (void *)i2,
+                                      fd->version, &fd->vv);
+                break;
+
+            case 'A': case 'c': case 'C': {
+                // byte array len, 1 byte
+                cram_byte_array_len_encoder e;
+                cram_stats st;
+
+                if (CRAM_MAJOR_VERS(fd->version) <= 3) {
+                    e.len_encoding = E_HUFFMAN;
+                    e.len_dat = NULL; // will get codes from st
+                } else {
+                    e.len_encoding = E_CONST_INT;
+                    e.len_dat = NULL; // will get codes from st
+                }
+                memset(&st, 0, sizeof(st));
+                if (cram_stats_add(&st, 1) < 0) goto block_err;
+                cram_stats_encoding(fd, &st);
+
+                e.val_encoding = E_EXTERNAL;
+                e.val_dat = (void *)sk;
+
+                c = cram_encoder_init(E_BYTE_ARRAY_LEN, &st,
+                                      E_BYTE_ARRAY, (void *)&e,
+                                      fd->version, &fd->vv);
+                break;
+            }
+
+            case 's': case 'S': {
+                // byte array len, 2 byte
+                cram_byte_array_len_encoder e;
+                cram_stats st;
+
+                if (CRAM_MAJOR_VERS(fd->version) <= 3) {
+                    e.len_encoding = E_HUFFMAN;
+                    e.len_dat = NULL; // will get codes from st
+                } else {
+                    e.len_encoding = E_CONST_INT;
+                    e.len_dat = NULL; // will get codes from st
+                }
+                memset(&st, 0, sizeof(st));
+                if (cram_stats_add(&st, 2) < 0) goto block_err;
+                cram_stats_encoding(fd, &st);
+
+                e.val_encoding = E_EXTERNAL;
+                e.val_dat = (void *)sk;
+
+                c = cram_encoder_init(E_BYTE_ARRAY_LEN, &st,
+                                      E_BYTE_ARRAY, (void *)&e,
+                                      fd->version, &fd->vv);
+                break;
+            }
+            case 'i': case 'I': case 'f': {
+                // byte array len, 4 byte
+                cram_byte_array_len_encoder e;
+                cram_stats st;
+
+                if (CRAM_MAJOR_VERS(fd->version) <= 3) {
+                    e.len_encoding = E_HUFFMAN;
+                    e.len_dat = NULL; // will get codes from st
+                } else {
+                    e.len_encoding = E_CONST_INT;
+                    e.len_dat = NULL; // will get codes from st
+                }
+                memset(&st, 0, sizeof(st));
+                if (cram_stats_add(&st, 4) < 0) goto block_err;
+                cram_stats_encoding(fd, &st);
+
+                e.val_encoding = E_EXTERNAL;
+                e.val_dat = (void *)sk;
+
+                c = cram_encoder_init(E_BYTE_ARRAY_LEN, &st,
+                                      E_BYTE_ARRAY, (void *)&e,
+                                      fd->version, &fd->vv);
+                break;
+            }
+
+            case 'B': {
+                // Byte array of variable size, but we generate our tag
+                // byte stream at the wrong stage (during reading and not
+                // after slice header construction). So we use
+                // BYTE_ARRAY_LEN with the length codec being external
+                // too.
+                cram_byte_array_len_encoder e;
+
+                e.len_encoding = CRAM_MAJOR_VERS(fd->version) >= 4
+                    ? E_VARINT_UNSIGNED
+                    : E_EXTERNAL;
+                e.len_dat = (void *)sk; // or key+128 for len?
+
+                e.val_encoding = E_EXTERNAL;
+                e.val_dat = (void *)sk;
+
+                c = cram_encoder_init(E_BYTE_ARRAY_LEN, NULL,
+                                      E_BYTE_ARRAY, (void *)&e,
+                                      fd->version, &fd->vv);
+                break;
+            }
+
+            default:
+                hts_log_error("Unsupported SAM aux type '%c'", aux[2]);
+                c = NULL;
+            }
+
+            if (!c)
+                goto_err;
+
+            m->codec = c;
+
+            // Link to fd-global tag metrics
+            pthread_mutex_lock(&fd->metrics_lock);
+            m->m = k_global ? (cram_metrics *)kh_val(fd->tags_used, k_global) : NULL;
+            pthread_mutex_unlock(&fd->metrics_lock);
+        }
+
+        cram_tag_map *tm = (cram_tag_map *)kh_val(c->tags_used, k);
+        if (!tm) goto_err;
+        cram_codec *codec = tm->codec;
+        if (!tm->codec) goto_err;
+
+        switch(aux[2]) {
+        case 'A': case 'C': case 'c':
+            if (aux_end - aux < 3+1)
+                goto err;
+
+            if (!tm->blk) {
+                if (!(tm->blk = cram_new_block(EXTERNAL, key)))
+                    goto err;
+                codec->u.e_byte_array_len.val_codec->out = tm->blk;
+            }
+
+            aux+=3;
+            //codec->encode(s, codec, aux, 1);
+            // Functionally equivalent, but less code.
+            BLOCK_APPEND_CHAR(tm->blk, *aux);
+            aux++;
+            break;
+
+        case 'S': case 's':
+            if (aux_end - aux < 3+2)
+                goto err;
+
+            if (!tm->blk) {
+                if (!(tm->blk = cram_new_block(EXTERNAL, key)))
+                    goto err;
+                codec->u.e_byte_array_len.val_codec->out = tm->blk;
+            }
+
+            aux+=3;
+            //codec->encode(s, codec, aux, 2);
+            BLOCK_APPEND(tm->blk, aux, 2);
+            aux+=2;
+            break;
+
+        case 'I': case 'i': case 'f':
+            if (aux_end - aux < 3+4)
+                goto err;
+
+            if (!tm->blk) {
+                if (!(tm->blk = cram_new_block(EXTERNAL, key)))
+                    goto err;
+                codec->u.e_byte_array_len.val_codec->out = tm->blk;
+            }
+
+            aux+=3;
+            //codec->encode(s, codec, aux, 4);
+            BLOCK_APPEND(tm->blk, aux, 4);
+            aux+=4;
+            break;
+
+        case 'd':
+            if (aux_end - aux < 3+8)
+                goto err;
+
+            if (!tm->blk) {
+                if (!(tm->blk = cram_new_block(EXTERNAL, key)))
+                    goto err;
+                codec->u.e_byte_array_len.val_codec->out = tm->blk;
+            }
+
+            aux+=3; //*tmp++=*aux++; *tmp++=*aux++; *tmp++=*aux++;
+            //codec->encode(s, codec, aux, 8);
+            BLOCK_APPEND(tm->blk, aux, 8);
+            aux+=8;
+            break;
+
+        case 'Z': case 'H': {
+            if (aux_end - aux < 3)
+                goto err;
+
+            if (!tm->blk) {
+                if (!(tm->blk = cram_new_block(EXTERNAL, key)))
+                    goto err;
+                codec->out = tm->blk;
+            }
+
+            char *aux_s;
+            aux += 3;
+            aux_s = aux;
+            while (aux < aux_end && *aux++);
+            if (aux == aux_end && aux[-1] != '\0') {
+                hts_log_error("Unterminated %c%c:%c tag for read \"%s\"",
+                              aux_s[-3], aux_s[-2], aux_s[-1],
+                              bam_get_qname(b));
+                goto err;
+            }
+            if (codec->encode(s, codec, aux_s, aux - aux_s) < 0)
+                goto err;
+            break;
+        }
+
+        case 'B': {
+            if (aux_end - aux < 4+4)
+                goto err;
+
+            int type = aux[3];
+            uint64_t count = (((uint64_t)((unsigned char *)aux)[4]) << 0 |
+                              ((uint64_t)((unsigned char *)aux)[5]) << 8 |
+                              ((uint64_t)((unsigned char *)aux)[6]) <<16 |
+                              ((uint64_t)((unsigned char *)aux)[7]) <<24);
+            uint64_t blen;
+            if (!tm->blk) {
+                if (!(tm->blk = cram_new_block(EXTERNAL, key)))
+                    goto err;
+                if (codec->u.e_byte_array_len.val_codec->codec == E_XDELTA) {
+                    if (!(tm->blk2 = cram_new_block(EXTERNAL, key+128)))
+                        goto err;
+                    codec->u.e_byte_array_len.len_codec->out = tm->blk2;
+                    codec->u.e_byte_array_len.val_codec->u.e_xdelta.sub_codec->out = tm->blk;
+                } else {
+                    codec->u.e_byte_array_len.len_codec->out = tm->blk;
+                    codec->u.e_byte_array_len.val_codec->out = tm->blk;
+                }
+            }
+
+            // skip TN field
+            aux+=3;
+
+            // We use BYTE_ARRAY_LEN with external length, so store that first
+            switch (type) {
+            case 'c': case 'C':
+                blen = count;
+                break;
+            case 's': case 'S':
+                blen = 2*count;
+                break;
+            case 'i': case 'I': case 'f':
+                blen = 4*count;
+                break;
+            default:
+                hts_log_error("Unknown sub-type '%c' for aux type 'B'", type);
+                goto err;
+            }
+
+            blen += 5; // sub-type & length
+            if (aux_end - aux < blen || blen > INT_MAX)
+                goto err;
+
+            if (codec->encode(s, codec, aux, (int) blen) < 0)
+                goto err;
+            aux += blen;
+            break;
+        }
+        default:
+            hts_log_error("Unknown aux type '%c'", aux_end - aux < 2 ? '?' : aux[2]);
+            goto err;
+        }
+        tm->blk->m = tm->m;
+    }
+
+    // FIXME: sort BLOCK_DATA(td_b) by char[3] triples
+
+    // And and increment TD hash entry
+    BLOCK_APPEND_CHAR(td_b, 0);
+
+    // Duplicate key as BLOCK_DATA() can be realloced to a new pointer.
+    key = string_ndup(c->comp_hdr->TD_keys,
+                      (char *)BLOCK_DATA(td_b) + TD_blk_size,
+                      BLOCK_SIZE(td_b) - TD_blk_size);
+    if (!key)
+        goto block_err;
+    k = kh_put(m_s2i, c->comp_hdr->TD_hash, key, &new);
+    if (new < 0) {
+        goto err;
+    } else if (new == 0) {
+        BLOCK_SIZE(td_b) = TD_blk_size;
+    } else {
+        kh_val(c->comp_hdr->TD_hash, k) = c->comp_hdr->nTL;
+        c->comp_hdr->nTL++;
+    }
+
+    cr->TL = kh_val(c->comp_hdr->TD_hash, k);
+    if (cram_stats_add(c->stats[DS_TL], cr->TL) < 0)
+        goto block_err;
+
+    if (orig != (char *)bam_aux(b))
+        free(orig);
+
+    if (err) *err = 0;
+
+    return brg;
+
+ err:
+ block_err:
+    if (orig != (char *)bam_aux(b))
+        free(orig);
+    return NULL;
+}
+
+/*
+ * During cram_next_container or before the final flush at end of
+ * file, we update the current slice headers and increment the slice
+ * number to the next slice.
+ *
+ * See cram_next_container() and cram_close().
+ */
+void cram_update_curr_slice(cram_container *c, int version) {
+    cram_slice *s = c->slice;
+    if (c->multi_seq) {
+        s->hdr->ref_seq_id    = -2;
+        s->hdr->ref_seq_start = 0;
+        s->hdr->ref_seq_span  = 0;
+    } else if (c->curr_ref == -1 && CRAM_ge31(version)) {
+        // Spec states span=0, but it broke our range queries.
+        // See commit message for this and prior.
+        s->hdr->ref_seq_id    = -1;
+        s->hdr->ref_seq_start = 0;
+        s->hdr->ref_seq_span  = 0;
+    } else {
+        s->hdr->ref_seq_id    = c->curr_ref;
+        s->hdr->ref_seq_start = c->first_base;
+        s->hdr->ref_seq_span  = MAX(0, c->last_base - c->first_base + 1);
+    }
+    s->hdr->num_records   = c->curr_rec;
+
+    if (c->curr_slice == 0) {
+        if (c->ref_seq_id != s->hdr->ref_seq_id)
+            c->ref_seq_id  = s->hdr->ref_seq_id;
+        c->ref_seq_start = c->first_base;
+    }
+
+    c->curr_slice++;
+}
+
+/*
+ * Handles creation of a new container or new slice, flushing any
+ * existing containers when appropriate.
+ *
+ * Really this is next slice, which may or may not lead to a new container.
+ *
+ * Returns cram_container pointer on success
+ *         NULL on failure.
+ */
+static cram_container *cram_next_container(cram_fd *fd, bam_seq_t *b) {
+    cram_container *c = fd->ctr;
+    int i;
+
+    /* First occurrence */
+    if (c->curr_ref == -2)
+        c->curr_ref = bam_ref(b);
+
+    if (c->slice)
+        cram_update_curr_slice(c, fd->version);
+
+    /* Flush container */
+    if (c->curr_slice == c->max_slice ||
+        (bam_ref(b) != c->curr_ref && !c->multi_seq)) {
+        c->ref_seq_span = fd->last_base - c->ref_seq_start + 1;
+        hts_log_info("Flush container %d/%"PRId64"..%"PRId64,
+                     c->ref_seq_id, c->ref_seq_start,
+                     c->ref_seq_start + c->ref_seq_span -1);
+
+        /* Encode slices */
+        if (-1 == cram_flush_container_mt(fd, c))
+            return NULL;
+        if (!fd->pool) {
+            // Move to sep func, as we need cram_flush_container for
+            // the closing phase to flush the partial container.
+            for (i = 0; i < c->max_slice; i++) {
+                cram_free_slice(c->slices[i]);
+                c->slices[i] = NULL;
+            }
+
+            c->slice = NULL;
+            c->curr_slice = 0;
+
+            /* Easy approach for purposes of freeing stats */
+            cram_free_container(c);
+        }
+
+        c = fd->ctr = cram_new_container(fd->seqs_per_slice,
+                                         fd->slices_per_container);
+        if (!c)
+            return NULL;
+
+        pthread_mutex_lock(&fd->ref_lock);
+        c->no_ref = fd->no_ref;
+        c->embed_ref = fd->embed_ref;
+        c->record_counter = fd->record_counter;
+        pthread_mutex_unlock(&fd->ref_lock);
+        c->curr_ref = bam_ref(b);
+    }
+
+    c->last_pos = c->first_base = c->last_base = bam_pos(b)+1;
+
+    /* New slice */
+    c->slice = c->slices[c->curr_slice] =
+        cram_new_slice(MAPPED_SLICE, c->max_rec);
+    if (!c->slice)
+        return NULL;
+
+    if (c->multi_seq) {
+        c->slice->hdr->ref_seq_id = -2;
+        c->slice->hdr->ref_seq_start = 0;
+        c->slice->last_apos = 1;
+    } else {
+        c->slice->hdr->ref_seq_id = bam_ref(b);
+        // wrong for unsorted data, will fix during encoding.
+        c->slice->hdr->ref_seq_start = bam_pos(b)+1;
+        c->slice->last_apos = bam_pos(b)+1;
+    }
+
+    c->curr_rec = 0;
+    c->s_num_bases = 0;
+    c->n_mapped = 0;
+
+    // QO field: 0 implies original orientation, 1 implies sequence orientation
+    // 1 is often preferable for NovaSeq, but impact is slight. ~0.5% diff.
+    // Conversely other data sets it's often better than 1% saving for 0.
+    // Short of trying both and learning, for now we use use 0 for V4, 1 for V3.
+    c->qs_seq_orient = CRAM_MAJOR_VERS(fd->version) >= 4 ? 0 : 1;
+
+    return c;
+}
+
+
+/*
+ * Converts a single bam record into a cram record.
+ * Possibly used within a thread.
+ *
+ * Returns 0 on success;
+ *        -1 on failure
+ */
+static int process_one_read(cram_fd *fd, cram_container *c,
+                            cram_slice *s, cram_record *cr,
+                            bam_seq_t *b, int rnum, kstring_t *MD,
+                            int embed_ref, int no_ref) {
+    int i, fake_qual = -1, NM = 0;
+    char *cp;
+    char *ref, *seq, *qual;
+
+    // Any places with N in seq and/or reference can lead to ambiguous
+    // interpretation of the SAM NM:i tag.  So we store these verbatim
+    // to ensure valid data round-trips the same regardless of who
+    // defines it as valid.
+    // Similarly when alignments go beyond end of the reference.
+    int verbatim_NM = fd->store_nm;
+    int verbatim_MD = fd->store_md;
+
+    // FIXME: multi-ref containers
+
+    cr->flags       = bam_flag(b);
+    cr->len         = bam_seq_len(b);
+    uint8_t *md;
+    if (!(md = bam_aux_get(b, "MD")))
+        MD = NULL;
+    else
+        MD->l = 0;
+
+    int cf_tag = 0;
+
+    if (embed_ref == 2) {
+        cf_tag  = MD ? 0 : 1;                   // No MD
+        cf_tag |= bam_aux_get(b, "NM") ? 0 : 2; // No NM
+    }
+
+    //fprintf(stderr, "%s => %d\n", rg ? rg : "\"\"", cr->rg);
+
+    ref = c->ref ? c->ref - (c->ref_start-1) : NULL;
+    cr->ref_id      = bam_ref(b);
+    if (cram_stats_add(c->stats[DS_RI], cr->ref_id) < 0)
+        goto block_err;
+    if (cram_stats_add(c->stats[DS_BF], fd->cram_flag_swap[cr->flags & 0xfff]) < 0)
+        goto block_err;
+
+    // Non reference based encoding means storing the bases verbatim as features, which in
+    // turn means every base also has a quality already stored.
+    if (!no_ref || CRAM_MAJOR_VERS(fd->version) >= 3)
+        cr->cram_flags |= CRAM_FLAG_PRESERVE_QUAL_SCORES;
+
+    if (cr->len <= 0 && CRAM_MAJOR_VERS(fd->version) >= 3)
+        cr->cram_flags |= CRAM_FLAG_NO_SEQ;
+    //cram_stats_add(c->stats[DS_CF], cr->cram_flags & CRAM_FLAG_MASK);
+
+    c->num_bases   += cr->len;
+    cr->apos        = bam_pos(b)+1;
+    if (cr->apos < 0 || cr->apos > INT64_MAX/2)
+        goto err;
+    if (c->pos_sorted) {
+        if (cr->apos < s->last_apos && !fd->ap_delta) {
+            c->pos_sorted = 0;
+        } else {
+            if (cram_stats_add(c->stats[DS_AP], cr->apos - s->last_apos) < 0)
+                goto block_err;
+            s->last_apos = cr->apos;
+        }
+    } else {
+        //cram_stats_add(c->stats[DS_AP], cr->apos);
+    }
+    c->max_apos += (cr->apos > c->max_apos) * (cr->apos - c->max_apos);
+
+    /*
+     * This seqs_ds is largely pointless and it could reuse the same memory
+     * over and over.
+     * s->base_blk is what we need for encoding.
+     */
+    cr->seq         = BLOCK_SIZE(s->seqs_blk);
+    cr->qual        = BLOCK_SIZE(s->qual_blk);
+    BLOCK_GROW(s->seqs_blk, cr->len+1);
+    BLOCK_GROW(s->qual_blk, cr->len);
+
+    // Convert BAM nibble encoded sequence to string of base pairs
+    seq = cp = (char *)BLOCK_END(s->seqs_blk);
+    *seq = 0;
+    nibble2base(bam_seq(b), cp, cr->len);
+    BLOCK_SIZE(s->seqs_blk) += cr->len;
+
+    qual = cp = (char *)bam_qual(b);
+
+
+    /* Copy and parse */
+    if (!(cr->flags & BAM_FUNMAP)) {
+        uint32_t *cig_to, *cig_from;
+        int64_t apos = cr->apos-1, spos = 0;
+        int64_t MD_last = apos; // last position of edit in MD tag
+
+        if (apos < 0) {
+            hts_log_error("Mapped read with position <= 0 is disallowed");
+            return -1;
+        }
+
+        cr->cigar       = s->ncigar;
+        cr->ncigar      = bam_cigar_len(b);
+        while (cr->cigar + cr->ncigar >= s->cigar_alloc) {
+            s->cigar_alloc = s->cigar_alloc ? s->cigar_alloc*2 : 1024;
+            s->cigar = realloc(s->cigar, s->cigar_alloc * sizeof(*s->cigar));
+            if (!s->cigar)
+                return -1;
+        }
+
+        cig_to = (uint32_t *)s->cigar;
+        cig_from = (uint32_t *)bam_cigar(b);
+
+        cr->feature = 0;
+        cr->nfeature = 0;
+        for (i = 0; i < cr->ncigar; i++) {
+            enum cigar_op cig_op = cig_from[i] & BAM_CIGAR_MASK;
+            uint32_t cig_len = cig_from[i] >> BAM_CIGAR_SHIFT;
+            cig_to[i] = cig_from[i];
+
+            /* Can also generate events from here for CRAM diffs */
+
+            switch (cig_op) {
+                int l;
+
+                // Don't trust = and X ops to be correct.
+            case BAM_CMATCH:
+            case BAM_CBASE_MATCH:
+            case BAM_CBASE_MISMATCH:
+                //fprintf(stderr, "\nBAM_CMATCH\nR: %.*s\nS: %.*s\n",
+                //      cig_len, &ref[apos], cig_len, &seq[spos]);
+                l = 0;
+                if (!no_ref && cr->len) {
+                    int end = cig_len+apos < c->ref_end
+                        ? cig_len : c->ref_end - apos;
+                    char *sp = &seq[spos];
+                    char *rp = &ref[apos];
+                    char *qp = &qual[spos];
+                    if (end > cr->len) {
+                        hts_log_error("CIGAR and query sequence are of different length");
+                        return -1;
+                    }
+                    for (l = 0; l < end; l++) {
+                        // This case is just too disputed and different tools
+                        // interpret these in different ways.  We give up and
+                        // store verbatim.
+                        if (rp[l] == 'N' && sp[l] == 'N')
+                            verbatim_NM = verbatim_MD = 1;
+                        if (rp[l] != sp[l]) {
+                            // Build our own MD tag if one is on the sequence, so
+                            // we can ensure it matches and thus can be discarded.
+                            if (MD && ref) {
+                                if (kputuw(apos+l - MD_last, MD) < 0) goto err;
+                                if (kputc(rp[l], MD) < 0) goto err;
+                                MD_last = apos+l+1;
+                            }
+                            NM++;
+                            if (!sp[l])
+                                break;
+                            if (0 && CRAM_MAJOR_VERS(fd->version) >= 3) {
+#if 0
+                                // Disabled for the time being as it doesn't
+                                // seem to gain us much.
+                                int ol=l;
+                                while (l<end && rp[l] != sp[l])
+                                    l++;
+                                if (l-ol > 1) {
+                                    if (cram_add_bases(fd, c, s, cr, spos+ol,
+                                                       l-ol, &seq[spos+ol]))
+                                        return -1;
+                                    l--;
+                                } else {
+                                    l = ol;
+                                    if (cram_add_substitution(fd, c, s, cr,
+                                                              spos+l, sp[l],
+                                                              qp[l], rp[l]))
+                                        return -1;
+                                }
+#else
+                                // With urmap pushed to the limit and lots
+                                // of unaligned data (should be soft-clipped)
+                                // this saves ~2-7%. Worth it?
+                                int nl = l;
+                                int max_end = nl, max_score = 0, score = 0;
+                                while (nl < end) {
+                                    if (rp[nl] != sp[nl]) {
+                                        score += 3;
+                                        if (max_score < score) {
+                                            max_score = score;
+                                            max_end = nl;
+                                        }
+                                    } else {
+                                        score--;
+                                        if (score < -2 ||
+                                            max_score - score > 7)
+                                            break;
+                                    }
+                                    nl++;
+                                }
+                                if (max_score > 20) {
+                                    cram_add_bases(fd, c, s, cr, spos+l,
+                                                   max_end-l, &seq[spos+l]);
+                                    l = max_end-1;
+                                } else {
+                                    while (l < nl) {
+                                        if (rp[l] != sp[l])
+                                            cram_add_substitution(fd, c, s,
+                                                                  cr, spos+l,
+                                                                  sp[l], qp[l],
+                                                                  rp[l]);
+                                        l++;
+                                    }
+                                    l--;
+                                }
+#endif
+                            } else {
+                                if (cram_add_substitution(fd, c, s, cr, spos+l,
+                                                          sp[l], qp[l], rp[l]))
+                                    return -1;
+                            }
+                        }
+                    }
+                    spos += l;
+                    apos += l;
+                }
+
+                if (l < cig_len && cr->len) {
+                    if (no_ref) {
+                        if (CRAM_MAJOR_VERS(fd->version) == 3) {
+                            if (cram_add_bases(fd, c, s, cr, spos,
+                                               cig_len-l, &seq[spos]))
+                                return -1;
+                            spos += cig_len-l;
+                        } else {
+                            for (; l < cig_len && seq[spos]; l++, spos++) {
+                                if (cram_add_base(fd, c, s, cr, spos,
+                                                  seq[spos], qual[spos]))
+                                    return -1;
+                            }
+                        }
+                    } else {
+                        /* off end of sequence or non-ref based output */
+                        verbatim_NM = verbatim_MD = 1;
+                        for (; l < cig_len && seq[spos]; l++, spos++) {
+                            if (cram_add_base(fd, c, s, cr, spos,
+                                              seq[spos], qual[spos]))
+                                return -1;
+                        }
+                    }
+                    apos += cig_len;
+                } else if (!cr->len) {
+                    /* Seq "*" */
+                    verbatim_NM = verbatim_MD = 1;
+                    apos += cig_len;
+                    spos += cig_len;
+                }
+                break;
+
+            case BAM_CDEL:
+                if (MD && ref) {
+                    if (kputuw(apos - MD_last, MD) < 0) goto err;
+                    if (apos < c->ref_end) {
+                        if (kputc_('^', MD) < 0) goto err;
+                        if (kputsn(&ref[apos], MIN(c->ref_end - apos, cig_len), MD) < 0)
+                            goto err;
+                    }
+                }
+                NM += cig_len;
+
+                if (cram_add_deletion(c, s, cr, spos, cig_len, &seq[spos]))
+                    return -1;
+                apos += cig_len;
+                MD_last = apos;
+                break;
+
+            case BAM_CREF_SKIP:
+                if (cram_add_skip(c, s, cr, spos, cig_len, &seq[spos]))
+                    return -1;
+                apos += cig_len;
+                MD_last += cig_len;
+                break;
+
+            case BAM_CINS:
+                if (cram_add_insertion(c, s, cr, spos, cig_len,
+                                       cr->len ? &seq[spos] : NULL))
+                    return -1;
+                if (no_ref && cr->len) {
+                    for (l = 0; l < cig_len; l++, spos++) {
+                        cram_add_quality(fd, c, s, cr, spos, qual[spos]);
+                    }
+                } else {
+                    spos += cig_len;
+                }
+                NM += cig_len;
+                break;
+
+            case BAM_CSOFT_CLIP:
+                if (cram_add_softclip(c, s, cr, spos, cig_len,
+                                      cr->len ? &seq[spos] : NULL,
+                                      fd->version))
+                    return -1;
+
+                if (no_ref &&
+                    !(cr->cram_flags & CRAM_FLAG_PRESERVE_QUAL_SCORES)) {
+                    if (cr->len) {
+                        for (l = 0; l < cig_len; l++, spos++) {
+                            cram_add_quality(fd, c, s, cr, spos, qual[spos]);
+                        }
+                    } else {
+                        for (l = 0; l < cig_len; l++, spos++) {
+                            cram_add_quality(fd, c, s, cr, spos, -1);
+                        }
+                    }
+                } else {
+                    spos += cig_len;
+                }
+                break;
+
+            case BAM_CHARD_CLIP:
+                if (cram_add_hardclip(c, s, cr, spos, cig_len, &seq[spos]))
+                    return -1;
+                break;
+
+            case BAM_CPAD:
+                if (cram_add_pad(c, s, cr, spos, cig_len, &seq[spos]))
+                    return -1;
+                break;
+
+            default:
+                hts_log_error("Unknown CIGAR op code %d", cig_op);
+                return -1;
+            }
+        }
+        if (cr->len && spos != cr->len) {
+            hts_log_error("CIGAR and query sequence are of different length");
+            return -1;
+        }
+        fake_qual = spos;
+        cr->aend = no_ref ? apos : MIN(apos, c->ref_end);
+        if (cram_stats_add(c->stats[DS_FN], cr->nfeature) < 0)
+            goto block_err;
+
+        if (MD && ref)
+            if (kputuw(apos - MD_last, MD) < 0) goto err;
+    } else {
+        // Unmapped
+        cr->cram_flags |= CRAM_FLAG_PRESERVE_QUAL_SCORES;
+        cr->cigar  = 0;
+        cr->ncigar = 0;
+        cr->nfeature = 0;
+        cr->aend = MIN(cr->apos, c->ref_end);
+        for (i = 0; i < cr->len; i++)
+            if (cram_stats_add(c->stats[DS_BA], seq[i]) < 0)
+                goto block_err;
+        fake_qual = 0;
+    }
+
+    cr->ntags      = 0; //cram_stats_add(c->stats[DS_TC], cr->ntags);
+    int err = 0;
+    sam_hrec_rg_t *brg =
+        cram_encode_aux(fd, b, c, s, cr, verbatim_NM, verbatim_MD, NM, MD,
+                        cf_tag, no_ref, &err);
+    if (err)
+        goto block_err;
+
+    /* Read group, identified earlier */
+    if (brg) {
+        cr->rg = brg->id;
+    } else if (CRAM_MAJOR_VERS(fd->version) == 1) {
+        sam_hrec_rg_t *brg = sam_hrecs_find_rg(fd->header->hrecs, "UNKNOWN");
+        if (!brg) goto block_err;
+        cr->rg = brg->id;
+    } else {
+        cr->rg = -1;
+    }
+    if (cram_stats_add(c->stats[DS_RG], cr->rg) < 0)
+        goto block_err;
+
+    /*
+     * Append to the qual block now. We do this here as
+     * cram_add_substitution() can generate BA/QS events which need to
+     * be in the qual block before we append the rest of the data.
+     */
+    if (cr->cram_flags & CRAM_FLAG_PRESERVE_QUAL_SCORES) {
+        /* Special case of seq "*" */
+        if (cr->len == 0) {
+            cr->len = fake_qual;
+            BLOCK_GROW(s->qual_blk, cr->len);
+            cp = (char *)BLOCK_END(s->qual_blk);
+            memset(cp, 255, cr->len);
+        } else {
+            BLOCK_GROW(s->qual_blk, cr->len);
+            cp = (char *)BLOCK_END(s->qual_blk);
+            char *from = (char *)&bam_qual(b)[0];
+            char *to = &cp[0];
+            memcpy(to, from, cr->len);
+
+            // Store quality in original orientation for better compression.
+            if (!c->qs_seq_orient) {
+                if (cr->flags & BAM_FREVERSE) {
+                    int i, j;
+                    for (i = 0, j = cr->len-1; i < j; i++, j--) {
+                        unsigned char c;
+                        c = to[i];
+                        to[i] = to[j];
+                        to[j] = c;
+                    }
+                }
+            }
+        }
+        BLOCK_SIZE(s->qual_blk) += cr->len;
+    } else {
+        if (cr->len == 0)
+            cr->len = fake_qual >= 0 ? fake_qual : cr->aend - cr->apos + 1;
+    }
+
+    if (cram_stats_add(c->stats[DS_RL], cr->len) < 0)
+        goto block_err;
+
+    /* Now we know apos and aend both, update mate-pair information */
+    {
+        int new;
+        khint_t k;
+        int sec = (cr->flags & BAM_FSECONDARY) ? 1 : 0;
+
+        //fprintf(stderr, "Checking %"PRId64"/%.*s\t", rnum,
+        //      cr->name_len, DSTRING_STR(s->name_ds)+cr->name);
+        if (cr->flags & BAM_FPAIRED) {
+            char *key = string_ndup(s->pair_keys, bam_name(b), bam_name_len(b));
+            if (!key)
+                return -1;
+
+            k = kh_put(m_s2i, s->pair[sec], key, &new);
+            if (-1 == new)
+                return -1;
+            else if (new > 0)
+                kh_val(s->pair[sec], k) = rnum;
+        } else {
+            new = 1;
+            k = 0; // Prevents false-positive warning from gcc -Og
+        }
+
+        if (new == 0) {
+            cram_record *p = &s->crecs[kh_val(s->pair[sec], k)];
+            int64_t aleft, aright;
+            int sign;
+
+            aleft = MIN(cr->apos, p->apos);
+            aright = MAX(cr->aend, p->aend);
+            if (cr->apos < p->apos) {
+                sign = 1;
+            } else if (cr->apos > p->apos) {
+                sign = -1;
+            } else if (cr->flags & BAM_FREAD1) {
+                sign = 1;
+            } else {
+                sign = -1;
+            }
+
+            // This vs p: tlen, matepos, flags. Permit TLEN 0 and/or TLEN +/-
+            // a small amount, if appropriate options set.
+            if ((!fd->tlen_zero && MAX(bam_mate_pos(b)+1, 0) != p->apos) &&
+                !(fd->tlen_zero && bam_mate_pos(b) == 0))
+                goto detached;
+
+            if (((bam_flag(b) & BAM_FMUNMAP) != 0) !=
+                ((p->flags & BAM_FUNMAP) != 0))
+                goto detached;
+
+            if (((bam_flag(b) & BAM_FMREVERSE) != 0) !=
+                ((p->flags & BAM_FREVERSE) != 0))
+                goto detached;
+
+
+            // p vs this: tlen, matepos, flags
+            if (p->ref_id != cr->ref_id &&
+                !(fd->tlen_zero && p->ref_id == -1))
+                goto detached;
+
+            if (p->mate_pos != cr->apos &&
+                !(fd->tlen_zero && p->mate_pos == 0))
+                goto detached;
+
+            if (((p->flags & BAM_FMUNMAP) != 0) !=
+                ((p->mate_flags & CRAM_M_UNMAP) != 0))
+                goto detached;
+
+            if (((p->flags & BAM_FMREVERSE) != 0) !=
+                ((p->mate_flags & CRAM_M_REVERSE) != 0))
+                goto detached;
+
+            // Supplementary reads are just too ill defined
+            if ((cr->flags & BAM_FSUPPLEMENTARY) ||
+                (p->flags & BAM_FSUPPLEMENTARY))
+                goto detached;
+
+            // When in lossy name mode, if a read isn't detached we
+            // cannot store the name.  The corollary is that when we
+            // must store the name, it must be detached (inefficient).
+            if (fd->lossy_read_names &&
+                (!(cr->cram_flags & CRAM_FLAG_DISCARD_NAME) ||
+                 !((p->cram_flags & CRAM_FLAG_DISCARD_NAME))))
+                goto detached;
+
+            // Now check TLEN.  We do this last as sometimes it's the
+            // only thing that differs.  In CRAM4 we have a better way
+            // of handling this that doesn't break detached status
+            int explicit_tlen = 0;
+            int tflag1 = ((bam_ins_size(b) &&
+                           llabs(bam_ins_size(b) - sign*(aright-aleft+1))
+                           > fd->tlen_approx)
+                          || (!bam_ins_size(b) && !fd->tlen_zero));
+
+            int tflag2 = ((p->tlen && llabs(p->tlen - -sign*(aright-aleft+1))
+                           > fd->tlen_approx)
+                          || (!p->tlen && !fd->tlen_zero));
+
+            if (tflag1 || tflag2) {
+                if (CRAM_MAJOR_VERS(fd->version) >= 4) {
+                    explicit_tlen = CRAM_FLAG_EXPLICIT_TLEN;
+                } else {
+                    // Stil do detached for unmapped data in CRAM4 as this
+                    // also impacts RNEXT calculation.
+                    goto detached;
+                }
+            }
+
+            /*
+             * The fields below are unused when encoding this read as it is
+             * no longer detached.  In theory they may get referred to when
+             * processing a 3rd or 4th read in this template?, so we set them
+             * here just to be sure.
+             *
+             * They do not need cram_stats_add() calls those as they are
+             * not emitted.
+             */
+            cr->mate_pos = p->apos;
+            cram_stats_add(c->stats[DS_NP], cr->mate_pos);
+            cr->tlen = explicit_tlen ? bam_ins_size(b) : sign*(aright-aleft+1);
+            cram_stats_add(c->stats[DS_TS], cr->tlen);
+            cr->mate_flags =
+                ((p->flags & BAM_FMUNMAP)   == BAM_FMUNMAP)   * CRAM_M_UNMAP +
+                ((p->flags & BAM_FMREVERSE) == BAM_FMREVERSE) * CRAM_M_REVERSE;
+
+            // Decrement statistics aggregated earlier
+            if (p->cram_flags & CRAM_FLAG_STATS_ADDED) {
+                cram_stats_del(c->stats[DS_NP], p->mate_pos);
+                cram_stats_del(c->stats[DS_MF], p->mate_flags);
+                if (!(p->cram_flags & CRAM_FLAG_EXPLICIT_TLEN))
+                    cram_stats_del(c->stats[DS_TS], p->tlen);
+                cram_stats_del(c->stats[DS_NS], p->mate_ref_id);
+            }
+
+            /* Similarly we could correct the p-> values too, but these will no
+             * longer have any code that refers back to them as the new 'p'
+             * for this template is our current 'cr'.
+             */
+            //p->mate_pos = cr->apos;
+            //p->mate_flags =
+            //  ((cr->flags & BAM_FMUNMAP)   == BAM_FMUNMAP)  * CRAM_M_UNMAP +
+            //  ((cr->flags & BAM_FMREVERSE) == BAM_FMREVERSE)* CRAM_M_REVERSE;
+            //p->tlen = p->apos - cr->aend;
+
+            // Clear detached from cr flags
+            cr->cram_flags &= ~CRAM_FLAG_DETACHED;
+            cr->cram_flags |= explicit_tlen;
+            if (cram_stats_add(c->stats[DS_CF], cr->cram_flags & CRAM_FLAG_MASK) < 0)
+                goto block_err;
+
+            // Clear detached from p flags and set downstream
+            if (p->cram_flags & CRAM_FLAG_STATS_ADDED) {
+                cram_stats_del(c->stats[DS_CF], p->cram_flags & CRAM_FLAG_MASK);
+                p->cram_flags &= ~CRAM_FLAG_STATS_ADDED;
+            }
+
+            p->cram_flags  &= ~CRAM_FLAG_DETACHED;
+            p->cram_flags  |=  CRAM_FLAG_MATE_DOWNSTREAM | explicit_tlen;;
+            if (cram_stats_add(c->stats[DS_CF], p->cram_flags & CRAM_FLAG_MASK) < 0)
+                goto block_err;
+
+            p->mate_line = rnum - (kh_val(s->pair[sec], k) + 1);
+            if (cram_stats_add(c->stats[DS_NF], p->mate_line) < 0)
+                goto block_err;
+
+            kh_val(s->pair[sec], k) = rnum;
+        } else {
+        detached:
+            //fprintf(stderr, "unpaired\n");
+
+            /* Derive mate flags from this flag */
+            cr->mate_flags = 0;
+            if (bam_flag(b) & BAM_FMUNMAP)
+                cr->mate_flags |= CRAM_M_UNMAP;
+            if (bam_flag(b) & BAM_FMREVERSE)
+                cr->mate_flags |= CRAM_M_REVERSE;
+
+            if (cram_stats_add(c->stats[DS_MF], cr->mate_flags) < 0)
+                goto block_err;
+
+            cr->mate_pos    = MAX(bam_mate_pos(b)+1, 0);
+            if (cram_stats_add(c->stats[DS_NP], cr->mate_pos) < 0)
+                goto block_err;
+
+            cr->tlen        = bam_ins_size(b);
+            if (cram_stats_add(c->stats[DS_TS], cr->tlen) < 0)
+                goto block_err;
+
+            cr->cram_flags |= CRAM_FLAG_DETACHED;
+            if (cram_stats_add(c->stats[DS_CF], cr->cram_flags & CRAM_FLAG_MASK) < 0)
+                goto block_err;
+            if (cram_stats_add(c->stats[DS_NS], bam_mate_ref(b)) < 0)
+                goto block_err;
+
+            cr->cram_flags |= CRAM_FLAG_STATS_ADDED;
+        }
+    }
+
+    cr->mqual       = bam_map_qual(b);
+    if (cram_stats_add(c->stats[DS_MQ], cr->mqual) < 0)
+        goto block_err;
+
+    cr->mate_ref_id = bam_mate_ref(b);
+
+    if (!(bam_flag(b) & BAM_FUNMAP)) {
+        if (c->first_base > cr->apos)
+            c->first_base = cr->apos;
+
+        if (c->last_base < cr->aend)
+            c->last_base = cr->aend;
+    }
+
+    return 0;
+
+ block_err:
+ err:
+    return -1;
+}
+
+/*
+ * Write iterator: put BAM format sequences into a CRAM file.
+ * We buffer up a containers worth of data at a time.
+ *
+ * Returns 0 on success
+ *        -1 on failure
+ */
+int cram_put_bam_seq(cram_fd *fd, bam_seq_t *b) {
+    cram_container *c;
+
+    if (!fd->ctr) {
+        fd->ctr = cram_new_container(fd->seqs_per_slice,
+                                     fd->slices_per_container);
+        if (!fd->ctr)
+            return -1;
+        fd->ctr->record_counter = fd->record_counter;
+
+        pthread_mutex_lock(&fd->ref_lock);
+        fd->ctr->no_ref = fd->no_ref;
+        fd->ctr->embed_ref = fd->embed_ref;
+        pthread_mutex_unlock(&fd->ref_lock);
+    }
+    c = fd->ctr;
+
+    int embed_ref = c->embed_ref;
+
+    if (!c->slice || c->curr_rec == c->max_rec ||
+        (bam_ref(b) != c->curr_ref && c->curr_ref >= -1) ||
+        (c->s_num_bases + c->s_aux_bytes >= fd->bases_per_slice)) {
+        int slice_rec, curr_rec, multi_seq = fd->multi_seq == 1;
+        int curr_ref = c->slice ? c->curr_ref : bam_ref(b);
+
+        /*
+         * Start packing slices when we routinely have under 1/4tr full.
+         *
+         * This option isn't available if we choose to embed references
+         * since we can only have one per slice.
+         *
+         * The multi_seq var here refers to our intention for the next slice.
+         * This slice has already been encoded so we output as-is.
+         */
+        if (fd->multi_seq == -1 && c->curr_rec < c->max_rec/4+10 &&
+            fd->last_slice && fd->last_slice < c->max_rec/4+10 &&
+            embed_ref<=0) {
+            if (!c->multi_seq)
+                hts_log_info("Multi-ref enabled for next container");
+            multi_seq = 1;
+        } else if (fd->multi_seq == 1) {
+            pthread_mutex_lock(&fd->metrics_lock);
+            if (fd->last_RI_count <= c->max_slice && fd->multi_seq_user != 1) {
+                multi_seq = 0;
+                hts_log_info("Multi-ref disabled for next container");
+            }
+            pthread_mutex_unlock(&fd->metrics_lock);
+        }
+
+        slice_rec = c->slice_rec;
+        curr_rec  = c->curr_rec;
+
+        if (CRAM_MAJOR_VERS(fd->version) == 1 ||
+            c->curr_rec == c->max_rec || fd->multi_seq != 1 || !c->slice ||
+            c->s_num_bases + c->s_aux_bytes >= fd->bases_per_slice) {
+            if (NULL == (c = cram_next_container(fd, b))) {
+                if (fd->ctr) {
+                    // prevent cram_close attempting to flush
+                    fd->ctr_mt = fd->ctr; // delay free when threading
+                    fd->ctr = NULL;
+                }
+                return -1;
+            }
+        }
+
+        /*
+         * Due to our processing order, some things we've already done we
+         * cannot easily undo. So when we first notice we should be packing
+         * multiple sequences per container we emit the small partial
+         * container as-is and then start a fresh one in a different mode.
+         */
+        if (multi_seq == 0 && fd->multi_seq == 1 && fd->multi_seq_user == -1) {
+            // User selected auto-mode, we're currently using multi-seq, but
+            // have detected we don't need to.  Switch back to auto.
+            fd->multi_seq = -1;
+        } else if (multi_seq) {
+            // We detected we need multi-seq
+            fd->multi_seq = 1;
+            c->multi_seq = 1;
+            c->pos_sorted = 0;
+
+            // Cram_next_container may end up flushing an existing one and
+            // triggering fd->embed_ref=2 if no reference is found.
+            // Embedded refs are incompatible with multi-seq, so we bail
+            // out and switch to no_ref in this scenario.  We do this
+            // within the container only, as multi_seq may be temporary
+            // and we switch back away from it again.
+            pthread_mutex_lock(&fd->ref_lock);
+            if (fd->embed_ref > 0 && c->curr_rec == 0 && c->curr_slice == 0) {
+                hts_log_warning("Changing from embed_ref to no_ref mode");
+                // Should we update fd->embed_ref and no_ref here too?
+                // Doing so means if we go into multi-seq and back out
+                // again, eg due a cluster of tiny refs in the middle of
+                // much larger ones, then we bake in no-ref mode.
+                //
+                // However for unsorted data we're realistically not
+                // going to switch back.
+                c->embed_ref = fd->embed_ref = 0; // or -1 for auto?
+                c->no_ref = fd->no_ref = 1;
+            }
+            pthread_mutex_unlock(&fd->ref_lock);
+
+            if (!c->refs_used) {
+                pthread_mutex_lock(&fd->ref_lock);
+                c->refs_used = calloc(fd->refs->nref, sizeof(int));
+                pthread_mutex_unlock(&fd->ref_lock);
+                if (!c->refs_used)
+                    return -1;
+            }
+        }
+
+        fd->last_slice = curr_rec - slice_rec;
+        c->slice_rec = c->curr_rec;
+
+        // Have we seen this reference before?
+        if (bam_ref(b) >= 0 && curr_ref >= 0 && bam_ref(b) != curr_ref &&
+            embed_ref<=0 && !fd->unsorted && multi_seq) {
+
+            if (!c->refs_used) {
+                pthread_mutex_lock(&fd->ref_lock);
+                c->refs_used = calloc(fd->refs->nref, sizeof(int));
+                pthread_mutex_unlock(&fd->ref_lock);
+                if (!c->refs_used)
+                    return -1;
+            } else if (c->refs_used && c->refs_used[bam_ref(b)]) {
+                pthread_mutex_lock(&fd->ref_lock);
+                fd->unsorted = 1;
+                fd->multi_seq = 1;
+                pthread_mutex_unlock(&fd->ref_lock);
+            }
+        }
+
+        c->curr_ref = bam_ref(b);
+        if (c->refs_used && c->curr_ref >= 0) c->refs_used[c->curr_ref]++;
+    }
+
+    if (!c->bams) {
+        /* First time through, allocate a set of bam pointers */
+        pthread_mutex_lock(&fd->bam_list_lock);
+        if (fd->bl) {
+            spare_bams *spare = fd->bl;
+            c->bams = spare->bams;
+            fd->bl = spare->next;
+            free(spare);
+        } else {
+            c->bams = calloc(c->max_c_rec, sizeof(bam_seq_t *));
+            if (!c->bams) {
+                pthread_mutex_unlock(&fd->bam_list_lock);
+                return -1;
+            }
+        }
+        pthread_mutex_unlock(&fd->bam_list_lock);
+    }
+
+    /* Copy or alloc+copy the bam record, for later encoding */
+    if (c->bams[c->curr_c_rec]) {
+        if (bam_copy1(c->bams[c->curr_c_rec], b) == NULL)
+            return -1;
+    } else {
+        c->bams[c->curr_c_rec] = bam_dup1(b);
+        if (c->bams[c->curr_c_rec] == NULL)
+            return -1;
+    }
+    if (bam_seq_len(b)) {
+        c->s_num_bases += bam_seq_len(b);
+    } else {
+        // No sequence in BAM record.  CRAM doesn't directly support this
+        // case, it ends up being stored as a string of N's for each query
+        // consuming CIGAR operation.  As this can become very inefficient
+        // in time and memory, data where the query length is excessively
+        // long are rejected.
+        hts_pos_t qlen = bam_cigar2qlen(b->core.n_cigar, bam_get_cigar(b));
+        if (qlen > 100000000) {
+            hts_log_error("CIGAR query length %"PRIhts_pos
+                          " for read \"%s\" is too long",
+                          qlen, bam_get_qname(b));
+            return -1;
+        }
+        c->s_num_bases += qlen;
+    }
+    c->curr_rec++;
+    c->curr_c_rec++;
+    c->s_aux_bytes += bam_get_l_aux(b);
+    c->n_mapped += (bam_flag(b) & BAM_FUNMAP) ? 0 : 1;
+    fd->record_counter++;
+
+    return 0;
+}
diff --git a/ext/htslib/cram/cram_encode.h b/ext/htslib/cram/cram_encode.h
new file mode 100644
index 0000000..03b8054
--- /dev/null
+++ b/ext/htslib/cram/cram_encode.h
@@ -0,0 +1,116 @@
+/*
+Copyright (c) 2012-2013, 2018 Genome Research Ltd.
+Author: James Bonfield <jkb@sanger.ac.uk>
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+Institute nor the names of its contributors may be used to endorse or promote
+products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+/*! \file
+ * Include cram.h instead.
+ *
+ * This is an internal part of the CRAM system and is automatically included
+ * when you #include cram.h.
+ *
+ * Implements the encoding portion of CRAM I/O. Also see
+ * cram_codecs.[ch] for the actual encoding functions themselves.
+ */
+
+#ifndef CRAM_ENCODE_H
+#define CRAM_ENCODE_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* ----------------------------------------------------------------------
+ * CRAM sequence iterators.
+ */
+
+/*! Write iterator: put BAM format sequences into a CRAM file.
+ *
+ * We buffer up a containers worth of data at a time.
+ *
+ * FIXME: break this into smaller pieces.
+ *
+ * @return
+ * Returns 0 on success;
+ *        -1 on failure
+ */
+int cram_put_bam_seq(cram_fd *fd, bam_seq_t *b);
+
+
+/* ----------------------------------------------------------------------
+ * Internal functions
+ */
+
+/*! INTERNAL:
+ * Encodes a compression header block into a generic cram_block structure.
+ *
+ * @return
+ * Returns cram_block ptr on success;
+ *         NULL on failure
+ */
+cram_block *cram_encode_compression_header(cram_fd *fd, cram_container *c,
+                                           cram_block_compression_hdr *h,
+                                           int embed_ref);
+
+/*! INTERNAL:
+ * Encodes a slice compression header.
+ *
+ * @return
+ * Returns cram_block on success;
+ *         NULL on failure
+ */
+cram_block *cram_encode_slice_header(cram_fd *fd, cram_slice *s);
+
+/*! INTERNAL:
+ * Encodes all slices in a container into blocks.
+ *
+ * @return
+ * Returns 0 on success;
+ *        -1 on failure
+ *
+ * FIXME: separate into encode_container and write_container. Ideally
+ * we should be able to do read_container / write_container or
+ * decode_container / encode_container.
+ */
+int cram_encode_container(cram_fd *fd, cram_container *c);
+
+/*! INTERNAL:
+ *
+ * During cram_next_container or before the final flush at end of
+ * file, we update the current slice headers and increment the slice
+ * number to the next slice.
+ *
+ * See cram_next_container() and cram_close().
+ */
+void cram_update_curr_slice(cram_container *c, int version);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/ext/htslib/cram/cram_external.c b/ext/htslib/cram/cram_external.c
new file mode 100644
index 0000000..4943750
--- /dev/null
+++ b/ext/htslib/cram/cram_external.c
@@ -0,0 +1,1033 @@
+/*
+Copyright (c) 2015, 2018-2020, 2022-2024 Genome Research Ltd.
+Author: James Bonfield <jkb@sanger.ac.uk>
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+Institute nor the names of its contributors may be used to endorse or promote
+products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+/*! \file
+ * External CRAM interface.
+ *
+ * Internally we're happy to use macros and to grub around in the cram
+ * structures.  This isn't very sustainable for an externally usable
+ * ABI though, so we have anonymous structs and accessor functions too
+ * to permit software such as samtools reheader to manipulate cram
+ * containers and blocks in a robust manner.
+ */
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+#include <stdint.h>
+
+#if defined(HAVE_EXTERNAL_LIBHTSCODECS)
+#include <htscodecs/rANS_static4x16.h>
+#else
+#include "../htscodecs/htscodecs/rANS_static4x16.h"
+#endif
+
+#include "../htslib/hfile.h"
+#include "cram.h"
+
+/*
+ *-----------------------------------------------------------------------------
+ * cram_fd
+ */
+sam_hdr_t *cram_fd_get_header(cram_fd *fd) { return fd->header; }
+void cram_fd_set_header(cram_fd *fd, sam_hdr_t *hdr) { fd->header = hdr; }
+
+int cram_fd_get_version(cram_fd *fd) { return fd->version; }
+void cram_fd_set_version(cram_fd *fd, int vers) { fd->version = vers; }
+
+int cram_major_vers(cram_fd *fd) { return CRAM_MAJOR_VERS(fd->version); }
+int cram_minor_vers(cram_fd *fd) { return CRAM_MINOR_VERS(fd->version); }
+
+hFILE *cram_fd_get_fp(cram_fd *fd) { return fd->fp; }
+void cram_fd_set_fp(cram_fd *fd, hFILE *fp) { fd->fp = fp; }
+
+
+/*
+ *-----------------------------------------------------------------------------
+ * cram_container
+ */
+int32_t cram_container_get_length(cram_container *c) {
+    return c->length;
+}
+
+void cram_container_set_length(cram_container *c, int32_t length) {
+    c->length = length;
+}
+
+
+int32_t cram_container_get_num_blocks(cram_container *c) {
+    return c->num_blocks;
+}
+
+void cram_container_set_num_blocks(cram_container *c, int32_t num_blocks) {
+    c->num_blocks = num_blocks;
+}
+
+int32_t cram_container_get_num_records(cram_container *c) {
+    return c->num_records;
+}
+
+int64_t cram_container_get_num_bases(cram_container *c) {
+    return c->num_bases;
+}
+
+
+/* Returns the landmarks[] array and the number of elements
+ * in num_landmarks.
+ */
+int32_t *cram_container_get_landmarks(cram_container *c, int32_t *num_landmarks) {
+    *num_landmarks = c->num_landmarks;
+    return c->landmark;
+}
+
+/* Sets the landmarks[] array (pointer copy, not a memory dup) and
+ * num_landmarks value.
+ */
+void cram_container_set_landmarks(cram_container *c, int32_t num_landmarks,
+                                  int32_t *landmarks) {
+    c->num_landmarks = num_landmarks;
+    c->landmark = landmarks;
+}
+
+
+/* Returns true if the container is empty (EOF marker) */
+int cram_container_is_empty(cram_fd *fd) {
+    return fd->empty_container;
+}
+
+void cram_container_get_coords(cram_container *c,
+                               int *refid, hts_pos_t *start, hts_pos_t *span) {
+    if (refid)
+        *refid = c->ref_seq_id;
+    if (start)
+        *start = c->ref_seq_start;
+    if (span)
+        *span  = c->ref_seq_span;
+}
+
+
+/*
+ *-----------------------------------------------------------------------------
+ * cram_block_compression_hdr
+ */
+
+/*
+ * Utility function to edit an RG id.
+ * This is only possible if there is one single RG value used and it
+ * is in the container compression header using HUFFMAN or BETA
+ * codec.  In this case it is essentially hard coded and needs no
+ * editing of external (or worse, CORE) blocks.
+ *
+ * Returns 0 on success
+ *        -1 on failure
+ */
+// Or arbitrary set compression header constant?
+
+static int cram_block_compression_hdr_set_DS(cram_block_compression_hdr *ch,
+                                             int ds, int new_rg) {
+    if (!ch || !ch->codecs[ds])
+        return -1;
+
+    switch (ch->codecs[ds]->codec) {
+    case E_HUFFMAN:
+        if (ch->codecs[ds]->u.huffman.ncodes != 1)
+            return -1;
+        ch->codecs[ds]->u.huffman.codes[0].symbol = new_rg;
+        return 0;
+
+    case E_BETA:
+        if (ch->codecs[ds]->u.beta.nbits != 0)
+            return -1;
+        ch->codecs[ds]->u.beta.offset = -new_rg;
+        return 0;
+
+    default:
+        break;
+    }
+
+    return -1;
+}
+
+int cram_block_compression_hdr_set_rg(cram_block_compression_hdr *ch, int new_rg) {
+    return cram_block_compression_hdr_set_DS(ch, DS_RG, new_rg);
+}
+
+/*
+ * Converts a cram_block_compression_hdr struct used for decoding to
+ * one used for encoding.  Maybe this should be a transparent
+ * operation applied on-demand.
+ *
+ * Returns 0 on success
+ *        -1 on failure
+ */
+int cram_block_compression_hdr_decoder2encoder(cram_fd *fd,
+                                               cram_block_compression_hdr *ch) {
+    int i;
+
+    if (!ch)
+        return -1;
+
+    for (i = 0; i < DS_END; i++) {
+        cram_codec *co = ch->codecs[i];
+        if (!co)
+            continue;
+
+        if (-1 == cram_codec_decoder2encoder(fd, co))
+            return -1;
+    }
+
+    return 0;
+}
+
+typedef struct {
+    cram_block_compression_hdr *hdr;
+    cram_map *curr_map;
+    int idx;
+    int is_tag; // phase 2 using tag_encoding_map
+} cram_codec_iter;
+
+static void cram_codec_iter_init(cram_block_compression_hdr *hdr,
+                                 cram_codec_iter *iter) {
+    iter->hdr = hdr;
+    iter->curr_map = NULL;
+    iter->idx = 0;
+    iter->is_tag = 0;
+}
+
+// See enum cram_DS_ID in cram/cram_structs
+static int cram_ds_to_key(enum cram_DS_ID ds) {
+    switch(ds) {
+    case DS_RN: return 256*'R'+'N';
+    case DS_QS: return 256*'Q'+'S';
+    case DS_IN: return 256*'I'+'N';
+    case DS_SC: return 256*'S'+'C';
+    case DS_BF: return 256*'B'+'F';
+    case DS_CF: return 256*'C'+'F';
+    case DS_AP: return 256*'A'+'P';
+    case DS_RG: return 256*'R'+'G';
+    case DS_MQ: return 256*'M'+'Q';
+    case DS_NS: return 256*'N'+'S';
+    case DS_MF: return 256*'M'+'F';
+    case DS_TS: return 256*'T'+'S';
+    case DS_NP: return 256*'N'+'P';
+    case DS_NF: return 256*'N'+'F';
+    case DS_RL: return 256*'R'+'L';
+    case DS_FN: return 256*'F'+'N';
+    case DS_FC: return 256*'F'+'C';
+    case DS_FP: return 256*'F'+'P';
+    case DS_DL: return 256*'D'+'L';
+    case DS_BA: return 256*'B'+'A';
+    case DS_BS: return 256*'B'+'S';
+    case DS_TL: return 256*'T'+'L';
+    case DS_RI: return 256*'R'+'I';
+    case DS_RS: return 256*'R'+'S';
+    case DS_PD: return 256*'P'+'D';
+    case DS_HC: return 256*'H'+'C';
+    case DS_BB: return 256*'B'+'B';
+    case DS_QQ: return 256*'Q'+'Q';
+    case DS_TN: return 256*'T'+'N';
+    case DS_TC: return 256*'T'+'C';
+    case DS_TM: return 256*'T'+'M';
+    case DS_TV: return 256*'T'+'V';
+    default: break;
+    }
+
+    return -1; // unknown
+}
+
+static cram_codec *cram_codec_iter_next(cram_codec_iter *iter,
+                                        int *key) {
+    cram_codec *cc = NULL;
+    cram_block_compression_hdr *hdr = iter->hdr;
+
+    if (!iter->is_tag) {
+        // 1: Iterating through main data-series
+        do {
+            cc = hdr->codecs[iter->idx++];
+        } while(!cc && iter->idx < DS_END);
+        if (cc) {
+            *key = cram_ds_to_key(iter->idx-1);
+            return cc;
+        }
+
+        // Reset index for phase 2
+        iter->idx = 0;
+        iter->is_tag = 1;
+    }
+
+    do {
+        if (!iter->curr_map)
+            iter->curr_map = hdr->tag_encoding_map[iter->idx++];
+
+        cc = iter->curr_map ? iter->curr_map->codec : NULL;
+        if (cc) {
+            *key = iter->curr_map->key;
+            iter->curr_map = iter->curr_map->next;
+            return cc;
+        }
+    } while (iter->idx < CRAM_MAP_HASH);
+
+    // End of codecs
+    return NULL;
+}
+
+/*
+ * A list of data-series, used to create a linked list threaded through
+ * a single array.
+ */
+typedef struct ds_list {
+    int data_series;
+    int next;
+} ds_list;
+
+KHASH_MAP_INIT_INT(cid, int64_t)
+
+// Opaque struct for the CRAM block content-id -> data-series map.
+struct cram_cid2ds_t {
+    ds_list *ds;        // array of data-series with linked lists threading through it
+    int ds_size;
+    int ds_idx;
+    khash_t(cid) *hash; // key=content_id,  value=index to ds array
+    int *ds_a;          // serialised array of data-series returned by queries.
+};
+
+void cram_cid2ds_free(cram_cid2ds_t *cid2ds) {
+    if (cid2ds) {
+        if (cid2ds->hash)
+            kh_destroy(cid, cid2ds->hash);
+        free(cid2ds->ds);
+        free(cid2ds->ds_a);
+        free(cid2ds);
+    }
+}
+
+/*
+ * Map cram block numbers to data-series.  It's normally a 1:1 mapping,
+ * but in rare cases it can be 1:many (or even many:many).
+ * The key is the block number and the value is an index into the data-series
+ * array, which we iterate over until reaching a negative value.
+ *
+ * Provide cid2ds as NULL to allocate a new map or pass in an existing one
+ * to append to this map.  The new (or existing) map is returned.
+ *
+ * Returns the cid2ds (newly allocated or as provided) on success,
+ *         NULL on failure.
+ */
+cram_cid2ds_t *cram_update_cid2ds_map(cram_block_compression_hdr *hdr,
+                                      cram_cid2ds_t *cid2ds) {
+    cram_cid2ds_t *c2d = cid2ds;
+    if (!c2d) {
+        c2d = calloc(1, sizeof(*c2d));
+        if (!c2d)
+            return NULL;
+
+        c2d->hash = kh_init(cid);
+        if (!c2d->hash)
+            goto err;
+    }
+
+    // Iterate through codecs.  Initially primary two-left ones in
+    // rec_encoding_map, and then the three letter in tag_encoding_map.
+    cram_codec_iter citer;
+    cram_codec_iter_init(hdr, &citer);
+    cram_codec *codec;
+    int key;
+
+    while ((codec = cram_codec_iter_next(&citer, &key))) {
+        // Having got a codec, we can then use cram_codec_to_id to get
+        // the block IDs utilised by that codec.  This is then our
+        // map for allocating data blocks to data series, but for shared
+        // blocks we can't separate out how much is used by each DS.
+        int bnum[2];
+        cram_codec_get_content_ids(codec, bnum);
+
+        khiter_t k;
+        int ret, i;
+        for (i = 0; i < 2; i++) {
+            if (bnum[i] > -2) {
+                k = kh_put(cid, c2d->hash, bnum[i], &ret);
+                if (ret < 0)
+                    goto err;
+
+                if (c2d->ds_idx >= c2d->ds_size) {
+                    c2d->ds_size += 100;
+                    c2d->ds_size *= 2;
+                    ds_list *ds_new = realloc(c2d->ds,
+                                              c2d->ds_size * sizeof(*ds_new));
+                    if (!ds_new)
+                        goto err;
+                    c2d->ds = ds_new;
+                }
+
+                if (ret == 0) {
+                    // Shared content_id, so add to list of DS
+
+                    // Maybe data-series should be part of the hash key?
+                    //
+                    // So top-32 bit is content-id, bot-32 bit is key.
+                    // Sort hash by key and then can group all the data-series
+                    // known together. ??
+                    //
+                    // Brute force for now, scan to see if recorded.
+                    // Typically this is minimal effort as we almost always
+                    // have 1 data-series per block content-id, so the list to
+                    // search is of size 1.
+                    int dsi = kh_value(c2d->hash, k);
+                    while (dsi >= 0) {
+                        if (c2d->ds[dsi].data_series == key)
+                            break;
+                        dsi = c2d->ds[dsi].next;
+                    }
+
+                    if (dsi == -1) {
+                        // Block content_id seen before, but not with this DS
+                        c2d->ds[c2d->ds_idx].data_series = key;
+                        c2d->ds[c2d->ds_idx].next = kh_value(c2d->hash, k);
+                        kh_value(c2d->hash, k) = c2d->ds_idx;
+                        c2d->ds_idx++;
+                    }
+                } else {
+                    // First time this content id has been used
+                    c2d->ds[c2d->ds_idx].data_series = key;
+                    c2d->ds[c2d->ds_idx].next = -1;
+                    kh_value(c2d->hash, k) = c2d->ds_idx;
+                    c2d->ds_idx++;
+                }
+            }
+        }
+    }
+
+    return c2d;
+
+ err:
+    if (c2d != cid2ds)
+        cram_cid2ds_free(c2d);
+    return NULL;
+}
+
+/*
+ * Return a list of data series observed as belonging to a block with
+ * the specified content_id.  *n is the number of data series
+ * returned, or 0 if block is unused.
+ * Block content_id of -1 is used to indicate the CORE block.
+ *
+ * The pointer returned is owned by the cram_cid2ds state and should
+ * not be freed by the caller.
+ */
+int *cram_cid2ds_query(cram_cid2ds_t *c2d, int content_id, int *n) {
+    *n = 0;
+    if (!c2d || !c2d->hash)
+        return NULL;
+
+    khiter_t k = kh_get(cid, c2d->hash, content_id);
+    if (k == kh_end(c2d->hash))
+        return NULL;
+
+    if (!c2d->ds_a) {
+        c2d->ds_a = malloc(c2d->ds_idx * sizeof(int));
+        if (!c2d->ds_a)
+            return NULL;
+    }
+
+    int dsi = kh_value(c2d->hash, k); // initial ds array index from hash
+    int idx = 0;
+    while (dsi >= 0) {
+        c2d->ds_a[idx++] = c2d->ds[dsi].data_series;
+        dsi = c2d->ds[dsi].next;      // iterate over list within ds array
+    }
+
+    *n = idx;
+    return c2d->ds_a;
+}
+
+/*
+ * Produces a description of the record and tag encodings held within
+ * a compression header and appends to 'ks'.
+ *
+ * Returns 0 on success,
+ *        <0 on failure.
+ */
+int cram_describe_encodings(cram_block_compression_hdr *hdr, kstring_t *ks) {
+    cram_codec_iter citer;
+    cram_codec_iter_init(hdr, &citer);
+    cram_codec *codec;
+    int key, r = 0;
+
+    while ((codec = cram_codec_iter_next(&citer, &key))) {
+        char key_s[4] = {0};
+        int key_i = 0;
+        if (key>>16) key_s[key_i++] = key>>16;
+        key_s[key_i++] = (key>>8)&0xff;
+        key_s[key_i++] = key&0xff;
+        r |= ksprintf(ks, "\t%s\t", key_s) < 0;
+        r |= cram_codec_describe(codec, ks) < 0;
+        r |= kputc('\n', ks) < 0;
+    }
+
+    return r ? -1 : 0;
+}
+
+/*
+ *-----------------------------------------------------------------------------
+ * cram_slice
+ */
+int32_t cram_slice_hdr_get_num_blocks(cram_block_slice_hdr *hdr) {
+    return hdr->num_blocks;
+}
+
+int cram_slice_hdr_get_embed_ref_id(cram_block_slice_hdr *h) {
+    return h->ref_base_id;
+}
+
+void cram_slice_hdr_get_coords(cram_block_slice_hdr *h,
+                               int *refid, hts_pos_t *start, hts_pos_t *span) {
+    if (refid)
+        *refid = h->ref_seq_id;
+    if (start)
+        *start = h->ref_seq_start;
+    if (span)
+        *span  = h->ref_seq_span;
+}
+
+/*
+ *-----------------------------------------------------------------------------
+ * cram_block
+ */
+int32_t cram_block_get_content_id(cram_block *b)  {
+    return b->content_type == CORE ? -1 : b->content_id;
+}
+int32_t cram_block_get_comp_size(cram_block *b)   { return b->comp_size; }
+int32_t cram_block_get_uncomp_size(cram_block *b) { return b->uncomp_size; }
+int32_t cram_block_get_crc32(cram_block *b)       { return b->crc32; }
+void *  cram_block_get_data(cram_block *b)        { return BLOCK_DATA(b); }
+int32_t cram_block_get_size(cram_block *b)        { return BLOCK_SIZE(b); }
+enum cram_block_method cram_block_get_method(cram_block *b) {
+    return (enum cram_block_method)b->orig_method;
+}
+enum cram_content_type cram_block_get_content_type(cram_block *b) {
+    return b->content_type;
+}
+
+void cram_block_set_content_id(cram_block *b, int32_t id) { b->content_id = id; }
+void cram_block_set_comp_size(cram_block *b, int32_t size) { b->comp_size = size; }
+void cram_block_set_uncomp_size(cram_block *b, int32_t size) { b->uncomp_size = size; }
+void cram_block_set_crc32(cram_block *b, int32_t crc) { b->crc32 = crc; }
+void cram_block_set_data(cram_block *b, void *data) { BLOCK_DATA(b) = data; }
+void cram_block_set_size(cram_block *b, int32_t size) { BLOCK_SIZE(b) = size; }
+
+int cram_block_append(cram_block *b, const void *data, int size) {
+    BLOCK_APPEND(b, data, size);
+    return 0;
+
+ block_err:
+    return -1;
+}
+void cram_block_update_size(cram_block *b) { BLOCK_UPLEN(b); }
+
+// Offset is known as "size" internally, but it can be confusing.
+size_t cram_block_get_offset(cram_block *b) { return BLOCK_SIZE(b); }
+void cram_block_set_offset(cram_block *b, size_t offset) { BLOCK_SIZE(b) = offset; }
+
+/*
+ * Given a compressed block of data in a specified compression method,
+ * fill out the 'cm' field with meta-data gleaned from the compressed
+ * block.
+ *
+ * If comp is CRAM_COMP_UNKNOWN, we attempt to auto-detect the compression
+ * format, but this doesn't work for all methods.
+ *
+ * Retuns the detected or specified comp method, and fills out *cm
+ * if non-NULL.
+ */
+cram_method_details *cram_expand_method(uint8_t *data, int32_t size,
+                                        enum cram_block_method comp) {
+    cram_method_details *cm = calloc(1, sizeof(*cm));
+    if (!cm)
+        return NULL;
+
+    const char *xz_header = "\xFD""7zXZ"; // including nul
+
+    if (comp == CRAM_COMP_UNKNOWN) {
+        // Auto-detect
+        if (size > 1 && data[0] == 0x1f && data[1] == 0x8b)
+            comp = CRAM_COMP_GZIP;
+        else if (size > 3 && data[1] == 'B' && data[2] == 'Z'
+                 && data[3] == 'h')
+            comp = CRAM_COMP_BZIP2;
+        else if (size > 6 && memcmp(xz_header, data, 6) == 0)
+            comp = CRAM_COMP_LZMA;
+        else
+            comp = CRAM_COMP_UNKNOWN;
+    }
+    cm->method = comp;
+
+    // Interrogate the compressed data stream to fill out additional fields.
+    switch (comp) {
+    case CRAM_COMP_GZIP:
+        if (size > 8) {
+            if (data[8] == 4)
+                cm->level = 1;
+            else if (data[8] == 2)
+                cm->level = 9;
+            else
+                cm->level = 5;
+        }
+        break;
+
+    case CRAM_COMP_BZIP2:
+        if (size > 3 && data[3] >= '1' && data[3] <= '9')
+            cm->level = data[3]-'0';
+        break;
+
+    case CRAM_COMP_RANS4x8:
+        cm->Nway = 4;
+        if (size > 0 && data[0] == 1)
+            cm->order = 1;
+        else
+            cm->order = 0;
+        break;
+
+    case CRAM_COMP_RANSNx16:
+        if (size > 0) {
+            cm->order  = data[0] & 1;
+            cm->Nway   = data[0] & RANS_ORDER_X32    ? 32 : 4;
+            cm->rle    = data[0] & RANS_ORDER_RLE    ?  1 : 0;
+            cm->pack   = data[0] & RANS_ORDER_PACK   ?  1 : 0;
+            cm->cat    = data[0] & RANS_ORDER_CAT    ?  1 : 0;
+            cm->stripe = data[0] & RANS_ORDER_STRIPE ?  1 : 0;
+            cm->nosz   = data[0] & RANS_ORDER_NOSZ   ?  1 : 0;
+        }
+        break;
+
+    case CRAM_COMP_ARITH:
+        if (size > 0) {
+            // Not in a public header, but the same transforms as rANSNx16
+            cm->order  = data[0] & 3;
+            cm->rle    = data[0] & RANS_ORDER_RLE    ?  1 : 0;
+            cm->pack   = data[0] & RANS_ORDER_PACK   ?  1 : 0;
+            cm->cat    = data[0] & RANS_ORDER_CAT    ?  1 : 0;
+            cm->stripe = data[0] & RANS_ORDER_STRIPE ?  1 : 0;
+            cm->nosz   = data[0] & RANS_ORDER_NOSZ   ?  1 : 0;
+            cm->ext    = data[0] & 4 /*external*/    ?  1 : 0;
+        }
+        break;
+
+    case CRAM_COMP_TOK3:
+        if (size > 8) {
+            if (data[8] == 1)
+                cm->level = 11;
+            else if (data[8] == 0)
+                cm->level = 1;
+        }
+        break;
+
+    default:
+        break;
+    }
+
+    return cm;
+}
+
+/*
+ *-----------------------------------------------------------------------------
+ * cram_codecs
+ */
+
+// -2 is unused.
+// -1 is CORE
+// >= 0 is the block with that Content ID
+void cram_codec_get_content_ids(cram_codec *c, int ids[2]) {
+    ids[0] = cram_codec_to_id(c, &ids[1]);
+}
+
+/*
+ *-----------------------------------------------------------------------------
+ * Utility functions
+ */
+
+/*
+ * Copies the blocks representing the next num_slice slices from a
+ * container from 'in' to 'out'.  It is expected that the file pointer
+ * is just after the read of the cram_container and cram compression
+ * header.
+ *
+ * Returns 0 on success
+ *        -1 on failure
+ */
+int cram_copy_slice(cram_fd *in, cram_fd *out, int32_t num_slice) {
+    int32_t i, j;
+
+    for (i = 0; i < num_slice; i++) {
+        cram_block *blk;
+        cram_block_slice_hdr *hdr;
+
+        if (!(blk = cram_read_block(in)))
+            return -1;
+        if (!(hdr = cram_decode_slice_header(in, blk))) {
+            cram_free_block(blk);
+            return -1;
+        }
+
+        if (cram_write_block(out, blk) != 0) {
+            cram_free_block(blk);
+            return -1;
+        }
+        cram_free_block(blk);
+
+        int num_blocks = cram_slice_hdr_get_num_blocks(hdr);
+        for (j = 0; j < num_blocks; j++) {
+            blk = cram_read_block(in);
+            if (!blk || cram_write_block(out, blk) != 0) {
+                if (blk) cram_free_block(blk);
+                return -1;
+            }
+            cram_free_block(blk);
+        }
+        cram_free_slice_header(hdr);
+    }
+
+    return 0;
+}
+
+/*
+ * Discards the next containers worth of data.
+ * Only the cram structure has been read so far.
+ *
+ * Returns 0 on success,
+ *        -1 on failure
+ */
+static int cram_skip_container(cram_fd *in, cram_container *c) {
+    // Compression header
+    cram_block *blk;
+    if (!(blk = cram_read_block(in)))
+        return -1;
+    cram_free_block(blk);
+
+    int i;
+    for (i = 0; i < c->num_landmarks; i++) {
+        cram_block_slice_hdr *hdr;
+
+        if (!(blk = cram_read_block(in)))
+            return -1;
+        if (!(hdr = cram_decode_slice_header(in, blk))) {
+            cram_free_block(blk);
+            return -1;
+        }
+        cram_free_block(blk);
+
+        int num_blocks = cram_slice_hdr_get_num_blocks(hdr), j;
+        for (j = 0; j < num_blocks; j++) {
+            blk = cram_read_block(in);
+            if (!blk) {
+                cram_free_slice_header(hdr);
+                return -1;
+            }
+            cram_free_block(blk);
+        }
+        cram_free_slice_header(hdr);
+    }
+
+    return 0;
+}
+
+
+/*
+ * Copies a container, but filtering it down to a specific region,
+ * which has already been set on the 'in' fd.
+ *
+ * This is used in e.g. samtools cat where we specified a region and discover
+ * that a region doesn't entirely span the container, so we have to select
+ * which reads we need to copy out of it.
+ *
+ * If ref_id is non-NULL we also return the last ref_id we filtered.
+ * This can be -2 if it's multi-ref and we observe more than one reference,
+ * and actual ref_id >= -1 if it's multi-ref and we observe just one ref or
+ * it's fixed reference.
+ *
+ * Returns 0 on success
+ *        -1 on error
+ */
+int cram_filter_container(cram_fd *in, cram_fd *out, cram_container *c,
+                          int *ref_id) {
+    int err = 0, fixed_ref = -3;
+
+    if (ref_id)
+        *ref_id = c->ref_seq_id;
+
+    int rid = in->range.refid == -2 ? -1 : in->range.refid;
+    if (rid != c->ref_seq_id ||
+        in->range.start > c->ref_seq_start + c->ref_seq_span-1)
+        // Except for multi-ref cases
+        if (c->ref_seq_id != -2)
+            return cram_skip_container(in, c);
+
+    // Container compression header
+    cram_block *blk = cram_read_block(in);
+    if (!blk)
+        return -1;
+    c->comp_hdr = cram_decode_compression_header(in, blk);
+    in->ctr = c;
+
+    // If it's multi-ref but a constant ref-id, then we can still do
+    // basic level chromosome filtering.  Similarly multi-ref where we're
+    // _already_ in ref "*" (unmapped) means we can just copy the container
+    // as there are no positions to filter on and "*" sorts to the end.
+    // TODO: how to tell "already in" though?
+    if (c->ref_seq_id == -2) {
+        cram_codec *cd = c->comp_hdr->codecs[DS_RI];
+        if (cd && cd->codec == E_HUFFMAN && cd->u.huffman.ncodes == 1 &&
+            // this check should be always true anyway
+            rid == cd->u.huffman.codes[0].symbol)
+            // We're in multi-ref mode, but actually the entire container
+            // matches.  So if we're in whole-chromosome mode we can just
+            // copy.
+            if (in->range.start <= 1 &&
+                in->range.end >= (INT64_MAX&(0xffffffffULL<<32))) {
+                if (ref_id)
+                    *ref_id = rid;
+                err |= cram_write_container(out, c) < 0;
+                err |= cram_write_block(out, blk);
+                return cram_copy_slice(in, out, c->num_landmarks) | -err;
+            }
+    }
+
+    // A simple read-write loop with region filtering automatically due to
+    // an earlier CRAM_OPT_RANGE request.
+    //
+    // We can hit EOF when reaching the end of the range, but we still need
+    // to manually check we don't attempt to read beyond this single container.
+
+    cram_range rng_copy = in->range;
+    in->range.start = INT64_MIN;
+    in->range.end = INT64_MAX;
+
+    bam1_t *b = bam_init1();
+    while ((c->curr_slice < c->max_slice ||
+            c->slice->curr_rec < c->slice->max_rec)) {
+        cram_slice *s;
+        if (c->slice && c->slice->curr_rec < c->slice->max_rec)
+            s = c->slice;
+        else if (c->curr_slice < c->max_slice)
+            s = cram_next_slice(in, &c);
+        else
+            break; // end of container
+        c->slice = s;
+
+        // This is more efficient if we check as a cram record instead of a
+        // bam record as we don't have to parse CIGAR end.
+        cram_record *cr = &c->slice->crecs[c->slice->curr_rec];
+        if (fixed_ref == -3)
+            fixed_ref = cr->ref_id;
+        else if (fixed_ref != cr->ref_id)
+            fixed_ref = -2;
+
+        if (rng_copy.refid != cr->ref_id) {
+            if (rng_copy.refid == -2) {
+                if (cr->ref_id > -1) {
+                    // Want unmapped, but have mapped
+                    c->slice->curr_rec++;
+                    continue;
+                }
+            } else {
+                if (rng_copy.refid > cr->ref_id || rng_copy.refid == -1) {
+                    // multi-ref and not at the correct ref yet
+                    c->slice->curr_rec++;
+                    continue;
+                } else {
+                    // multi-ref and beyond the desired ref
+                    break;
+                }
+            }
+        }
+
+        // Correct ref, but check the desired region
+        if (cr->aend < rng_copy.start) {
+            c->slice->curr_rec++;
+            continue;
+        }
+        if (cr->apos > rng_copy.end)
+            break;
+
+        // Broadly rquivalent to cram_get_bam_seq, but starting from 'cr'
+        err |= cram_to_bam(in->header, in, s, cr, s->curr_rec++, &b) < 0;
+
+        if (cram_put_bam_seq(out, b) < 0) {
+            err |= 1;
+            break;
+        }
+    }
+    bam_destroy1(b);
+
+    if (ref_id)
+        *ref_id = fixed_ref;
+
+    in->range = rng_copy;
+
+    // Avoids double frees as we stole the container from our other
+    // file descriptor.
+    in->ctr    = NULL;
+    in->ctr_mt = NULL;
+
+    err |= cram_flush(out);
+    cram_free_block(blk);
+
+    return -err;
+}
+
+
+/*
+ * Renumbers RG numbers in a cram compression header.
+ *
+ * CRAM stores RG as the Nth number in the header, rather than a
+ * string holding the ID: tag.  This is smaller in space, but means
+ * "samtools cat" to join files together that contain single but
+ * different RG lines needs a way of renumbering them.
+ *
+ * The file descriptor is expected to be immediately after the
+ * cram_container structure (ie before the cram compression header).
+ * Due to the nature of the CRAM format, this needs to read and write
+ * the blocks itself.  Note that there may be multiple slices within
+ * the container, meaning multiple compression headers to manipulate.
+ * Changing RG may change the size of the compression header and
+ * therefore the length field in the container.  Hence we rewrite all
+ * blocks just in case and also emit the adjusted container.
+ *
+ * The current implementation can only cope with renumbering a single
+ * RG (and only then if it is using HUFFMAN or BETA codecs).  In
+ * theory it *may* be possible to renumber multiple RGs if they use
+ * HUFFMAN to the CORE block or use an external block unshared by any
+ * other data series.  So we have an API that can be upgraded to
+ * support this, but do not implement it for now.  An example
+ * implementation of RG as an EXTERNAL block would be to find that
+ * block and rewrite it, returning the number of blocks consumed.
+ *
+ * Returns 0 on success;
+ *        -1 if unable to edit;
+ *        -2 on other errors (eg I/O).
+ */
+int cram_transcode_rg(cram_fd *in, cram_fd *out,
+                      cram_container *c,
+                      int nrg, int *in_rg, int *out_rg) {
+    int new_rg = *out_rg, old_size, new_size;
+    cram_block *o_blk, *n_blk;
+    cram_block_compression_hdr *ch;
+
+    if (nrg != 1) {
+        hts_log_error("CRAM transcode supports only a single RG");
+        return -2;
+    }
+
+    // Produce a new block holding the updated compression header,
+    // with RG transcoded to a new value. (Single only supported.)
+    o_blk = cram_read_block(in);
+    old_size = cram_block_size(o_blk);
+    ch = cram_decode_compression_header(in, o_blk);
+    if (cram_block_compression_hdr_set_rg(ch, new_rg) != 0)
+        return -1;
+    if (cram_block_compression_hdr_decoder2encoder(in, ch) != 0)
+        return -1;
+    n_blk = cram_encode_compression_header(in, c, ch, in->embed_ref);
+    cram_free_compression_header(ch);
+
+    /*
+     * Warning: this has internal knowledge of the cram compression
+     * header format.
+     *
+     * The decoder doesn't set c->tags_used, so the encoder puts a two
+     * byte blank segment.  This means n_blk is too short.  We skip
+     * through the decoded old block (o_blk) and copy from there.
+     */
+    char *cp = cram_block_get_data(o_blk);
+    char *op = cp;
+    char *endp = cp + cram_block_get_uncomp_size(o_blk);
+    //fprintf(stderr, "sz = %d\n", (int)(endp-cp));
+    int32_t i32, err = 0;
+
+    i32 = in->vv.varint_get32(&cp, endp, &err);
+    cp += i32;
+    i32 = in->vv.varint_get32(&cp, endp, &err);
+    cp += i32;
+    op = cp;
+    i32 = in->vv.varint_get32(&cp, endp, &err);
+    i32 += (cp-op);
+    if (err)
+        return -2;
+
+    //fprintf(stderr, "remaining %d bytes\n", i32);
+    cram_block_set_size(n_blk, cram_block_get_size(n_blk)-2);
+    cram_block_append(n_blk, op, i32);
+    cram_block_update_size(n_blk);
+
+    new_size = cram_block_size(n_blk);
+
+    //fprintf(stderr, "size %d -> %d\n", old_size, new_size);
+
+    // Now we've constructedthe updated compression header,
+    // amend the container too (it may have changed size).
+    int32_t *landmarks, num_landmarks;
+    landmarks = cram_container_get_landmarks(c, &num_landmarks);
+
+    if (old_size != new_size) {
+        int diff = new_size - old_size, j;
+
+        for (j = 0; j < num_landmarks; j++)
+            landmarks[j] += diff;
+        //cram_container_set_landmarks(c, num_landmarks, landmarks);
+        cram_container_set_length(c, cram_container_get_length(c) + diff);
+    }
+
+    // Finally write it all out; container, compression header,
+    // and then all the remaining slice blocks.
+    if (cram_write_container(out, c) != 0)
+        return -2;
+
+    cram_write_block(out, n_blk);
+    cram_free_block(o_blk);
+    cram_free_block(n_blk);
+
+    // Container num_blocks can be invalid, due to a bug.
+    // Instead we iterate in slice context instead.
+    return cram_copy_slice(in, out, num_landmarks);
+}
+
+
+/*!
+ * Returns the refs_t structure used by a cram file handle.
+ *
+ * This may be used in conjunction with option CRAM_OPT_SHARED_REF to
+ * share reference memory between multiple file handles.
+ *
+ * @return
+ * Returns NULL if none exists or the file handle is not a CRAM file.
+ */
+refs_t *cram_get_refs(htsFile *fd) {
+    return fd->format.format == cram
+        ? fd->fp.cram->refs
+        : NULL;
+}
diff --git a/ext/htslib/cram/cram_index.c b/ext/htslib/cram/cram_index.c
new file mode 100644
index 0000000..77c953d
--- /dev/null
+++ b/ext/htslib/cram/cram_index.c
@@ -0,0 +1,1040 @@
+/*
+Copyright (c) 2013-2020, 2023-2024 Genome Research Ltd.
+Author: James Bonfield <jkb@sanger.ac.uk>
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+Institute nor the names of its contributors may be used to endorse or promote
+products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+/*
+ * The index is a gzipped tab-delimited text file with one line per slice.
+ * The columns are:
+ * 1: reference number (0 to N-1, as per BAM ref_id)
+ * 2: reference position of 1st read in slice (1..?)
+ * 3: number of reads in slice
+ * 4: offset of container start (relative to end of SAM header, so 1st
+ *    container is offset 0).
+ * 5: slice number within container (ie which landmark).
+ *
+ * In memory, we hold this in a nested containment list. Each list element is
+ * a cram_index struct. Each element in turn can contain its own list of
+ * cram_index structs.
+ *
+ * Any start..end range which is entirely contained within another (and
+ * earlier as it is sorted) range will be held within it. This ensures that
+ * the outer list will never have containments and we can safely do a
+ * binary search to find the first range which overlaps any given coordinate.
+ */
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+
+#include <stdio.h>
+#include <errno.h>
+#include <assert.h>
+#include <inttypes.h>
+#include <stdlib.h>
+#include <string.h>
+#include <zlib.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <math.h>
+
+#include "../htslib/bgzf.h"
+#include "../htslib/hfile.h"
+#include "../hts_internal.h"
+#include "cram.h"
+#include "os.h"
+
+#if 0
+static void dump_index_(cram_index *e, int level) {
+    int i, n;
+    n = printf("%*s%d / %d .. %d, ", level*4, "", e->refid, e->start, e->end);
+    printf("%*soffset %"PRId64" %p %p\n", MAX(0,50-n), "", e->offset, e, e->e_next);
+    for (i = 0; i < e->nslice; i++) {
+        dump_index_(&e->e[i], level+1);
+    }
+}
+
+static void dump_index(cram_fd *fd) {
+    int i;
+    for (i = 0; i < fd->index_sz; i++) {
+        dump_index_(&fd->index[i], 0);
+    }
+}
+#endif
+
+// Thread a linked list through the nested containment list.
+// This makes navigating it and finding the "next" index entry
+// trivial.
+static cram_index *link_index_(cram_index *e, cram_index *e_last) {
+    int i;
+    if (e_last)
+        e_last->e_next = e;
+
+    // We don't want to link in the top-level cram_index with
+    // offset=0 and start/end = INT_MIN/INT_MAX.
+    if (e->offset)
+        e_last = e;
+
+    for (i = 0; i < e->nslice; i++)
+        e_last = link_index_(&e->e[i], e_last);
+
+    return e_last;
+}
+
+static void link_index(cram_fd *fd) {
+    int i;
+    cram_index *e_last = NULL;
+
+    for (i = 0; i < fd->index_sz; i++) {
+        e_last = link_index_(&fd->index[i], e_last);
+    }
+
+    if (e_last)
+        e_last->e_next = NULL;
+}
+
+static int kget_int32(kstring_t *k, size_t *pos, int32_t *val_p) {
+    int sign = 1;
+    int32_t val = 0;
+    size_t p = *pos;
+
+    while (p < k->l && (k->s[p] == ' ' || k->s[p] == '\t'))
+        p++;
+
+    if (p < k->l && k->s[p] == '-')
+        sign = -1, p++;
+
+    if (p >= k->l || !(k->s[p] >= '0' && k->s[p] <= '9'))
+        return -1;
+
+    while (p < k->l && k->s[p] >= '0' && k->s[p] <= '9') {
+        int digit = k->s[p++]-'0';
+        val = val*10 + digit;
+    }
+
+    *pos = p;
+    *val_p = sign*val;
+
+    return 0;
+}
+
+static int kget_int64(kstring_t *k, size_t *pos, int64_t *val_p) {
+    int sign = 1;
+    int64_t val = 0;
+    size_t p = *pos;
+
+    while (p < k->l && (k->s[p] == ' ' || k->s[p] == '\t'))
+        p++;
+
+    if (p < k->l && k->s[p] == '-')
+        sign = -1, p++;
+
+    if (p >= k->l || !(k->s[p] >= '0' && k->s[p] <= '9'))
+        return -1;
+
+    while (p < k->l && k->s[p] >= '0' && k->s[p] <= '9') {
+        int digit = k->s[p++]-'0';
+        val = val*10 + digit;
+    }
+
+    *pos = p;
+    *val_p = sign*val;
+
+    return 0;
+}
+
+/*
+ * Loads a CRAM .crai index into memory.
+ *
+ * Returns 0 for success
+ *        -1 for failure
+ */
+int cram_index_load(cram_fd *fd, const char *fn, const char *fn_idx) {
+
+    char *tfn_idx = NULL;
+    char buf[65536];
+    ssize_t len;
+    kstring_t kstr = {0};
+    hFILE *fp;
+    cram_index *idx;
+    cram_index **idx_stack = NULL, *ep, e;
+    int idx_stack_alloc = 0, idx_stack_ptr = 0;
+    size_t pos = 0;
+
+    /* Check if already loaded */
+    if (fd->index)
+        return 0;
+
+    fd->index = calloc((fd->index_sz = 1), sizeof(*fd->index));
+    if (!fd->index)
+        return -1;
+
+    idx = &fd->index[0];
+    idx->refid = -1;
+    idx->start = INT_MIN;
+    idx->end   = INT_MAX;
+
+    idx_stack = calloc(++idx_stack_alloc, sizeof(*idx_stack));
+    if (!idx_stack)
+        goto fail;
+
+    idx_stack[idx_stack_ptr] = idx;
+
+    // Support pathX.cram##idx##pathY.crai
+    const char *fn_delim = strstr(fn, HTS_IDX_DELIM);
+    if (fn_delim && !fn_idx)
+        fn_idx = fn_delim + strlen(HTS_IDX_DELIM);
+
+    if (!fn_idx) {
+        if (hts_idx_check_local(fn, HTS_FMT_CRAI, &tfn_idx) == 0 && hisremote(fn))
+            tfn_idx = hts_idx_getfn(fn, ".crai");
+
+        if (!tfn_idx) {
+            hts_log_error("Could not retrieve index file for '%s'", fn);
+            goto fail;
+        }
+        fn_idx = tfn_idx;
+    }
+
+    if (!(fp = hopen(fn_idx, "r"))) {
+        hts_log_error("Could not open index file '%s'", fn_idx);
+        goto fail;
+    }
+
+    // Load the file into memory
+    while ((len = hread(fp, buf, sizeof(buf))) > 0) {
+        if (kputsn(buf, len, &kstr) < 0)
+            goto fail;
+    }
+
+    if (len < 0 || kstr.l < 2)
+        goto fail;
+
+    if (hclose(fp) < 0)
+        goto fail;
+
+    // Uncompress if required
+    if (kstr.s[0] == 31 && (uc)kstr.s[1] == 139) {
+        size_t l = 0;
+        char *s = zlib_mem_inflate(kstr.s, kstr.l, &l);
+        if (!s)
+            goto fail;
+
+        free(kstr.s);
+        kstr.s = s;
+        kstr.l = l;
+        kstr.m = l; // conservative estimate of the size allocated
+        if (kputsn("", 0, &kstr) < 0) // ensure kstr.s is NUL-terminated
+            goto fail;
+    }
+
+
+    // Parse it line at a time
+    while (pos < kstr.l) {
+        /* 1.1 layout */
+        if (kget_int32(&kstr, &pos, &e.refid) == -1)
+            goto fail;
+
+        if (kget_int32(&kstr, &pos, &e.start) == -1)
+            goto fail;
+
+        if (kget_int32(&kstr, &pos, &e.end) == -1)
+            goto fail;
+
+        if (kget_int64(&kstr, &pos, &e.offset) == -1)
+            goto fail;
+
+        if (kget_int32(&kstr, &pos, &e.slice) == -1)
+            goto fail;
+
+        if (kget_int32(&kstr, &pos, &e.len) == -1)
+            goto fail;
+
+        e.end += e.start-1;
+        //printf("%d/%d..%d-offset=%" PRIu64 ",len=%d,slice=%d\n", e.refid, e.start, e.end, e.offset, e.len, e.slice);
+
+        if (e.refid < -1) {
+            hts_log_error("Malformed index file, refid %d", e.refid);
+            goto fail;
+        }
+
+        if (e.refid != idx->refid) {
+            if (fd->index_sz < e.refid+2) {
+                cram_index *new_idx;
+                int new_sz = e.refid+2;
+                size_t index_end = fd->index_sz * sizeof(*fd->index);
+                new_idx = realloc(fd->index,
+                                  new_sz * sizeof(*fd->index));
+                if (!new_idx)
+                    goto fail;
+
+                fd->index = new_idx;
+                fd->index_sz = new_sz;
+                memset(((char *)fd->index) + index_end, 0,
+                       fd->index_sz * sizeof(*fd->index) - index_end);
+            }
+            idx = &fd->index[e.refid+1];
+            idx->refid = e.refid;
+            idx->start = INT_MIN;
+            idx->end   = INT_MAX;
+            idx->nslice = idx->nalloc = 0;
+            idx->e = NULL;
+            idx_stack[(idx_stack_ptr = 0)] = idx;
+        }
+
+        while (!(e.start >= idx->start && e.end <= idx->end) ||
+               (idx->start == 0 && idx->refid == -1)) {
+            idx = idx_stack[--idx_stack_ptr];
+        }
+
+        // Now contains, so append
+        if (idx->nslice+1 >= idx->nalloc) {
+            cram_index *new_e;
+            idx->nalloc = idx->nalloc ? idx->nalloc*2 : 16;
+            new_e = realloc(idx->e, idx->nalloc * sizeof(*idx->e));
+            if (!new_e)
+                goto fail;
+
+            idx->e = new_e;
+        }
+
+        e.nalloc = e.nslice = 0; e.e = NULL;
+        *(ep = &idx->e[idx->nslice++]) = e;
+        idx = ep;
+
+        if (++idx_stack_ptr >= idx_stack_alloc) {
+            cram_index **new_stack;
+            idx_stack_alloc *= 2;
+            new_stack = realloc(idx_stack, idx_stack_alloc*sizeof(*idx_stack));
+            if (!new_stack)
+                goto fail;
+            idx_stack = new_stack;
+        }
+        idx_stack[idx_stack_ptr] = idx;
+
+        while (pos < kstr.l && kstr.s[pos] != '\n')
+            pos++;
+        pos++;
+    }
+
+    free(idx_stack);
+    free(kstr.s);
+    free(tfn_idx);
+
+    // Convert NCList to linear linked list
+    link_index(fd);
+
+    //dump_index(fd);
+
+    return 0;
+
+ fail:
+    free(kstr.s);
+    free(idx_stack);
+    free(tfn_idx);
+    cram_index_free(fd); // Also sets fd->index = NULL
+    return -1;
+}
+
+static void cram_index_free_recurse(cram_index *e) {
+    if (e->e) {
+        int i;
+        for (i = 0; i < e->nslice; i++) {
+            cram_index_free_recurse(&e->e[i]);
+        }
+        free(e->e);
+    }
+}
+
+void cram_index_free(cram_fd *fd) {
+    int i;
+
+    if (!fd->index)
+        return;
+
+    for (i = 0; i < fd->index_sz; i++) {
+        cram_index_free_recurse(&fd->index[i]);
+    }
+    free(fd->index);
+
+    fd->index = NULL;
+}
+
+/*
+ * Searches the index for the first slice overlapping a reference ID
+ * and position, or one immediately preceding it if none is found in
+ * the index to overlap this position. (Our index may have missing
+ * entries, but we require at least one per reference.)
+ *
+ * If the index finds multiple slices overlapping this position we
+ * return the first one only. Subsequent calls should specify
+ * "from" as the last slice we checked to find the next one. Otherwise
+ * set "from" to be NULL to find the first one.
+ *
+ * Refid can also be any of the special HTS_IDX_ values.
+ * For backwards compatibility, refid -1 is equivalent to HTS_IDX_NOCOOR.
+ *
+ * Returns the cram_index pointer on success
+ *         NULL on failure
+ */
+cram_index *cram_index_query(cram_fd *fd, int refid, hts_pos_t pos,
+                             cram_index *from) {
+    int i, j, k;
+    cram_index *e;
+
+    if (from) {
+        // Continue from a previous search.
+        // We switch to just scanning the linked list, as the nested
+        // lists are typically short.
+        if (refid == HTS_IDX_NOCOOR)
+            refid = -1;
+
+        e = from->e_next;
+        if (e && e->refid == refid && e->start <= pos)
+            return e;
+        else
+            return NULL;
+    }
+
+    switch(refid) {
+    case HTS_IDX_NONE:
+    case HTS_IDX_REST:
+        // fail, or already there, dealt with elsewhere.
+        return NULL;
+
+    case -1:
+    case HTS_IDX_NOCOOR:
+        refid = -1;
+        pos = 0;
+        break;
+
+    case HTS_IDX_START: {
+        int64_t min_idx = INT64_MAX;
+        for (i = 0, j = -1; i < fd->index_sz; i++) {
+            if (fd->index[i].e && fd->index[i].e[0].offset < min_idx) {
+                min_idx = fd->index[i].e[0].offset;
+                j = i;
+            }
+        }
+        if (j < 0)
+            return NULL;
+        return fd->index[j].e;
+    }
+
+    default:
+        if (refid < HTS_IDX_NONE || refid+1 >= fd->index_sz)
+            return NULL;
+    }
+
+    from = &fd->index[refid+1];
+
+    // Ref with nothing aligned against it.
+    if (!from->e)
+        return NULL;
+
+    // This sequence is covered by the index, so binary search to find
+    // the optimal starting block.
+    i = 0, j = fd->index[refid+1].nslice-1;
+    for (k = j/2; k != i; k = (j-i)/2 + i) {
+        if (from->e[k].refid > refid) {
+            j = k;
+            continue;
+        }
+
+        if (from->e[k].refid < refid) {
+            i = k;
+            continue;
+        }
+
+        if (from->e[k].start >= pos) {
+            j = k;
+            continue;
+        }
+
+        if (from->e[k].start < pos) {
+            i = k;
+            continue;
+        }
+    }
+    // i==j or i==j-1. Check if j is better.
+    if (j >= 0 && from->e[j].start < pos && from->e[j].refid == refid)
+        i = j;
+
+    /* The above found *a* bin overlapping, but not necessarily the first */
+    while (i > 0 && from->e[i-1].end >= pos)
+        i--;
+
+    /* We may be one bin before the optimum, so check */
+    while (i+1 < from->nslice &&
+           (from->e[i].refid < refid ||
+            from->e[i].end < pos))
+        i++;
+
+    e = &from->e[i];
+
+    return e;
+}
+
+// Return the index entry for last slice on a specific reference.
+cram_index *cram_index_last(cram_fd *fd, int refid, cram_index *from) {
+    int slice;
+
+    if (refid+1 < 0 || refid+1 >= fd->index_sz)
+        return NULL;
+
+    if (!from)
+        from = &fd->index[refid+1];
+
+    // Ref with nothing aligned against it.
+    if (!from->e)
+        return NULL;
+
+    slice = fd->index[refid+1].nslice - 1;
+
+    // e is the last entry in the nested containment list, but it may
+    // contain further slices within it.
+    cram_index *e = &from->e[slice];
+    while (e->e_next)
+        e = e->e_next;
+
+    return e;
+}
+
+/*
+ * Find the last container overlapping pos 'end', and the file offset of
+ * its end (equivalent to the start offset of the container following it).
+ */
+cram_index *cram_index_query_last(cram_fd *fd, int refid, hts_pos_t end) {
+    cram_index *e = NULL, *prev_e;
+    do {
+        prev_e = e;
+        e = cram_index_query(fd, refid, end, prev_e);
+    } while (e);
+
+    if (!prev_e)
+        return NULL;
+    e = prev_e;
+
+    // Note: offset of e and e->e_next may be the same if we're using a
+    // multi-ref container where a single container generates multiple
+    // index entries.
+    //
+    // We need to keep iterating until offset differs in order to find
+    // the genuine file offset for the end of container.
+    do {
+        prev_e = e;
+        e = e->e_next;
+    } while (e && e->offset == prev_e->offset);
+
+    return prev_e;
+}
+
+/*
+ * Skips to a container overlapping the start coordinate listed in
+ * cram_range.
+ *
+ * In theory we call cram_index_query multiple times, once per slice
+ * overlapping the range. However slices may be absent from the index
+ * which makes this problematic. Instead we find the left-most slice
+ * and then read from then on, skipping decoding of slices and/or
+ * whole containers when they don't overlap the specified cram_range.
+ *
+ * This function also updates the cram_fd range field.
+ *
+ * Returns 0 on success
+ *        -1 on general failure
+ *        -2 on no-data (empty chromosome)
+ */
+int cram_seek_to_refpos(cram_fd *fd, cram_range *r) {
+    int ret = 0;
+    cram_index *e;
+
+    if (r->refid == HTS_IDX_NONE) {
+        ret = -2; goto err;
+    }
+
+    // Ideally use an index, so see if we have one.
+    if ((e = cram_index_query(fd, r->refid, r->start, NULL))) {
+        if (0 != cram_seek(fd, e->offset, SEEK_SET)) {
+            if (0 != cram_seek(fd, e->offset - fd->first_container, SEEK_CUR)) {
+                ret = -1; goto err;
+            }
+        }
+    } else {
+        // Absent from index, but this most likely means it simply has no data.
+        ret = -2; goto err;
+    }
+
+    pthread_mutex_lock(&fd->range_lock);
+    fd->range = *r;
+    if (r->refid == HTS_IDX_NOCOOR) {
+        fd->range.refid = -1;
+        fd->range.start = 0;
+    } else if (r->refid == HTS_IDX_START || r->refid == HTS_IDX_REST) {
+        fd->range.refid = -2; // special case in cram_next_slice
+    }
+    pthread_mutex_unlock(&fd->range_lock);
+
+    if (fd->ctr) {
+        cram_free_container(fd->ctr);
+        if (fd->ctr_mt && fd->ctr_mt != fd->ctr)
+            cram_free_container(fd->ctr_mt);
+        fd->ctr = NULL;
+        fd->ctr_mt = NULL;
+        fd->ooc = 0;
+        fd->eof = 0;
+    }
+
+    return 0;
+
+ err:
+    // It's unlikely fd->range will be accessed after EOF or error,
+    // but this maintains identical behaviour to the previous code.
+    pthread_mutex_lock(&fd->range_lock);
+    fd->range = *r;
+    pthread_mutex_unlock(&fd->range_lock);
+    return ret;
+}
+
+
+/*
+ * A specialised form of cram_index_build (below) that deals with slices
+ * having multiple references in this (ref_id -2). In this scenario we
+ * decode the slice to look at the RI data series instead.
+ *
+ * Returns 0 on success
+ *        -1 on read failure
+ *        -2 on wrong sort order
+ *        -4 on write failure
+ */
+static int cram_index_build_multiref(cram_fd *fd,
+                                     cram_container *c,
+                                     cram_slice *s,
+                                     BGZF *fp,
+                                     off_t cpos,
+                                     int32_t landmark,
+                                     int sz) {
+    int i, ref = -2;
+    int64_t ref_start = 0, ref_end;
+    char buf[1024];
+
+    if (fd->mode != 'w') {
+        if (0 != cram_decode_slice(fd, c, s, fd->header))
+            return -1;
+    }
+
+    ref_end = INT_MIN;
+
+    int32_t last_ref = -9;
+    int32_t last_pos = -9;
+    for (i = 0; i < s->hdr->num_records; i++) {
+        if (s->crecs[i].ref_id == last_ref && s->crecs[i].apos < last_pos) {
+            hts_log_error("CRAM file is not sorted by chromosome / position");
+            return -2;
+        }
+        last_ref = s->crecs[i].ref_id;
+        last_pos = s->crecs[i].apos;
+
+        if (s->crecs[i].ref_id == ref) {
+            if (ref_end < s->crecs[i].aend)
+                ref_end = s->crecs[i].aend;
+            continue;
+        }
+
+        if (ref != -2) {
+            snprintf(buf, sizeof(buf),
+                     "%d\t%"PRId64"\t%"PRId64"\t%"PRId64"\t%d\t%d\n",
+                     ref, ref_start, ref_end - ref_start + 1,
+                     (int64_t)cpos, landmark, sz);
+            if (bgzf_write(fp, buf, strlen(buf)) < 0)
+                return -4;
+        }
+
+        ref = s->crecs[i].ref_id;
+        ref_start = s->crecs[i].apos;
+        ref_end   = s->crecs[i].aend;
+    }
+
+    if (ref != -2) {
+        snprintf(buf, sizeof(buf),
+                 "%d\t%"PRId64"\t%"PRId64"\t%"PRId64"\t%d\t%d\n",
+                 ref, ref_start, ref_end - ref_start + 1,
+                 (int64_t)cpos, landmark, sz);
+        if (bgzf_write(fp, buf, strlen(buf)) < 0)
+            return -4;
+    }
+
+    return 0;
+}
+
+/*
+ * Adds a single slice to the index.
+ */
+int cram_index_slice(cram_fd *fd,
+                     cram_container *c,
+                     cram_slice *s,
+                     BGZF *fp,
+                     off_t cpos,
+                     off_t spos, // relative to cpos
+                     off_t sz) {
+    int ret;
+    char buf[1024];
+
+    if (sz > INT_MAX) {
+        hts_log_error("CRAM slice is too big (%"PRId64" bytes)",
+                      (int64_t) sz);
+        return -1;
+    }
+
+    if (s->hdr->ref_seq_id == -2) {
+        ret = cram_index_build_multiref(fd, c, s, fp, cpos, spos, sz);
+    } else {
+        snprintf(buf, sizeof(buf),
+                 "%d\t%"PRId64"\t%"PRId64"\t%"PRId64"\t%d\t%d\n",
+                 s->hdr->ref_seq_id, s->hdr->ref_seq_start,
+                 s->hdr->ref_seq_span, (int64_t)cpos, (int)spos, (int)sz);
+        ret = (bgzf_write(fp, buf, strlen(buf)) >= 0)? 0 : -4;
+    }
+
+    return ret;
+}
+
+/*
+ * Adds a single container to the index.
+ */
+static
+int cram_index_container(cram_fd *fd,
+                         cram_container *c,
+                         BGZF *fp,
+                         off_t cpos) {
+    int j;
+    off_t spos;
+
+    // 2.0 format
+    for (j = 0; j < c->num_landmarks; j++) {
+        cram_slice *s;
+        off_t sz;
+        int ret;
+
+        spos = htell(fd->fp);
+        if (spos - cpos - (off_t) c->offset != c->landmark[j]) {
+            hts_log_error("CRAM slice offset %"PRId64" does not match"
+                          " landmark %d in container header (%"PRId32")",
+                          (int64_t) (spos - cpos - (off_t) c->offset),
+                          j, c->landmark[j]);
+            return -1;
+        }
+
+        if (!(s = cram_read_slice(fd))) {
+            return -1;
+        }
+
+        sz = htell(fd->fp) - spos;
+        ret = cram_index_slice(fd, c, s, fp, cpos, c->landmark[j], sz);
+
+        cram_free_slice(s);
+
+        if (ret < 0) {
+            return ret;
+        }
+    }
+
+    return 0;
+}
+
+
+/*
+ * Builds an index file.
+ *
+ * fd is a newly opened cram file that we wish to index.
+ * fn_base is the filename of the associated CRAM file.
+ * fn_idx is the filename of the index file to be written;
+ * if NULL, we add ".crai" to fn_base to get the index filename.
+ *
+ * Returns 0 on success,
+ *         negative on failure (-1 for read failure, -4 for write failure)
+ */
+int cram_index_build(cram_fd *fd, const char *fn_base, const char *fn_idx) {
+    cram_container *c;
+    off_t cpos, hpos;
+    BGZF *fp;
+    kstring_t fn_idx_str = {0};
+    int64_t last_ref = -9, last_start = -9;
+
+    // Useful for cram_index_build_multiref
+    cram_set_option(fd, CRAM_OPT_REQUIRED_FIELDS, SAM_RNAME | SAM_POS | SAM_CIGAR);
+
+    if (! fn_idx) {
+        kputs(fn_base, &fn_idx_str);
+        kputs(".crai", &fn_idx_str);
+        fn_idx = fn_idx_str.s;
+    }
+
+    if (!(fp = bgzf_open(fn_idx, "wg"))) {
+        perror(fn_idx);
+        free(fn_idx_str.s);
+        return -4;
+    }
+
+    free(fn_idx_str.s);
+
+    cpos = htell(fd->fp);
+    while ((c = cram_read_container(fd))) {
+        if (fd->err) {
+            perror("Cram container read");
+            return -1;
+        }
+
+        hpos = htell(fd->fp);
+
+        if (!(c->comp_hdr_block = cram_read_block(fd)))
+            return -1;
+        assert(c->comp_hdr_block->content_type == COMPRESSION_HEADER);
+
+        c->comp_hdr = cram_decode_compression_header(fd, c->comp_hdr_block);
+        if (!c->comp_hdr)
+            return -1;
+
+        if (c->ref_seq_id == last_ref && c->ref_seq_start < last_start) {
+            hts_log_error("CRAM file is not sorted by chromosome / position");
+            return -2;
+        }
+        last_ref = c->ref_seq_id;
+        last_start = c->ref_seq_start;
+
+        if (cram_index_container(fd, c, fp, cpos) < 0) {
+            bgzf_close(fp);
+            return -1;
+        }
+
+        off_t next_cpos = htell(fd->fp);
+        if (next_cpos != hpos + c->length) {
+            hts_log_error("Length %"PRId32" in container header at offset %lld does not match block lengths (%lld)",
+                          c->length, (long long) cpos, (long long) next_cpos - hpos);
+            return -1;
+        }
+        cpos = next_cpos;
+
+        cram_free_container(c);
+    }
+    if (fd->err) {
+        bgzf_close(fp);
+        return -1;
+    }
+
+    return (bgzf_close(fp) >= 0)? 0 : -4;
+}
+
+// internal recursive step
+static int64_t cram_num_containers_between_(cram_index *e, int64_t *last_pos,
+                                            int64_t nct,
+                                            off_t cstart, off_t cend,
+                                            int64_t *first, int64_t *last) {
+    int64_t nc = 0, i;
+
+    if (e->offset) {
+        if (e->offset != *last_pos) {
+            if (e->offset >= cstart && (!cend || e->offset <= cend)) {
+                if (first && *first < 0)
+                    *first = nct;
+                if (last)
+                    *last = nct;
+            }
+            nc++;
+        }
+        // else a new multi-ref in same container
+        *last_pos = e->offset;
+    }
+
+    for (i = 0; i < e->nslice; i++)
+        nc += cram_num_containers_between_(&e->e[i], last_pos, nc + nct,
+                                           cstart, cend, first, last);
+
+    return nc;
+}
+
+/*! Returns the number of containers in the CRAM file within given offsets.
+ *
+ * The cstart and cend offsets are the locations of the start of containers
+ * as returned by index_container_offset.
+ *
+ * If non-NULL, first and last will hold the inclusive range of container
+ * numbers, counting from zero.
+ *
+ * @return
+ * Returns the number of containers, equivalent to *last-*first+1.
+ */
+int64_t cram_num_containers_between(cram_fd *fd,
+                                    off_t cstart, off_t cend,
+                                    int64_t *first, int64_t *last) {
+    int64_t nc = 0, i;
+    int64_t last_pos = -99;
+    int64_t l_first = -1, l_last = -1;
+
+    for (i = 0; i < fd->index_sz; i++) {
+        int j = i+1 == fd->index_sz ? 0 : i+1; // maps "*" to end
+        nc += cram_num_containers_between_(&fd->index[j], &last_pos, nc,
+                                           cstart, cend, &l_first, &l_last);
+    }
+
+    if (first)
+        *first = l_first;
+    if (last)
+        *last = l_last;
+
+    return l_last - l_first + 1;
+}
+
+/*
+ * Queries the total number of distinct containers in the index.
+ * Note there may be more containers in the file than in the index, as we
+ * are not required to have an index entry for every one.
+ */
+int64_t cram_num_containers(cram_fd *fd) {
+    return cram_num_containers_between(fd, 0, 0, NULL, NULL);
+}
+
+
+/*! Returns the byte offset for the start of the n^th container.
+ *
+ * The index must have previously been loaded, otherwise <0 is returned.
+ */
+static cram_index *cram_container_num2offset_(cram_index *e, int num,
+                                              int64_t *last_pos, int *nc) {
+    if (e->offset) {
+        if (e->offset != *last_pos) {
+            if (*nc == num)
+                return e;
+            (*nc)++;
+        }
+        // else a new multi-ref in same container
+        *last_pos = e->offset;
+    }
+
+    int i;
+    for (i = 0; i < e->nslice; i++) {
+        cram_index *tmp = cram_container_num2offset_(&e->e[i], num,
+                                                     last_pos, nc);
+        if (tmp)
+            return tmp;
+    }
+
+
+    return NULL;
+}
+
+off_t cram_container_num2offset(cram_fd *fd, int64_t num) {
+    int nc = 0, i;
+    int64_t last_pos = -9;
+    cram_index *e = NULL;
+
+    for (i = 0; i < fd->index_sz; i++) {
+        int j = i+1 == fd->index_sz ? 0 : i+1; // maps "*" to end
+        if (!fd->index[j].nslice)
+            continue;
+        if ((e = cram_container_num2offset_(&fd->index[j], num,
+                                            &last_pos, &nc)))
+            break;
+    }
+
+    return e ? e->offset : -1;
+}
+
+
+/*! Returns the container number for the first container at offset >= pos.
+ *
+ * The index must have previously been loaded, otherwise <0 is returned.
+ */
+static cram_index *cram_container_offset2num_(cram_index *e, off_t pos,
+                                              int64_t *last_pos, int *nc) {
+    if (e->offset) {
+        if (e->offset != *last_pos) {
+            if (e->offset >= pos)
+                return e;
+            (*nc)++;
+        }
+        // else a new multi-ref in same container
+        *last_pos = e->offset;
+    }
+
+    int i;
+    for (i = 0; i < e->nslice; i++) {
+        cram_index *tmp = cram_container_offset2num_(&e->e[i], pos,
+                                                     last_pos, nc);
+        if (tmp)
+            return tmp;
+    }
+
+
+    return NULL;
+}
+
+int64_t cram_container_offset2num(cram_fd *fd, off_t pos) {
+    int nc = 0, i;
+    int64_t last_pos = -9;
+    cram_index *e = NULL;
+
+    for (i = 0; i < fd->index_sz; i++) {
+        int j = i+1 == fd->index_sz ? 0 : i+1; // maps "*" to end
+        if (!fd->index[j].nslice)
+            continue;
+        if ((e = cram_container_offset2num_(&fd->index[j], pos,
+                                            &last_pos, &nc)))
+            break;
+    }
+
+    return e ? nc : -1;
+}
+
+/*!
+ * Returns the file offsets of CRAM containers covering a specific region
+ * query.  Note both offsets are the START of the container.
+ *
+ * first will point to the start of the first overlapping container
+ * last will point to the start of the last overlapping container
+ *
+ * Returns 0 on success
+ *        <0 on failure
+ */
+int cram_index_extents(cram_fd *fd, int refid, hts_pos_t start, hts_pos_t end,
+                       off_t *first, off_t *last) {
+    cram_index *ci;
+
+    if (first) {
+        if (!(ci = cram_index_query(fd, refid, start, NULL)))
+            return -1;
+        *first = ci->offset;
+    }
+
+    if (last) {
+        if (!(ci = cram_index_query_last(fd, refid, end)))
+            return -1;
+        *last = ci->offset;
+    }
+
+    return 0;
+}
diff --git a/ext/htslib/cram/cram_index.h b/ext/htslib/cram/cram_index.h
new file mode 100644
index 0000000..5fa1154
--- /dev/null
+++ b/ext/htslib/cram/cram_index.h
@@ -0,0 +1,115 @@
+/*
+Copyright (c) 2013, 2018 Genome Research Ltd.
+Author: James Bonfield <jkb@sanger.ac.uk>
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+Institute nor the names of its contributors may be used to endorse or promote
+products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#ifndef CRAM_INDEX_H
+#define CRAM_INDEX_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Loads a CRAM .crai index into memory.
+ * Returns 0 for success
+ *        -1 for failure
+ */
+int cram_index_load(cram_fd *fd, const char *fn, const char *fn_idx);
+
+void cram_index_free(cram_fd *fd);
+
+/*
+ * Searches the index for the first slice overlapping a reference ID
+ * and position.
+ *
+ * Returns the cram_index pointer on success
+ *         NULL on failure
+ */
+cram_index *cram_index_query(cram_fd *fd, int refid, hts_pos_t pos, cram_index *frm);
+cram_index *cram_index_last(cram_fd *fd, int refid, cram_index *from);
+cram_index *cram_index_query_last(cram_fd *fd, int refid, hts_pos_t end);
+
+/*
+ * Skips to a container overlapping the start coordinate listed in
+ * cram_range.
+ *
+ * Returns 0 on success
+ *        -1 on failure
+ */
+int cram_seek_to_refpos(cram_fd *fd, cram_range *r);
+
+void cram_index_free(cram_fd *fd);
+
+/*
+ * Skips to a container overlapping the start coordinate listed in
+ * cram_range.
+ *
+ * In theory we call cram_index_query multiple times, once per slice
+ * overlapping the range. However slices may be absent from the index
+ * which makes this problematic. Instead we find the left-most slice
+ * and then read from then on, skipping decoding of slices and/or
+ * whole containers when they don't overlap the specified cram_range.
+ *
+ * Returns 0 on success
+ *        -1 on failure
+ */
+int cram_seek_to_refpos(cram_fd *fd, cram_range *r);
+
+/*
+ * Builds an index file.
+ *
+ * fd is a newly opened cram file that we wish to index.
+ * fn_base is the filename of the associated CRAM file.
+ * fn_idx is the filename of the index file to be written;
+ * if NULL, we add ".crai" to fn_base to get the index filename.
+ *
+ * Returns 0 on success,
+ *         negative on failure (-1 for read failure, -4 for write failure)
+ */
+int cram_index_build(cram_fd *fd, const char *fn_base, const char *fn_idx);
+
+/*
+ * Adds a single slice to the index.
+ *
+ * Returns 0 on success,
+ *        -1 on failure
+ */
+int cram_index_slice(cram_fd *fd,
+                     cram_container *c,
+                     cram_slice *s,
+                     BGZF *fp,
+                     off_t cpos,
+                     off_t spos, // relative to cpos
+                     off_t sz);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/ext/htslib/cram/cram_io.c b/ext/htslib/cram/cram_io.c
new file mode 100644
index 0000000..7f7ffca
--- /dev/null
+++ b/ext/htslib/cram/cram_io.c
@@ -0,0 +1,6025 @@
+/*
+Copyright (c) 2012-2024 Genome Research Ltd.
+Author: James Bonfield <jkb@sanger.ac.uk>
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+Institute nor the names of its contributors may be used to endorse or promote
+products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+/*
+ * CRAM I/O primitives.
+ *
+ * - ITF8 encoding and decoding.
+ * - Block based I/O
+ * - Zlib inflating and deflating (memory)
+ * - CRAM basic data structure reading and writing
+ * - File opening / closing
+ * - Reference sequence handling
+ */
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+
+#include <stdio.h>
+#include <errno.h>
+#include <assert.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <zlib.h>
+#ifdef HAVE_LIBBZ2
+#include <bzlib.h>
+#endif
+#ifdef HAVE_LIBLZMA
+#ifdef HAVE_LZMA_H
+#include <lzma.h>
+#else
+#include "../os/lzma_stub.h"
+#endif
+#endif
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <math.h>
+#include <stdint.h>
+
+#ifdef HAVE_LIBDEFLATE
+#include <libdeflate.h>
+#define crc32(a,b,c) libdeflate_crc32((a),(b),(c))
+#endif
+
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+#include "../fuzz_settings.h"
+#endif
+
+#include "cram.h"
+#include "os.h"
+#include "../htslib/hts.h"
+#include "open_trace_file.h"
+
+#if defined(HAVE_EXTERNAL_LIBHTSCODECS)
+#include <htscodecs/rANS_static.h>
+#include <htscodecs/rANS_static4x16.h>
+#include <htscodecs/arith_dynamic.h>
+#include <htscodecs/tokenise_name3.h>
+#include <htscodecs/fqzcomp_qual.h>
+#include <htscodecs/varint.h> // CRAM v4.0 variable-size integers
+#else
+#include "../htscodecs/htscodecs/rANS_static.h"
+#include "../htscodecs/htscodecs/rANS_static4x16.h"
+#include "../htscodecs/htscodecs/arith_dynamic.h"
+#include "../htscodecs/htscodecs/tokenise_name3.h"
+#include "../htscodecs/htscodecs/fqzcomp_qual.h"
+#include "../htscodecs/htscodecs/varint.h"
+#endif
+
+//#define REF_DEBUG
+
+#ifdef REF_DEBUG
+#include <sys/syscall.h>
+#define gettid() (int)syscall(SYS_gettid)
+
+#define RP(...) fprintf (stderr, __VA_ARGS__)
+#else
+#define RP(...)
+#endif
+
+#include "../htslib/hfile.h"
+#include "../htslib/bgzf.h"
+#include "../htslib/faidx.h"
+#include "../hts_internal.h"
+
+#ifndef PATH_MAX
+#define PATH_MAX FILENAME_MAX
+#endif
+
+#define TRIAL_SPAN 70
+#define NTRIALS 3
+
+#define CRAM_DEFAULT_LEVEL 5
+
+/* ----------------------------------------------------------------------
+ * ITF8 encoding and decoding.
+ *
+ * Also see the itf8_get and itf8_put macros in cram_io.h
+ */
+
+/*
+ * LEGACY: consider using itf8_decode_crc.
+ *
+ * Reads an integer in ITF-8 encoding from 'cp' and stores it in
+ * *val.
+ *
+ * Returns the number of bytes read on success
+ *        -1 on failure
+ */
+int itf8_decode(cram_fd *fd, int32_t *val_p) {
+    static int nbytes[16] = {
+        0,0,0,0, 0,0,0,0,                               // 0000xxxx - 0111xxxx
+        1,1,1,1,                                        // 1000xxxx - 1011xxxx
+        2,2,                                            // 1100xxxx - 1101xxxx
+        3,                                              // 1110xxxx
+        4,                                              // 1111xxxx
+    };
+
+    static int nbits[16] = {
+        0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, // 0000xxxx - 0111xxxx
+        0x3f, 0x3f, 0x3f, 0x3f,                         // 1000xxxx - 1011xxxx
+        0x1f, 0x1f,                                     // 1100xxxx - 1101xxxx
+        0x0f,                                           // 1110xxxx
+        0x0f,                                           // 1111xxxx
+    };
+
+    int32_t val = hgetc(fd->fp);
+    if (val == -1)
+        return -1;
+
+    int i = nbytes[val>>4];
+    val &= nbits[val>>4];
+
+    switch(i) {
+    case 0:
+        *val_p = val;
+        return 1;
+
+    case 1:
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        *val_p = val;
+        return 2;
+
+    case 2:
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        *val_p = val;
+        return 3;
+
+    case 3:
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        *val_p = val;
+        return 4;
+
+    case 4: // really 3.5 more, why make it different?
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        val = (val<<4) | (((unsigned char)hgetc(fd->fp)) & 0x0f);
+        *val_p = val;
+    }
+
+    return 5;
+}
+
+int itf8_decode_crc(cram_fd *fd, int32_t *val_p, uint32_t *crc) {
+    static int nbytes[16] = {
+        0,0,0,0, 0,0,0,0,                               // 0000xxxx - 0111xxxx
+        1,1,1,1,                                        // 1000xxxx - 1011xxxx
+        2,2,                                            // 1100xxxx - 1101xxxx
+        3,                                              // 1110xxxx
+        4,                                              // 1111xxxx
+    };
+
+    static int nbits[16] = {
+        0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, // 0000xxxx - 0111xxxx
+        0x3f, 0x3f, 0x3f, 0x3f,                         // 1000xxxx - 1011xxxx
+        0x1f, 0x1f,                                     // 1100xxxx - 1101xxxx
+        0x0f,                                           // 1110xxxx
+        0x0f,                                           // 1111xxxx
+    };
+    unsigned char c[5];
+
+    int32_t val = hgetc(fd->fp);
+    if (val == -1)
+        return -1;
+
+    c[0]=val;
+
+    int i = nbytes[val>>4];
+    val &= nbits[val>>4];
+
+    if (i > 0) {
+        if (hread(fd->fp, &c[1], i) < i)
+            return -1;
+    }
+
+    switch(i) {
+    case 0:
+        *val_p = val;
+        *crc = crc32(*crc, c, 1);
+        return 1;
+
+    case 1:
+        val = (val<<8) | c[1];
+        *val_p = val;
+        *crc = crc32(*crc, c, 2);
+        return 2;
+
+    case 2:
+        val = (val<<8) | c[1];
+        val = (val<<8) | c[2];
+        *val_p = val;
+        *crc = crc32(*crc, c, 3);
+        return 3;
+
+    case 3:
+        val = (val<<8) | c[1];
+        val = (val<<8) | c[2];
+        val = (val<<8) | c[3];
+        *val_p = val;
+        *crc = crc32(*crc, c, 4);
+        return 4;
+
+    case 4: // really 3.5 more, why make it different?
+        {
+            uint32_t uv = val;
+            uv = (uv<<8) |  c[1];
+            uv = (uv<<8) |  c[2];
+            uv = (uv<<8) |  c[3];
+            uv = (uv<<4) | (c[4] & 0x0f);
+            // Avoid implementation-defined behaviour on negative values
+            *val_p = uv < 0x80000000UL ? (int32_t) uv : -((int32_t) (0xffffffffUL - uv)) - 1;
+            *crc = crc32(*crc, c, 5);
+        }
+    }
+
+    return 5;
+}
+
+/*
+ * Stores a value to memory in ITF-8 format.
+ *
+ * Returns the number of bytes required to store the number.
+ * This is a maximum of 5 bytes.
+ */
+static inline int itf8_put(char *cp, int32_t val) {
+    unsigned char *up = (unsigned char *)cp;
+    if        (!(val & ~0x00000007f)) { // 1 byte
+        *up = val;
+        return 1;
+    } else if (!(val & ~0x00003fff)) { // 2 byte
+        *up++ = (val >> 8 ) | 0x80;
+        *up   = val & 0xff;
+        return 2;
+    } else if (!(val & ~0x01fffff)) { // 3 byte
+        *up++ = (val >> 16) | 0xc0;
+        *up++ = (val >> 8 ) & 0xff;
+        *up   = val & 0xff;
+        return 3;
+    } else if (!(val & ~0x0fffffff)) { // 4 byte
+        *up++ = (val >> 24) | 0xe0;
+        *up++ = (val >> 16) & 0xff;
+        *up++ = (val >> 8 ) & 0xff;
+        *up   = val & 0xff;
+        return 4;
+    } else {                           // 5 byte
+        *up++ = 0xf0 | ((val>>28) & 0xff);
+        *up++ = (val >> 20) & 0xff;
+        *up++ = (val >> 12) & 0xff;
+        *up++ = (val >> 4 ) & 0xff;
+        *up = val & 0x0f;
+        return 5;
+    }
+}
+
+
+/* 64-bit itf8 variant */
+static inline int ltf8_put(char *cp, int64_t val) {
+    unsigned char *up = (unsigned char *)cp;
+    if        (!(val & ~((1LL<<7)-1))) {
+        *up = val;
+        return 1;
+    } else if (!(val & ~((1LL<<(6+8))-1))) {
+        *up++ = (val >> 8 ) | 0x80;
+        *up   = val & 0xff;
+        return 2;
+    } else if (!(val & ~((1LL<<(5+2*8))-1))) {
+        *up++ = (val >> 16) | 0xc0;
+        *up++ = (val >> 8 ) & 0xff;
+        *up   = val & 0xff;
+        return 3;
+    } else if (!(val & ~((1LL<<(4+3*8))-1))) {
+        *up++ = (val >> 24) | 0xe0;
+        *up++ = (val >> 16) & 0xff;
+        *up++ = (val >> 8 ) & 0xff;
+        *up   = val & 0xff;
+        return 4;
+    } else if (!(val & ~((1LL<<(3+4*8))-1))) {
+        *up++ = (val >> 32) | 0xf0;
+        *up++ = (val >> 24) & 0xff;
+        *up++ = (val >> 16) & 0xff;
+        *up++ = (val >> 8 ) & 0xff;
+        *up   = val & 0xff;
+        return 5;
+    } else if (!(val & ~((1LL<<(2+5*8))-1))) {
+        *up++ = (val >> 40) | 0xf8;
+        *up++ = (val >> 32) & 0xff;
+        *up++ = (val >> 24) & 0xff;
+        *up++ = (val >> 16) & 0xff;
+        *up++ = (val >> 8 ) & 0xff;
+        *up   = val & 0xff;
+        return 6;
+    } else if (!(val & ~((1LL<<(1+6*8))-1))) {
+        *up++ = (val >> 48) | 0xfc;
+        *up++ = (val >> 40) & 0xff;
+        *up++ = (val >> 32) & 0xff;
+        *up++ = (val >> 24) & 0xff;
+        *up++ = (val >> 16) & 0xff;
+        *up++ = (val >> 8 ) & 0xff;
+        *up   = val & 0xff;
+        return 7;
+    } else if (!(val & ~((1LL<<(7*8))-1))) {
+        *up++ = (val >> 56) | 0xfe;
+        *up++ = (val >> 48) & 0xff;
+        *up++ = (val >> 40) & 0xff;
+        *up++ = (val >> 32) & 0xff;
+        *up++ = (val >> 24) & 0xff;
+        *up++ = (val >> 16) & 0xff;
+        *up++ = (val >> 8 ) & 0xff;
+        *up   = val & 0xff;
+        return 8;
+    } else {
+        *up++ = 0xff;
+        *up++ = (val >> 56) & 0xff;
+        *up++ = (val >> 48) & 0xff;
+        *up++ = (val >> 40) & 0xff;
+        *up++ = (val >> 32) & 0xff;
+        *up++ = (val >> 24) & 0xff;
+        *up++ = (val >> 16) & 0xff;
+        *up++ = (val >> 8 ) & 0xff;
+        *up   = val & 0xff;
+        return 9;
+    }
+}
+
+/*
+ * Encodes and writes a single integer in ITF-8 format.
+ * Returns 0 on success
+ *        -1 on failure
+ */
+int itf8_encode(cram_fd *fd, int32_t val) {
+    char buf[5];
+    int len = itf8_put(buf, val);
+    return hwrite(fd->fp, buf, len) == len ? 0 : -1;
+}
+
+const int itf8_bytes[16] = {
+    1, 1, 1, 1,  1, 1, 1, 1,
+    2, 2, 2, 2,  3, 3, 4, 5
+};
+
+const int ltf8_bytes[256] = {
+    1, 1, 1, 1,  1, 1, 1, 1,  1, 1, 1, 1,  1, 1, 1, 1,
+    1, 1, 1, 1,  1, 1, 1, 1,  1, 1, 1, 1,  1, 1, 1, 1,
+    1, 1, 1, 1,  1, 1, 1, 1,  1, 1, 1, 1,  1, 1, 1, 1,
+    1, 1, 1, 1,  1, 1, 1, 1,  1, 1, 1, 1,  1, 1, 1, 1,
+
+    1, 1, 1, 1,  1, 1, 1, 1,  1, 1, 1, 1,  1, 1, 1, 1,
+    1, 1, 1, 1,  1, 1, 1, 1,  1, 1, 1, 1,  1, 1, 1, 1,
+    1, 1, 1, 1,  1, 1, 1, 1,  1, 1, 1, 1,  1, 1, 1, 1,
+    1, 1, 1, 1,  1, 1, 1, 1,  1, 1, 1, 1,  1, 1, 1, 1,
+
+    2, 2, 2, 2,  2, 2, 2, 2,  2, 2, 2, 2,  2, 2, 2, 2,
+    2, 2, 2, 2,  2, 2, 2, 2,  2, 2, 2, 2,  2, 2, 2, 2,
+    2, 2, 2, 2,  2, 2, 2, 2,  2, 2, 2, 2,  2, 2, 2, 2,
+    2, 2, 2, 2,  2, 2, 2, 2,  2, 2, 2, 2,  2, 2, 2, 2,
+
+    3, 3, 3, 3,  3, 3, 3, 3,  3, 3, 3, 3,  3, 3, 3, 3,
+    3, 3, 3, 3,  3, 3, 3, 3,  3, 3, 3, 3,  3, 3, 3, 3,
+
+    4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
+
+    5, 5, 5, 5,  5, 5, 5, 5,  6, 6, 6, 6,  7, 7, 8, 9
+};
+
+/*
+ * LEGACY: consider using ltf8_decode_crc.
+ */
+int ltf8_decode(cram_fd *fd, int64_t *val_p) {
+    int c = hgetc(fd->fp);
+    int64_t val = (unsigned char)c;
+    if (c == -1)
+        return -1;
+
+    if (val < 0x80) {
+        *val_p =   val;
+        return 1;
+
+    } else if (val < 0xc0) {
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        *val_p = val & (((1LL<<(6+8)))-1);
+        return 2;
+
+    } else if (val < 0xe0) {
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        *val_p = val & ((1LL<<(5+2*8))-1);
+        return 3;
+
+    } else if (val < 0xf0) {
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        *val_p = val & ((1LL<<(4+3*8))-1);
+        return 4;
+
+    } else if (val < 0xf8) {
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        *val_p = val & ((1LL<<(3+4*8))-1);
+        return 5;
+
+    } else if (val < 0xfc) {
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        *val_p = val & ((1LL<<(2+5*8))-1);
+        return 6;
+
+    } else if (val < 0xfe) {
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        *val_p = val & ((1LL<<(1+6*8))-1);
+        return 7;
+
+    } else if (val < 0xff) {
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        *val_p = val & ((1LL<<(7*8))-1);
+        return 8;
+
+    } else {
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        val = (val<<8) | (unsigned char)hgetc(fd->fp);
+        *val_p = val;
+    }
+
+    return 9;
+}
+
+int ltf8_decode_crc(cram_fd *fd, int64_t *val_p, uint32_t *crc) {
+    unsigned char c[9];
+    int64_t val = hgetc(fd->fp);
+    if (val < 0)
+        return -1;
+
+    c[0] = val;
+
+    if (val < 0x80) {
+        *val_p =   val;
+        *crc = crc32(*crc, c, 1);
+        return 1;
+
+    } else if (val < 0xc0) {
+        int v = hgetc(fd->fp);
+        if (v < 0)
+            return -1;
+        val = (val<<8) | (c[1]=v);
+        *val_p = val & (((1LL<<(6+8)))-1);
+        *crc = crc32(*crc, c, 2);
+        return 2;
+
+    } else if (val < 0xe0) {
+        if (hread(fd->fp, &c[1], 2) < 2)
+            return -1;
+        val = (val<<8) | c[1];
+        val = (val<<8) | c[2];
+        *val_p = val & ((1LL<<(5+2*8))-1);
+        *crc = crc32(*crc, c, 3);
+        return 3;
+
+    } else if (val < 0xf0) {
+        if (hread(fd->fp, &c[1], 3) < 3)
+            return -1;
+        val = (val<<8) | c[1];
+        val = (val<<8) | c[2];
+        val = (val<<8) | c[3];
+        *val_p = val & ((1LL<<(4+3*8))-1);
+        *crc = crc32(*crc, c, 4);
+        return 4;
+
+    } else if (val < 0xf8) {
+        if (hread(fd->fp, &c[1], 4) < 4)
+            return -1;
+        val = (val<<8) | c[1];
+        val = (val<<8) | c[2];
+        val = (val<<8) | c[3];
+        val = (val<<8) | c[4];
+        *val_p = val & ((1LL<<(3+4*8))-1);
+        *crc = crc32(*crc, c, 5);
+        return 5;
+
+    } else if (val < 0xfc) {
+        if (hread(fd->fp, &c[1], 5) < 5)
+            return -1;
+        val = (val<<8) | c[1];
+        val = (val<<8) | c[2];
+        val = (val<<8) | c[3];
+        val = (val<<8) | c[4];
+        val = (val<<8) | c[5];
+        *val_p = val & ((1LL<<(2+5*8))-1);
+        *crc = crc32(*crc, c, 6);
+        return 6;
+
+    } else if (val < 0xfe) {
+        if (hread(fd->fp, &c[1], 6) < 6)
+            return -1;
+        val = (val<<8) | c[1];
+        val = (val<<8) | c[2];
+        val = (val<<8) | c[3];
+        val = (val<<8) | c[4];
+        val = (val<<8) | c[5];
+        val = (val<<8) | c[6];
+        *val_p = val & ((1LL<<(1+6*8))-1);
+        *crc = crc32(*crc, c, 7);
+        return 7;
+
+    } else if (val < 0xff) {
+        uint64_t uval = val;
+        if (hread(fd->fp, &c[1], 7) < 7)
+            return -1;
+        uval = (uval<<8) | c[1];
+        uval = (uval<<8) | c[2];
+        uval = (uval<<8) | c[3];
+        uval = (uval<<8) | c[4];
+        uval = (uval<<8) | c[5];
+        uval = (uval<<8) | c[6];
+        uval = (uval<<8) | c[7];
+        *val_p = uval & ((1ULL<<(7*8))-1);
+        *crc = crc32(*crc, c, 8);
+        return 8;
+
+    } else {
+        uint64_t uval;
+        if (hread(fd->fp, &c[1], 8) < 8)
+            return -1;
+        uval =             c[1];
+        uval = (uval<<8) | c[2];
+        uval = (uval<<8) | c[3];
+        uval = (uval<<8) | c[4];
+        uval = (uval<<8) | c[5];
+        uval = (uval<<8) | c[6];
+        uval = (uval<<8) | c[7];
+        uval = (uval<<8) | c[8];
+        *crc = crc32(*crc, c, 9);
+        // Avoid implementation-defined behaviour on negative values
+        *val_p = c[1] < 0x80 ? (int64_t) uval : -((int64_t) (0xffffffffffffffffULL - uval)) - 1;
+    }
+
+    return 9;
+}
+
+/*
+ * Pushes a value in ITF8 format onto the end of a block.
+ * This shouldn't be used for high-volume data as it is not the fastest
+ * method.
+ *
+ * Returns the number of bytes written
+ */
+int itf8_put_blk(cram_block *blk, int32_t val) {
+    char buf[5];
+    int sz;
+
+    sz = itf8_put(buf, val);
+    BLOCK_APPEND(blk, buf, sz);
+    return sz;
+
+ block_err:
+    return -1;
+}
+
+int ltf8_put_blk(cram_block *blk, int64_t val) {
+    char buf[9];
+    int sz;
+
+    sz = ltf8_put(buf, val);
+    BLOCK_APPEND(blk, buf, sz);
+    return sz;
+
+ block_err:
+    return -1;
+}
+
+static int64_t safe_itf8_get(char **cp, const char *endp, int *err) {
+    const unsigned char *up = (unsigned char *)*cp;
+
+    if (endp && endp - *cp < 5 &&
+        (*cp >= endp || endp - *cp < itf8_bytes[up[0]>>4])) {
+        if (err) *err = 1;
+        return 0;
+    }
+
+    if (up[0] < 0x80) {
+        (*cp)++;
+        return up[0];
+    } else if (up[0] < 0xc0) {
+        (*cp)+=2;
+        return ((up[0] <<8) |  up[1])                           & 0x3fff;
+    } else if (up[0] < 0xe0) {
+        (*cp)+=3;
+        return ((up[0]<<16) | (up[1]<< 8) |  up[2])             & 0x1fffff;
+    } else if (up[0] < 0xf0) {
+        (*cp)+=4;
+        uint32_t uv = (((uint32_t)up[0]<<24) | (up[1]<<16) | (up[2]<<8) | up[3]) & 0x0fffffff;
+        return (int32_t)uv;
+    } else {
+        (*cp)+=5;
+        uint32_t uv = (((uint32_t)up[0] & 0x0f)<<28) | (up[1]<<20) | (up[2]<<12) | (up[3]<<4) | (up[4] & 0x0f);
+        return (int32_t)uv;
+    }
+}
+
+static int64_t safe_ltf8_get(char **cp, const char *endp, int *err) {
+    unsigned char *up = (unsigned char *)*cp;
+
+    if (endp && endp - *cp < 9 &&
+        (*cp >= endp || endp - *cp < ltf8_bytes[up[0]])) {
+        if (err) *err = 1;
+        return 0;
+    }
+
+    if (up[0] < 0x80) {
+        (*cp)++;
+        return up[0];
+    } else if (up[0] < 0xc0) {
+        (*cp)+=2;
+        return (((uint64_t)up[0]<< 8) |
+                 (uint64_t)up[1]) & (((1LL<<(6+8)))-1);
+    } else if (up[0] < 0xe0) {
+        (*cp)+=3;
+        return (((uint64_t)up[0]<<16) |
+                ((uint64_t)up[1]<< 8) |
+                 (uint64_t)up[2]) & ((1LL<<(5+2*8))-1);
+    } else if (up[0] < 0xf0) {
+        (*cp)+=4;
+        return (((uint64_t)up[0]<<24) |
+                ((uint64_t)up[1]<<16) |
+                ((uint64_t)up[2]<< 8) |
+                 (uint64_t)up[3]) & ((1LL<<(4+3*8))-1);
+    } else if (up[0] < 0xf8) {
+        (*cp)+=5;
+        return (((uint64_t)up[0]<<32) |
+                ((uint64_t)up[1]<<24) |
+                ((uint64_t)up[2]<<16) |
+                ((uint64_t)up[3]<< 8) |
+                 (uint64_t)up[4]) & ((1LL<<(3+4*8))-1);
+    } else if (up[0] < 0xfc) {
+        (*cp)+=6;
+        return (((uint64_t)up[0]<<40) |
+                ((uint64_t)up[1]<<32) |
+                ((uint64_t)up[2]<<24) |
+                ((uint64_t)up[3]<<16) |
+                ((uint64_t)up[4]<< 8) |
+                 (uint64_t)up[5]) & ((1LL<<(2+5*8))-1);
+    } else if (up[0] < 0xfe) {
+        (*cp)+=7;
+        return (((uint64_t)up[0]<<48) |
+                ((uint64_t)up[1]<<40) |
+                ((uint64_t)up[2]<<32) |
+                ((uint64_t)up[3]<<24) |
+                ((uint64_t)up[4]<<16) |
+                ((uint64_t)up[5]<< 8) |
+                 (uint64_t)up[6]) & ((1LL<<(1+6*8))-1);
+    } else if (up[0] < 0xff) {
+        (*cp)+=8;
+        return (((uint64_t)up[1]<<48) |
+                ((uint64_t)up[2]<<40) |
+                ((uint64_t)up[3]<<32) |
+                ((uint64_t)up[4]<<24) |
+                ((uint64_t)up[5]<<16) |
+                ((uint64_t)up[6]<< 8) |
+                 (uint64_t)up[7]) & ((1LL<<(7*8))-1);
+    } else {
+        (*cp)+=9;
+        return (((uint64_t)up[1]<<56) |
+                ((uint64_t)up[2]<<48) |
+                ((uint64_t)up[3]<<40) |
+                ((uint64_t)up[4]<<32) |
+                ((uint64_t)up[5]<<24) |
+                ((uint64_t)up[6]<<16) |
+                ((uint64_t)up[7]<< 8) |
+                 (uint64_t)up[8]);
+    }
+}
+
+// Wrapper for now
+static int safe_itf8_put(char *cp, char *cp_end, int32_t val) {
+    return itf8_put(cp, val);
+}
+
+static int safe_ltf8_put(char *cp, char *cp_end, int64_t val) {
+    return ltf8_put(cp, val);
+}
+
+static int itf8_size(int64_t v) {
+    return ((!((v)&~0x7f))?1:(!((v)&~0x3fff))?2:(!((v)&~0x1fffff))?3:(!((v)&~0xfffffff))?4:5);
+}
+
+//-----------------------------------------------------------------------------
+
+// CRAM v4.0 onwards uses a different variable sized integer encoding
+// that is size agnostic.
+
+// Local interface to varint.h inline version, so we can use in func ptr.
+// Note a lot of these use the unsigned interface but take signed int64_t.
+// This is because the old CRAM ITF8 inteface had signed -1 as unsigned
+// 0xffffffff.
+static int uint7_size(int64_t v) {
+    return var_size_u64(v);
+}
+
+static int64_t uint7_get_32(char **cp, const char *endp, int *err) {
+    uint32_t val;
+    int nb = var_get_u32((uint8_t *)(*cp), (const uint8_t *)endp, &val);
+    (*cp) += nb;
+    if (!nb && err) *err = 1;
+    return val;
+}
+
+static int64_t sint7_get_32(char **cp, const char *endp, int *err) {
+    int32_t val;
+    int nb = var_get_s32((uint8_t *)(*cp), (const uint8_t *)endp, &val);
+    (*cp) += nb;
+    if (!nb && err) *err = 1;
+    return val;
+}
+
+static int64_t uint7_get_64(char **cp, const char *endp, int *err) {
+    uint64_t val;
+    int nb = var_get_u64((uint8_t *)(*cp), (const uint8_t *)endp, &val);
+    (*cp) += nb;
+    if (!nb && err) *err = 1;
+    return val;
+}
+
+static int64_t sint7_get_64(char **cp, const char *endp, int *err) {
+    int64_t val;
+    int nb = var_get_s64((uint8_t *)(*cp), (const uint8_t *)endp, &val);
+    (*cp) += nb;
+    if (!nb && err) *err = 1;
+    return val;
+}
+
+static int uint7_put_32(char *cp, char *endp, int32_t val) {
+    return var_put_u32((uint8_t *)cp, (uint8_t *)endp, val);
+}
+
+static int sint7_put_32(char *cp, char *endp, int32_t val) {
+    return var_put_s32((uint8_t *)cp, (uint8_t *)endp, val);
+}
+
+static int uint7_put_64(char *cp, char *endp, int64_t val) {
+    return var_put_u64((uint8_t *)cp, (uint8_t *)endp, val);
+}
+
+static int sint7_put_64(char *cp, char *endp, int64_t val) {
+    return var_put_s64((uint8_t *)cp, (uint8_t *)endp, val);
+}
+
+// Put direct to to cram_block
+static int uint7_put_blk_32(cram_block *blk, int32_t v) {
+    uint8_t buf[10];
+    int sz = var_put_u32(buf, buf+10, v);
+    BLOCK_APPEND(blk, buf, sz);
+    return sz;
+
+ block_err:
+    return -1;
+}
+
+static int sint7_put_blk_32(cram_block *blk, int32_t v) {
+    uint8_t buf[10];
+    int sz = var_put_s32(buf, buf+10, v);
+    BLOCK_APPEND(blk, buf, sz);
+    return sz;
+
+ block_err:
+    return -1;
+}
+
+static int uint7_put_blk_64(cram_block *blk, int64_t v) {
+    uint8_t buf[10];
+    int sz = var_put_u64(buf, buf+10, v);
+    BLOCK_APPEND(blk, buf, sz);
+    return sz;
+
+ block_err:
+    return -1;
+}
+
+static int sint7_put_blk_64(cram_block *blk, int64_t v) {
+    uint8_t buf[10];
+    int sz = var_put_s64(buf, buf+10, v);
+    BLOCK_APPEND(blk, buf, sz);
+    return sz;
+
+ block_err:
+    return -1;
+}
+
+// Decode 32-bits with CRC update from cram_fd
+static int uint7_decode_crc32(cram_fd *fd, int32_t *val_p, uint32_t *crc) {
+    uint8_t b[5], i = 0;
+    int c;
+    uint32_t v = 0;
+
+#ifdef VARINT2
+    b[0] = hgetc(fd->fp);
+    if (b[0] < 177) {
+    } else if (b[0] < 241) {
+        b[1] = hgetc(fd->fp);
+    } else if (b[0] < 249) {
+        b[1] = hgetc(fd->fp);
+        b[2] = hgetc(fd->fp);
+    } else {
+        int n = b[0]+2, z = 1;
+        while (n-- >= 249)
+            b[z++] = hgetc(fd->fp);
+    }
+    i = var_get_u32(b, NULL, &v);
+#else
+//    // Little endian
+//    int s = 0;
+//    do {
+//        b[i++] = c = hgetc(fd->fp);
+//        if (c < 0)
+//            return -1;
+//        v |= (c & 0x7f) << s;
+//      s += 7;
+//    } while (i < 5 && (c & 0x80));
+
+    // Big endian, see also htscodecs/varint.h
+    do {
+        b[i++] = c = hgetc(fd->fp);
+        if (c < 0)
+            return -1;
+        v = (v<<7) | (c & 0x7f);
+    } while (i < 5 && (c & 0x80));
+#endif
+    *crc = crc32(*crc, b, i);
+
+    *val_p = v;
+    return i;
+}
+
+// Decode 32-bits with CRC update from cram_fd
+static int sint7_decode_crc32(cram_fd *fd, int32_t *val_p, uint32_t *crc) {
+    uint8_t b[5], i = 0;
+    int c;
+    uint32_t v = 0;
+
+#ifdef VARINT2
+    b[0] = hgetc(fd->fp);
+    if (b[0] < 177) {
+    } else if (b[0] < 241) {
+        b[1] = hgetc(fd->fp);
+    } else if (b[0] < 249) {
+        b[1] = hgetc(fd->fp);
+        b[2] = hgetc(fd->fp);
+    } else {
+        int n = b[0]+2, z = 1;
+        while (n-- >= 249)
+            b[z++] = hgetc(fd->fp);
+    }
+    i = var_get_u32(b, NULL, &v);
+#else
+//    // Little endian
+//    int s = 0;
+//    do {
+//        b[i++] = c = hgetc(fd->fp);
+//        if (c < 0)
+//            return -1;
+//        v |= (c & 0x7f) << s;
+//      s += 7;
+//    } while (i < 5 && (c & 0x80));
+
+    // Big endian, see also htscodecs/varint.h
+    do {
+        b[i++] = c = hgetc(fd->fp);
+        if (c < 0)
+            return -1;
+        v = (v<<7) | (c & 0x7f);
+    } while (i < 5 && (c & 0x80));
+#endif
+    *crc = crc32(*crc, b, i);
+
+    *val_p = (v>>1) ^ -(v&1);
+    return i;
+}
+
+
+// Decode 64-bits with CRC update from cram_fd
+static int uint7_decode_crc64(cram_fd *fd, int64_t *val_p, uint32_t *crc) {
+    uint8_t b[10], i = 0;
+    int c;
+    uint64_t v = 0;
+
+#ifdef VARINT2
+    b[0] = hgetc(fd->fp);
+    if (b[0] < 177) {
+    } else if (b[0] < 241) {
+        b[1] = hgetc(fd->fp);
+    } else if (b[0] < 249) {
+        b[1] = hgetc(fd->fp);
+        b[2] = hgetc(fd->fp);
+    } else {
+        int n = b[0]+2, z = 1;
+        while (n-- >= 249)
+            b[z++] = hgetc(fd->fp);
+    }
+    i = var_get_u64(b, NULL, &v);
+#else
+//    // Little endian
+//    int s = 0;
+//    do {
+//        b[i++] = c = hgetc(fd->fp);
+//        if (c < 0)
+//            return -1;
+//        v |= (c & 0x7f) << s;
+//      s += 7;
+//    } while (i < 10 && (c & 0x80));
+
+    // Big endian, see also htscodecs/varint.h
+    do {
+        b[i++] = c = hgetc(fd->fp);
+        if (c < 0)
+            return -1;
+        v = (v<<7) | (c & 0x7f);
+    } while (i < 5 && (c & 0x80));
+#endif
+    *crc = crc32(*crc, b, i);
+
+    *val_p = v;
+    return i;
+}
+
+//-----------------------------------------------------------------------------
+
+/*
+ * Decodes a 32-bit little endian value from fd and stores in val.
+ *
+ * Returns the number of bytes read on success
+ *         -1 on failure
+ */
+static int int32_decode(cram_fd *fd, int32_t *val) {
+    int32_t i;
+    if (4 != hread(fd->fp, &i, 4))
+        return -1;
+
+    *val = le_int4(i);
+    return 4;
+}
+
+/*
+ * Encodes a 32-bit little endian value 'val' and writes to fd.
+ *
+ * Returns the number of bytes written on success
+ *         -1 on failure
+ */
+static int int32_encode(cram_fd *fd, int32_t val) {
+    uint32_t v = le_int4(val);
+    if (4 != hwrite(fd->fp, &v, 4))
+        return -1;
+
+    return 4;
+}
+
+/* As int32_decoded/encode, but from/to blocks instead of cram_fd */
+int int32_get_blk(cram_block *b, int32_t *val) {
+    if (b->uncomp_size - BLOCK_SIZE(b) < 4)
+        return -1;
+
+    uint32_t v =
+         ((uint32_t) b->data[b->byte  ])        |
+        (((uint32_t) b->data[b->byte+1]) <<  8) |
+        (((uint32_t) b->data[b->byte+2]) << 16) |
+        (((uint32_t) b->data[b->byte+3]) << 24);
+    // Avoid implementation-defined behaviour on negative values
+    *val = v < 0x80000000U ? (int32_t) v : -((int32_t) (0xffffffffU - v)) - 1;
+    BLOCK_SIZE(b) += 4;
+    return 4;
+}
+
+/* As int32_decoded/encode, but from/to blocks instead of cram_fd */
+int int32_put_blk(cram_block *b, int32_t val) {
+    unsigned char cp[4];
+    uint32_t v = val;
+    cp[0] = ( v      & 0xff);
+    cp[1] = ((v>>8)  & 0xff);
+    cp[2] = ((v>>16) & 0xff);
+    cp[3] = ((v>>24) & 0xff);
+
+    BLOCK_APPEND(b, cp, 4);
+    return 0;
+
+ block_err:
+    return -1;
+}
+
+#ifdef HAVE_LIBDEFLATE
+/* ----------------------------------------------------------------------
+ * libdeflate compression code, with interface to match
+ * zlib_mem_{in,de}flate for simplicity elsewhere.
+ */
+
+// Named the same as the version that uses zlib as we always use libdeflate for
+// decompression when available.
+char *zlib_mem_inflate(char *cdata, size_t csize, size_t *size) {
+    struct libdeflate_decompressor *z = libdeflate_alloc_decompressor();
+    if (!z) {
+        hts_log_error("Call to libdeflate_alloc_decompressor failed");
+        return NULL;
+    }
+
+    uint8_t *data = NULL, *new_data;
+    if (!*size)
+        *size = csize*2;
+    for(;;) {
+        new_data = realloc(data, *size);
+        if (!new_data) {
+            hts_log_error("Memory allocation failure");
+            goto fail;
+        }
+        data = new_data;
+
+        int ret = libdeflate_gzip_decompress(z, cdata, csize, data, *size, size);
+
+        // Auto grow output buffer size if needed and try again.
+        // Fortunately for all bar one call of this we know the size already.
+        if (ret == LIBDEFLATE_INSUFFICIENT_SPACE) {
+            (*size) *= 1.5;
+            continue;
+        }
+
+        if (ret != LIBDEFLATE_SUCCESS) {
+            hts_log_error("Inflate operation failed: %d", ret);
+            goto fail;
+        } else {
+            break;
+        }
+    }
+
+    libdeflate_free_decompressor(z);
+    return (char *)data;
+
+ fail:
+    libdeflate_free_decompressor(z);
+    free(data);
+    return NULL;
+}
+
+// Named differently as we use both zlib/libdeflate for compression.
+static char *libdeflate_deflate(char *data, size_t size, size_t *cdata_size,
+                                int level, int strat) {
+    level = level > 0 ? level : 6; // libdeflate doesn't honour -1 as default
+    level *= 1.23;     // NB levels go up to 12 here; 5 onwards is +1
+    level += level>=8; // 5,6,7->6,7,8  8->10  9->12
+    if (level > 12) level = 12;
+
+    if (strat == Z_RLE) // not supported by libdeflate
+        level = 1;
+
+    struct libdeflate_compressor *z = libdeflate_alloc_compressor(level);
+    if (!z) {
+        hts_log_error("Call to libdeflate_alloc_compressor failed");
+        return NULL;
+    }
+
+    unsigned char *cdata = NULL; /* Compressed output */
+    size_t cdata_alloc;
+    cdata = malloc(cdata_alloc = size*1.05+100);
+    if (!cdata) {
+        hts_log_error("Memory allocation failure");
+        libdeflate_free_compressor(z);
+        return NULL;
+    }
+
+    *cdata_size = libdeflate_gzip_compress(z, data, size, cdata, cdata_alloc);
+    libdeflate_free_compressor(z);
+
+    if (*cdata_size == 0) {
+        hts_log_error("Call to libdeflate_gzip_compress failed");
+        free(cdata);
+        return NULL;
+    }
+
+    return (char *)cdata;
+}
+
+#else
+
+/* ----------------------------------------------------------------------
+ * zlib compression code - from Gap5's tg_iface_g.c
+ * They're static here as they're only used within the cram_compress_block
+ * and cram_uncompress_block functions, which are the external interface.
+ */
+char *zlib_mem_inflate(char *cdata, size_t csize, size_t *size) {
+    z_stream s;
+    unsigned char *data = NULL; /* Uncompressed output */
+    int data_alloc = 0;
+    int err;
+
+    /* Starting point at uncompressed size, and scale after that */
+    data = malloc(data_alloc = csize*1.2+100);
+    if (!data)
+        return NULL;
+
+    /* Initialise zlib stream */
+    s.zalloc = Z_NULL; /* use default allocation functions */
+    s.zfree  = Z_NULL;
+    s.opaque = Z_NULL;
+    s.next_in  = (unsigned char *)cdata;
+    s.avail_in = csize;
+    s.total_in = 0;
+    s.next_out  = data;
+    s.avail_out = data_alloc;
+    s.total_out = 0;
+
+    //err = inflateInit(&s);
+    err = inflateInit2(&s, 15 + 32);
+    if (err != Z_OK) {
+        hts_log_error("Call to zlib inflateInit failed: %s", s.msg);
+        free(data);
+        return NULL;
+    }
+
+    /* Decode to 'data' array */
+    for (;s.avail_in;) {
+        unsigned char *data_tmp;
+        int alloc_inc;
+
+        s.next_out = &data[s.total_out];
+        err = inflate(&s, Z_NO_FLUSH);
+        if (err == Z_STREAM_END)
+            break;
+
+        if (err != Z_OK) {
+            hts_log_error("Call to zlib inflate failed: %s", s.msg);
+            free(data);
+            inflateEnd(&s);
+            return NULL;
+        }
+
+        /* More to come, so realloc based on growth so far */
+        alloc_inc = (double)s.avail_in/s.total_in * s.total_out + 100;
+        data = realloc((data_tmp = data), data_alloc += alloc_inc);
+        if (!data) {
+            free(data_tmp);
+            inflateEnd(&s);
+            return NULL;
+        }
+        s.avail_out += alloc_inc;
+    }
+    inflateEnd(&s);
+
+    *size = s.total_out;
+    return (char *)data;
+}
+#endif
+
+#if !defined(HAVE_LIBDEFLATE) || LIBDEFLATE_VERSION_MAJOR < 1 || (LIBDEFLATE_VERSION_MAJOR ==  1 && LIBDEFLATE_VERSION_MINOR <= 8)
+static char *zlib_mem_deflate(char *data, size_t size, size_t *cdata_size,
+                              int level, int strat) {
+    z_stream s;
+    unsigned char *cdata = NULL; /* Compressed output */
+    int cdata_alloc = 0;
+    int cdata_pos = 0;
+    int err;
+
+    cdata = malloc(cdata_alloc = size*1.05+100);
+    if (!cdata)
+        return NULL;
+    cdata_pos = 0;
+
+    /* Initialise zlib stream */
+    s.zalloc = Z_NULL; /* use default allocation functions */
+    s.zfree  = Z_NULL;
+    s.opaque = Z_NULL;
+    s.next_in  = (unsigned char *)data;
+    s.avail_in = size;
+    s.total_in = 0;
+    s.next_out  = cdata;
+    s.avail_out = cdata_alloc;
+    s.total_out = 0;
+    s.data_type = Z_BINARY;
+
+    err = deflateInit2(&s, level, Z_DEFLATED, 15|16, 9, strat);
+    if (err != Z_OK) {
+        hts_log_error("Call to zlib deflateInit2 failed: %s", s.msg);
+        return NULL;
+    }
+
+    /* Encode to 'cdata' array */
+    for (;s.avail_in;) {
+        s.next_out = &cdata[cdata_pos];
+        s.avail_out = cdata_alloc - cdata_pos;
+        if (cdata_alloc - cdata_pos <= 0) {
+            hts_log_error("Deflate produced larger output than expected");
+            return NULL;
+        }
+        err = deflate(&s, Z_NO_FLUSH);
+        cdata_pos = cdata_alloc - s.avail_out;
+        if (err != Z_OK) {
+            hts_log_error("Call to zlib deflate failed: %s", s.msg);
+            break;
+        }
+    }
+    if (deflate(&s, Z_FINISH) != Z_STREAM_END) {
+        hts_log_error("Call to zlib deflate failed: %s", s.msg);
+    }
+    *cdata_size = s.total_out;
+
+    if (deflateEnd(&s) != Z_OK) {
+        hts_log_error("Call to zlib deflate failed: %s", s.msg);
+    }
+    return (char *)cdata;
+}
+#endif
+
+#ifdef HAVE_LIBLZMA
+/* ------------------------------------------------------------------------ */
+/*
+ * Data compression routines using liblzma (xz)
+ *
+ * On a test set this shrunk the main db from 136157104 bytes to 114796168, but
+ * caused tg_index to grow from 2m43.707s to 15m3.961s. Exporting as bfastq
+ * went from 18.3s to 36.3s. So decompression suffers too, but not as bad
+ * as compression times.
+ *
+ * For now we disable this functionality. If it's to be reenabled make sure you
+ * improve the mem_inflate implementation as it's just a test hack at the
+ * moment.
+ */
+
+static char *lzma_mem_deflate(char *data, size_t size, size_t *cdata_size,
+                              int level) {
+    char *out;
+    size_t out_size = lzma_stream_buffer_bound(size);
+    *cdata_size = 0;
+
+    out = malloc(out_size);
+
+    /* Single call compression */
+    if (LZMA_OK != lzma_easy_buffer_encode(level, LZMA_CHECK_CRC32, NULL,
+                                           (uint8_t *)data, size,
+                                           (uint8_t *)out, cdata_size,
+                                           out_size))
+        return NULL;
+
+    return out;
+}
+
+static char *lzma_mem_inflate(char *cdata, size_t csize, size_t *size) {
+    lzma_stream strm = LZMA_STREAM_INIT;
+    size_t out_size = 0, out_pos = 0;
+    char *out = NULL, *new_out;
+    int r;
+
+    /* Initiate the decoder */
+    if (LZMA_OK != lzma_stream_decoder(&strm, lzma_easy_decoder_memusage(9), 0))
+        return NULL;
+
+    /* Decode loop */
+    strm.avail_in = csize;
+    strm.next_in = (uint8_t *)cdata;
+
+    for (;strm.avail_in;) {
+        if (strm.avail_in > out_size - out_pos) {
+            out_size += strm.avail_in * 4 + 32768;
+            new_out = realloc(out, out_size);
+            if (!new_out)
+                goto fail;
+            out = new_out;
+        }
+        strm.avail_out = out_size - out_pos;
+        strm.next_out = (uint8_t *)&out[out_pos];
+
+        r = lzma_code(&strm, LZMA_RUN);
+        if (LZMA_OK != r && LZMA_STREAM_END != r) {
+            hts_log_error("LZMA decode failure (error %d)", r);
+            goto fail;
+        }
+
+        out_pos = strm.total_out;
+
+        if (r == LZMA_STREAM_END)
+            break;
+    }
+
+    /* finish up any unflushed data; necessary? */
+    r = lzma_code(&strm, LZMA_FINISH);
+    if (r != LZMA_OK && r != LZMA_STREAM_END) {
+        hts_log_error("Call to lzma_code failed with error %d", r);
+        goto fail;
+    }
+
+    new_out = realloc(out, strm.total_out > 0 ? strm.total_out : 1);
+    if (new_out)
+        out = new_out;
+    *size = strm.total_out;
+
+    lzma_end(&strm);
+
+    return out;
+
+ fail:
+    lzma_end(&strm);
+    free(out);
+    return NULL;
+}
+#endif
+
+/* ----------------------------------------------------------------------
+ * CRAM blocks - the dynamically growable data block. We have code to
+ * create, update, (un)compress and read/write.
+ *
+ * These are derived from the deflate_interlaced.c blocks, but with the
+ * CRAM extension of content types and IDs.
+ */
+
+/*
+ * Allocates a new cram_block structure with a specified content_type and
+ * id.
+ *
+ * Returns block pointer on success
+ *         NULL on failure
+ */
+cram_block *cram_new_block(enum cram_content_type content_type,
+                           int content_id) {
+    cram_block *b = malloc(sizeof(*b));
+    if (!b)
+        return NULL;
+    b->method = b->orig_method = RAW;
+    b->content_type = content_type;
+    b->content_id = content_id;
+    b->comp_size = 0;
+    b->uncomp_size = 0;
+    b->data = NULL;
+    b->alloc = 0;
+    b->byte = 0;
+    b->bit = 7; // MSB
+    b->crc32 = 0;
+    b->idx = 0;
+    b->m = NULL;
+
+    return b;
+}
+
+/*
+ * Reads a block from a cram file.
+ * Returns cram_block pointer on success.
+ *         NULL on failure
+ */
+cram_block *cram_read_block(cram_fd *fd) {
+    cram_block *b = malloc(sizeof(*b));
+    unsigned char c;
+    uint32_t crc = 0;
+    if (!b)
+        return NULL;
+
+    //fprintf(stderr, "Block at %d\n", (int)ftell(fd->fp));
+
+    if (-1 == (b->method      = hgetc(fd->fp))) { free(b); return NULL; }
+    c = b->method; crc = crc32(crc, &c, 1);
+    if (-1 == (b->content_type= hgetc(fd->fp))) { free(b); return NULL; }
+    c = b->content_type; crc = crc32(crc, &c, 1);
+    if (-1 == fd->vv.varint_decode32_crc(fd, &b->content_id, &crc))  { free(b); return NULL; }
+    if (-1 == fd->vv.varint_decode32_crc(fd, &b->comp_size, &crc))   { free(b); return NULL; }
+    if (-1 == fd->vv.varint_decode32_crc(fd, &b->uncomp_size, &crc)) { free(b); return NULL; }
+
+    //fprintf(stderr, "  method %d, ctype %d, cid %d, csize %d, ucsize %d\n",
+    //      b->method, b->content_type, b->content_id, b->comp_size, b->uncomp_size);
+
+    if (b->method == RAW) {
+        if (b->uncomp_size < 0 || b->comp_size != b->uncomp_size) {
+            free(b);
+            return NULL;
+        }
+        b->alloc = b->uncomp_size;
+        if (!(b->data = malloc(b->uncomp_size))){ free(b); return NULL; }
+        if (b->uncomp_size != hread(fd->fp, b->data, b->uncomp_size)) {
+            free(b->data);
+            free(b);
+            return NULL;
+        }
+    } else {
+        if (b->comp_size < 0 || b->uncomp_size < 0) {
+            free(b);
+            return NULL;
+        }
+        b->alloc = b->comp_size;
+        if (!(b->data = malloc(b->comp_size)))  { free(b); return NULL; }
+        if (b->comp_size != hread(fd->fp, b->data, b->comp_size)) {
+            free(b->data);
+            free(b);
+            return NULL;
+        }
+    }
+
+    if (CRAM_MAJOR_VERS(fd->version) >= 3) {
+        if (-1 == int32_decode(fd, (int32_t *)&b->crc32)) {
+            free(b->data);
+            free(b);
+            return NULL;
+        }
+
+        b->crc32_checked = fd->ignore_md5;
+        b->crc_part = crc;
+    } else {
+        b->crc32_checked = 1; // CRC not present
+    }
+
+    b->orig_method = b->method;
+    b->idx = 0;
+    b->byte = 0;
+    b->bit = 7; // MSB
+
+    return b;
+}
+
+
+/*
+ * Computes the size of a cram block, including the block
+ * header itself.
+ */
+uint32_t cram_block_size(cram_block *b) {
+    unsigned char dat[100], *cp = dat;;
+    uint32_t sz;
+
+    *cp++ = b->method;
+    *cp++ = b->content_type;
+    cp += itf8_put((char*)cp, b->content_id);
+    cp += itf8_put((char*)cp, b->comp_size);
+    cp += itf8_put((char*)cp, b->uncomp_size);
+
+    sz = cp-dat + 4;
+    sz += b->method == RAW ? b->uncomp_size : b->comp_size;
+
+    return sz;
+}
+
+/*
+ * Writes a CRAM block.
+ * Returns 0 on success
+ *        -1 on failure
+ */
+int cram_write_block(cram_fd *fd, cram_block *b) {
+    char vardata[100];
+    int vardata_o = 0;
+
+    assert(b->method != RAW || (b->comp_size == b->uncomp_size));
+
+    if (hputc(b->method,       fd->fp)  == EOF) return -1;
+    if (hputc(b->content_type, fd->fp)  == EOF) return -1;
+    vardata_o += fd->vv.varint_put32(vardata          , vardata+100, b->content_id);
+    vardata_o += fd->vv.varint_put32(vardata+vardata_o, vardata+100, b->comp_size);
+    vardata_o += fd->vv.varint_put32(vardata+vardata_o, vardata+100, b->uncomp_size);
+    if (vardata_o != hwrite(fd->fp, vardata, vardata_o))
+        return -1;
+
+    if (b->data) {
+        if (b->method == RAW) {
+            if (b->uncomp_size != hwrite(fd->fp, b->data, b->uncomp_size))
+                return -1;
+        } else {
+            if (b->comp_size != hwrite(fd->fp, b->data, b->comp_size))
+                return -1;
+        }
+    } else {
+        // Absent blocks should be size 0
+        assert(b->method == RAW && b->uncomp_size == 0);
+    }
+
+    if (CRAM_MAJOR_VERS(fd->version) >= 3) {
+        char dat[100], *cp = (char *)dat;
+        uint32_t crc;
+
+        *cp++ = b->method;
+        *cp++ = b->content_type;
+        cp += fd->vv.varint_put32(cp, dat+100, b->content_id);
+        cp += fd->vv.varint_put32(cp, dat+100, b->comp_size);
+        cp += fd->vv.varint_put32(cp, dat+100, b->uncomp_size);
+        crc = crc32(0L, (uc *)dat, cp-dat);
+
+        if (b->method == RAW) {
+            b->crc32 = crc32(crc, b->data ? b->data : (uc*)"", b->uncomp_size);
+        } else {
+            b->crc32 = crc32(crc, b->data ? b->data : (uc*)"", b->comp_size);
+        }
+
+        if (-1 == int32_encode(fd, b->crc32))
+            return -1;
+    }
+
+    return 0;
+}
+
+/*
+ * Frees a CRAM block, deallocating internal data too.
+ */
+void cram_free_block(cram_block *b) {
+    if (!b)
+        return;
+    if (b->data)
+        free(b->data);
+    free(b);
+}
+
+/*
+ * Uncompresses a CRAM block, if compressed.
+ */
+int cram_uncompress_block(cram_block *b) {
+    char *uncomp;
+    size_t uncomp_size = 0;
+
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    // Pretend the CRC was OK so the fuzzer doesn't have to get it right
+    b->crc32_checked = 1;
+#endif
+
+    if (b->crc32_checked == 0) {
+        uint32_t crc = crc32(b->crc_part, b->data ? b->data : (uc *)"", b->alloc);
+        b->crc32_checked = 1;
+        if (crc != b->crc32) {
+            hts_log_error("Block CRC32 failure");
+            return -1;
+        }
+    }
+
+    if (b->uncomp_size == 0) {
+        // blank block
+        b->method = RAW;
+        return 0;
+    }
+    assert(b->uncomp_size >= 0); // cram_read_block should ensure this
+
+    switch (b->method) {
+    case RAW:
+        return 0;
+
+    case GZIP:
+        uncomp_size = b->uncomp_size;
+        uncomp = zlib_mem_inflate((char *)b->data, b->comp_size, &uncomp_size);
+
+        if (!uncomp)
+            return -1;
+        if (uncomp_size != b->uncomp_size) {
+            free(uncomp);
+            return -1;
+        }
+        free(b->data);
+        b->data = (unsigned char *)uncomp;
+        b->alloc = uncomp_size;
+        b->method = RAW;
+        break;
+
+#ifdef HAVE_LIBBZ2
+    case BZIP2: {
+        unsigned int usize = b->uncomp_size;
+        if (!(uncomp = malloc(usize)))
+            return -1;
+        if (BZ_OK != BZ2_bzBuffToBuffDecompress(uncomp, &usize,
+                                                (char *)b->data, b->comp_size,
+                                                0, 0)) {
+            free(uncomp);
+            return -1;
+        }
+        free(b->data);
+        b->data = (unsigned char *)uncomp;
+        b->alloc = usize;
+        b->method = RAW;
+        b->uncomp_size = usize; // Just in case it differs
+        break;
+    }
+#else
+    case BZIP2:
+        hts_log_error("Bzip2 compression is not compiled into this version. Please rebuild and try again");
+        return -1;
+#endif
+
+#ifdef HAVE_LIBLZMA
+    case LZMA:
+        uncomp = lzma_mem_inflate((char *)b->data, b->comp_size, &uncomp_size);
+        if (!uncomp)
+            return -1;
+        if (uncomp_size != b->uncomp_size) {
+            free(uncomp);
+            return -1;
+        }
+        free(b->data);
+        b->data = (unsigned char *)uncomp;
+        b->alloc = uncomp_size;
+        b->method = RAW;
+        break;
+#else
+    case LZMA:
+        hts_log_error("Lzma compression is not compiled into this version. Please rebuild and try again");
+        return -1;
+        break;
+#endif
+
+    case RANS: {
+        unsigned int usize = b->uncomp_size, usize2;
+        uncomp = (char *)rans_uncompress(b->data, b->comp_size, &usize2);
+        if (!uncomp)
+            return -1;
+        if (usize != usize2) {
+            free(uncomp);
+            return -1;
+        }
+        free(b->data);
+        b->data = (unsigned char *)uncomp;
+        b->alloc = usize2;
+        b->method = RAW;
+        b->uncomp_size = usize2; // Just in case it differs
+        //fprintf(stderr, "Expanded %d to %d\n", b->comp_size, b->uncomp_size);
+        break;
+    }
+
+    case FQZ: {
+        uncomp_size = b->uncomp_size;
+        uncomp = fqz_decompress((char *)b->data, b->comp_size, &uncomp_size, NULL, 0);
+        if (!uncomp)
+            return -1;
+        free(b->data);
+        b->data = (unsigned char *)uncomp;
+        b->alloc = uncomp_size;
+        b->method = RAW;
+        b->uncomp_size = uncomp_size;
+        break;
+    }
+
+    case RANS_PR0: {
+        unsigned int usize = b->uncomp_size, usize2;
+        uncomp = (char *)rans_uncompress_4x16(b->data, b->comp_size, &usize2);
+        if (!uncomp)
+            return -1;
+        if (usize != usize2) {
+            free(uncomp);
+            return -1;
+        }
+        b->orig_method = RANS_PR0 + (b->data[0]&1)
+            + 2*((b->data[0]&0x40)>0) + 4*((b->data[0]&0x80)>0);
+        free(b->data);
+        b->data = (unsigned char *)uncomp;
+        b->alloc = usize2;
+        b->method = RAW;
+        b->uncomp_size = usize2; // Just incase it differs
+        //fprintf(stderr, "Expanded %d to %d\n", b->comp_size, b->uncomp_size);
+        break;
+    }
+
+    case ARITH_PR0: {
+        unsigned int usize = b->uncomp_size, usize2;
+        uncomp = (char *)arith_uncompress_to(b->data, b->comp_size, NULL, &usize2);
+        if (!uncomp)
+            return -1;
+        if (usize != usize2) {
+            free(uncomp);
+            return -1;
+        }
+        b->orig_method = ARITH_PR0 + (b->data[0]&1)
+            + 2*((b->data[0]&0x40)>0) + 4*((b->data[0]&0x80)>0);
+        free(b->data);
+        b->data = (unsigned char *)uncomp;
+        b->alloc = usize2;
+        b->method = RAW;
+        b->uncomp_size = usize2; // Just incase it differs
+        //fprintf(stderr, "Expanded %d to %d\n", b->comp_size, b->uncomp_size);
+        break;
+    }
+
+    case TOK3: {
+        uint32_t out_len;
+        uint8_t *cp = tok3_decode_names(b->data, b->comp_size, &out_len);
+        if (!cp)
+            return -1;
+        b->orig_method = TOK3;
+        b->method = RAW;
+        free(b->data);
+        b->data = cp;
+        b->alloc = out_len;
+        b->uncomp_size = out_len;
+        break;
+    }
+
+    default:
+        return -1;
+    }
+
+    return 0;
+}
+
+static char *cram_compress_by_method(cram_slice *s, char *in, size_t in_size,
+                                     int content_id, size_t *out_size,
+                                     enum cram_block_method_int method,
+                                     int level, int strat) {
+    switch (method) {
+    case GZIP:
+    case GZIP_RLE:
+    case GZIP_1:
+        // Read names bizarrely benefit from zlib over libdeflate for
+        // mid-range compression levels.  Focusing purely of ratio or
+        // speed, libdeflate still wins.  It also seems to win for
+        // other data series too.
+        //
+        // Eg RN at level 5;  libdeflate=55.9MB  zlib=51.6MB
+#ifdef HAVE_LIBDEFLATE
+#  if (LIBDEFLATE_VERSION_MAJOR < 1 || (LIBDEFLATE_VERSION_MAJOR == 1 && LIBDEFLATE_VERSION_MINOR <= 8))
+        if (content_id == DS_RN && level >= 4 && level <= 7)
+            return zlib_mem_deflate(in, in_size, out_size, level, strat);
+        else
+#  endif
+            return libdeflate_deflate(in, in_size, out_size, level, strat);
+#else
+        return zlib_mem_deflate(in, in_size, out_size, level, strat);
+#endif
+
+    case BZIP2: {
+#ifdef HAVE_LIBBZ2
+        unsigned int comp_size = in_size*1.01 + 600;
+        char *comp = malloc(comp_size);
+        if (!comp)
+            return NULL;
+
+        if (BZ_OK != BZ2_bzBuffToBuffCompress(comp, &comp_size,
+                                              in, in_size,
+                                              level, 0, 30)) {
+            free(comp);
+            return NULL;
+        }
+        *out_size = comp_size;
+        return comp;
+#else
+        return NULL;
+#endif
+    }
+
+    case FQZ:
+    case FQZ_b:
+    case FQZ_c:
+    case FQZ_d: {
+        // Extract the necessary portion of the slice into an fqz_slice struct.
+        // These previously were the same thing, but this permits us to detach
+        // the codec from the rest of this CRAM implementation.
+        fqz_slice *f = malloc(2*s->hdr->num_records * sizeof(uint32_t) + sizeof(fqz_slice));
+        if (!f)
+            return NULL;
+        f->num_records = s->hdr->num_records;
+        f->len = (uint32_t *)(((char *)f) + sizeof(fqz_slice));
+        f->flags = f->len + s->hdr->num_records;
+        int i;
+        for (i = 0; i < s->hdr->num_records; i++) {
+            f->flags[i] = s->crecs[i].flags;
+            f->len[i] = (i+1 < s->hdr->num_records
+                         ? s->crecs[i+1].qual - s->crecs[i].qual
+                         : s->block[DS_QS]->uncomp_size - s->crecs[i].qual);
+        }
+        char *comp = fqz_compress(strat & 0xff /* cram vers */, f,
+                                  in, in_size, out_size, strat >> 8, NULL);
+        free(f);
+        return comp;
+    }
+
+    case LZMA:
+#ifdef HAVE_LIBLZMA
+        return lzma_mem_deflate(in, in_size, out_size, level);
+#else
+        return NULL;
+#endif
+
+    case RANS0:
+    case RANS1: {
+        unsigned int out_size_i;
+        unsigned char *cp;
+        cp = rans_compress((unsigned char *)in, in_size, &out_size_i,
+                           method == RANS0 ? 0 : 1);
+        *out_size = out_size_i;
+        return (char *)cp;
+    }
+
+    case RANS_PR0:
+    case RANS_PR1:
+    case RANS_PR64:
+    case RANS_PR9:
+    case RANS_PR128:
+    case RANS_PR129:
+    case RANS_PR192:
+    case RANS_PR193: {
+        unsigned int out_size_i;
+        unsigned char *cp;
+
+        // see enum cram_block. We map RANS_* methods to order bit-fields
+        static int methmap[] = { 1, 64,9, 128,129, 192,193 };
+
+        int m = method == RANS_PR0 ? 0 : methmap[method - RANS_PR1];
+        cp = rans_compress_4x16((unsigned char *)in, in_size, &out_size_i,
+                                m | RANS_ORDER_SIMD_AUTO);
+        *out_size = out_size_i;
+        return (char *)cp;
+    }
+
+    case ARITH_PR0:
+    case ARITH_PR1:
+    case ARITH_PR64:
+    case ARITH_PR9:
+    case ARITH_PR128:
+    case ARITH_PR129:
+    case ARITH_PR192:
+    case ARITH_PR193: {
+        unsigned int out_size_i;
+        unsigned char *cp;
+
+        // see enum cram_block. We map ARITH_* methods to order bit-fields
+        static int methmap[] = { 1, 64,9, 128,129, 192,193 };
+
+        cp = arith_compress_to((unsigned char *)in, in_size, NULL, &out_size_i,
+                               method == ARITH_PR0 ? 0 : methmap[method - ARITH_PR1]);
+        *out_size = out_size_i;
+        return (char *)cp;
+    }
+
+    case TOK3:
+    case TOKA: {
+        int out_len;
+        int lev = level;
+        if (method == TOK3 && lev > 3)
+            lev = 3;
+        uint8_t *cp = tok3_encode_names(in, in_size, lev, strat, &out_len, NULL);
+        *out_size = out_len;
+        return (char *)cp;
+    }
+
+    case RAW:
+        break;
+
+    default:
+        return NULL;
+    }
+
+    return NULL;
+}
+
+
+/*
+ * Compresses a block using one of two different zlib strategies. If we only
+ * want one choice set strat2 to be -1.
+ *
+ * The logic here is that sometimes Z_RLE does a better job than Z_FILTERED
+ * or Z_DEFAULT_STRATEGY on quality data. If so, we'd rather use it as it is
+ * significantly faster.
+ *
+ * Method and level -1 implies defaults, as specified in cram_fd.
+ */
+int cram_compress_block2(cram_fd *fd, cram_slice *s,
+                         cram_block *b, cram_metrics *metrics,
+                         int method, int level) {
+
+    if (!b)
+        return 0;
+
+    char *comp = NULL;
+    size_t comp_size = 0;
+    int strat;
+
+    // Internally we have parameterised methods that externally map
+    // to the same CRAM method value.
+    // See enum_cram_block_method_int in cram_structs.h.
+    int methmap[] = {
+        // Externally defined values
+        RAW, GZIP, BZIP2, LZMA, RANS, RANSPR, ARITH, FQZ, TOK3,
+
+        // Reserved for possible expansion
+        0, 0,
+
+        // Internally parameterised versions matching back to above
+        // external values
+        GZIP, GZIP,
+        FQZ, FQZ, FQZ,
+        RANS,
+        RANSPR, RANSPR, RANSPR, RANSPR, RANSPR, RANSPR, RANSPR,
+        TOK3,
+        ARITH,  ARITH,  ARITH,  ARITH,  ARITH,  ARITH,  ARITH,
+    };
+
+    if (b->method != RAW) {
+        // Maybe already compressed if s->block[0] was compressed and
+        // we have e.g. s->block[DS_BA] set to s->block[0] due to only
+        // one base type present and hence using E_HUFFMAN on block 0.
+        // A second explicit attempt to compress the same block then
+        // occurs.
+        return 0;
+    }
+
+    if (method == -1) {
+        method = 1<<GZIP;
+        if (fd->use_bz2)
+            method |= 1<<BZIP2;
+        if (fd->use_lzma)
+            method |= 1<<LZMA;
+    }
+
+    if (level == -1)
+        level = fd->level;
+
+    //fprintf(stderr, "IN: block %d, sz %d\n", b->content_id, b->uncomp_size);
+
+    if (method == RAW || level == 0 || b->uncomp_size == 0) {
+        b->method = RAW;
+        b->comp_size = b->uncomp_size;
+        //fprintf(stderr, "Skip block id %d\n", b->content_id);
+        return 0;
+    }
+
+#ifndef ABS
+#    define ABS(a) ((a)>=0?(a):-(a))
+#endif
+
+    if (metrics) {
+        pthread_mutex_lock(&fd->metrics_lock);
+        // Sudden changes in size trigger a retrial.  These are mainly
+        // triggered when switching to sorted / unsorted, where the number
+        // of elements in a slice radically changes.
+        //
+        // We also get large fluctuations based on genome coordinate for
+        // e.g. SA:Z and SC series, but we consider the typical scale of
+        // delta between blocks and use this to look for abnormality.
+
+        // Equivalent to (but minus possible integer overflow)
+        //   (b->uncomp_size + 1000)/4 > metrics->input_avg_sz+1000 ||
+        //    b->uncomp_size + 1000    < (metrics->input_avg_sz+1000)/4)
+        if (metrics->input_avg_sz &&
+            (b->uncomp_size/4 - 750 > metrics->input_avg_sz ||
+             b->uncomp_size         < metrics->input_avg_sz/4 - 750) &&
+            ABS(b->uncomp_size-metrics->input_avg_sz)/10
+                > metrics->input_avg_delta) {
+            metrics->next_trial = 0;
+        }
+
+        if (metrics->trial > 0 || --metrics->next_trial <= 0) {
+            int m, unpackable = metrics->unpackable;
+            size_t sz_best = b->uncomp_size;
+            size_t sz[CRAM_MAX_METHOD] = {0};
+            int method_best = 0; // RAW
+            char *c_best = NULL, *c = NULL;
+
+            metrics->input_avg_delta =
+                0.9 * (metrics->input_avg_delta +
+                       ABS(b->uncomp_size - metrics->input_avg_sz));
+
+            metrics->input_avg_sz += b->uncomp_size*.2;
+            metrics->input_avg_sz *= 0.8;
+
+            if (metrics->revised_method)
+                method = metrics->revised_method;
+            else
+                metrics->revised_method = method;
+
+            if (metrics->next_trial <= 0) {
+                metrics->next_trial = TRIAL_SPAN;
+                metrics->trial = NTRIALS;
+                for (m = 0; m < CRAM_MAX_METHOD; m++)
+                    metrics->sz[m] /= 2;
+                metrics->unpackable = 0;
+            }
+
+            // Compress this block using the best method
+            if (unpackable && CRAM_MAJOR_VERS(fd->version) > 3) {
+                // No point trying bit-pack if 17+ symbols.
+                if (method & (1<<RANS_PR128))
+                    method = (method|(1<<RANS_PR0))&~(1<<RANS_PR128);
+                if (method & (1<<RANS_PR129))
+                    method = (method|(1<<RANS_PR1))&~(1<<RANS_PR129);
+                if (method & (1<<RANS_PR192))
+                    method = (method|(1<<RANS_PR64))&~(1<<RANS_PR192);
+                if (method & (1<<RANS_PR193))
+                    method = (method|(1<<RANS_PR64)|(1<<RANS_PR1))&~(1<<RANS_PR193);
+
+                if (method & (1<<ARITH_PR128))
+                    method = (method|(1<<ARITH_PR0))&~(1<<ARITH_PR128);
+                if (method & (1<<ARITH_PR129))
+                    method = (method|(1<<ARITH_PR1))&~(1<<ARITH_PR129);
+                if (method & (1<<ARITH_PR192))
+                    method = (method|(1<<ARITH_PR64))&~(1<<ARITH_PR192);
+                if (method & (1u<<ARITH_PR193))
+                    method = (method|(1<<ARITH_PR64)|(1<<ARITH_PR1))&~(1u<<ARITH_PR193);
+            }
+
+            // Libdeflate doesn't have a Z_RLE strategy.
+            // We treat it as level 1, but iff we haven't also
+            // explicitly listed that in the method list.
+#ifdef HAVE_LIBDEFLATE
+            if ((method & (1<<GZIP_RLE)) && (method & (1<<GZIP_1)))
+                method &= ~(1<<GZIP_RLE);
+#endif
+
+            pthread_mutex_unlock(&fd->metrics_lock);
+
+            for (m = 0; m < CRAM_MAX_METHOD; m++) {
+                if (method & (1u<<m)) {
+                    int lvl = level;
+                    switch (m) {
+                    case GZIP:     strat = Z_FILTERED; break;
+                    case GZIP_1:   strat = Z_DEFAULT_STRATEGY; lvl = 1; break;
+                    case GZIP_RLE: strat = Z_RLE; break;
+                    case FQZ:      strat = CRAM_MAJOR_VERS(fd->version); break;
+                    case FQZ_b:    strat = CRAM_MAJOR_VERS(fd->version)+256; break;
+                    case FQZ_c:    strat = CRAM_MAJOR_VERS(fd->version)+2*256; break;
+                    case FQZ_d:    strat = CRAM_MAJOR_VERS(fd->version)+3*256; break;
+                    case TOK3:     strat = 0; break;
+                    case TOKA:     strat = 1; break;
+                    default:       strat = 0;
+                    }
+
+                    c = cram_compress_by_method(s, (char *)b->data, b->uncomp_size,
+                                                b->content_id, &sz[m], m, lvl, strat);
+
+                    if (c && sz_best > sz[m]) {
+                        sz_best = sz[m];
+                        method_best = m;
+                        if (c_best)
+                            free(c_best);
+                        c_best = c;
+                    } else if (c) {
+                        free(c);
+                    } else {
+                        sz[m] = UINT_MAX; // arbitrarily worse than raw
+                    }
+                } else {
+                    sz[m] = UINT_MAX; // arbitrarily worse than raw
+                }
+            }
+
+            if (c_best) {
+                free(b->data);
+                b->data = (unsigned char *)c_best;
+                b->method = method_best; // adjusted to methmap[method_best] later
+                b->comp_size = sz_best;
+            }
+
+            // Accumulate stats for all methods tried
+            pthread_mutex_lock(&fd->metrics_lock);
+            for (m = 0; m < CRAM_MAX_METHOD; m++)
+                // don't be overly sure on small blocks.
+                // +2000 means eg bzip2 vs gzip (1.07 to 1.04) or gz vs rans1
+                // needs to be at least 60 bytes smaller to overcome the
+                // fixed size addition.
+                metrics->sz[m] += sz[m]+2000;
+
+            // When enough trials performed, find the best on average
+            if (--metrics->trial == 0) {
+                int best_method = RAW;
+                int best_sz = INT_MAX;
+
+                // Relative costs of methods. See enum_cram_block_method_int
+                // and methmap
+                double meth_cost[32] = {
+                    // Externally defined methods
+                    1,    // 0  raw
+                    1.04, // 1  gzip (Z_FILTERED)
+                    1.07, // 2  bzip2
+                    1.08, // 3  lzma
+                    1.00, // 4  rans    (O0)
+                    1.00, // 5  ranspr  (O0)
+                    1.04, // 6  arithpr (O0)
+                    1.05, // 7  fqz
+                    1.05, // 8  tok3 (rans)
+                    1.00, 1.00, // 9,10 reserved
+
+                    // Paramterised versions of above
+                    1.01, // gzip rle
+                    1.01, // gzip -1
+
+                    1.05, 1.05, 1.05, // FQZ_b,c,d
+
+                    1.01, // rans O1
+
+                    1.01, // rans_pr1
+                    1.00, // rans_pr64; if smaller, usually fast
+                    1.03, // rans_pr65/9
+                    1.00, // rans_pr128
+                    1.01, // rans_pr129
+                    1.00, // rans_pr192
+                    1.01, // rans_pr193
+
+                    1.07, // tok3 arith
+
+                    1.04, // arith_pr1
+                    1.04, // arith_pr64
+                    1.04, // arith_pr9
+                    1.03, // arith_pr128
+                    1.04, // arith_pr129
+                    1.04, // arith_pr192
+                    1.04, // arith_pr193
+                };
+
+                // Scale methods by cost based on compression level
+                if (fd->level <= 1) {
+                    for (m = 0; m < CRAM_MAX_METHOD; m++)
+                        metrics->sz[m] *= 1+(meth_cost[m]-1)*4;
+                } else if (fd->level <= 3) {
+                    for (m = 0; m < CRAM_MAX_METHOD; m++)
+                        metrics->sz[m] *= 1+(meth_cost[m]-1);
+                } else if (fd->level <= 6) {
+                    for (m = 0; m < CRAM_MAX_METHOD; m++)
+                        metrics->sz[m] *= 1+(meth_cost[m]-1)/2;
+                } else if (fd->level <= 7) {
+                    for (m = 0; m < CRAM_MAX_METHOD; m++)
+                        metrics->sz[m] *= 1+(meth_cost[m]-1)/3;
+                } // else cost is ignored
+
+                // Ensure these are never used; BSC and ZSTD
+                metrics->sz[9] = metrics->sz[10] = INT_MAX;
+
+                for (m = 0; m < CRAM_MAX_METHOD; m++) {
+                    if ((!metrics->sz[m]) || (!(method & (1u<<m))))
+                        continue;
+
+                    if (best_sz > metrics->sz[m])
+                        best_sz = metrics->sz[m], best_method = m;
+                }
+
+                if (best_method != metrics->method) {
+                    //metrics->trial = (NTRIALS+1)/2; // be sure
+                    //metrics->next_trial /= 1.5;
+                    metrics->consistency = 0;
+                } else {
+                    metrics->next_trial *= MIN(2, 1+metrics->consistency/4.0);
+                    metrics->consistency++;
+                }
+
+                metrics->method = best_method;
+                switch (best_method) {
+                case GZIP:     strat = Z_FILTERED; break;
+                case GZIP_1:   strat = Z_DEFAULT_STRATEGY; break;
+                case GZIP_RLE: strat = Z_RLE; break;
+                case FQZ:      strat = CRAM_MAJOR_VERS(fd->version); break;
+                case FQZ_b:    strat = CRAM_MAJOR_VERS(fd->version)+256; break;
+                case FQZ_c:    strat = CRAM_MAJOR_VERS(fd->version)+2*256; break;
+                case FQZ_d:    strat = CRAM_MAJOR_VERS(fd->version)+3*256; break;
+                case TOK3:     strat = 0; break;
+                case TOKA:     strat = 1; break;
+                default:       strat = 0;
+                }
+                metrics->strat  = strat;
+
+                // If we see at least MAXFAIL trials in a row for a specific
+                // compression method with more than MAXDELTA aggregate
+                // size then we drop this from the list of methods used
+                // for this block type.
+#define MAXDELTA 0.20
+#define MAXFAILS 4
+                for (m = 0; m < CRAM_MAX_METHOD; m++) {
+                    if (best_method == m) {
+                        metrics->cnt[m] = 0;
+                        metrics->extra[m] = 0;
+                    } else if (best_sz < metrics->sz[m]) {
+                        double r = (double)metrics->sz[m] / best_sz - 1;
+                        int mul = 1+(fd->level>=7);
+                        if (++metrics->cnt[m] >= MAXFAILS*mul &&
+                            (metrics->extra[m] += r) >= MAXDELTA*mul)
+                            method &= ~(1u<<m);
+
+                        // Special case for fqzcomp as it rarely changes
+                        if (m == FQZ || m == FQZ_b || m == FQZ_c || m == FQZ_d) {
+                            if (metrics->sz[m] > best_sz)
+                                method &= ~(1u<<m);
+                        }
+                    }
+                }
+
+                //if (fd->verbose > 1 && method != metrics->revised_method)
+                //    fprintf(stderr, "%d: revising method from %x to %x\n",
+                //          b->content_id, metrics->revised_method, method);
+                metrics->revised_method = method;
+            }
+            pthread_mutex_unlock(&fd->metrics_lock);
+        } else {
+            metrics->input_avg_delta =
+                0.9 * (metrics->input_avg_delta +
+                       ABS(b->uncomp_size - metrics->input_avg_sz));
+
+            metrics->input_avg_sz += b->uncomp_size*.2;
+            metrics->input_avg_sz *= 0.8;
+
+            strat = metrics->strat;
+            method = metrics->method;
+
+            pthread_mutex_unlock(&fd->metrics_lock);
+            comp = cram_compress_by_method(s, (char *)b->data, b->uncomp_size,
+                                           b->content_id, &comp_size, method,
+                                           method == GZIP_1 ? 1 : level,
+                                           strat);
+            if (!comp)
+                return -1;
+
+            if (comp_size < b->uncomp_size) {
+                free(b->data);
+                b->data = (unsigned char *)comp;
+                b->comp_size = comp_size;
+                b->method = method;
+            } else {
+                free(comp);
+            }
+        }
+
+    } else {
+        // no cached metrics, so just do zlib?
+        comp = cram_compress_by_method(s, (char *)b->data, b->uncomp_size,
+                                       b->content_id, &comp_size, GZIP, level, Z_FILTERED);
+        if (!comp) {
+            hts_log_error("Compression failed!");
+            return -1;
+        }
+
+        if (comp_size < b->uncomp_size) {
+            free(b->data);
+            b->data = (unsigned char *)comp;
+            b->comp_size = comp_size;
+            b->method = GZIP;
+        } else {
+            free(comp);
+        }
+        strat = Z_FILTERED;
+    }
+
+    hts_log_info("Compressed block ID %d from %d to %d by method %s",
+                 b->content_id, b->uncomp_size, b->comp_size,
+                 cram_block_method2str(b->method));
+
+    b->method = methmap[b->method];
+
+    return 0;
+}
+int cram_compress_block(cram_fd *fd, cram_block *b, cram_metrics *metrics,
+                        int method, int level) {
+    return cram_compress_block2(fd, NULL, b, metrics, method, level);
+}
+
+cram_metrics *cram_new_metrics(void) {
+    cram_metrics *m = calloc(1, sizeof(*m));
+    if (!m)
+        return NULL;
+    m->trial = NTRIALS-1;
+    m->next_trial = TRIAL_SPAN/2; // learn quicker at start
+    m->method = RAW;
+    m->strat = 0;
+    m->revised_method = 0;
+    m->unpackable = 0;
+
+    return m;
+}
+
+char *cram_block_method2str(enum cram_block_method_int m) {
+    switch(m) {
+    case RAW:         return "RAW";
+    case GZIP:        return "GZIP";
+    case BZIP2:       return "BZIP2";
+    case LZMA:        return "LZMA";
+    case RANS0:       return "RANS0";
+    case RANS1:       return "RANS1";
+    case GZIP_RLE:    return "GZIP_RLE";
+    case GZIP_1:      return "GZIP_1";
+    case FQZ:         return "FQZ";
+    case FQZ_b:       return "FQZ_b";
+    case FQZ_c:       return "FQZ_c";
+    case FQZ_d:       return "FQZ_d";
+    case RANS_PR0:    return "RANS_PR0";
+    case RANS_PR1:    return "RANS_PR1";
+    case RANS_PR64:   return "RANS_PR64";
+    case RANS_PR9:    return "RANS_PR9";
+    case RANS_PR128:  return "RANS_PR128";
+    case RANS_PR129:  return "RANS_PR129";
+    case RANS_PR192:  return "RANS_PR192";
+    case RANS_PR193:  return "RANS_PR193";
+    case TOK3:        return "TOK3_R";
+    case TOKA:        return "TOK3_A";
+    case ARITH_PR0:   return "ARITH_PR0";
+    case ARITH_PR1:   return "ARITH_PR1";
+    case ARITH_PR64:  return "ARITH_PR64";
+    case ARITH_PR9:   return "ARITH_PR9";
+    case ARITH_PR128: return "ARITH_PR128";
+    case ARITH_PR129: return "ARITH_PR129";
+    case ARITH_PR192: return "ARITH_PR192";
+    case ARITH_PR193: return "ARITH_PR193";
+    case BM_ERROR: break;
+    }
+    return "?";
+}
+
+char *cram_content_type2str(enum cram_content_type t) {
+    switch (t) {
+    case FILE_HEADER:         return "FILE_HEADER";
+    case COMPRESSION_HEADER:  return "COMPRESSION_HEADER";
+    case MAPPED_SLICE:        return "MAPPED_SLICE";
+    case UNMAPPED_SLICE:      return "UNMAPPED_SLICE";
+    case EXTERNAL:            return "EXTERNAL";
+    case CORE:                return "CORE";
+    case CT_ERROR:            break;
+    }
+    return "?";
+}
+
+/* ----------------------------------------------------------------------
+ * Reference sequence handling
+ *
+ * These revolve around the refs_t structure, which may potentially be
+ * shared between multiple cram_fd.
+ *
+ * We start with refs_create() to allocate an empty refs_t and then
+ * populate it with @SQ line data using refs_from_header(). This is done on
+ * cram_open().  Also at start up we can call cram_load_reference() which
+ * is used with "scramble -r foo.fa". This replaces the fd->refs with the
+ * new one specified. In either case refs2id() is then called which
+ * maps ref_entry names to @SQ ids (refs_t->ref_id[]).
+ *
+ * Later, possibly within a thread, we will want to know the actual ref
+ * seq itself, obtained by calling cram_get_ref().  This may use the
+ * UR: or M5: fields or the filename specified in the original
+ * cram_load_reference() call.
+ *
+ * Given the potential for multi-threaded reference usage, we have
+ * reference counting (sorry for the confusing double use of "ref") to
+ * track the number of callers interested in any specific reference.
+ */
+
+/*
+ * Frees/unmaps a reference sequence and associated file handles.
+ */
+static void ref_entry_free_seq(ref_entry *e) {
+    if (e->mf)
+        mfclose(e->mf);
+    if (e->seq && !e->mf)
+        free(e->seq);
+
+    e->seq = NULL;
+    e->mf = NULL;
+}
+
+void refs_free(refs_t *r) {
+    RP("refs_free()\n");
+
+    if (--r->count > 0)
+        return;
+
+    if (!r)
+        return;
+
+    if (r->pool)
+        string_pool_destroy(r->pool);
+
+    if (r->h_meta) {
+        khint_t k;
+
+        for (k = kh_begin(r->h_meta); k != kh_end(r->h_meta); k++) {
+            ref_entry *e;
+
+            if (!kh_exist(r->h_meta, k))
+                continue;
+            if (!(e = kh_val(r->h_meta, k)))
+                continue;
+            ref_entry_free_seq(e);
+            free(e);
+        }
+
+        kh_destroy(refs, r->h_meta);
+    }
+
+    if (r->ref_id)
+        free(r->ref_id);
+
+    if (r->fp)
+        bgzf_close(r->fp);
+
+    pthread_mutex_destroy(&r->lock);
+
+    free(r);
+}
+
+static refs_t *refs_create(void) {
+    refs_t *r = calloc(1, sizeof(*r));
+
+    RP("refs_create()\n");
+
+    if (!r)
+        return NULL;
+
+    if (!(r->pool = string_pool_create(8192)))
+        goto err;
+
+    r->ref_id = NULL; // see refs2id() to populate.
+    r->count = 1;
+    r->last = NULL;
+    r->last_id = -1;
+
+    if (!(r->h_meta = kh_init(refs)))
+        goto err;
+
+    pthread_mutex_init(&r->lock, NULL);
+
+    return r;
+
+ err:
+    refs_free(r);
+    return NULL;
+}
+
+/*
+ * Opens a reference fasta file as a BGZF stream, allowing for
+ * compressed files.  It automatically builds a .fai file if
+ * required and if compressed a .gzi bgzf index too.
+ *
+ * Returns a BGZF handle on success;
+ *         NULL on failure.
+ */
+static BGZF *bgzf_open_ref(char *fn, char *mode, int is_md5) {
+    BGZF *fp;
+
+    if (!is_md5 && !hisremote(fn)) {
+        char fai_file[PATH_MAX];
+
+        snprintf(fai_file, PATH_MAX, "%s.fai", fn);
+        if (access(fai_file, R_OK) != 0)
+            if (fai_build(fn) != 0)
+                return NULL;
+    }
+
+    if (!(fp = bgzf_open(fn, mode))) {
+        perror(fn);
+        return NULL;
+    }
+
+    if (fp->is_compressed == 1 && bgzf_index_load(fp, fn, ".gzi") < 0) {
+        hts_log_error("Unable to load .gzi index '%s.gzi'", fn);
+        bgzf_close(fp);
+        return NULL;
+    }
+
+    return fp;
+}
+
+/*
+ * Loads a FAI file for a reference.fasta.
+ * "is_err" indicates whether failure to load is worthy of emitting an
+ * error message. In some cases (eg with embedded references) we
+ * speculatively load, just in case, and silently ignore errors.
+ *
+ * Returns the refs_t struct on success (maybe newly allocated);
+ *         NULL on failure
+ */
+static refs_t *refs_load_fai(refs_t *r_orig, const char *fn, int is_err) {
+    hFILE *fp = NULL;
+    char fai_fn[PATH_MAX];
+    char line[8192];
+    refs_t *r = r_orig;
+    size_t fn_l = strlen(fn);
+    int id = 0, id_alloc = 0;
+
+    RP("refs_load_fai %s\n", fn);
+
+    if (!r)
+        if (!(r = refs_create()))
+            goto err;
+
+    if (r->fp)
+        if (bgzf_close(r->fp) != 0)
+            goto err;
+    r->fp = NULL;
+
+    /* Look for a FASTA##idx##FAI format */
+    char *fn_delim = strstr(fn, HTS_IDX_DELIM);
+    if (fn_delim) {
+        if (!(r->fn = string_ndup(r->pool, fn, fn_delim - fn)))
+            goto err;
+        fn_delim += strlen(HTS_IDX_DELIM);
+        snprintf(fai_fn, PATH_MAX, "%s", fn_delim);
+    } else {
+        /* An index file was provided, instead of the actual reference file */
+        if (fn_l > 4 && strcmp(&fn[fn_l-4], ".fai") == 0) {
+            if (!r->fn) {
+                if (!(r->fn = string_ndup(r->pool, fn, fn_l-4)))
+                    goto err;
+            }
+            snprintf(fai_fn, PATH_MAX, "%s", fn);
+        } else {
+        /* Only the reference file provided. Get the index file name from it */
+            if (!(r->fn = string_dup(r->pool, fn)))
+                goto err;
+            snprintf(fai_fn, PATH_MAX, "%.*s.fai", PATH_MAX-5, fn);
+        }
+    }
+
+    if (!(r->fp = bgzf_open_ref(r->fn, "r", 0))) {
+        hts_log_error("Failed to open reference file '%s'", r->fn);
+        goto err;
+    }
+
+    if (!(fp = hopen(fai_fn, "r"))) {
+        hts_log_error("Failed to open index file '%s'", fai_fn);
+        if (is_err)
+            perror(fai_fn);
+        goto err;
+    }
+    while (hgets(line, 8192, fp) != NULL) {
+        ref_entry *e = malloc(sizeof(*e));
+        char *cp;
+        int n;
+        khint_t k;
+
+        if (!e)
+            return NULL;
+
+        // id
+        for (cp = line; *cp && !isspace_c(*cp); cp++)
+            ;
+        *cp++ = 0;
+        e->name = string_dup(r->pool, line);
+
+        // length
+        while (*cp && isspace_c(*cp))
+            cp++;
+        e->length = strtoll(cp, &cp, 10);
+
+        // offset
+        while (*cp && isspace_c(*cp))
+            cp++;
+        e->offset = strtoll(cp, &cp, 10);
+
+        // bases per line
+        while (*cp && isspace_c(*cp))
+            cp++;
+        e->bases_per_line = strtol(cp, &cp, 10);
+
+        // line length
+        while (*cp && isspace_c(*cp))
+            cp++;
+        e->line_length = strtol(cp, &cp, 10);
+
+        // filename
+        e->fn = r->fn;
+
+        e->count = 0;
+        e->seq = NULL;
+        e->mf = NULL;
+        e->is_md5 = 0;
+        e->validated_md5 = 0;
+
+        k = kh_put(refs, r->h_meta, e->name, &n);
+        if (-1 == n)  {
+            free(e);
+            return NULL;
+        }
+
+        if (n) {
+            kh_val(r->h_meta, k) = e;
+        } else {
+            ref_entry *re = kh_val(r->h_meta, k);
+            if (re && (re->count != 0 || re->length != 0)) {
+                /* Keep old */
+                free(e);
+            } else {
+                /* Replace old */
+                if (re)
+                    free(re);
+                kh_val(r->h_meta, k) = e;
+            }
+        }
+
+        if (id >= id_alloc) {
+            ref_entry **new_refs;
+            int x;
+
+            id_alloc = id_alloc ?id_alloc*2 : 16;
+            new_refs = realloc(r->ref_id, id_alloc * sizeof(*r->ref_id));
+            if (!new_refs)
+                goto err;
+            r->ref_id = new_refs;
+
+            for (x = id; x < id_alloc; x++)
+                r->ref_id[x] = NULL;
+        }
+        r->ref_id[id] = e;
+        r->nref = ++id;
+    }
+
+    if(hclose(fp) < 0)
+        goto err;
+    return r;
+
+ err:
+    if (fp)
+        hclose_abruptly(fp);
+
+    if (!r_orig)
+        refs_free(r);
+
+    return NULL;
+}
+
+/*
+ * Verifies that the CRAM @SQ lines and .fai files match.
+ */
+static void sanitise_SQ_lines(cram_fd *fd) {
+    int i;
+
+    if (!fd->header || !fd->header->hrecs)
+        return;
+
+    if (!fd->refs || !fd->refs->h_meta)
+        return;
+
+    for (i = 0; i < fd->header->hrecs->nref; i++) {
+        const char *name = fd->header->hrecs->ref[i].name;
+        khint_t k = kh_get(refs, fd->refs->h_meta, name);
+        ref_entry *r;
+
+        // We may have @SQ lines which have no known .fai, but do not
+        // in themselves pose a problem because they are unused in the file.
+        if (k == kh_end(fd->refs->h_meta))
+            continue;
+
+        if (!(r = (ref_entry *)kh_val(fd->refs->h_meta, k)))
+            continue;
+
+        if (r->length && r->length != fd->header->hrecs->ref[i].len) {
+            assert(strcmp(r->name, fd->header->hrecs->ref[i].name) == 0);
+
+            // Should we also check MD5sums here to ensure the correct
+            // reference was given?
+            hts_log_warning("Header @SQ length mismatch for ref %s, %"PRIhts_pos" vs %d",
+                            r->name, fd->header->hrecs->ref[i].len, (int)r->length);
+
+            // Fixing the parsed @SQ header will make MD:Z: strings work
+            // and also stop it producing N for the sequence.
+            fd->header->hrecs->ref[i].len = r->length;
+        }
+    }
+}
+
+/*
+ * Indexes references by the order they appear in a BAM file. This may not
+ * necessarily be the same order they appear in the fasta reference file.
+ *
+ * Returns 0 on success
+ *        -1 on failure
+ */
+int refs2id(refs_t *r, sam_hdr_t *hdr) {
+    int i;
+    sam_hrecs_t *h = hdr->hrecs;
+
+    if (r->ref_id)
+        free(r->ref_id);
+    if (r->last)
+        r->last = NULL;
+
+    r->ref_id = calloc(h->nref, sizeof(*r->ref_id));
+    if (!r->ref_id)
+        return -1;
+
+    r->nref = h->nref;
+    for (i = 0; i < h->nref; i++) {
+        khint_t k = kh_get(refs, r->h_meta, h->ref[i].name);
+        if (k != kh_end(r->h_meta)) {
+            r->ref_id[i] = kh_val(r->h_meta, k);
+        } else {
+            hts_log_warning("Unable to find ref name '%s'", h->ref[i].name);
+        }
+    }
+
+    return 0;
+}
+
+/*
+ * Generates refs_t entries based on @SQ lines in the header.
+ * Returns 0 on success
+ *         -1 on failure
+ */
+static int refs_from_header(cram_fd *fd) {
+    if (!fd)
+        return -1;
+
+    refs_t *r = fd->refs;
+    if (!r)
+        return -1;
+
+    sam_hdr_t *h = fd->header;
+    if (!h)
+        return 0;
+
+    if (!h->hrecs) {
+        if (-1 == sam_hdr_fill_hrecs(h))
+            return -1;
+    }
+
+    if (h->hrecs->nref == 0)
+        return 0;
+
+    //fprintf(stderr, "refs_from_header for %p mode %c\n", fd, fd->mode);
+
+    /* Existing refs are fine, as long as they're compatible with the hdr. */
+    ref_entry **new_ref_id = realloc(r->ref_id, (r->nref + h->hrecs->nref) * sizeof(*r->ref_id));
+    if (!new_ref_id)
+        return -1;
+    r->ref_id = new_ref_id;
+
+    int i, j;
+    /* Copy info from h->ref[i] over to r */
+    for (i = 0, j = r->nref; i < h->hrecs->nref; i++) {
+        sam_hrec_type_t *ty;
+        sam_hrec_tag_t *tag;
+        khint_t k;
+        int n;
+
+        k = kh_get(refs, r->h_meta, h->hrecs->ref[i].name);
+        if (k != kh_end(r->h_meta))
+            // Ref already known about
+            continue;
+
+        if (!(r->ref_id[j] = calloc(1, sizeof(ref_entry))))
+            return -1;
+
+        if (!h->hrecs->ref[i].name)
+            return -1;
+
+        r->ref_id[j]->name = string_dup(r->pool, h->hrecs->ref[i].name);
+        if (!r->ref_id[j]->name) return -1;
+        r->ref_id[j]->length = 0; // marker for not yet loaded
+
+        /* Initialise likely filename if known */
+        if ((ty = sam_hrecs_find_type_id(h->hrecs, "SQ", "SN", h->hrecs->ref[i].name))) {
+            if ((tag = sam_hrecs_find_key(ty, "M5", NULL))) {
+                r->ref_id[j]->fn = string_dup(r->pool, tag->str+3);
+                //fprintf(stderr, "Tagging @SQ %s / %s\n", r->ref_id[h]->name, r->ref_id[h]->fn);
+            }
+        }
+
+        k = kh_put(refs, r->h_meta, r->ref_id[j]->name, &n);
+        if (n <= 0) // already exists or error
+            return -1;
+        kh_val(r->h_meta, k) = r->ref_id[j];
+
+        j++;
+    }
+    r->nref = j;
+
+    return 0;
+}
+
+/*
+ * Attaches a header to a cram_fd.
+ *
+ * This should be used when creating a new cram_fd for writing where
+ * we have a header already constructed (eg from a file we've read
+ * in).
+ */
+int cram_set_header2(cram_fd *fd, const sam_hdr_t *hdr) {
+    if (!fd || !hdr )
+        return -1;
+
+    if (fd->header != hdr) {
+        if (fd->header)
+            sam_hdr_destroy(fd->header);
+        fd->header = sam_hdr_dup(hdr);
+        if (!fd->header)
+            return -1;
+    }
+    return refs_from_header(fd);
+}
+
+int cram_set_header(cram_fd *fd, sam_hdr_t *hdr) {
+    return cram_set_header2(fd, hdr);
+}
+
+/*
+ * Returns whether the path refers to a directory.
+ */
+static int is_directory(char *fn) {
+    struct stat buf;
+    if ( stat(fn,&buf) ) return 0;
+    return S_ISDIR(buf.st_mode);
+}
+
+/*
+ * Converts a directory and a filename into an expanded path, replacing %s
+ * in directory with the filename and %[0-9]+s with portions of the filename
+ * Any remaining parts of filename are added to the end with /%s.
+ */
+static int expand_cache_path(char *path, char *dir, const char *fn) {
+    char *cp, *start = path;
+    size_t len;
+    size_t sz = PATH_MAX;
+
+    while ((cp = strchr(dir, '%'))) {
+        if (cp-dir >= sz) return -1;
+        strncpy(path, dir, cp-dir);
+        path += cp-dir;
+        sz -= cp-dir;
+
+        if (*++cp == 's') {
+            len = strlen(fn);
+            if (len >= sz) return -1;
+            strcpy(path, fn);
+            path += len;
+            sz -= len;
+            fn += len;
+            cp++;
+        } else if (*cp >= '0' && *cp <= '9') {
+            char *endp;
+            long l;
+
+            l = strtol(cp, &endp, 10);
+            l = MIN(l, strlen(fn));
+            if (*endp == 's') {
+                if (l >= sz) return -1;
+                strncpy(path, fn, l);
+                path += l;
+                fn += l;
+                sz -= l;
+                *path = 0;
+                cp = endp+1;
+            } else {
+                if (sz < 3) return -1;
+                *path++ = '%';
+                *path++ = *cp++;
+            }
+        } else {
+            if (sz < 3) return -1;
+            *path++ = '%';
+            *path++ = *cp++;
+        }
+        dir = cp;
+    }
+
+    len = strlen(dir);
+    if (len >= sz) return -1;
+    strcpy(path, dir);
+    path += len;
+    sz -= len;
+
+    len = strlen(fn) + ((*fn && path > start && path[-1] != '/') ? 1 : 0);
+    if (len >= sz) return -1;
+    if (*fn && path > start && path[-1] != '/')
+        *path++ = '/';
+    strcpy(path, fn);
+    return 0;
+}
+
+/*
+ * Make the directory containing path and any prefix directories.
+ */
+static void mkdir_prefix(char *path, int mode) {
+    char *cp = strrchr(path, '/');
+    if (!cp)
+        return;
+
+    *cp = 0;
+    if (is_directory(path)) {
+        *cp = '/';
+        return;
+    }
+
+    if (mkdir(path, mode) == 0) {
+        chmod(path, mode);
+        *cp = '/';
+        return;
+    }
+
+    mkdir_prefix(path, mode);
+    mkdir(path, mode);
+    chmod(path, mode);
+    *cp = '/';
+}
+
+/*
+ * Return the cache directory to use, based on the first of these
+ * environment variables to be set to a non-empty value.
+ */
+static const char *get_cache_basedir(const char **extra) {
+    char *base;
+
+    *extra = "";
+
+    base = getenv("XDG_CACHE_HOME");
+    if (base && *base) return base;
+
+    base = getenv("HOME");
+    if (base && *base) { *extra = "/.cache"; return base; }
+
+    base = getenv("TMPDIR");
+    if (base && *base) return base;
+
+    base = getenv("TEMP");
+    if (base && *base) return base;
+
+    return "/tmp";
+}
+
+/*
+ * Queries the M5 string from the header and attempts to populate the
+ * reference from this using the REF_PATH environment.
+ *
+ * Returns 0 on success
+ *        -1 on failure
+ */
+static int cram_populate_ref(cram_fd *fd, int id, ref_entry *r) {
+    char *ref_path = getenv("REF_PATH");
+    sam_hrec_type_t *ty;
+    sam_hrec_tag_t *tag;
+    char path[PATH_MAX];
+    kstring_t path_tmp = KS_INITIALIZE;
+    char cache[PATH_MAX], cache_root[PATH_MAX];
+    char *local_cache = getenv("REF_CACHE");
+    mFILE *mf;
+    int local_path = 0;
+
+    hts_log_info("Running cram_populate_ref on fd %p, id %d", (void *)fd, id);
+
+    cache_root[0] = '\0';
+
+    if (!ref_path || *ref_path == '\0') {
+        /*
+         * If we have no ref path, we use the EBI server.
+         * However to avoid spamming it we require a local ref cache too.
+         */
+        ref_path = "https://www.ebi.ac.uk/ena/cram/md5/%s";
+        if (!local_cache || *local_cache == '\0') {
+            const char *extra;
+            const char *base = get_cache_basedir(&extra);
+            snprintf(cache_root, PATH_MAX, "%s%s/hts-ref", base, extra);
+            snprintf(cache,PATH_MAX, "%s%s/hts-ref/%%2s/%%2s/%%s", base, extra);
+            local_cache = cache;
+            hts_log_info("Populating local cache: %s", local_cache);
+        }
+    }
+
+    if (!r->name)
+        return -1;
+
+    if (!(ty = sam_hrecs_find_type_id(fd->header->hrecs, "SQ", "SN", r->name)))
+        return -1;
+
+    if (!(tag = sam_hrecs_find_key(ty, "M5", NULL)))
+        goto no_M5;
+
+    hts_log_info("Querying ref %s", tag->str+3);
+
+    /* Use cache if available */
+    if (local_cache && *local_cache) {
+        if (expand_cache_path(path, local_cache, tag->str+3) == 0)
+            local_path = 1;
+    }
+
+#ifndef HAVE_MMAP
+    char *path2;
+    /* Search local files in REF_PATH; we can open them and return as above */
+    if (!local_path && (path2 = find_path(tag->str+3, ref_path))) {
+        int len = snprintf(path, PATH_MAX, "%s", path2);
+        free(path2);
+        if (len > 0 && len < PATH_MAX) // in case it's too long
+            local_path = 1;
+    }
+#endif
+
+    /* Found via REF_CACHE or local REF_PATH file */
+    if (local_path) {
+        struct stat sb;
+        BGZF *fp;
+
+        if (0 == stat(path, &sb)
+            && S_ISREG(sb.st_mode)
+            && (fp = bgzf_open(path, "r"))) {
+            r->length = sb.st_size;
+            r->offset = r->line_length = r->bases_per_line = 0;
+
+            r->fn = string_dup(fd->refs->pool, path);
+
+            if (fd->refs->fp)
+                if (bgzf_close(fd->refs->fp) != 0)
+                    return -1;
+            fd->refs->fp = fp;
+            fd->refs->fn = r->fn;
+            r->is_md5 = 1;
+            r->validated_md5 = 1;
+
+            // Fall back to cram_get_ref() where it'll do the actual
+            // reading of the file.
+            return 0;
+        }
+    }
+
+
+    /* Otherwise search full REF_PATH; slower as loads entire file */
+    if ((mf = open_path_mfile(tag->str+3, ref_path, NULL))) {
+        size_t sz;
+        r->seq = mfsteal(mf, &sz);
+        if (r->seq) {
+            r->mf = NULL;
+        } else {
+            // keep mf around as we couldn't detach
+            r->seq = mf->data;
+            r->mf = mf;
+        }
+        r->length = sz;
+        r->is_md5 = 1;
+        r->validated_md5 = 1;
+    } else {
+        refs_t *refs;
+        const char *fn;
+
+    no_M5:
+        /* Failed to find in search path or M5 cache, see if @SQ UR: tag? */
+        if (!(tag = sam_hrecs_find_key(ty, "UR", NULL)))
+            return -1;
+
+        fn = (strncmp(tag->str+3, "file:", 5) == 0)
+            ? tag->str+8
+            : tag->str+3;
+
+        if (fd->refs->fp) {
+            if (bgzf_close(fd->refs->fp) != 0)
+                return -1;
+            fd->refs->fp = NULL;
+        }
+        if (!(refs = refs_load_fai(fd->refs, fn, 0)))
+            return -1;
+        sanitise_SQ_lines(fd);
+
+        fd->refs = refs;
+        if (fd->refs->fp) {
+            if (bgzf_close(fd->refs->fp) != 0)
+                return -1;
+            fd->refs->fp = NULL;
+        }
+
+        if (!fd->refs->fn)
+            return -1;
+
+        if (-1 == refs2id(fd->refs, fd->header))
+            return -1;
+        if (!fd->refs->ref_id || !fd->refs->ref_id[id])
+            return -1;
+
+        // Local copy already, so fall back to cram_get_ref().
+        return 0;
+    }
+
+    /* Populate the local disk cache if required */
+    if (local_cache && *local_cache) {
+        hFILE *fp;
+
+        if (*cache_root && !is_directory(cache_root)) {
+            hts_log_warning("Creating reference cache directory %s\n"
+                            "This may become large; see the samtools(1) manual page REF_CACHE discussion",
+                            cache_root);
+        }
+
+        if (expand_cache_path(path, local_cache, tag->str+3) < 0) {
+            return 0; // Not fatal - we have the data already so keep going.
+        }
+        hts_log_info("Writing cache file '%s'", path);
+        mkdir_prefix(path, 01777);
+
+        fp = hts_open_tmpfile(path, "wx", &path_tmp);
+        if (!fp) {
+            perror(path_tmp.s);
+            free(path_tmp.s);
+
+            // Not fatal - we have the data already so keep going.
+            return 0;
+        }
+
+        // Check md5sum
+        hts_md5_context *md5;
+        char unsigned md5_buf1[16];
+        char md5_buf2[33];
+
+        if (!(md5 = hts_md5_init())) {
+            hclose_abruptly(fp);
+            unlink(path_tmp.s);
+            free(path_tmp.s);
+            return -1;
+        }
+        hts_md5_update(md5, r->seq, r->length);
+        hts_md5_final(md5_buf1, md5);
+        hts_md5_destroy(md5);
+        hts_md5_hex(md5_buf2, md5_buf1);
+
+        if (strncmp(tag->str+3, md5_buf2, 32) != 0) {
+            hts_log_error("Mismatching md5sum for downloaded reference");
+            hclose_abruptly(fp);
+            unlink(path_tmp.s);
+            free(path_tmp.s);
+            return -1;
+        }
+
+        ssize_t length_written = hwrite(fp, r->seq, r->length);
+        if (hclose(fp) < 0 || length_written != r->length ||
+            chmod(path_tmp.s, 0444) < 0 ||
+            rename(path_tmp.s, path) < 0) {
+            hts_log_error("Creating reference at %s failed: %s",
+                          path, strerror(errno));
+            unlink(path_tmp.s);
+        }
+    }
+
+    free(path_tmp.s);
+    return 0;
+}
+
+static void cram_ref_incr_locked(refs_t *r, int id) {
+    RP("%d INC REF %d, %d %p\n", gettid(), id,
+       (int)(id>=0 && r->ref_id[id]?r->ref_id[id]->count+1:-999),
+       id>=0 && r->ref_id[id]?r->ref_id[id]->seq:(char *)1);
+
+    if (id < 0 || !r->ref_id[id] || !r->ref_id[id]->seq)
+        return;
+
+    if (r->last_id == id)
+        r->last_id = -1;
+
+    ++r->ref_id[id]->count;
+}
+
+void cram_ref_incr(refs_t *r, int id) {
+    pthread_mutex_lock(&r->lock);
+    cram_ref_incr_locked(r, id);
+    pthread_mutex_unlock(&r->lock);
+}
+
+static void cram_ref_decr_locked(refs_t *r, int id) {
+    RP("%d DEC REF %d, %d %p\n", gettid(), id,
+       (int)(id>=0 && r->ref_id[id]?r->ref_id[id]->count-1:-999),
+       id>=0 && r->ref_id[id]?r->ref_id[id]->seq:(char *)1);
+
+    if (id < 0 || !r->ref_id[id] || !r->ref_id[id]->seq) {
+        return;
+    }
+
+    if (--r->ref_id[id]->count <= 0) {
+        assert(r->ref_id[id]->count == 0);
+        if (r->last_id >= 0) {
+            if (r->ref_id[r->last_id]->count <= 0 &&
+                r->ref_id[r->last_id]->seq) {
+                RP("%d FREE REF %d (%p)\n", gettid(),
+                   r->last_id, r->ref_id[r->last_id]->seq);
+                ref_entry_free_seq(r->ref_id[r->last_id]);
+                if (r->ref_id[r->last_id]->is_md5) r->ref_id[r->last_id]->length = 0;
+            }
+        }
+        r->last_id = id;
+    }
+}
+
+void cram_ref_decr(refs_t *r, int id) {
+    pthread_mutex_lock(&r->lock);
+    cram_ref_decr_locked(r, id);
+    pthread_mutex_unlock(&r->lock);
+}
+
+/*
+ * Used by cram_ref_load and cram_get_ref. The file handle will have
+ * already been opened, so we can catch it. The ref_entry *e informs us
+ * of whether this is a multi-line fasta file or a raw MD5 style file.
+ * Either way we create a single contiguous sequence.
+ *
+ * Returns all or part of a reference sequence on success (malloced);
+ *         NULL on failure.
+ */
+static char *load_ref_portion(BGZF *fp, ref_entry *e,
+                              hts_pos_t start, hts_pos_t end) {
+    off_t offset, len;
+    char *seq;
+
+    if (end < start)
+        end = start;
+
+    /*
+     * Compute locations in file. This is trivial for the MD5 files, but
+     * is still necessary for the fasta variants.
+     *
+     * Note the offset here, as with faidx, has the assumption that white-
+     * space (the diff between line_length and bases_per_line) only occurs
+     * at the end of a line of text.
+     */
+    offset = e->line_length
+        ? e->offset + (start-1)/e->bases_per_line * e->line_length +
+          (start-1) % e->bases_per_line
+        : start-1;
+
+    len = (e->line_length
+           ? e->offset + (end-1)/e->bases_per_line * e->line_length +
+             (end-1) % e->bases_per_line
+           : end-1) - offset + 1;
+
+    if (bgzf_useek(fp, offset, SEEK_SET) < 0) {
+        perror("bgzf_useek() on reference file");
+        return NULL;
+    }
+
+    if (len == 0 || !(seq = malloc(len))) {
+        return NULL;
+    }
+
+    if (len != bgzf_read(fp, seq, len)) {
+        perror("bgzf_read() on reference file");
+        free(seq);
+        return NULL;
+    }
+
+    /* Strip white-space if required. */
+    if (len != end-start+1) {
+        hts_pos_t i, j;
+        char *cp = seq;
+        char *cp_to;
+
+        // Copy up to the first white-space, and then repeatedly just copy
+        // bases_per_line verbatim, and use the slow method to end again.
+        //
+        // This may seem excessive, but this code can be a significant
+        // portion of total CRAM decode CPU time for shallow data sets.
+        for (i = j = 0; i < len; i++) {
+            if (!isspace_c(cp[i]))
+                cp[j++] = cp[i] & ~0x20;
+            else
+                break;
+        }
+        while (i < len && isspace_c(cp[i]))
+            i++;
+        while (i < len - e->line_length) {
+            hts_pos_t j_end = j + e->bases_per_line;
+            while (j < j_end)
+                cp[j++] = cp[i++] & ~0x20; // toupper equiv
+            i += e->line_length - e->bases_per_line;
+        }
+        for (; i < len; i++) {
+            if (!isspace_c(cp[i]))
+                cp[j++] = cp[i] & ~0x20;
+        }
+
+        cp_to = cp+j;
+
+        if (cp_to - seq != end-start+1) {
+            hts_log_error("Malformed reference file");
+            free(seq);
+            return NULL;
+        }
+    } else {
+        int i;
+        for (i = 0; i < len; i++) {
+            seq[i] = toupper_c(seq[i]);
+        }
+    }
+
+    return seq;
+}
+
+/*
+ * Load the entire reference 'id'.
+ * This also increments the reference count by 1.
+ *
+ * Returns ref_entry on success;
+ *         NULL on failure
+ */
+ref_entry *cram_ref_load(refs_t *r, int id, int is_md5) {
+    ref_entry *e = r->ref_id[id];
+    hts_pos_t start = 1, end = e->length;
+    char *seq;
+
+    if (e->seq) {
+        return e;
+    }
+
+    assert(e->count == 0);
+
+    if (r->last) {
+#ifdef REF_DEBUG
+        int idx = 0;
+        for (idx = 0; idx < r->nref; idx++)
+            if (r->last == r->ref_id[idx])
+                break;
+        RP("%d cram_ref_load DECR %d\n", gettid(), idx);
+#endif
+        assert(r->last->count > 0);
+        if (--r->last->count <= 0) {
+            RP("%d FREE REF %d (%p)\n", gettid(), id, r->ref_id[id]->seq);
+            if (r->last->seq)
+                ref_entry_free_seq(r->last);
+        }
+    }
+
+    if (!r->fn)
+        return NULL;
+
+    /* Open file if it's not already the current open reference */
+    if (strcmp(r->fn, e->fn) || r->fp == NULL) {
+        if (r->fp)
+            if (bgzf_close(r->fp) != 0)
+                return NULL;
+        r->fn = e->fn;
+        if (!(r->fp = bgzf_open_ref(r->fn, "r", is_md5)))
+            return NULL;
+    }
+
+    RP("%d Loading ref %d (%d..%d)\n", gettid(), id, start, end);
+
+    if (!(seq = load_ref_portion(r->fp, e, start, end))) {
+        return NULL;
+    }
+
+    RP("%d Loaded ref %d (%d..%d) = %p\n", gettid(), id, start, end, seq);
+
+    RP("%d INC REF %d, %"PRId64"\n", gettid(), id, (e->count+1));
+    e->seq = seq;
+    e->mf = NULL;
+    e->count++;
+
+    /*
+     * Also keep track of last used ref so incr/decr loops on the same
+     * sequence don't cause load/free loops.
+     */
+    RP("%d cram_ref_load INCR %d => %"PRId64"\n", gettid(), id, e->count+1);
+    r->last = e;
+    e->count++;
+
+    return e;
+}
+
+/*
+ * Returns a portion of a reference sequence from start to end inclusive.
+ * The returned pointer is owned by either the cram_file fd or by the
+ * internal refs_t structure and should not be freed  by the caller.
+ *
+ * The difference is whether or not this refs_t is in use by just the one
+ * cram_fd or by multiples, or whether we have multiple threads accessing
+ * references. In either case fd->shared will be true and we start using
+ * reference counting to track the number of users of a specific reference
+ * sequence.
+ *
+ * Otherwise the ref seq returned is allocated as part of cram_fd itself
+ * and will be freed up on the next call to cram_get_ref or cram_close.
+ *
+ * To return the entire reference sequence, specify start as 1 and end
+ * as 0.
+ *
+ * To cease using a reference, call cram_ref_decr().
+ *
+ * Returns reference on success,
+ *         NULL on failure
+ */
+char *cram_get_ref(cram_fd *fd, int id, hts_pos_t start, hts_pos_t end) {
+    ref_entry *r;
+    char *seq;
+    int ostart = start;
+
+    if (id == -1 || start < 1)
+        return NULL;
+
+    /* FIXME: axiomatic query of r->seq being true?
+     * Or shortcut for unsorted data where we load once and never free?
+     */
+
+    //fd->shared_ref = 1; // hard code for now to simplify things
+
+    pthread_mutex_lock(&fd->ref_lock);
+
+    RP("%d cram_get_ref on fd %p, id %d, range %d..%d\n", gettid(), fd, id, start, end);
+
+    /*
+     * Unsorted data implies we want to fetch an entire reference at a time.
+     * We just deal with this at the moment by claiming we're sharing
+     * references instead, which has the same requirement.
+     */
+    if (fd->unsorted)
+        fd->shared_ref = 1;
+
+
+    /* Sanity checking: does this ID exist? */
+    if (id >= fd->refs->nref) {
+        hts_log_error("No reference found for id %d", id);
+        pthread_mutex_unlock(&fd->ref_lock);
+        return NULL;
+    }
+
+    if (!fd->refs || !fd->refs->ref_id[id]) {
+        hts_log_error("No reference found for id %d", id);
+        pthread_mutex_unlock(&fd->ref_lock);
+        return NULL;
+    }
+
+    if (!(r = fd->refs->ref_id[id])) {
+        hts_log_error("No reference found for id %d", id);
+        pthread_mutex_unlock(&fd->ref_lock);
+        return NULL;
+    }
+
+
+    /*
+     * It has an entry, but may not have been populated yet.
+     * Any manually loaded .fai files have their lengths known.
+     * A ref entry computed from @SQ lines (M5 or UR field) will have
+     * r->length == 0 unless it's been loaded once and verified that we have
+     * an on-disk filename for it.
+     *
+     * 19 Sep 2013: Moved the lock here as the cram_populate_ref code calls
+     * open_path_mfile and libcurl, which isn't multi-thread safe unless I
+     * rewrite my code to have one curl handle per thread.
+     */
+    pthread_mutex_lock(&fd->refs->lock);
+    if (r->length == 0) {
+        if (fd->ref_fn)
+            hts_log_warning("Reference file given, but ref '%s' not present",
+                            r->name);
+        if (cram_populate_ref(fd, id, r) == -1) {
+            hts_log_warning("Failed to populate reference for id %d", id);
+            pthread_mutex_unlock(&fd->refs->lock);
+            pthread_mutex_unlock(&fd->ref_lock);
+            return NULL;
+        }
+        r = fd->refs->ref_id[id];
+        if (fd->unsorted)
+            cram_ref_incr_locked(fd->refs, id);
+    }
+
+
+    /*
+     * We now know that we the filename containing the reference, so check
+     * for limits. If it's over half the reference we'll load all of it in
+     * memory as this will speed up subsequent calls.
+     */
+    if (end < 1)
+        end = r->length;
+    if (end >= r->length)
+        end  = r->length;
+
+    if (end - start >= 0.5*r->length || fd->shared_ref) {
+        start = 1;
+        end = r->length;
+    }
+
+    /*
+     * Maybe we have it cached already? If so use it.
+     *
+     * Alternatively if we don't have the sequence but we're sharing
+     * references and/or are asking for the entire length of it, then
+     * load the full reference into the refs structure and return
+     * a pointer to that one instead.
+     */
+    if (fd->shared_ref || r->seq || (start == 1 && end == r->length)) {
+        char *cp;
+
+        if (id >= 0) {
+            if (r->seq) {
+                cram_ref_incr_locked(fd->refs, id);
+            } else {
+                ref_entry *e;
+                if (!(e = cram_ref_load(fd->refs, id, r->is_md5))) {
+                    pthread_mutex_unlock(&fd->refs->lock);
+                    pthread_mutex_unlock(&fd->ref_lock);
+                    return NULL;
+                }
+
+                /* unsorted data implies cache ref indefinitely, to avoid
+                 * continually loading and unloading.
+                 */
+                if (fd->unsorted)
+                    cram_ref_incr_locked(fd->refs, id);
+            }
+
+            fd->ref = NULL; /* We never access it directly */
+            fd->ref_start = 1;
+            fd->ref_end   = r->length;
+            fd->ref_id    = id;
+
+            cp = fd->refs->ref_id[id]->seq + ostart-1;
+        } else {
+            fd->ref = NULL;
+            cp = NULL;
+        }
+
+        RP("%d cram_get_ref returning for id %d, count %d\n", gettid(), id, (int)r->count);
+
+        pthread_mutex_unlock(&fd->refs->lock);
+        pthread_mutex_unlock(&fd->ref_lock);
+        return cp;
+    }
+
+    /*
+     * Otherwise we're not sharing, we don't have a copy of it already and
+     * we're only asking for a small portion of it.
+     *
+     * In this case load up just that segment ourselves, freeing any old
+     * small segments in the process.
+     */
+
+    /* Unmapped ref ID */
+    if (id < 0 || !fd->refs->fn) {
+        if (fd->ref_free) {
+            free(fd->ref_free);
+            fd->ref_free = NULL;
+        }
+        fd->ref = NULL;
+        fd->ref_id = id;
+        pthread_mutex_unlock(&fd->refs->lock);
+        pthread_mutex_unlock(&fd->ref_lock);
+        return NULL;
+    }
+
+    /* Open file if it's not already the current open reference */
+    if (strcmp(fd->refs->fn, r->fn) || fd->refs->fp == NULL) {
+        if (fd->refs->fp)
+            if (bgzf_close(fd->refs->fp) != 0)
+                return NULL;
+        fd->refs->fn = r->fn;
+        if (!(fd->refs->fp = bgzf_open_ref(fd->refs->fn, "r", r->is_md5))) {
+            pthread_mutex_unlock(&fd->refs->lock);
+            pthread_mutex_unlock(&fd->ref_lock);
+            return NULL;
+        }
+    }
+
+    if (!(fd->ref = load_ref_portion(fd->refs->fp, r, start, end))) {
+        pthread_mutex_unlock(&fd->refs->lock);
+        pthread_mutex_unlock(&fd->ref_lock);
+        return NULL;
+    }
+
+    if (fd->ref_free)
+        free(fd->ref_free);
+
+    fd->ref_id    = id;
+    fd->ref_start = start;
+    fd->ref_end   = end;
+    fd->ref_free = fd->ref;
+    seq = fd->ref;
+
+    pthread_mutex_unlock(&fd->refs->lock);
+    pthread_mutex_unlock(&fd->ref_lock);
+
+    return seq ? seq + ostart - start : NULL;
+}
+
+/*
+ * If fd has been opened for reading, it may be permitted to specify 'fn'
+ * as NULL and let the code auto-detect the reference by parsing the
+ * SAM header @SQ lines.
+ */
+int cram_load_reference(cram_fd *fd, char *fn) {
+    int ret = 0;
+
+    if (fn) {
+        fd->refs = refs_load_fai(fd->refs, fn,
+                                 !(fd->embed_ref>0 && fd->mode == 'r'));
+        fn = fd->refs ? fd->refs->fn : NULL;
+        if (!fn)
+            ret = -1;
+        sanitise_SQ_lines(fd);
+    }
+    fd->ref_fn = fn;
+
+    if ((!fd->refs || (fd->refs->nref == 0 && !fn)) && fd->header) {
+        if (fd->refs)
+            refs_free(fd->refs);
+        if (!(fd->refs = refs_create()))
+            return -1;
+        if (-1 == refs_from_header(fd))
+            return -1;
+    }
+
+    if (fd->header)
+        if (-1 == refs2id(fd->refs, fd->header))
+            return -1;
+
+    return ret;
+}
+
+/* ----------------------------------------------------------------------
+ * Containers
+ */
+
+/*
+ * Creates a new container, specifying the maximum number of slices
+ * and records permitted.
+ *
+ * Returns cram_container ptr on success
+ *         NULL on failure
+ */
+cram_container *cram_new_container(int nrec, int nslice) {
+    cram_container *c = calloc(1, sizeof(*c));
+    enum cram_DS_ID id;
+
+    if (!c)
+        return NULL;
+
+    c->curr_ref = -2;
+
+    c->max_c_rec = nrec * nslice;
+    c->curr_c_rec = 0;
+
+    c->max_rec = nrec;
+    c->record_counter = 0;
+    c->num_bases = 0;
+    c->s_num_bases = 0;
+
+    c->max_slice = nslice;
+    c->curr_slice = 0;
+
+    c->pos_sorted = 1;
+    c->max_apos   = 0;
+    c->multi_seq  = 0;
+    c->qs_seq_orient = 1;
+    c->no_ref = 0;
+    c->embed_ref = -1; // automatic selection
+
+    c->bams = NULL;
+
+    if (!(c->slices = calloc(nslice != 0 ? nslice : 1, sizeof(cram_slice *))))
+        goto err;
+    c->slice = NULL;
+
+    if (!(c->comp_hdr = cram_new_compression_header()))
+        goto err;
+    c->comp_hdr_block = NULL;
+
+    for (id = DS_RN; id < DS_TN; id++)
+        if (!(c->stats[id] = cram_stats_create())) goto err;
+
+    //c->aux_B_stats = cram_stats_create();
+
+    if (!(c->tags_used = kh_init(m_tagmap)))
+        goto err;
+    c->refs_used = 0;
+    c->ref_free = 0;
+
+    return c;
+
+ err:
+    if (c) {
+        if (c->slices)
+            free(c->slices);
+        free(c);
+    }
+    return NULL;
+}
+
+static void free_bam_list(bam_seq_t **bams, int max_rec) {
+    int i;
+    for (i = 0; i < max_rec; i++)
+        bam_free(bams[i]);
+
+    free(bams);
+}
+
+void cram_free_container(cram_container *c) {
+    enum cram_DS_ID id;
+    int i;
+
+    if (!c)
+        return;
+
+    if (c->refs_used)
+        free(c->refs_used);
+
+    if (c->landmark)
+        free(c->landmark);
+
+    if (c->comp_hdr)
+        cram_free_compression_header(c->comp_hdr);
+
+    if (c->comp_hdr_block)
+        cram_free_block(c->comp_hdr_block);
+
+    // Free the slices; filled out by encoder only
+    if (c->slices) {
+        for (i = 0; i < c->max_slice; i++) {
+            if (c->slices[i])
+                cram_free_slice(c->slices[i]);
+            if (c->slices[i] == c->slice)
+                c->slice = NULL;
+        }
+        free(c->slices);
+    }
+
+    // Free the current slice; set by both encoder & decoder
+    if (c->slice) {
+        cram_free_slice(c->slice);
+        c->slice = NULL;
+    }
+
+    for (id = DS_RN; id < DS_TN; id++)
+        if (c->stats[id]) cram_stats_free(c->stats[id]);
+
+    //if (c->aux_B_stats) cram_stats_free(c->aux_B_stats);
+
+    if (c->tags_used) {
+        khint_t k;
+
+        for (k = kh_begin(c->tags_used); k != kh_end(c->tags_used); k++) {
+            if (!kh_exist(c->tags_used, k))
+                continue;
+
+            cram_tag_map *tm = (cram_tag_map *)kh_val(c->tags_used, k);
+            if (tm) {
+                cram_codec *c = tm->codec;
+
+                if (c) c->free(c);
+
+                // If tm->blk or tm->blk2 is set, then we haven't yet got to
+                // cram_encode_container which copies the blocks to s->aux_block
+                // and NULLifies tm->blk*.  In this case we failed to complete
+                // the container construction, so we have to free up our partially
+                // converted CRAM.
+                cram_free_block(tm->blk);
+                cram_free_block(tm->blk2);
+                free(tm);
+            }
+        }
+
+        kh_destroy(m_tagmap, c->tags_used);
+    }
+
+    if (c->ref_free)
+        free(c->ref);
+
+    if (c->bams)
+        free_bam_list(c->bams, c->max_c_rec);
+
+    free(c);
+}
+
+/*
+ * Reads a container header.
+ *
+ * Returns cram_container on success
+ *         NULL on failure or no container left (fd->err == 0).
+ */
+cram_container *cram_read_container(cram_fd *fd) {
+    cram_container c2, *c;
+    int i, s;
+    size_t rd = 0;
+    uint32_t crc = 0;
+
+    fd->err = 0;
+    fd->eof = 0;
+
+    memset(&c2, 0, sizeof(c2));
+    if (CRAM_MAJOR_VERS(fd->version) == 1) {
+        if ((s = fd->vv.varint_decode32_crc(fd, &c2.length, &crc)) == -1) {
+            fd->eof = fd->empty_container ? 1 : 2;
+            return NULL;
+        } else {
+            rd+=s;
+        }
+    } else if (CRAM_MAJOR_VERS(fd->version) < 4) {
+        uint32_t len;
+        if ((s = int32_decode(fd, &c2.length)) == -1) {
+            if (CRAM_MAJOR_VERS(fd->version) == 2 &&
+                CRAM_MINOR_VERS(fd->version) == 0)
+                fd->eof = 1; // EOF blocks arrived in v2.1
+            else
+                fd->eof = fd->empty_container ? 1 : 2;
+            return NULL;
+        } else {
+            rd+=s;
+        }
+        len = le_int4(c2.length);
+        crc = crc32(0L, (unsigned char *)&len, 4);
+    } else {
+        if ((s = fd->vv.varint_decode32_crc(fd, &c2.length, &crc))   == -1) {
+            fd->eof = fd->empty_container ? 1 : 2;
+            return NULL;
+        } else {
+            rd+=s;
+        }
+    }
+    if ((s = fd->vv.varint_decode32s_crc(fd, &c2.ref_seq_id, &crc))   == -1) return NULL; else rd+=s;
+    if (CRAM_MAJOR_VERS(fd->version) >= 4) {
+        int64_t i64;
+        if ((s = fd->vv.varint_decode64_crc(fd, &i64, &crc))== -1) return NULL; else rd+=s;
+        c2.ref_seq_start = i64;
+        if ((s = fd->vv.varint_decode64_crc(fd, &i64, &crc)) == -1) return NULL; else rd+=s;
+        c2.ref_seq_span = i64;
+    } else {
+        int32_t i32;
+        if ((s = fd->vv.varint_decode32_crc(fd, &i32, &crc))== -1) return NULL; else rd+=s;
+        c2.ref_seq_start = i32;
+        if ((s = fd->vv.varint_decode32_crc(fd, &i32, &crc)) == -1) return NULL; else rd+=s;
+        c2.ref_seq_span = i32;
+    }
+    if ((s = fd->vv.varint_decode32_crc(fd, &c2.num_records, &crc))  == -1) return NULL; else rd+=s;
+
+    if (CRAM_MAJOR_VERS(fd->version) == 1) {
+        c2.record_counter = 0;
+        c2.num_bases = 0;
+    } else {
+        if (CRAM_MAJOR_VERS(fd->version) >= 3) {
+            if ((s = fd->vv.varint_decode64_crc(fd, &c2.record_counter, &crc)) == -1)
+                return NULL;
+            else
+                rd += s;
+        } else {
+            int32_t i32;
+            if ((s = fd->vv.varint_decode32_crc(fd, &i32, &crc)) == -1)
+                return NULL;
+            else
+                rd += s;
+            c2.record_counter = i32;
+        }
+
+        if ((s = fd->vv.varint_decode64_crc(fd, &c2.num_bases, &crc))== -1)
+            return NULL;
+        else
+            rd += s;
+    }
+    if ((s = fd->vv.varint_decode32_crc(fd, &c2.num_blocks, &crc))   == -1)
+        return NULL;
+    else
+        rd+=s;
+    if ((s = fd->vv.varint_decode32_crc(fd, &c2.num_landmarks, &crc))== -1)
+        return NULL;
+    else
+        rd+=s;
+
+    if (c2.num_landmarks < 0 || c2.num_landmarks >= SIZE_MAX / sizeof(int32_t))
+        return NULL;
+
+    if (!(c = calloc(1, sizeof(*c))))
+        return NULL;
+
+    *c = c2;
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    if (c->num_landmarks > FUZZ_ALLOC_LIMIT/sizeof(int32_t)) {
+        fd->err = errno = ENOMEM;
+        cram_free_container(c);
+        return NULL;
+    }
+#endif
+    if (c->num_landmarks && !(c->landmark = malloc(c->num_landmarks * sizeof(int32_t)))) {
+        fd->err = errno;
+        cram_free_container(c);
+        return NULL;
+    }
+    for (i = 0; i < c->num_landmarks; i++) {
+        if ((s = fd->vv.varint_decode32_crc(fd, &c->landmark[i], &crc)) == -1) {
+            cram_free_container(c);
+            return NULL;
+        } else {
+            rd += s;
+        }
+    }
+
+    if (CRAM_MAJOR_VERS(fd->version) >= 3) {
+        if (-1 == int32_decode(fd, (int32_t *)&c->crc32)) {
+            cram_free_container(c);
+            return NULL;
+        } else {
+            rd+=4;
+        }
+
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+        // Pretend the CRC was OK so the fuzzer doesn't have to get it right
+        crc = c->crc32;
+#endif
+
+        if (crc != c->crc32) {
+            hts_log_error("Container header CRC32 failure");
+            cram_free_container(c);
+            return NULL;
+        }
+    }
+
+    c->offset = rd;
+    c->slices = NULL;
+    c->slice = NULL;
+    c->curr_slice = 0;
+    c->max_slice = c->num_landmarks;
+    c->slice_rec = 0;
+    c->curr_rec = 0;
+    c->max_rec = 0;
+
+    if (c->ref_seq_id == -2) {
+        c->multi_seq = 1;
+        fd->multi_seq = 1;
+    }
+
+    fd->empty_container =
+        (c->num_records == 0 &&
+         c->ref_seq_id == -1 &&
+         c->ref_seq_start == 0x454f46 /* EOF */) ? 1 : 0;
+
+    return c;
+}
+
+
+/* MAXIMUM storage size needed for the container. */
+int cram_container_size(cram_container *c) {
+    return 55 + 5*c->num_landmarks;
+}
+
+
+/*
+ * Stores the container structure in dat and returns *size as the
+ * number of bytes written to dat[].  The input size of dat is also
+ * held in *size and should be initialised to cram_container_size(c).
+ *
+ * Returns 0 on success;
+ *        -1 on failure
+ */
+int cram_store_container(cram_fd *fd, cram_container *c, char *dat, int *size)
+{
+    char *cp = (char *)dat;
+    int i;
+
+    // Check the input buffer is large enough according to our stated
+    // requirements. (NOTE: it may actually take less.)
+    if (cram_container_size(c) > *size)
+        return -1;
+
+    if (CRAM_MAJOR_VERS(fd->version) == 1) {
+        cp += itf8_put(cp, c->length);
+    } else {
+        *(int32_t *)cp = le_int4(c->length);
+        cp += 4;
+    }
+    if (c->multi_seq) {
+        cp += fd->vv.varint_put32(cp, NULL, -2);
+        cp += fd->vv.varint_put32(cp, NULL, 0);
+        cp += fd->vv.varint_put32(cp, NULL, 0);
+    } else {
+        cp += fd->vv.varint_put32s(cp, NULL, c->ref_seq_id);
+        if (CRAM_MAJOR_VERS(fd->version) >= 4) {
+            cp += fd->vv.varint_put64(cp, NULL, c->ref_seq_start);
+            cp += fd->vv.varint_put64(cp, NULL, c->ref_seq_span);
+        } else {
+            cp += fd->vv.varint_put32(cp, NULL, c->ref_seq_start);
+            cp += fd->vv.varint_put32(cp, NULL, c->ref_seq_span);
+        }
+    }
+    cp += fd->vv.varint_put32(cp, NULL, c->num_records);
+    if (CRAM_MAJOR_VERS(fd->version) == 2) {
+        cp += fd->vv.varint_put64(cp, NULL, c->record_counter);
+    } else if (CRAM_MAJOR_VERS(fd->version) >= 3) {
+        cp += fd->vv.varint_put32(cp, NULL, c->record_counter);
+    }
+    cp += fd->vv.varint_put64(cp, NULL, c->num_bases);
+    cp += fd->vv.varint_put32(cp, NULL, c->num_blocks);
+    cp += fd->vv.varint_put32(cp, NULL, c->num_landmarks);
+    for (i = 0; i < c->num_landmarks; i++)
+        cp += fd->vv.varint_put32(cp, NULL, c->landmark[i]);
+
+    if (CRAM_MAJOR_VERS(fd->version) >= 3) {
+        c->crc32 = crc32(0L, (uc *)dat, cp-dat);
+        cp[0] =  c->crc32        & 0xff;
+        cp[1] = (c->crc32 >>  8) & 0xff;
+        cp[2] = (c->crc32 >> 16) & 0xff;
+        cp[3] = (c->crc32 >> 24) & 0xff;
+        cp += 4;
+    }
+
+    *size = cp-dat; // actual used size
+
+    return 0;
+}
+
+
+/*
+ * Writes a container structure.
+ *
+ * Returns 0 on success
+ *        -1 on failure
+ */
+int cram_write_container(cram_fd *fd, cram_container *c) {
+    char buf_a[1024], *buf = buf_a, *cp;
+    int i;
+
+    if (61 + c->num_landmarks * 10 >= 1024) {
+        buf = malloc(61 + c->num_landmarks * 10);
+        if (!buf)
+            return -1;
+    }
+    cp = buf;
+
+    if (CRAM_MAJOR_VERS(fd->version) == 1) {
+        cp += itf8_put(cp, c->length);
+    } else if (CRAM_MAJOR_VERS(fd->version) <= 3) {
+        *(int32_t *)cp = le_int4(c->length);
+        cp += 4;
+    } else {
+        cp += fd->vv.varint_put32(cp, NULL, c->length);
+    }
+    if (c->multi_seq) {
+        cp += fd->vv.varint_put32(cp, NULL, (uint32_t)-2);
+        cp += fd->vv.varint_put32(cp, NULL, 0);
+        cp += fd->vv.varint_put32(cp, NULL, 0);
+    } else {
+        cp += fd->vv.varint_put32s(cp, NULL, c->ref_seq_id);
+        if (CRAM_MAJOR_VERS(fd->version) >= 4) {
+            cp += fd->vv.varint_put64(cp, NULL, c->ref_seq_start);
+            cp += fd->vv.varint_put64(cp, NULL, c->ref_seq_span);
+        } else {
+            cp += fd->vv.varint_put32(cp, NULL, c->ref_seq_start);
+            cp += fd->vv.varint_put32(cp, NULL, c->ref_seq_span);
+        }
+    }
+    cp += fd->vv.varint_put32(cp, NULL, c->num_records);
+    if (CRAM_MAJOR_VERS(fd->version) >= 3)
+        cp += fd->vv.varint_put64(cp, NULL, c->record_counter);
+    else
+        cp += fd->vv.varint_put32(cp, NULL, c->record_counter);
+    cp += fd->vv.varint_put64(cp, NULL, c->num_bases);
+    cp += fd->vv.varint_put32(cp, NULL, c->num_blocks);
+    cp += fd->vv.varint_put32(cp, NULL, c->num_landmarks);
+    for (i = 0; i < c->num_landmarks; i++)
+        cp += fd->vv.varint_put32(cp, NULL, c->landmark[i]);
+
+    if (CRAM_MAJOR_VERS(fd->version) >= 3) {
+        c->crc32 = crc32(0L, (uc *)buf, cp-buf);
+        cp[0] =  c->crc32        & 0xff;
+        cp[1] = (c->crc32 >>  8) & 0xff;
+        cp[2] = (c->crc32 >> 16) & 0xff;
+        cp[3] = (c->crc32 >> 24) & 0xff;
+        cp += 4;
+    }
+
+    if (cp-buf != hwrite(fd->fp, buf, cp-buf)) {
+        if (buf != buf_a)
+            free(buf);
+        return -1;
+    }
+
+    if (buf != buf_a)
+        free(buf);
+
+    return 0;
+}
+
+// common component shared by cram_flush_container{,_mt}
+static int cram_flush_container2(cram_fd *fd, cram_container *c) {
+    int i, j;
+
+    if (c->curr_slice > 0 && !c->slices)
+        return -1;
+
+    //fprintf(stderr, "Writing container %d, sum %u\n", c->record_counter, sum);
+
+    off_t c_offset = htell(fd->fp); // File offset of container
+
+    /* Write the container struct itself */
+    if (0 != cram_write_container(fd, c))
+        return -1;
+
+    off_t hdr_size = htell(fd->fp) - c_offset;
+
+    /* And the compression header */
+    if (0 != cram_write_block(fd, c->comp_hdr_block))
+        return -1;
+
+    /* Followed by the slice blocks */
+    off_t file_offset = htell(fd->fp);
+    for (i = 0; i < c->curr_slice; i++) {
+        cram_slice *s = c->slices[i];
+        off_t spos = file_offset - c_offset - hdr_size;
+
+        if (0 != cram_write_block(fd, s->hdr_block))
+            return -1;
+
+        for (j = 0; j < s->hdr->num_blocks; j++) {
+            if (0 != cram_write_block(fd, s->block[j]))
+                return -1;
+        }
+
+        file_offset = htell(fd->fp);
+        off_t sz = file_offset - c_offset - hdr_size - spos;
+
+        if (fd->idxfp) {
+            if (cram_index_slice(fd, c, s, fd->idxfp, c_offset, spos, sz) < 0)
+                return -1;
+        }
+    }
+
+    return 0;
+}
+
+/*
+ * Flushes a completely or partially full container to disk, writing
+ * container structure, header and blocks. This also calls the encoder
+ * functions.
+ *
+ * Returns 0 on success
+ *        -1 on failure
+ */
+int cram_flush_container(cram_fd *fd, cram_container *c) {
+    /* Encode the container blocks and generate compression header */
+    if (0 != cram_encode_container(fd, c))
+        return -1;
+
+    return cram_flush_container2(fd, c);
+}
+
+typedef struct {
+    cram_fd *fd;
+    cram_container *c;
+} cram_job;
+
+void *cram_flush_thread(void *arg) {
+    cram_job *j = (cram_job *)arg;
+
+    /* Encode the container blocks and generate compression header */
+    if (0 != cram_encode_container(j->fd, j->c)) {
+        hts_log_error("Call to cram_encode_container failed");
+        return NULL;
+    }
+
+    return arg;
+}
+
+static int cram_flush_result(cram_fd *fd) {
+    int i, ret = 0;
+    hts_tpool_result *r;
+    cram_container *lc = NULL;
+
+    // NB: we can have one result per slice, not per container,
+    // so we need to free the container only after all slices
+    // within it have been freed.  (Automatic via reference counting.)
+    while ((r = hts_tpool_next_result(fd->rqueue))) {
+        cram_job *j = (cram_job *)hts_tpool_result_data(r);
+        cram_container *c;
+
+        if (!j) {
+            hts_tpool_delete_result(r, 0);
+            return -1;
+        }
+
+        fd = j->fd;
+        c = j->c;
+
+        if (fd->mode == 'w')
+            if (0 != cram_flush_container2(fd, c))
+                return -1;
+
+        // Free the slices; filled out by encoder only
+        if (c->slices) {
+            for (i = 0; i < c->max_slice; i++) {
+                if (c->slices[i])
+                    cram_free_slice(c->slices[i]);
+                if (c->slices[i] == c->slice)
+                    c->slice = NULL;
+                c->slices[i] = NULL;
+            }
+        }
+
+        // Free the current slice; set by both encoder & decoder
+        if (c->slice) {
+            cram_free_slice(c->slice);
+            c->slice = NULL;
+        }
+        c->curr_slice = 0;
+
+        // Our jobs will be in order, so we free the last
+        // container when our job has switched to a new one.
+        if (c != lc) {
+            if (lc) {
+                if (fd->ctr == lc)
+                    fd->ctr = NULL;
+                if (fd->ctr_mt == lc)
+                    fd->ctr_mt = NULL;
+                cram_free_container(lc);
+            }
+            lc = c;
+        }
+
+        hts_tpool_delete_result(r, 1);
+    }
+    if (lc) {
+        if (fd->ctr == lc)
+            fd->ctr = NULL;
+        if (fd->ctr_mt == lc)
+            fd->ctr_mt = NULL;
+        cram_free_container(lc);
+    }
+
+    return ret;
+}
+
+// Note: called while metrics_lock is held.
+// Will be left in this state too, but may temporarily unlock.
+void reset_metrics(cram_fd *fd) {
+    int i;
+
+    if (fd->pool) {
+        // If multi-threaded we have multiple blocks being
+        // compressed already and several on the to-do list
+        // (fd->rqueue->pending).  It's tricky to reset the
+        // metrics exactly the correct point, so instead we
+        // just flush the pool, reset, and then continue again.
+
+        // Don't bother starting a new trial before then though.
+        for (i = 0; i < DS_END; i++) {
+            cram_metrics *m = fd->m[i];
+            if (!m)
+                continue;
+            m->next_trial = 999;
+        }
+
+        pthread_mutex_unlock(&fd->metrics_lock);
+        hts_tpool_process_flush(fd->rqueue);
+        pthread_mutex_lock(&fd->metrics_lock);
+    }
+
+    for (i = 0; i < DS_END; i++) {
+        cram_metrics *m = fd->m[i];
+        if (!m)
+            continue;
+
+        m->trial = NTRIALS;
+        m->next_trial = TRIAL_SPAN;
+        m->revised_method = 0;
+        m->unpackable = 0;
+
+        memset(m->sz, 0, sizeof(m->sz));
+    }
+}
+
+int cram_flush_container_mt(cram_fd *fd, cram_container *c) {
+    cram_job *j;
+
+    // At the junction of mapped to unmapped data the compression
+    // methods may need to change due to very different statistical
+    // properties; particularly BA if minhash sorted.
+    //
+    // However with threading we'll have several in-flight blocks
+    // arriving out of order.
+    //
+    // So we do one trial reset of NThreads to last for NThreads
+    // duration to get us over this transition period, followed
+    // by another retrial of the usual ntrials & trial span.
+    pthread_mutex_lock(&fd->metrics_lock);
+    if (c->n_mapped < 0.3*c->curr_rec &&
+        fd->last_mapped > 0.7*c->max_rec) {
+        reset_metrics(fd);
+    }
+    fd->last_mapped = c->n_mapped * (c->max_rec+1)/(c->curr_rec+1) ;
+    pthread_mutex_unlock(&fd->metrics_lock);
+
+    if (!fd->pool)
+        return cram_flush_container(fd, c);
+
+    if (!(j = malloc(sizeof(*j))))
+        return -1;
+    j->fd = fd;
+    j->c = c;
+
+    // Flush the job.  Note our encoder queue may be full, so we
+    // either have to keep trying in non-blocking mode (what we do) or
+    // use a dedicated separate thread for draining the queue.
+    for (;;) {
+        errno = 0;
+        hts_tpool_dispatch2(fd->pool, fd->rqueue, cram_flush_thread, j, 1);
+        int pending = (errno == EAGAIN);
+        if (cram_flush_result(fd) != 0)
+            return -1;
+        if (!pending)
+            break;
+
+        usleep(1000);
+    }
+
+    return 0;
+}
+
+/* ----------------------------------------------------------------------
+ * Compression headers; the first part of the container
+ */
+
+/*
+ * Creates a new blank container compression header
+ *
+ * Returns header ptr on success
+ *         NULL on failure
+ */
+cram_block_compression_hdr *cram_new_compression_header(void) {
+    cram_block_compression_hdr *hdr = calloc(1, sizeof(*hdr));
+    if (!hdr)
+        return NULL;
+
+    if (!(hdr->TD_blk = cram_new_block(CORE, 0))) {
+        free(hdr);
+        return NULL;
+    }
+
+    if (!(hdr->TD_hash = kh_init(m_s2i))) {
+        cram_free_block(hdr->TD_blk);
+        free(hdr);
+        return NULL;
+    }
+
+    if (!(hdr->TD_keys = string_pool_create(8192))) {
+        kh_destroy(m_s2i, hdr->TD_hash);
+        cram_free_block(hdr->TD_blk);
+        free(hdr);
+        return NULL;
+    }
+
+    return hdr;
+}
+
+void cram_free_compression_header(cram_block_compression_hdr *hdr) {
+    int i;
+
+    if (hdr->landmark)
+        free(hdr->landmark);
+
+    if (hdr->preservation_map)
+        kh_destroy(map, hdr->preservation_map);
+
+    for (i = 0; i < CRAM_MAP_HASH; i++) {
+        cram_map *m, *m2;
+        for (m = hdr->rec_encoding_map[i]; m; m = m2) {
+            m2 = m->next;
+            if (m->codec)
+                m->codec->free(m->codec);
+            free(m);
+        }
+    }
+
+    for (i = 0; i < CRAM_MAP_HASH; i++) {
+        cram_map *m, *m2;
+        for (m = hdr->tag_encoding_map[i]; m; m = m2) {
+            m2 = m->next;
+            if (m->codec)
+                m->codec->free(m->codec);
+            free(m);
+        }
+    }
+
+    for (i = 0; i < DS_END; i++) {
+        if (hdr->codecs[i])
+            hdr->codecs[i]->free(hdr->codecs[i]);
+    }
+
+    if (hdr->TL)
+        free(hdr->TL);
+    if (hdr->TD_blk)
+        cram_free_block(hdr->TD_blk);
+    if (hdr->TD_hash)
+        kh_destroy(m_s2i, hdr->TD_hash);
+    if (hdr->TD_keys)
+        string_pool_destroy(hdr->TD_keys);
+
+    free(hdr);
+}
+
+
+/* ----------------------------------------------------------------------
+ * Slices and slice headers
+ */
+
+void cram_free_slice_header(cram_block_slice_hdr *hdr) {
+    if (!hdr)
+        return;
+
+    if (hdr->block_content_ids)
+        free(hdr->block_content_ids);
+
+    free(hdr);
+
+    return;
+}
+
+void cram_free_slice(cram_slice *s) {
+    if (!s)
+        return;
+
+    if (s->hdr_block)
+        cram_free_block(s->hdr_block);
+
+    if (s->block) {
+        int i;
+
+        if (s->hdr) {
+            for (i = 0; i < s->hdr->num_blocks; i++) {
+                if (i > 0 && s->block[i] == s->block[0])
+                    continue;
+                cram_free_block(s->block[i]);
+            }
+        }
+        free(s->block);
+    }
+
+    {
+        // Normally already copied into s->block[], but potentially still
+        // here if we error part way through cram_encode_slice.
+        int i;
+        for (i = 0; i < s->naux_block; i++)
+            cram_free_block(s->aux_block[i]);
+    }
+
+    if (s->block_by_id)
+        free(s->block_by_id);
+
+    if (s->hdr)
+        cram_free_slice_header(s->hdr);
+
+    if (s->seqs_blk)
+        cram_free_block(s->seqs_blk);
+
+    if (s->qual_blk)
+        cram_free_block(s->qual_blk);
+
+    if (s->name_blk)
+        cram_free_block(s->name_blk);
+
+    if (s->aux_blk)
+        cram_free_block(s->aux_blk);
+
+    if (s->base_blk)
+        cram_free_block(s->base_blk);
+
+    if (s->soft_blk)
+        cram_free_block(s->soft_blk);
+
+    if (s->cigar)
+        free(s->cigar);
+
+    if (s->crecs)
+        free(s->crecs);
+
+    if (s->features)
+        free(s->features);
+
+    if (s->TN)
+        free(s->TN);
+
+    if (s->pair_keys)
+        string_pool_destroy(s->pair_keys);
+
+    if (s->pair[0])
+        kh_destroy(m_s2i, s->pair[0]);
+    if (s->pair[1])
+        kh_destroy(m_s2i, s->pair[1]);
+
+    if (s->aux_block)
+        free(s->aux_block);
+
+    free(s);
+}
+
+/*
+ * Creates a new empty slice in memory, for subsequent writing to
+ * disk.
+ *
+ * Returns cram_slice ptr on success
+ *         NULL on failure
+ */
+cram_slice *cram_new_slice(enum cram_content_type type, int nrecs) {
+    cram_slice *s = calloc(1, sizeof(*s));
+    if (!s)
+        return NULL;
+
+    if (!(s->hdr = (cram_block_slice_hdr *)calloc(1, sizeof(*s->hdr))))
+        goto err;
+    s->hdr->content_type = type;
+
+    s->hdr_block = NULL;
+    s->block = NULL;
+    s->block_by_id = NULL;
+    s->last_apos = 0;
+    if (!(s->crecs = malloc(nrecs * sizeof(cram_record))))  goto err;
+    s->cigar_alloc = 1024;
+    if (!(s->cigar = malloc(s->cigar_alloc * sizeof(*s->cigar)))) goto err;
+    s->ncigar = 0;
+
+    if (!(s->seqs_blk = cram_new_block(EXTERNAL, 0)))       goto err;
+    if (!(s->qual_blk = cram_new_block(EXTERNAL, DS_QS)))   goto err;
+    if (!(s->name_blk = cram_new_block(EXTERNAL, DS_RN)))   goto err;
+    if (!(s->aux_blk  = cram_new_block(EXTERNAL, DS_aux)))  goto err;
+    if (!(s->base_blk = cram_new_block(EXTERNAL, DS_IN)))   goto err;
+    if (!(s->soft_blk = cram_new_block(EXTERNAL, DS_SC)))   goto err;
+
+    s->features = NULL;
+    s->nfeatures = s->afeatures = 0;
+
+#ifndef TN_external
+    s->TN = NULL;
+    s->nTN = s->aTN = 0;
+#endif
+
+    // Volatile keys as we do realloc in dstring
+    if (!(s->pair_keys = string_pool_create(8192))) goto err;
+    if (!(s->pair[0] = kh_init(m_s2i)))             goto err;
+    if (!(s->pair[1] = kh_init(m_s2i)))             goto err;
+
+#ifdef BA_external
+    s->BA_len = 0;
+#endif
+
+    return s;
+
+ err:
+    if (s)
+        cram_free_slice(s);
+
+    return NULL;
+}
+
+/*
+ * Loads an entire slice.
+ * FIXME: In 1.0 the native unit of slices within CRAM is broken
+ * as slices contain references to objects in other slices.
+ * To work around this while keeping the slice oriented outer loop
+ * we read all slices and stitch them together into a fake large
+ * slice instead.
+ *
+ * Returns cram_slice ptr on success
+ *         NULL on failure
+ */
+cram_slice *cram_read_slice(cram_fd *fd) {
+    cram_block *b = cram_read_block(fd);
+    cram_slice *s = calloc(1, sizeof(*s));
+    int i, n, max_id, min_id;
+
+    if (!b || !s)
+        goto err;
+
+    s->hdr_block = b;
+    switch (b->content_type) {
+    case MAPPED_SLICE:
+    case UNMAPPED_SLICE:
+        if (!(s->hdr = cram_decode_slice_header(fd, b)))
+            goto err;
+        break;
+
+    default:
+        hts_log_error("Unexpected block of type %s",
+                      cram_content_type2str(b->content_type));
+        goto err;
+    }
+
+    if (s->hdr->num_blocks < 1) {
+        hts_log_error("Slice does not include any data blocks");
+        goto err;
+    }
+
+    s->block = calloc(n = s->hdr->num_blocks, sizeof(*s->block));
+    if (!s->block)
+        goto err;
+
+    for (max_id = i = 0, min_id = INT_MAX; i < n; i++) {
+        if (!(s->block[i] = cram_read_block(fd)))
+            goto err;
+
+        if (s->block[i]->content_type == EXTERNAL) {
+            if (max_id < s->block[i]->content_id)
+                max_id = s->block[i]->content_id;
+            if (min_id > s->block[i]->content_id)
+                min_id = s->block[i]->content_id;
+        }
+    }
+
+    if (!(s->block_by_id = calloc(512, sizeof(s->block[0]))))
+        goto err;
+
+    for (i = 0; i < n; i++) {
+        if (s->block[i]->content_type != EXTERNAL)
+            continue;
+        uint32_t v = s->block[i]->content_id;
+        if (v >= 256)
+            v = 256 + v % 251;
+        s->block_by_id[v] = s->block[i];
+    }
+
+    /* Initialise encoding/decoding tables */
+    s->cigar_alloc = 1024;
+    if (!(s->cigar = malloc(s->cigar_alloc * sizeof(*s->cigar)))) goto err;
+    s->ncigar = 0;
+
+    if (!(s->seqs_blk = cram_new_block(EXTERNAL, 0)))      goto err;
+    if (!(s->qual_blk = cram_new_block(EXTERNAL, DS_QS)))  goto err;
+    if (!(s->name_blk = cram_new_block(EXTERNAL, DS_RN)))  goto err;
+    if (!(s->aux_blk  = cram_new_block(EXTERNAL, DS_aux))) goto err;
+    if (!(s->base_blk = cram_new_block(EXTERNAL, DS_IN)))  goto err;
+    if (!(s->soft_blk = cram_new_block(EXTERNAL, DS_SC)))  goto err;
+
+    s->crecs = NULL;
+
+    s->last_apos = s->hdr->ref_seq_start;
+    s->decode_md = fd->decode_md;
+
+    return s;
+
+ err:
+    if (b)
+        cram_free_block(b);
+    if (s) {
+        s->hdr_block = NULL;
+        cram_free_slice(s);
+    }
+    return NULL;
+}
+
+
+/* ----------------------------------------------------------------------
+ * CRAM file definition (header)
+ */
+
+/*
+ * Reads a CRAM file definition structure.
+ * Returns file_def ptr on success
+ *         NULL on failure
+ */
+cram_file_def *cram_read_file_def(cram_fd *fd) {
+    cram_file_def *def = malloc(sizeof(*def));
+    if (!def)
+        return NULL;
+
+    if (26 != hread(fd->fp, &def->magic[0], 26)) {
+        free(def);
+        return NULL;
+    }
+
+    if (memcmp(def->magic, "CRAM", 4) != 0) {
+        free(def);
+        return NULL;
+    }
+
+    if (def->major_version > 4) {
+        hts_log_error("CRAM version number mismatch. Expected 1.x, 2.x, 3.x or 4.x, got %d.%d",
+                      def->major_version, def->minor_version);
+        free(def);
+        return NULL;
+    }
+
+    fd->first_container += 26;
+    fd->curr_position = fd->first_container;
+    fd->last_slice = 0;
+
+    return def;
+}
+
+/*
+ * Writes a cram_file_def structure to cram_fd.
+ * Returns 0 on success
+ *        -1 on failure
+ */
+int cram_write_file_def(cram_fd *fd, cram_file_def *def) {
+    return (hwrite(fd->fp, &def->magic[0], 26) == 26) ? 0 : -1;
+}
+
+void cram_free_file_def(cram_file_def *def) {
+    if (def) free(def);
+}
+
+/* ----------------------------------------------------------------------
+ * SAM header I/O
+ */
+
+
+/*
+ * Reads the SAM header from the first CRAM data block.
+ * Also performs minimal parsing to extract read-group
+ * and sample information.
+
+ * Returns SAM hdr ptr on success
+ *         NULL on failure
+ */
+sam_hdr_t *cram_read_SAM_hdr(cram_fd *fd) {
+    int32_t header_len;
+    char *header;
+    sam_hdr_t *hdr;
+
+    /* 1.1 onwards stores the header in the first block of a container */
+    if (CRAM_MAJOR_VERS(fd->version) == 1) {
+        /* Length */
+        if (-1 == int32_decode(fd, &header_len))
+            return NULL;
+
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+        if (header_len > FUZZ_ALLOC_LIMIT)
+            return NULL;
+#endif
+
+        /* Alloc and read */
+        if (header_len < 0 || NULL == (header = malloc((size_t) header_len+1)))
+            return NULL;
+
+        if (header_len != hread(fd->fp, header, header_len)) {
+            free(header);
+            return NULL;
+        }
+        header[header_len] = '\0';
+
+        fd->first_container += 4 + header_len;
+    } else {
+        cram_container *c = cram_read_container(fd);
+        cram_block *b;
+        int i;
+        int64_t len;
+
+        if (!c)
+            return NULL;
+
+        fd->first_container += c->length + c->offset;
+        fd->curr_position = fd->first_container;
+
+        if (c->num_blocks < 1) {
+            cram_free_container(c);
+            return NULL;
+        }
+
+        if (!(b = cram_read_block(fd))) {
+            cram_free_container(c);
+            return NULL;
+        }
+        if (cram_uncompress_block(b) != 0) {
+            cram_free_container(c);
+            cram_free_block(b);
+            return NULL;
+        }
+
+        len = b->comp_size + 2 + 4*(CRAM_MAJOR_VERS(fd->version) >= 3) +
+            fd->vv.varint_size(b->content_id) +
+            fd->vv.varint_size(b->uncomp_size) +
+            fd->vv.varint_size(b->comp_size);
+
+        /* Extract header from 1st block */
+        if (-1 == int32_get_blk(b, &header_len) ||
+            header_len < 0 || /* Spec. says signed...  why? */
+            b->uncomp_size - 4 < header_len) {
+            cram_free_container(c);
+            cram_free_block(b);
+            return NULL;
+        }
+        if (NULL == (header = malloc((size_t) header_len+1))) {
+            cram_free_container(c);
+            cram_free_block(b);
+            return NULL;
+        }
+        memcpy(header, BLOCK_END(b), header_len);
+        header[header_len] = '\0';
+        cram_free_block(b);
+
+        /* Consume any remaining blocks */
+        for (i = 1; i < c->num_blocks; i++) {
+            if (!(b = cram_read_block(fd))) {
+                cram_free_container(c);
+                free(header);
+                return NULL;
+            }
+            len += b->comp_size + 2 + 4*(CRAM_MAJOR_VERS(fd->version) >= 3) +
+                fd->vv.varint_size(b->content_id) +
+                fd->vv.varint_size(b->uncomp_size) +
+                fd->vv.varint_size(b->comp_size);
+            cram_free_block(b);
+        }
+
+        if (c->length > 0 && len > 0 && c->length > len) {
+            // Consume padding
+            char *pads = malloc(c->length - len);
+            if (!pads) {
+                cram_free_container(c);
+                free(header);
+                return NULL;
+            }
+
+            if (c->length - len != hread(fd->fp, pads, c->length - len)) {
+                cram_free_container(c);
+                free(header);
+                free(pads);
+                return NULL;
+            }
+            free(pads);
+        }
+
+        cram_free_container(c);
+    }
+
+    /* Parse */
+    hdr = sam_hdr_init();
+    if (!hdr) {
+        free(header);
+        return NULL;
+    }
+
+    if (-1 == sam_hdr_add_lines(hdr, header, header_len)) {
+        free(header);
+        sam_hdr_destroy(hdr);
+        return NULL;
+    }
+
+    hdr->l_text = header_len;
+    hdr->text = header;
+
+    return hdr;
+
+}
+
+/*
+ * Converts 'in' to a full pathname to store in out.
+ * Out must be at least PATH_MAX bytes long.
+ */
+static void full_path(char *out, char *in) {
+    size_t in_l = strlen(in);
+    if (hisremote(in)) {
+        if (in_l > PATH_MAX) {
+            hts_log_error("Reference path is longer than %d", PATH_MAX);
+            return;
+        }
+        strncpy(out, in, PATH_MAX-1);
+        out[PATH_MAX-1] = 0;
+        return;
+    }
+    if (*in == '/' ||
+        // Windows paths
+        (in_l > 3 && toupper_c(*in) >= 'A'  && toupper_c(*in) <= 'Z' &&
+         in[1] == ':' && (in[2] == '/' || in[2] == '\\'))) {
+        strncpy(out, in, PATH_MAX-1);
+        out[PATH_MAX-1] = 0;
+    } else {
+        size_t len;
+
+        // unable to get dir or out+in is too long
+        if (!getcwd(out, PATH_MAX) ||
+            (len = strlen(out))+1+strlen(in) >= PATH_MAX) {
+            strncpy(out, in, PATH_MAX-1);
+            out[PATH_MAX-1] = 0;
+            return;
+        }
+
+        snprintf(out+len, PATH_MAX - len, "/%s", in);
+
+        // FIXME: cope with `pwd`/../../../foo.fa ?
+    }
+}
+
+/*
+ * Writes a CRAM SAM header.
+ * Returns 0 on success
+ *        -1 on failure
+ */
+int cram_write_SAM_hdr(cram_fd *fd, sam_hdr_t *hdr) {
+    size_t header_len;
+    int blank_block = (CRAM_MAJOR_VERS(fd->version) >= 3);
+
+    /* Write CRAM MAGIC if not yet written. */
+    if (fd->file_def->major_version == 0) {
+        fd->file_def->major_version = CRAM_MAJOR_VERS(fd->version);
+        fd->file_def->minor_version = CRAM_MINOR_VERS(fd->version);
+        if (0 != cram_write_file_def(fd, fd->file_def))
+            return -1;
+    }
+
+    /* 1.0 requires an UNKNOWN read-group */
+    if (CRAM_MAJOR_VERS(fd->version) == 1) {
+        if (!sam_hrecs_find_rg(hdr->hrecs, "UNKNOWN"))
+            if (sam_hdr_add_line(hdr, "RG",
+                            "ID", "UNKNOWN", "SM", "UNKNOWN", NULL))
+                return -1;
+    }
+
+    if (-1 == refs_from_header(fd))
+        return -1;
+    if (-1 == refs2id(fd->refs, fd->header))
+        return -1;
+
+    /* Fix M5 strings */
+    if (fd->refs && !fd->no_ref && fd->embed_ref <= 1) {
+        int i;
+        for (i = 0; i < hdr->hrecs->nref; i++) {
+            sam_hrec_type_t *ty;
+            char *ref;
+
+            if (!(ty = sam_hrecs_find_type_id(hdr->hrecs, "SQ", "SN", hdr->hrecs->ref[i].name)))
+                return -1;
+
+            if (!sam_hrecs_find_key(ty, "M5", NULL)) {
+                char unsigned buf[16];
+                char buf2[33];
+                hts_pos_t rlen;
+                hts_md5_context *md5;
+
+                if (!fd->refs ||
+                    !fd->refs->ref_id ||
+                    !fd->refs->ref_id[i]) {
+                    return -1;
+                }
+                rlen = fd->refs->ref_id[i]->length;
+                ref = cram_get_ref(fd, i, 1, rlen);
+                if (NULL == ref) {
+                    if (fd->embed_ref == -1) {
+                        // auto embed-ref
+                        hts_log_warning("No M5 tags present and could not "
+                                        "find reference");
+                        hts_log_warning("Enabling embed_ref=2 option");
+                        hts_log_warning("NOTE: the CRAM file will be bigger "
+                                        "than using an external reference");
+                        pthread_mutex_lock(&fd->ref_lock);
+                        fd->embed_ref = 2;
+                        pthread_mutex_unlock(&fd->ref_lock);
+                        break;
+                    }
+                    return -1;
+                }
+                rlen = fd->refs->ref_id[i]->length; /* In case it just loaded */
+                if (!(md5 = hts_md5_init()))
+                    return -1;
+                if (HTS_POS_MAX <= ULONG_MAX) {
+                    // Platforms with 64-bit unsigned long update in one go
+                    hts_md5_update(md5, ref, rlen);
+                } else {
+                    // Those with 32-bit ulong (Windows) may have to loop
+                    // over epic references
+                    hts_pos_t pos = 0;
+                    while (rlen - pos > ULONG_MAX) {
+                        hts_md5_update(md5, ref + pos, ULONG_MAX);
+                        pos += ULONG_MAX;
+                    }
+                    hts_md5_update(md5, ref + pos, (unsigned long)(rlen - pos));
+                }
+                hts_md5_final(buf, md5);
+                hts_md5_destroy(md5);
+                cram_ref_decr(fd->refs, i);
+
+                hts_md5_hex(buf2, buf);
+                fd->refs->ref_id[i]->validated_md5 = 1;
+                if (sam_hdr_update_line(hdr, "SQ", "SN", hdr->hrecs->ref[i].name, "M5", buf2, NULL))
+                    return -1;
+            }
+
+            if (fd->ref_fn) {
+                char ref_fn[PATH_MAX];
+                full_path(ref_fn, fd->ref_fn);
+                if (sam_hdr_update_line(hdr, "SQ", "SN", hdr->hrecs->ref[i].name, "UR", ref_fn, NULL))
+                    return -1;
+            }
+        }
+    }
+
+    /* Length */
+    header_len = sam_hdr_length(hdr);
+    if (header_len > INT32_MAX) {
+        hts_log_error("Header is too long for CRAM format");
+        return -1;
+    }
+    if (CRAM_MAJOR_VERS(fd->version) == 1) {
+        if (-1 == int32_encode(fd, header_len))
+            return -1;
+
+        /* Text data */
+        if (header_len != hwrite(fd->fp, sam_hdr_str(hdr), header_len))
+            return -1;
+    } else {
+        /* Create block(s) inside a container */
+        cram_block *b = cram_new_block(FILE_HEADER, 0);
+        cram_container *c = cram_new_container(0, 0);
+        int padded_length;
+        char *pads;
+        int is_cram_3 = (CRAM_MAJOR_VERS(fd->version) >= 3);
+
+        if (!b || !c) {
+            if (b) cram_free_block(b);
+            if (c) cram_free_container(c);
+            return -1;
+        }
+
+        if (int32_put_blk(b, header_len) < 0)
+            return -1;
+        if (header_len)
+            BLOCK_APPEND(b, sam_hdr_str(hdr), header_len);
+        BLOCK_UPLEN(b);
+
+        // Compress header block if V3.0 and above
+        if (CRAM_MAJOR_VERS(fd->version) >= 3)
+            if (cram_compress_block(fd, b, NULL, -1, -1) < 0)
+                return -1;
+
+        if (blank_block) {
+            c->length = b->comp_size + 2 + 4*is_cram_3 +
+                fd->vv.varint_size(b->content_id)   +
+                fd->vv.varint_size(b->uncomp_size)  +
+                fd->vv.varint_size(b->comp_size);
+
+            c->num_blocks = 2;
+            c->num_landmarks = 2;
+            if (!(c->landmark = malloc(2*sizeof(*c->landmark)))) {
+                cram_free_block(b);
+                cram_free_container(c);
+                return -1;
+            }
+            c->landmark[0] = 0;
+            c->landmark[1] = c->length;
+
+            // Plus extra storage for uncompressed secondary blank block
+            padded_length = MIN(c->length*.5, 10000);
+            c->length += padded_length + 2 + 4*is_cram_3 +
+                fd->vv.varint_size(b->content_id) +
+                fd->vv.varint_size(padded_length)*2;
+        } else {
+            // Pad the block instead.
+            c->num_blocks = 1;
+            c->num_landmarks = 1;
+            if (!(c->landmark = malloc(sizeof(*c->landmark))))
+                return -1;
+            c->landmark[0] = 0;
+
+            padded_length = MAX(c->length*1.5, 10000) - c->length;
+
+            c->length = b->comp_size + padded_length +
+                2 + 4*is_cram_3 +
+                fd->vv.varint_size(b->content_id)   +
+                fd->vv.varint_size(b->uncomp_size)  +
+                fd->vv.varint_size(b->comp_size);
+
+            if (NULL == (pads = calloc(1, padded_length))) {
+                cram_free_block(b);
+                cram_free_container(c);
+                return -1;
+            }
+            BLOCK_APPEND(b, pads, padded_length);
+            BLOCK_UPLEN(b);
+            free(pads);
+        }
+
+        if (-1 == cram_write_container(fd, c)) {
+            cram_free_block(b);
+            cram_free_container(c);
+            return -1;
+        }
+
+        if (-1 == cram_write_block(fd, b)) {
+            cram_free_block(b);
+            cram_free_container(c);
+            return -1;
+        }
+
+        if (blank_block) {
+            BLOCK_RESIZE(b, padded_length);
+            memset(BLOCK_DATA(b), 0, padded_length);
+            BLOCK_SIZE(b) = padded_length;
+            BLOCK_UPLEN(b);
+            b->method = RAW;
+            if (-1 == cram_write_block(fd, b)) {
+                cram_free_block(b);
+                cram_free_container(c);
+                return -1;
+            }
+        }
+
+        cram_free_block(b);
+        cram_free_container(c);
+    }
+
+    if (0 != hflush(fd->fp))
+        return -1;
+
+    RP("=== Finishing saving header ===\n");
+
+    return 0;
+
+ block_err:
+    return -1;
+}
+
+/* ----------------------------------------------------------------------
+ * The top-level cram opening, closing and option handling
+ */
+
+/*
+ * Sets CRAM variable sized integer decode function tables.
+ * CRAM 1, 2, and 3.x all used ITF8 for uint32 and UTF8 for uint64.
+ * CRAM 4.x uses the same encoding mechanism for 32-bit and 64-bit
+ * (or anything inbetween), but also now supports signed values.
+ *
+ * Version is the CRAM major version number.
+ * vv is the vector table (probably &cram_fd->vv)
+ */
+static void cram_init_varint(varint_vec *vv, int version) {
+    if (version >= 4) {
+        vv->varint_get32 = uint7_get_32; // FIXME: varint.h API should be size agnostic
+        vv->varint_get32s = sint7_get_32;
+        vv->varint_get64 = uint7_get_64;
+        vv->varint_get64s = sint7_get_64;
+        vv->varint_put32 = uint7_put_32;
+        vv->varint_put32s = sint7_put_32;
+        vv->varint_put64 = uint7_put_64;
+        vv->varint_put64s = sint7_put_64;
+        vv->varint_put32_blk = uint7_put_blk_32;
+        vv->varint_put32s_blk = sint7_put_blk_32;
+        vv->varint_put64_blk = uint7_put_blk_64;
+        vv->varint_put64s_blk = sint7_put_blk_64;
+        vv->varint_size = uint7_size;
+        vv->varint_decode32_crc = uint7_decode_crc32;
+        vv->varint_decode32s_crc = sint7_decode_crc32;
+        vv->varint_decode64_crc = uint7_decode_crc64;
+    } else {
+        vv->varint_get32 = safe_itf8_get;
+        vv->varint_get32s = safe_itf8_get;
+        vv->varint_get64 = safe_ltf8_get;
+        vv->varint_get64s = safe_ltf8_get;
+        vv->varint_put32 = safe_itf8_put;
+        vv->varint_put32s = safe_itf8_put;
+        vv->varint_put64 = safe_ltf8_put;
+        vv->varint_put64s = safe_ltf8_put;
+        vv->varint_put32_blk = itf8_put_blk;
+        vv->varint_put32s_blk = itf8_put_blk;
+        vv->varint_put64_blk = ltf8_put_blk;
+        vv->varint_put64s_blk = ltf8_put_blk;
+        vv->varint_size = itf8_size;
+        vv->varint_decode32_crc = itf8_decode_crc;
+        vv->varint_decode32s_crc = itf8_decode_crc;
+        vv->varint_decode64_crc = ltf8_decode_crc;
+    }
+}
+
+/*
+ * Initialises the lookup tables. These could be global statics, but they're
+ * clumsy to setup in a multi-threaded environment unless we generate
+ * verbatim code and include that.
+ */
+static void cram_init_tables(cram_fd *fd) {
+    int i;
+
+    memset(fd->L1, 4, 256);
+    fd->L1['A'] = 0; fd->L1['a'] = 0;
+    fd->L1['C'] = 1; fd->L1['c'] = 1;
+    fd->L1['G'] = 2; fd->L1['g'] = 2;
+    fd->L1['T'] = 3; fd->L1['t'] = 3;
+
+    memset(fd->L2, 5, 256);
+    fd->L2['A'] = 0; fd->L2['a'] = 0;
+    fd->L2['C'] = 1; fd->L2['c'] = 1;
+    fd->L2['G'] = 2; fd->L2['g'] = 2;
+    fd->L2['T'] = 3; fd->L2['t'] = 3;
+    fd->L2['N'] = 4; fd->L2['n'] = 4;
+
+    if (CRAM_MAJOR_VERS(fd->version) == 1) {
+        for (i = 0; i < 0x200; i++) {
+            int f = 0;
+
+            if (i & CRAM_FPAIRED)      f |= BAM_FPAIRED;
+            if (i & CRAM_FPROPER_PAIR) f |= BAM_FPROPER_PAIR;
+            if (i & CRAM_FUNMAP)       f |= BAM_FUNMAP;
+            if (i & CRAM_FREVERSE)     f |= BAM_FREVERSE;
+            if (i & CRAM_FREAD1)       f |= BAM_FREAD1;
+            if (i & CRAM_FREAD2)       f |= BAM_FREAD2;
+            if (i & CRAM_FSECONDARY)   f |= BAM_FSECONDARY;
+            if (i & CRAM_FQCFAIL)      f |= BAM_FQCFAIL;
+            if (i & CRAM_FDUP)         f |= BAM_FDUP;
+
+            fd->bam_flag_swap[i]  = f;
+        }
+
+        for (i = 0; i < 0x1000; i++) {
+            int g = 0;
+
+            if (i & BAM_FPAIRED)           g |= CRAM_FPAIRED;
+            if (i & BAM_FPROPER_PAIR)  g |= CRAM_FPROPER_PAIR;
+            if (i & BAM_FUNMAP)        g |= CRAM_FUNMAP;
+            if (i & BAM_FREVERSE)      g |= CRAM_FREVERSE;
+            if (i & BAM_FREAD1)        g |= CRAM_FREAD1;
+            if (i & BAM_FREAD2)        g |= CRAM_FREAD2;
+            if (i & BAM_FSECONDARY)    g |= CRAM_FSECONDARY;
+            if (i & BAM_FQCFAIL)       g |= CRAM_FQCFAIL;
+            if (i & BAM_FDUP)          g |= CRAM_FDUP;
+
+            fd->cram_flag_swap[i] = g;
+        }
+    } else {
+        /* NOP */
+        for (i = 0; i < 0x1000; i++)
+            fd->bam_flag_swap[i] = i;
+        for (i = 0; i < 0x1000; i++)
+            fd->cram_flag_swap[i] = i;
+    }
+
+    memset(fd->cram_sub_matrix, 4, 32*32);
+    for (i = 0; i < 32; i++) {
+        fd->cram_sub_matrix[i]['A'&0x1f]=0;
+        fd->cram_sub_matrix[i]['C'&0x1f]=1;
+        fd->cram_sub_matrix[i]['G'&0x1f]=2;
+        fd->cram_sub_matrix[i]['T'&0x1f]=3;
+        fd->cram_sub_matrix[i]['N'&0x1f]=4;
+    }
+    for (i = 0; i < 20; i+=4) {
+        int j;
+        for (j = 0; j < 20; j++) {
+            fd->cram_sub_matrix["ACGTN"[i>>2]&0x1f][j]=3;
+            fd->cram_sub_matrix["ACGTN"[i>>2]&0x1f][j]=3;
+            fd->cram_sub_matrix["ACGTN"[i>>2]&0x1f][j]=3;
+            fd->cram_sub_matrix["ACGTN"[i>>2]&0x1f][j]=3;
+        }
+        fd->cram_sub_matrix["ACGTN"[i>>2]&0x1f][CRAM_SUBST_MATRIX[i+0]&0x1f]=0;
+        fd->cram_sub_matrix["ACGTN"[i>>2]&0x1f][CRAM_SUBST_MATRIX[i+1]&0x1f]=1;
+        fd->cram_sub_matrix["ACGTN"[i>>2]&0x1f][CRAM_SUBST_MATRIX[i+2]&0x1f]=2;
+        fd->cram_sub_matrix["ACGTN"[i>>2]&0x1f][CRAM_SUBST_MATRIX[i+3]&0x1f]=3;
+    }
+
+    cram_init_varint(&fd->vv, CRAM_MAJOR_VERS(fd->version));
+}
+
+// Default version numbers for CRAM
+static int major_version = 3;
+static int minor_version = 0;
+
+/*
+ * Opens a CRAM file for read (mode "rb") or write ("wb").
+ * The filename may be "-" to indicate stdin or stdout.
+ *
+ * Returns file handle on success
+ *         NULL on failure.
+ */
+cram_fd *cram_open(const char *filename, const char *mode) {
+    hFILE *fp;
+    cram_fd *fd;
+    char fmode[3]= { mode[0], '\0', '\0' };
+
+    if (strlen(mode) > 1 && (mode[1] == 'b' || mode[1] == 'c')) {
+        fmode[1] = 'b';
+    }
+
+    fp = hopen(filename, fmode);
+    if (!fp)
+        return NULL;
+
+    fd = cram_dopen(fp, filename, mode);
+    if (!fd)
+        hclose_abruptly(fp);
+
+    return fd;
+}
+
+/* Opens an existing stream for reading or writing.
+ *
+ * Returns file handle on success;
+ *         NULL on failure.
+ */
+cram_fd *cram_dopen(hFILE *fp, const char *filename, const char *mode) {
+    int i;
+    char *cp;
+    cram_fd *fd = calloc(1, sizeof(*fd));
+    if (!fd)
+        return NULL;
+
+    fd->level = CRAM_DEFAULT_LEVEL;
+    for (i = 0; mode[i]; i++) {
+        if (mode[i] >= '0' && mode[i] <= '9') {
+            fd->level = mode[i] - '0';
+            break;
+        }
+    }
+
+    fd->fp = fp;
+    fd->mode = *mode;
+    fd->first_container = 0;
+    fd->curr_position = 0;
+
+    if (fd->mode == 'r') {
+        /* Reader */
+
+        if (!(fd->file_def = cram_read_file_def(fd)))
+            goto err;
+
+        fd->version = fd->file_def->major_version * 256 +
+            fd->file_def->minor_version;
+
+        cram_init_tables(fd);
+
+        if (!(fd->header = cram_read_SAM_hdr(fd))) {
+            cram_free_file_def(fd->file_def);
+            goto err;
+        }
+
+    } else {
+        /* Writer */
+        cram_file_def *def = calloc(1, sizeof(*def));
+        if (!def)
+            return NULL;
+
+        fd->file_def = def;
+
+        def->magic[0] = 'C';
+        def->magic[1] = 'R';
+        def->magic[2] = 'A';
+        def->magic[3] = 'M';
+        def->major_version = 0; // Indicator to write file def later.
+        def->minor_version = 0;
+        memset(def->file_id, 0, 20);
+        strncpy(def->file_id, filename, 20);
+
+        fd->version = major_version * 256 + minor_version;
+        cram_init_tables(fd);
+
+        /* SAM header written later along with this file_def */
+    }
+
+    fd->prefix = strdup((cp = strrchr(filename, '/')) ? cp+1 : filename);
+    if (!fd->prefix)
+        goto err;
+    fd->first_base = fd->last_base = -1;
+    fd->record_counter = 0;
+
+    fd->ctr = NULL;
+    fd->ctr_mt = NULL;
+    fd->refs  = refs_create();
+    if (!fd->refs)
+        goto err;
+    fd->ref_id = -2;
+    fd->ref = NULL;
+
+    fd->decode_md = 0;
+    fd->seqs_per_slice = SEQS_PER_SLICE;
+    fd->bases_per_slice = BASES_PER_SLICE;
+    fd->slices_per_container = SLICE_PER_CNT;
+    fd->embed_ref = -1; // automatic selection
+    fd->no_ref = 0;
+    fd->no_ref_counter = 0;
+    fd->ap_delta = 0;
+    fd->ignore_md5 = 0;
+    fd->lossy_read_names = 0;
+    fd->use_bz2 = 0;
+    fd->use_rans = (CRAM_MAJOR_VERS(fd->version) >= 3);
+    fd->use_tok = (CRAM_MAJOR_VERS(fd->version) >= 3) && (CRAM_MINOR_VERS(fd->version) >= 1);
+    fd->use_lzma = 0;
+    fd->multi_seq = -1;
+    fd->multi_seq_user = -1;
+    fd->unsorted   = 0;
+    fd->shared_ref = 0;
+    fd->store_md = 0;
+    fd->store_nm = 0;
+    fd->last_RI_count = 0;
+
+    fd->index       = NULL;
+    fd->own_pool    = 0;
+    fd->pool        = NULL;
+    fd->rqueue      = NULL;
+    fd->job_pending = NULL;
+    fd->ooc         = 0;
+    fd->required_fields = INT_MAX;
+
+    pthread_mutex_init(&fd->metrics_lock, NULL);
+    pthread_mutex_init(&fd->ref_lock, NULL);
+    pthread_mutex_init(&fd->range_lock, NULL);
+    pthread_mutex_init(&fd->bam_list_lock, NULL);
+
+    for (i = 0; i < DS_END; i++) {
+        fd->m[i] = cram_new_metrics();
+        if (!fd->m[i])
+            goto err;
+    }
+
+    if (!(fd->tags_used = kh_init(m_metrics)))
+        goto err;
+
+    fd->range.refid = -2; // no ref.
+    fd->eof = 1;          // See samtools issue #150
+    fd->ref_fn = NULL;
+
+    fd->bl = NULL;
+
+    /* Initialise dummy refs from the @SQ headers */
+    if (-1 == refs_from_header(fd))
+        goto err;
+
+    return fd;
+
+ err:
+    if (fd)
+        free(fd);
+
+    return NULL;
+}
+
+/*
+ * Seek within a CRAM file.
+ *
+ * Returns 0 on success
+ *        -1 on failure
+ */
+int cram_seek(cram_fd *fd, off_t offset, int whence) {
+    char buf[65536];
+
+    fd->ooc = 0;
+
+    cram_drain_rqueue(fd);
+
+    if (hseek(fd->fp, offset, whence) >= 0) {
+        return 0;
+    }
+
+    if (!(whence == SEEK_CUR && offset >= 0))
+        return -1;
+
+    /* Couldn't fseek, but we're in SEEK_CUR mode so read instead */
+    while (offset > 0) {
+        int len = MIN(65536, offset);
+        if (len != hread(fd->fp, buf, len))
+            return -1;
+        offset -= len;
+    }
+
+    return 0;
+}
+
+/*
+ * Flushes a CRAM file.
+ * Useful for when writing to stdout without wishing to close the stream.
+ *
+ * Returns 0 on success
+ *        -1 on failure
+ */
+int cram_flush(cram_fd *fd) {
+    if (!fd)
+        return -1;
+
+    int ret = 0;
+
+    if (fd->mode == 'w' && fd->ctr) {
+        if(fd->ctr->slice)
+            cram_update_curr_slice(fd->ctr, fd->version);
+
+        if (-1 == cram_flush_container_mt(fd, fd->ctr))
+            ret = -1;
+
+        cram_free_container(fd->ctr);
+        if (fd->ctr_mt == fd->ctr)
+            fd->ctr_mt = NULL;
+        fd->ctr = NULL;
+    }
+
+    return ret;
+}
+
+/*
+ * Writes an EOF block to a CRAM file.
+ *
+ * Returns 0 on success
+ *        -1 on failure
+ */
+int cram_write_eof_block(cram_fd *fd) {
+    // EOF block is a container with special values to aid detection
+    if (CRAM_MAJOR_VERS(fd->version) >= 2) {
+        // Empty container with
+        //   ref_seq_id -1
+        //   start pos 0x454f46 ("EOF")
+        //   span 0
+        //   nrec 0
+        //   counter 0
+        //   nbases 0
+        //   1 block (landmark 0)
+        //   (CRC32)
+        cram_container c;
+        memset(&c, 0, sizeof(c));
+        c.ref_seq_id = -1;
+        c.ref_seq_start = 0x454f46; // "EOF"
+        c.ref_seq_span = 0;
+        c.record_counter = 0;
+        c.num_bases = 0;
+        c.num_blocks = 1;
+        int32_t land[1] = {0};
+        c.landmark = land;
+
+        // An empty compression header block with
+        //   method raw (0)
+        //   type comp header (1)
+        //   content id 0
+        //   block contents size 6
+        //   raw size 6
+        //     empty preservation map (01 00)
+        //     empty data series map (01 00)
+        //     empty tag map (01 00)
+        //   block CRC
+        cram_block_compression_hdr ch;
+        memset(&ch, 0, sizeof(ch));
+        c.comp_hdr_block = cram_encode_compression_header(fd, &c, &ch, 0);
+
+        c.length = c.comp_hdr_block->byte            // Landmark[0]
+            + 5                                      // block struct
+            + 4*(CRAM_MAJOR_VERS(fd->version) >= 3); // CRC
+        if (cram_write_container(fd, &c) < 0 ||
+            cram_write_block(fd, c.comp_hdr_block) < 0) {
+            cram_close(fd);
+            cram_free_block(c.comp_hdr_block);
+            return -1;
+        }
+        if (ch.preservation_map)
+            kh_destroy(map, ch.preservation_map);
+        cram_free_block(c.comp_hdr_block);
+
+        // V2.1 bytes
+        // 0b 00 00 00 ff ff ff ff 0f // Cont HDR: size, ref seq id
+        // e0 45 4f 46 00 00 00       // Cont HDR: pos, span, nrec, counter
+        // 00 01 00                   // Cont HDR: nbase, nblk, landmark
+        // 00 01 00 06 06             // Comp.HDR blk
+        // 01 00 01 00 01 00          // Comp.HDR blk
+
+        // V3.0 bytes:
+        // 0f 00 00 00 ff ff ff ff 0f // Cont HDR: size, ref seq id
+        // e0 45 4f 46 00 00 00       // Cont HDR: pos, span, nrec, counter
+        // 00 01 00                   // Cont HDR: nbase, nblk, landmark
+        // 05 bd d9 4f                // CRC32
+        // 00 01 00 06 06             // Comp.HDR blk
+        // 01 00 01 00 01 00          // Comp.HDR blk
+        // ee 63 01 4b                // CRC32
+
+        // V4.0 bytes:
+        // 0f 00 00 00 8f ff ff ff    // Cont HDR: size, ref seq id
+        // 82 95 9e 46 00 00 00       // Cont HDR: pos, span, nrec, counter
+        // 00 01 00                   // Cont HDR: nbase, nblk, landmark
+        // ac d6 05 bc                // CRC32
+        // 00 01 00 06 06             // Comp.HDR blk
+        // 01 00 01 00 01 00          // Comp.HDR blk
+        // ee 63 01 4b                // CRC32
+    }
+
+    return 0;
+}
+
+/*
+ * Closes a CRAM file.
+ * Returns 0 on success
+ *        -1 on failure
+ */
+int cram_close(cram_fd *fd) {
+    spare_bams *bl, *next;
+    int i, ret = 0;
+
+    if (!fd)
+        return -1;
+
+    if (fd->mode == 'w' && fd->ctr) {
+        if(fd->ctr->slice)
+            cram_update_curr_slice(fd->ctr, fd->version);
+
+        if (-1 == cram_flush_container_mt(fd, fd->ctr))
+            ret = -1;
+    }
+
+    if (fd->mode != 'w')
+        cram_drain_rqueue(fd);
+
+    if (fd->pool && fd->eof >= 0 && fd->rqueue) {
+        hts_tpool_process_flush(fd->rqueue);
+
+        if (0 != cram_flush_result(fd))
+            ret = -1;
+
+        if (fd->mode == 'w')
+            fd->ctr = NULL; // prevent double freeing
+
+        //fprintf(stderr, "CRAM: destroy queue %p\n", fd->rqueue);
+
+        hts_tpool_process_destroy(fd->rqueue);
+    }
+
+    pthread_mutex_destroy(&fd->metrics_lock);
+    pthread_mutex_destroy(&fd->ref_lock);
+    pthread_mutex_destroy(&fd->range_lock);
+    pthread_mutex_destroy(&fd->bam_list_lock);
+
+    if (ret == 0 && fd->mode == 'w') {
+        /* Write EOF block */
+        if (0 != cram_write_eof_block(fd))
+            ret = -1;
+    }
+
+    for (bl = fd->bl; bl; bl = next) {
+        int max_rec = fd->seqs_per_slice * fd->slices_per_container;
+
+        next = bl->next;
+        free_bam_list(bl->bams, max_rec);
+        free(bl);
+    }
+
+    if (hclose(fd->fp) != 0)
+        ret = -1;
+
+    if (fd->file_def)
+        cram_free_file_def(fd->file_def);
+
+    if (fd->header)
+        sam_hdr_destroy(fd->header);
+
+    free(fd->prefix);
+
+    if (fd->ctr)
+        cram_free_container(fd->ctr);
+
+    if (fd->ctr_mt && fd->ctr_mt != fd->ctr)
+        cram_free_container(fd->ctr_mt);
+
+    if (fd->refs)
+        refs_free(fd->refs);
+    if (fd->ref_free)
+        free(fd->ref_free);
+
+    for (i = 0; i < DS_END; i++)
+        if (fd->m[i])
+            free(fd->m[i]);
+
+    if (fd->tags_used) {
+        khint_t k;
+
+        for (k = kh_begin(fd->tags_used); k != kh_end(fd->tags_used); k++) {
+            if (kh_exist(fd->tags_used, k))
+                free(kh_val(fd->tags_used, k));
+        }
+
+        kh_destroy(m_metrics, fd->tags_used);
+    }
+
+    if (fd->index)
+        cram_index_free(fd);
+
+    if (fd->own_pool && fd->pool)
+        hts_tpool_destroy(fd->pool);
+
+    if (fd->idxfp)
+        if (bgzf_close(fd->idxfp) < 0)
+            ret = -1;
+
+    free(fd);
+
+    return ret;
+}
+
+/*
+ * Returns 1 if we hit an EOF while reading.
+ */
+int cram_eof(cram_fd *fd) {
+    return fd->eof;
+}
+
+
+/*
+ * Sets options on the cram_fd. See CRAM_OPT_* definitions in cram_structs.h.
+ * Use this immediately after opening.
+ *
+ * Returns 0 on success
+ *        -1 on failure
+ */
+int cram_set_option(cram_fd *fd, enum hts_fmt_option opt, ...) {
+    int r;
+    va_list args;
+
+    va_start(args, opt);
+    r = cram_set_voption(fd, opt, args);
+    va_end(args);
+
+    return r;
+}
+
+/*
+ * Sets options on the cram_fd. See CRAM_OPT_* definitions in cram_structs.h.
+ * Use this immediately after opening.
+ *
+ * Returns 0 on success
+ *        -1 on failure
+ */
+int cram_set_voption(cram_fd *fd, enum hts_fmt_option opt, va_list args) {
+    refs_t *refs;
+
+    if (!fd) {
+        errno = EBADF;
+        return -1;
+    }
+
+    switch (opt) {
+    case CRAM_OPT_DECODE_MD:
+        fd->decode_md = va_arg(args, int);
+        break;
+
+    case CRAM_OPT_PREFIX:
+        if (fd->prefix)
+            free(fd->prefix);
+        if (!(fd->prefix = strdup(va_arg(args, char *))))
+            return -1;
+        break;
+
+    case CRAM_OPT_VERBOSITY:
+        break;
+
+    case CRAM_OPT_SEQS_PER_SLICE:
+        fd->seqs_per_slice = va_arg(args, int);
+        if (fd->bases_per_slice == BASES_PER_SLICE)
+            fd->bases_per_slice = fd->seqs_per_slice * 500;
+        break;
+
+    case CRAM_OPT_BASES_PER_SLICE:
+        fd->bases_per_slice = va_arg(args, int);
+        break;
+
+    case CRAM_OPT_SLICES_PER_CONTAINER:
+        fd->slices_per_container = va_arg(args, int);
+        break;
+
+    case CRAM_OPT_EMBED_REF:
+        fd->embed_ref = va_arg(args, int);
+        break;
+
+    case CRAM_OPT_NO_REF:
+        fd->no_ref = va_arg(args, int);
+        break;
+
+    case CRAM_OPT_POS_DELTA:
+        fd->ap_delta = va_arg(args, int);
+        break;
+
+    case CRAM_OPT_IGNORE_MD5:
+        fd->ignore_md5 = va_arg(args, int);
+        break;
+
+    case CRAM_OPT_LOSSY_NAMES:
+        fd->lossy_read_names = va_arg(args, int);
+        // Currently lossy read names required paired (attached) reads.
+        // TLEN 0 or being 1 out causes read pairs to be detached, breaking
+        // the lossy read name compression, so we have extra options to
+        // slacken the exact TLEN round-trip checks.
+        fd->tlen_approx = fd->lossy_read_names;
+        fd->tlen_zero = fd->lossy_read_names;
+        break;
+
+    case CRAM_OPT_USE_BZIP2:
+        fd->use_bz2 = va_arg(args, int);
+        break;
+
+    case CRAM_OPT_USE_RANS:
+        fd->use_rans = va_arg(args, int);
+        break;
+
+    case CRAM_OPT_USE_TOK:
+        fd->use_tok = va_arg(args, int);
+        break;
+
+    case CRAM_OPT_USE_FQZ:
+        fd->use_fqz = va_arg(args, int);
+        break;
+
+    case CRAM_OPT_USE_ARITH:
+        fd->use_arith = va_arg(args, int);
+        break;
+
+    case CRAM_OPT_USE_LZMA:
+        fd->use_lzma = va_arg(args, int);
+        break;
+
+    case CRAM_OPT_SHARED_REF:
+        fd->shared_ref = 1;
+        refs = va_arg(args, refs_t *);
+        if (refs != fd->refs) {
+            if (fd->refs)
+                refs_free(fd->refs);
+            fd->refs = refs;
+            fd->refs->count++;
+        }
+        break;
+
+    case CRAM_OPT_RANGE: {
+        int r = cram_seek_to_refpos(fd, va_arg(args, cram_range *));
+        pthread_mutex_lock(&fd->range_lock);
+        if (fd->range.refid != -2)
+            fd->required_fields |= SAM_POS;
+        pthread_mutex_unlock(&fd->range_lock);
+        return r;
+    }
+
+    case CRAM_OPT_RANGE_NOSEEK: {
+        // As per CRAM_OPT_RANGE, but no seeking
+        pthread_mutex_lock(&fd->range_lock);
+        cram_range *r = va_arg(args, cram_range *);
+        fd->range = *r;
+        if (r->refid == HTS_IDX_NOCOOR) {
+            fd->range.refid = -1;
+            fd->range.start = 0;
+        } else if (r->refid == HTS_IDX_START || r->refid == HTS_IDX_REST) {
+            fd->range.refid = -2; // special case in cram_next_slice
+        }
+        if (fd->range.refid != -2)
+            fd->required_fields |= SAM_POS;
+        fd->ooc = 0;
+        fd->eof = 0;
+        pthread_mutex_unlock(&fd->range_lock);
+        return 0;
+    }
+
+    case CRAM_OPT_REFERENCE:
+        return cram_load_reference(fd, va_arg(args, char *));
+
+    case CRAM_OPT_VERSION: {
+        int major, minor;
+        char *s = va_arg(args, char *);
+        if (2 != sscanf(s, "%d.%d", &major, &minor)) {
+            hts_log_error("Malformed version string %s", s);
+            return -1;
+        }
+        if (!((major == 1 &&  minor == 0) ||
+              (major == 2 && (minor == 0 || minor == 1)) ||
+              (major == 3 && (minor == 0 || minor == 1)) ||
+              (major == 4 &&  minor == 0))) {
+            hts_log_error("Unknown version string; use 1.0, 2.0, 2.1, 3.0, 3.1 or 4.0");
+            errno = EINVAL;
+            return -1;
+        }
+
+        if (major > 3 || (major == 3 && minor > 1)) {
+            hts_log_warning(
+                "CRAM version %s is still a draft and subject to change.\n"
+                "This is a technology demonstration that should not be "
+                "used for archival data.", s);
+        }
+
+        fd->version = major*256 + minor;
+
+        fd->use_rans = (CRAM_MAJOR_VERS(fd->version) >= 3) ? 1 : 0;
+
+        fd->use_tok = ((CRAM_MAJOR_VERS(fd->version) == 3 &&
+                        CRAM_MINOR_VERS(fd->version) >= 1) ||
+                        CRAM_MAJOR_VERS(fd->version) >= 4) ? 1 : 0;
+        cram_init_tables(fd);
+
+        break;
+    }
+
+    case CRAM_OPT_MULTI_SEQ_PER_SLICE:
+        fd->multi_seq_user = fd->multi_seq = va_arg(args, int);
+        break;
+
+    case CRAM_OPT_NTHREADS: {
+        int nthreads =  va_arg(args, int);
+        if (nthreads >= 1) {
+            if (!(fd->pool = hts_tpool_init(nthreads)))
+                return -1;
+
+            fd->rqueue = hts_tpool_process_init(fd->pool, nthreads*2, 0);
+            fd->shared_ref = 1;
+            fd->own_pool = 1;
+        }
+        break;
+    }
+
+    case CRAM_OPT_THREAD_POOL: {
+        htsThreadPool *p = va_arg(args, htsThreadPool *);
+        fd->pool = p ? p->pool : NULL;
+        if (fd->pool) {
+            fd->rqueue = hts_tpool_process_init(fd->pool,
+                                                p->qsize ? p->qsize : hts_tpool_size(fd->pool)*2,
+                                                0);
+        }
+        fd->shared_ref = 1; // Needed to avoid clobbering ref between threads
+        fd->own_pool = 0;
+
+        //fd->qsize = 1;
+        //fd->decoded = calloc(fd->qsize, sizeof(cram_container *));
+        //hts_tpool_dispatch(fd->pool, cram_decoder_thread, fd);
+        break;
+    }
+
+    case CRAM_OPT_REQUIRED_FIELDS:
+        fd->required_fields = va_arg(args, int);
+        if (fd->range.refid != -2)
+            fd->required_fields |= SAM_POS;
+        break;
+
+    case CRAM_OPT_STORE_MD:
+        fd->store_md = va_arg(args, int);
+        break;
+
+    case CRAM_OPT_STORE_NM:
+        fd->store_nm = va_arg(args, int);
+        break;
+
+    case HTS_OPT_COMPRESSION_LEVEL:
+        fd->level = va_arg(args, int);
+        break;
+
+    case HTS_OPT_PROFILE: {
+        enum hts_profile_option prof = va_arg(args, int);
+        switch (prof) {
+        case HTS_PROFILE_FAST:
+            if (fd->level == CRAM_DEFAULT_LEVEL) fd->level = 1;
+            fd->use_tok = 0;
+            fd->seqs_per_slice = 10000;
+            break;
+
+        case HTS_PROFILE_NORMAL:
+            break;
+
+        case HTS_PROFILE_SMALL:
+            if (fd->level == CRAM_DEFAULT_LEVEL) fd->level = 6;
+            fd->use_bz2 = 1;
+            fd->use_fqz = 1;
+            fd->seqs_per_slice = 25000;
+            break;
+
+        case HTS_PROFILE_ARCHIVE:
+            if (fd->level == CRAM_DEFAULT_LEVEL) fd->level = 7;
+            fd->use_bz2 = 1;
+            fd->use_fqz = 1;
+            fd->use_arith = 1;
+            if (fd->level > 7)
+                fd->use_lzma = 1;
+            fd->seqs_per_slice = 100000;
+            break;
+        }
+
+        if (fd->bases_per_slice == BASES_PER_SLICE)
+            fd->bases_per_slice = fd->seqs_per_slice * 500;
+        break;
+    }
+
+    default:
+        hts_log_error("Unknown CRAM option code %d", opt);
+        errno = EINVAL;
+        return -1;
+    }
+
+    return 0;
+}
+
+int cram_check_EOF(cram_fd *fd)
+{
+    // Byte 9 in these templates is & with 0x0f to resolve differences
+    // between ITF-8 interpretations between early Java and C
+    // implementations of CRAM
+    static const unsigned char TEMPLATE_2_1[30] = {
+        0x0b, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0xe0,
+        0x45, 0x4f, 0x46, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00,
+        0x01, 0x00, 0x06, 0x06, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00
+    };
+    static const unsigned char TEMPLATE_3[38] = {
+        0x0f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0xe0,
+        0x45, 0x4f, 0x46, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x05,
+        0xbd, 0xd9, 0x4f, 0x00, 0x01, 0x00, 0x06, 0x06, 0x01, 0x00,
+        0x01, 0x00, 0x01, 0x00, 0xee, 0x63, 0x01, 0x4b
+    };
+
+    unsigned char buf[38]; // max(sizeof TEMPLATE_*)
+
+    uint8_t major = CRAM_MAJOR_VERS(fd->version);
+    uint8_t minor = CRAM_MINOR_VERS(fd->version);
+
+    const unsigned char *template;
+    ssize_t template_len;
+    if ((major < 2) ||
+        (major == 2 && minor == 0)) {
+        return 3; // No EOF support in cram versions less than 2.1
+    } else if (major == 2 && minor == 1) {
+        template = TEMPLATE_2_1;
+        template_len = sizeof TEMPLATE_2_1;
+    } else {
+        template = TEMPLATE_3;
+        template_len = sizeof TEMPLATE_3;
+    }
+
+    off_t offset = htell(fd->fp);
+    if (hseek(fd->fp, -template_len, SEEK_END) < 0) {
+        if (errno == ESPIPE) {
+            hclearerr(fd->fp);
+            return 2;
+        }
+        else {
+            return -1;
+        }
+    }
+    if (hread(fd->fp, buf, template_len) != template_len) return -1;
+    if (hseek(fd->fp, offset, SEEK_SET) < 0) return -1;
+    buf[8] &= 0x0f;
+    return (memcmp(template, buf, template_len) == 0)? 1 : 0;
+}
diff --git a/ext/htslib/cram/cram_io.h b/ext/htslib/cram/cram_io.h
new file mode 100644
index 0000000..d2d583d
--- /dev/null
+++ b/ext/htslib/cram/cram_io.h
@@ -0,0 +1,648 @@
+/*
+Copyright (c) 2012-2020 Genome Research Ltd.
+Author: James Bonfield <jkb@sanger.ac.uk>
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+Institute nor the names of its contributors may be used to endorse or promote
+products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+/*! \file
+ * Include cram.h instead.
+ *
+ * This is an internal part of the CRAM system and is automatically included
+ * when you #include cram.h.
+ *
+ * Implements the low level CRAM I/O primitives.
+ * This includes basic data types such as byte, int, ITF-8,
+ * maps, bitwise I/O, etc.
+ */
+
+#ifndef CRAM_IO_H
+#define CRAM_IO_H
+
+#include <stdint.h>
+
+#include "misc.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**@{ ----------------------------------------------------------------------
+ * ITF8 encoding and decoding.
+ *
+ * Also see the itf8_get and itf8_put macros.
+ */
+
+/*! INTERNAL: Converts two characters into an integer for use in switch{} */
+#define CRAM_KEY(a,b) ((((unsigned char) a)<<8)|(((unsigned char) b)))
+
+/*! Reads an integer in ITF-8 encoding from 'fd' and stores it in
+ * *val.
+ *
+ * @return
+ * Returns the number of bytes read on success;
+ *        -1 on failure
+ */
+int itf8_decode(cram_fd *fd, int32_t *val);
+
+extern const int itf8_bytes[16];
+extern const int ltf8_bytes[256];
+
+/*! Pushes a value in ITF8 format onto the end of a block.
+ *
+ * This shouldn't be used for high-volume data as it is not the fastest
+ * method.
+ *
+ * @return
+ * Returns the number of bytes written
+ */
+int itf8_put_blk(cram_block *blk, int32_t val);
+int ltf8_put_blk(cram_block *blk, int64_t val);
+
+/*! Pulls a literal 32-bit value from a block.
+ *
+ * @returns the number of bytes decoded;
+ *         -1 on failure.
+ */
+int int32_get_blk(cram_block *b, int32_t *val);
+
+/*! Pushes a literal 32-bit value onto the end of a block.
+ *
+ * @return
+ * Returns 0 on success;
+ *        -1 on failure.
+ */
+int int32_put_blk(cram_block *blk, int32_t val);
+
+
+/**@}*/
+/**@{ ----------------------------------------------------------------------
+ * CRAM blocks - the dynamically growable data block. We have code to
+ * create, update, (un)compress and read/write.
+ *
+ * These are derived from the deflate_interlaced.c blocks, but with the
+ * CRAM extension of content types and IDs.
+ */
+
+/*! Allocates a new cram_block structure with a specified content_type and
+ * id.
+ *
+ * @return
+ * Returns block pointer on success;
+ *         NULL on failure
+ */
+cram_block *cram_new_block(enum cram_content_type content_type,
+                           int content_id);
+
+/*! Reads a block from a cram file.
+ *
+ * @return
+ * Returns cram_block pointer on success;
+ *         NULL on failure
+ */
+cram_block *cram_read_block(cram_fd *fd);
+
+/*! Writes a CRAM block.
+ *
+ * @return
+ * Returns 0 on success;
+ *        -1 on failure
+ */
+int cram_write_block(cram_fd *fd, cram_block *b);
+
+/*! Frees a CRAM block, deallocating internal data too.
+ */
+void cram_free_block(cram_block *b);
+
+/*! Uncompress a memory block using Zlib.
+ *
+ * @return
+ * Returns 0 on success;
+ *        -1 on failure
+ */
+char *zlib_mem_inflate(char *cdata, size_t csize, size_t *size);
+
+/*! Uncompresses a CRAM block, if compressed.
+ *
+ * @return
+ * Returns 0 on success;
+ *        -1 on failure
+ */
+int cram_uncompress_block(cram_block *b);
+
+/*! Compresses a block.
+ *
+ * Compresses a block using one of two different zlib strategies. If we only
+ * want one choice set strat2 to be -1.
+ *
+ * The logic here is that sometimes Z_RLE does a better job than Z_FILTERED
+ * or Z_DEFAULT_STRATEGY on quality data. If so, we'd rather use it as it is
+ * significantly faster.
+ *
+ * @return
+ * Returns 0 on success;
+ *        -1 on failure
+ */
+int cram_compress_block(cram_fd *fd, cram_block *b, cram_metrics *metrics,
+                        int method, int level);
+int cram_compress_block2(cram_fd *fd, cram_slice *s,
+                         cram_block *b, cram_metrics *metrics,
+                         int method, int level);
+
+cram_metrics *cram_new_metrics(void);
+char *cram_block_method2str(enum cram_block_method_int m);
+char *cram_content_type2str(enum cram_content_type t);
+
+/*
+ * Find an external block by its content_id
+ */
+
+static inline cram_block *cram_get_block_by_id(cram_slice *slice, int id) {
+  //fprintf(stderr, "%d\t%p\n", id, slice->block_by_id);
+    uint32_t v = id;
+    if (slice->block_by_id && v < 256) {
+        return slice->block_by_id[v];
+    } else {
+        v = 256 + v % 251;
+        if (slice->block_by_id &&
+            slice->block_by_id[v] &&
+            slice->block_by_id[v]->content_id == id)
+            return slice->block_by_id[v];
+
+        // Otherwise a linear search in case of collision
+        int i;
+        for (i = 0; i < slice->hdr->num_blocks; i++) {
+            cram_block *b = slice->block[i];
+            if (b && b->content_type == EXTERNAL && b->content_id == id)
+                return b;
+        }
+    }
+    return NULL;
+}
+
+/* --- Accessor macros for manipulating blocks on a byte by byte basis --- */
+
+/* Block size and data pointer. */
+#define BLOCK_SIZE(b) ((b)->byte)
+#define BLOCK_DATA(b) ((b)->data)
+
+/* Returns the address one past the end of the block */
+#define BLOCK_END(b) (&(b)->data[(b)->byte])
+
+/* Make block exactly 'l' bytes long */
+static inline int block_resize_exact(cram_block *b, size_t len) {
+    unsigned char *tmp = realloc(b->data, len);
+    if (!tmp)
+        return -1;
+    b->alloc = len;
+    b->data = tmp;
+    return 0;
+}
+
+/* Request block to be at least 'l' bytes long */
+static inline int block_resize(cram_block *b, size_t len) {
+    if (b->alloc > len)
+        return 0;
+
+    size_t alloc = b->alloc+800;
+    alloc = MAX(alloc + (alloc>>2), len);
+    return block_resize_exact(b, alloc);
+}
+
+
+/* Ensure the block can hold at least another 'l' bytes */
+static inline int block_grow(cram_block *b, size_t len) {
+    return block_resize(b, BLOCK_SIZE(b) + len);
+}
+
+/* Append string 's' of length 'l'. */
+static inline int block_append(cram_block *b, const void *s, size_t len) {
+    if (block_grow(b, len) < 0)
+        return -1;
+
+    if (len) {
+        memcpy(BLOCK_END(b), s, len);
+        BLOCK_SIZE(b) += len;
+    }
+
+    return 0;
+}
+
+/* Append as single character 'c' */
+static inline int block_append_char(cram_block *b, char c) {
+    if (block_grow(b, 1) < 0)
+        return -1;
+
+    b->data[b->byte++] = c;
+    return 0;
+}
+
+/* Append a single unsigned integer */
+static inline unsigned char *append_uint32(unsigned char *cp, uint32_t i);
+static inline int block_append_uint(cram_block *b, unsigned int i) {
+    if (block_grow(b, 11) < 0)
+        return -1;
+
+    unsigned char *cp = &b->data[b->byte];
+    b->byte += append_uint32(cp, i) - cp;
+    return 0;
+}
+
+// Versions of above with built in goto block_err calls.
+#define BLOCK_RESIZE_EXACT(b,l) if (block_resize_exact((b),(l))<0) goto block_err
+#define BLOCK_RESIZE(b,l)       if (block_resize((b),(l))      <0) goto block_err
+#define BLOCK_GROW(b,l)         if (block_grow((b),(l))        <0) goto block_err
+#define BLOCK_APPEND(b,s,l)     if (block_append((b),(s),(l))  <0) goto block_err
+#define BLOCK_APPEND_CHAR(b,c)  if (block_append_char((b),(c)) <0) goto block_err
+#define BLOCK_APPEND_UINT(b,i)  if (block_append_uint((b),(i)) <0) goto block_err
+
+static inline unsigned char *append_uint32(unsigned char *cp, uint32_t i) {
+    uint32_t j;
+
+    if (i == 0) {
+        *cp++ = '0';
+        return cp;
+    }
+
+    if (i < 100)        goto b1;
+    if (i < 10000)      goto b3;
+    if (i < 1000000)    goto b5;
+    if (i < 100000000)  goto b7;
+
+    if ((j = i / 1000000000)) {*cp++ = j + '0'; i -= j*1000000000; goto x8;}
+    if ((j = i / 100000000))  {*cp++ = j + '0'; i -= j*100000000;  goto x7;}
+ b7:if ((j = i / 10000000))   {*cp++ = j + '0'; i -= j*10000000;   goto x6;}
+    if ((j = i / 1000000))    {*cp++ = j + '0', i -= j*1000000;    goto x5;}
+ b5:if ((j = i / 100000))     {*cp++ = j + '0', i -= j*100000;     goto x4;}
+    if ((j = i / 10000))      {*cp++ = j + '0', i -= j*10000;      goto x3;}
+ b3:if ((j = i / 1000))       {*cp++ = j + '0', i -= j*1000;       goto x2;}
+    if ((j = i / 100))        {*cp++ = j + '0', i -= j*100;        goto x1;}
+ b1:if ((j = i / 10))         {*cp++ = j + '0', i -= j*10;         goto x0;}
+    if (i)                     *cp++ = i + '0';
+    return cp;
+
+ x8: *cp++ = i / 100000000 + '0', i %= 100000000;
+ x7: *cp++ = i / 10000000  + '0', i %= 10000000;
+ x6: *cp++ = i / 1000000   + '0', i %= 1000000;
+ x5: *cp++ = i / 100000    + '0', i %= 100000;
+ x4: *cp++ = i / 10000     + '0', i %= 10000;
+ x3: *cp++ = i / 1000      + '0', i %= 1000;
+ x2: *cp++ = i / 100       + '0', i %= 100;
+ x1: *cp++ = i / 10        + '0', i %= 10;
+ x0: *cp++ = i             + '0';
+
+    return cp;
+}
+
+static inline unsigned char *append_sub32(unsigned char *cp, uint32_t i) {
+    *cp++ = i / 100000000 + '0', i %= 100000000;
+    *cp++ = i / 10000000  + '0', i %= 10000000;
+    *cp++ = i / 1000000   + '0', i %= 1000000;
+    *cp++ = i / 100000    + '0', i %= 100000;
+    *cp++ = i / 10000     + '0', i %= 10000;
+    *cp++ = i / 1000      + '0', i %= 1000;
+    *cp++ = i / 100       + '0', i %= 100;
+    *cp++ = i / 10        + '0', i %= 10;
+    *cp++ = i             + '0';
+
+    return cp;
+}
+
+static inline unsigned char *append_uint64(unsigned char *cp, uint64_t i) {
+    uint64_t j;
+
+    if (i <= 0xffffffff)
+        return append_uint32(cp, i);
+
+    if ((j = i/1000000000) > 1000000000) {
+        cp = append_uint32(cp, j/1000000000);
+        j %= 1000000000;
+        cp = append_sub32(cp, j);
+    } else {
+        cp = append_uint32(cp, i / 1000000000);
+    }
+    cp = append_sub32(cp, i % 1000000000);
+
+    return cp;
+}
+
+#define BLOCK_UPLEN(b) \
+    (b)->comp_size = (b)->uncomp_size = BLOCK_SIZE((b))
+
+/**@}*/
+/**@{ ----------------------------------------------------------------------
+ * Reference sequence handling
+ */
+
+/*! Loads a reference set from fn and stores in the cram_fd.
+ *
+ * @return
+ * Returns 0 on success;
+ *        -1 on failure
+ */
+int cram_load_reference(cram_fd *fd, char *fn);
+
+/*! Generates a lookup table in refs based on the SQ headers in sam_hdr_t.
+ *
+ * Indexes references by the order they appear in a BAM file. This may not
+ * necessarily be the same order they appear in the fasta reference file.
+ *
+ * @return
+ * Returns 0 on success;
+ *        -1 on failure
+ */
+int refs2id(refs_t *r, sam_hdr_t *hdr);
+
+void refs_free(refs_t *r);
+
+/*! Returns a portion of a reference sequence from start to end inclusive.
+ *
+ * The returned pointer is owned by the cram_file fd and should not be freed
+ * by the caller. It is valid only until the next cram_get_ref is called
+ * with the same fd parameter (so is thread-safe if given multiple files).
+ *
+ * To return the entire reference sequence, specify start as 1 and end
+ * as 0.
+ *
+ * @return
+ * Returns reference on success;
+ *         NULL on failure
+ */
+char *cram_get_ref(cram_fd *fd, int id, hts_pos_t start, hts_pos_t end);
+void cram_ref_incr(refs_t *r, int id);
+void cram_ref_decr(refs_t *r, int id);
+/**@}*/
+/**@{ ----------------------------------------------------------------------
+ * Containers
+ */
+
+/*! Creates a new container, specifying the maximum number of slices
+ * and records permitted.
+ *
+ * @return
+ * Returns cram_container ptr on success;
+ *         NULL on failure
+ */
+cram_container *cram_new_container(int nrec, int nslice);
+void cram_free_container(cram_container *c);
+
+/*! Reads a container header.
+ *
+ * @return
+ * Returns cram_container on success;
+ *         NULL on failure or no container left (fd->err == 0).
+ */
+cram_container *cram_read_container(cram_fd *fd);
+
+/*! Writes a container structure.
+ *
+ * @return
+ * Returns 0 on success;
+ *        -1 on failure
+ */
+int cram_write_container(cram_fd *fd, cram_container *h);
+
+/*! Flushes a container to disk.
+ *
+ * Flushes a completely or partially full container to disk, writing
+ * container structure, header and blocks. This also calls the encoder
+ * functions.
+ *
+ * @return
+ * Returns 0 on success;
+ *        -1 on failure
+ */
+int cram_flush_container(cram_fd *fd, cram_container *c);
+int cram_flush_container_mt(cram_fd *fd, cram_container *c);
+
+
+/**@}*/
+/**@{ ----------------------------------------------------------------------
+ * Compression headers; the first part of the container
+ */
+
+/*! Creates a new blank container compression header
+ *
+ * @return
+ * Returns header ptr on success;
+ *         NULL on failure
+ */
+cram_block_compression_hdr *cram_new_compression_header(void);
+
+/*! Frees a cram_block_compression_hdr */
+void cram_free_compression_header(cram_block_compression_hdr *hdr);
+
+
+/**@}*/
+/**@{ ----------------------------------------------------------------------
+ * Slices and slice headers
+ */
+
+/*! Frees a slice header */
+void cram_free_slice_header(cram_block_slice_hdr *hdr);
+
+/*! Frees a slice */
+void cram_free_slice(cram_slice *s);
+
+/*! Creates a new empty slice in memory, for subsequent writing to
+ * disk.
+ *
+ * @return
+ * Returns cram_slice ptr on success;
+ *         NULL on failure
+ */
+cram_slice *cram_new_slice(enum cram_content_type type, int nrecs);
+
+/*! Loads an entire slice.
+ *
+ * FIXME: In 1.0 the native unit of slices within CRAM is broken
+ * as slices contain references to objects in other slices.
+ * To work around this while keeping the slice oriented outer loop
+ * we read all slices and stitch them together into a fake large
+ * slice instead.
+ *
+ * @return
+ * Returns cram_slice ptr on success;
+ *         NULL on failure
+ */
+cram_slice *cram_read_slice(cram_fd *fd);
+
+
+
+/**@}*/
+/**@{ ----------------------------------------------------------------------
+ * CRAM file definition (header)
+ */
+
+/*! Reads a CRAM file definition structure.
+ *
+ * @return
+ * Returns file_def ptr on success;
+ *         NULL on failure
+ */
+cram_file_def *cram_read_file_def(cram_fd *fd);
+
+/*! Writes a cram_file_def structure to cram_fd.
+ *
+ * @return
+ * Returns 0 on success;
+ *        -1 on failure
+ */
+int cram_write_file_def(cram_fd *fd, cram_file_def *def);
+
+/*! Frees a cram_file_def structure. */
+void cram_free_file_def(cram_file_def *def);
+
+
+/**@}*/
+/**@{ ----------------------------------------------------------------------
+ * SAM header I/O
+ */
+
+/*! Reads the SAM header from the first CRAM data block.
+ *
+ * Also performs minimal parsing to extract read-group
+ * and sample information.
+ *
+ * @return
+ * Returns SAM hdr ptr on success;
+ *         NULL on failure
+ */
+sam_hdr_t *cram_read_SAM_hdr(cram_fd *fd);
+
+/*! Writes a CRAM SAM header.
+ *
+ * @return
+ * Returns 0 on success;
+ *        -1 on failure
+ */
+int cram_write_SAM_hdr(cram_fd *fd, sam_hdr_t *hdr);
+
+
+/**@}*/
+/**@{ ----------------------------------------------------------------------
+ * The top-level cram opening, closing and option handling
+ */
+
+/*! Opens a CRAM file for read (mode "rb") or write ("wb").
+ *
+ * The filename may be "-" to indicate stdin or stdout.
+ *
+ * @return
+ * Returns file handle on success;
+ *         NULL on failure.
+ */
+cram_fd *cram_open(const char *filename, const char *mode);
+
+/*! Opens an existing stream for reading or writing.
+ *
+ * @return
+ * Returns file handle on success;
+ *         NULL on failure.
+ */
+cram_fd *cram_dopen(struct hFILE *fp, const char *filename, const char *mode);
+
+/*! Closes a CRAM file.
+ *
+ * @return
+ * Returns 0 on success;
+ *        -1 on failure
+ */
+int cram_close(cram_fd *fd);
+
+/*
+ * Seek within a CRAM file.
+ *
+ * Returns 0 on success
+ *        -1 on failure
+ */
+int cram_seek(cram_fd *fd, off_t offset, int whence);
+
+/*
+ * Flushes a CRAM file.
+ * Useful for when writing to stdout without wishing to close the stream.
+ *
+ * Returns 0 on success
+ *        -1 on failure
+ */
+int cram_flush(cram_fd *fd);
+
+/*! Checks for end of file on a cram_fd stream.
+ *
+ * @return
+ * Returns 0 if not at end of file
+ *         1 if we hit an expected EOF (end of range or EOF block)
+ *         2 for other EOF (end of stream without EOF block)
+ */
+int cram_eof(cram_fd *fd);
+
+/*! Sets options on the cram_fd.
+ *
+ * See CRAM_OPT_* definitions in cram_structs.h.
+ * Use this immediately after opening.
+ *
+ * @return
+ * Returns 0 on success;
+ *        -1 on failure
+ */
+int cram_set_option(cram_fd *fd, enum hts_fmt_option opt, ...);
+
+/*! Sets options on the cram_fd.
+ *
+ * See CRAM_OPT_* definitions in cram_structs.h.
+ * Use this immediately after opening.
+ *
+ * @return
+ * Returns 0 on success;
+ *        -1 on failure
+ */
+int cram_set_voption(cram_fd *fd, enum hts_fmt_option opt, va_list args);
+
+/*!
+ * Attaches a header to a cram_fd.
+ *
+ * This should be used when creating a new cram_fd for writing where
+ * we have an sam_hdr_t already constructed (eg from a file we've read
+ * in).
+ *
+ * @return
+ * Returns 0 on success;
+ *        -1 on failure
+ */
+int cram_set_header2(cram_fd *fd, const sam_hdr_t *hdr);
+
+/*!
+ * Returns the hFILE connected to a cram_fd.
+ */
+static inline struct hFILE *cram_hfile(cram_fd *fd) {
+    return fd->fp;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* CRAM_IO_H */
diff --git a/ext/htslib/cram/cram_samtools.h b/ext/htslib/cram/cram_samtools.h
new file mode 100644
index 0000000..a4c9bf5
--- /dev/null
+++ b/ext/htslib/cram/cram_samtools.h
@@ -0,0 +1,75 @@
+/*
+Copyright (c) 2010-2013, 2018, 2020 Genome Research Ltd.
+Author: James Bonfield <jkb@sanger.ac.uk>
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+Institute nor the names of its contributors may be used to endorse or promote
+products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#ifndef CRAM_SAMTOOLS_H
+#define CRAM_SAMTOOLS_H
+
+/* Samtools compatible API */
+#define bam_blk_size(b)  ((b)->l_data)
+#define bam_set_blk_size(b,v) ((b)->data_len = (v))
+
+#define bam_ref(b)       (b)->core.tid
+#define bam_pos(b)       (b)->core.pos
+#define bam_mate_pos(b)  (b)->core.mpos
+#define bam_mate_ref(b)  (b)->core.mtid
+#define bam_ins_size(b)  (b)->core.isize
+#define bam_seq_len(b)   (b)->core.l_qseq
+#define bam_cigar_len(b) (b)->core.n_cigar
+#define bam_flag(b)      (b)->core.flag
+#define bam_bin(b)       (b)->core.bin
+#define bam_map_qual(b)  (b)->core.qual
+#define bam_name_len(b)  ((b)->core.l_qname - (b)->core.l_extranul)
+#define bam_name(b)      bam_get_qname((b))
+#define bam_qual(b)      bam_get_qual((b))
+#define bam_seq(b)       bam_get_seq((b))
+#define bam_cigar(b)     bam_get_cigar((b))
+#define bam_aux(b)       bam_get_aux((b))
+
+#define bam_free(b)      bam_destroy1((b))
+
+#define bam_reg2bin(beg,end) hts_reg2bin((beg),(end),14,5)
+
+#include "../htslib/sam.h"
+
+enum cigar_op {
+    BAM_CMATCH_=BAM_CMATCH,
+    BAM_CINS_=BAM_CINS,
+    BAM_CDEL_=BAM_CDEL,
+    BAM_CREF_SKIP_=BAM_CREF_SKIP,
+    BAM_CSOFT_CLIP_=BAM_CSOFT_CLIP,
+    BAM_CHARD_CLIP_=BAM_CHARD_CLIP,
+    BAM_CPAD_=BAM_CPAD,
+    BAM_CBASE_MATCH=BAM_CEQUAL,
+    BAM_CBASE_MISMATCH=BAM_CDIFF
+};
+
+typedef bam1_t bam_seq_t;
+
+#endif /* CRAM_SAMTOOLS_H */
diff --git a/ext/htslib/cram/cram_stats.c b/ext/htslib/cram/cram_stats.c
new file mode 100644
index 0000000..d06b8ff
--- /dev/null
+++ b/ext/htslib/cram/cram_stats.c
@@ -0,0 +1,227 @@
+/*
+Copyright (c) 2012-2014, 2016, 2018, 2020 Genome Research Ltd.
+Author: James Bonfield <jkb@sanger.ac.uk>
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+Institute nor the names of its contributors may be used to endorse or promote
+products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+
+#include <stdio.h>
+#include <errno.h>
+#include <assert.h>
+#include <stdlib.h>
+#include <string.h>
+#include <zlib.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <math.h>
+#include <inttypes.h>
+
+#include "cram.h"
+#include "os.h"
+
+cram_stats *cram_stats_create(void) {
+    return calloc(1, sizeof(cram_stats));
+}
+
+int cram_stats_add(cram_stats *st, int64_t val) {
+    st->nsamp++;
+
+    //assert(val >= 0);
+
+    if (val < MAX_STAT_VAL && val >= 0) {
+        st->freqs[val]++;
+    } else {
+        khint_t k;
+        int r;
+
+        if (!st->h) {
+            st->h = kh_init(m_i2i);
+            if (!st->h)
+                return -1;
+        }
+
+        k = kh_put(m_i2i, st->h, val, &r);
+        if (r == 0)
+            kh_val(st->h, k)++;
+        else if (r != -1)
+            kh_val(st->h, k) = 1;
+        else
+            return -1;
+    }
+    return 0;
+}
+
+void cram_stats_del(cram_stats *st, int64_t val) {
+    st->nsamp--;
+
+    //assert(val >= 0);
+
+    if (val < MAX_STAT_VAL && val >= 0) {
+        st->freqs[val]--;
+        assert(st->freqs[val] >= 0);
+    } else if (st->h) {
+        khint_t k = kh_get(m_i2i, st->h, val);
+
+        if (k != kh_end(st->h)) {
+            if (--kh_val(st->h, k) == 0)
+                kh_del(m_i2i, st->h, k);
+        } else {
+            hts_log_warning("Failed to remove val %"PRId64" from cram_stats", val);
+            st->nsamp++;
+        }
+    } else {
+        hts_log_warning("Failed to remove val %"PRId64" from cram_stats", val);
+        st->nsamp++;
+    }
+}
+
+#if DEBUG_CRAM_STATS
+void cram_stats_dump(cram_stats *st) {
+    int i;
+    fprintf(stderr, "cram_stats:\n");
+    for (i = 0; i < MAX_STAT_VAL; i++) {
+        if (!st->freqs[i])
+            continue;
+        fprintf(stderr, "\t%d\t%d\n", i, st->freqs[i]);
+    }
+    if (st->h) {
+        khint_t k;
+        for (k = kh_begin(st->h); k != kh_end(st->h); k++) {
+            if (!kh_exist(st->h, k))
+                continue;
+
+            fprintf(stderr, "\t%d\t%d\n", kh_key(st->h, k), kh_val(st->h, k));
+        }
+    }
+}
+#endif
+
+/*
+ * Computes entropy from integer frequencies for various encoding methods and
+ * picks the best encoding.
+ *
+ * FIXME: we could reuse some of the code here for the actual encoding
+ * parameters too. Eg the best 'k' for SUBEXP or the code lengths for huffman.
+ *
+ * Returns the best codec to use.
+ */
+enum cram_encoding cram_stats_encoding(cram_fd *fd, cram_stats *st) {
+    int nvals, i, max_val = 0, min_val = INT_MAX;
+    int *vals = NULL, *freqs = NULL, vals_alloc = 0;
+    int ntot HTS_UNUSED = 0;
+
+#if DEBUG_CRAM_STATS
+    cram_stats_dump(st);
+#endif
+
+    /* Count number of unique symbols */
+    for (nvals = i = 0; i < MAX_STAT_VAL; i++) {
+        if (!st->freqs[i])
+            continue;
+        if (nvals >= vals_alloc) {
+            vals_alloc = vals_alloc ? vals_alloc*2 : 1024;
+            int *vals_tmp  = realloc(vals,  vals_alloc * sizeof(int));
+            int *freqs_tmp = realloc(freqs, vals_alloc * sizeof(int));
+            if (!vals_tmp || !freqs_tmp) {
+                free(vals_tmp  ? vals_tmp  : vals);
+                free(freqs_tmp ? freqs_tmp : freqs);
+                return E_HUFFMAN; // Cannot do much else atm
+            }
+            vals = vals_tmp;
+            freqs = freqs_tmp;
+        }
+        vals[nvals] = i;
+        freqs[nvals] = st->freqs[i];
+        ntot += freqs[nvals];
+        if (max_val < i) max_val = i;
+        if (min_val > i) min_val = i;
+        nvals++;
+    }
+    if (st->h) {
+        khint_t k;
+        int i;
+
+        for (k = kh_begin(st->h); k != kh_end(st->h); k++) {
+            if (!kh_exist(st->h, k))
+                continue;
+
+            if (nvals >= vals_alloc) {
+                vals_alloc = vals_alloc ? vals_alloc*2 : 1024;
+                int *vals_tmp  = realloc(vals,  vals_alloc * sizeof(int));
+                int *freqs_tmp = realloc(freqs, vals_alloc * sizeof(int));
+                if (!vals_tmp || !freqs_tmp) {
+                    free(vals_tmp  ? vals_tmp  : vals);
+                    free(freqs_tmp ? freqs_tmp : freqs);
+                    return E_HUFFMAN; // Cannot do much else atm
+                }
+                vals = vals_tmp;
+                freqs = freqs_tmp;
+            }
+            i = kh_key(st->h, k);
+            vals[nvals]=i;
+            freqs[nvals] = kh_val(st->h, k);
+            ntot += freqs[nvals];
+            if (max_val < i) max_val = i;
+            if (min_val > i) min_val = i;
+            nvals++;
+        }
+    }
+
+    st->nvals = nvals;
+    st->min_val = min_val;
+    st->max_val = max_val;
+    assert(ntot == st->nsamp);
+
+    free(vals);
+    free(freqs);
+
+    /*
+     * Simple policy that everything is external unless it can be
+     * encoded using zero bits as a unary item huffman table.
+     */
+    if (CRAM_MAJOR_VERS(fd->version) >= 4) {
+        // Note, we're assuming integer data here as we don't have the
+        // type passed in.  Cram_encoder_init does know the type and
+        // will convert to E_CONST_BYTE or E_EXTERNAL as appropriate.
+        if (nvals == 1)
+            return E_CONST_INT;
+        else if (nvals == 0 || min_val < 0)
+            return E_VARINT_SIGNED;
+        else
+            return E_VARINT_UNSIGNED;
+    } else {
+        return nvals <= 1 ? E_HUFFMAN : E_EXTERNAL;
+    }
+}
+
+void cram_stats_free(cram_stats *st) {
+    if (st->h)
+        kh_destroy(m_i2i, st->h);
+    free(st);
+}
diff --git a/ext/htslib/cram/cram_stats.h b/ext/htslib/cram/cram_stats.h
new file mode 100644
index 0000000..5f8cfec
--- /dev/null
+++ b/ext/htslib/cram/cram_stats.h
@@ -0,0 +1,59 @@
+/*
+Copyright (c) 2012-2013, 2018 Genome Research Ltd.
+Author: James Bonfield <jkb@sanger.ac.uk>
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+Institute nor the names of its contributors may be used to endorse or promote
+products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#ifndef CRAM_STATS_H
+#define CRAM_STATS_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+cram_stats *cram_stats_create(void);
+int cram_stats_add(cram_stats *st, int64_t val);
+void cram_stats_del(cram_stats *st, int64_t val);
+void cram_stats_dump(cram_stats *st);
+void cram_stats_free(cram_stats *st);
+
+/*
+ * Computes entropy from integer frequencies for various encoding methods and
+ * picks the best encoding.
+ *
+ * FIXME: we could reuse some of the code here for the actual encoding
+ * parameters too. Eg the best 'k' for SUBEXP or the code lengths for huffman.
+ *
+ * Returns the best codec to use.
+ */
+enum cram_encoding cram_stats_encoding(cram_fd *fd, cram_stats *st);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/ext/htslib/cram/cram_structs.h b/ext/htslib/cram/cram_structs.h
new file mode 100644
index 0000000..9540b56
--- /dev/null
+++ b/ext/htslib/cram/cram_structs.h
@@ -0,0 +1,978 @@
+/*
+Copyright (c) 2012-2016, 2018-2020, 2023 Genome Research Ltd.
+Author: James Bonfield <jkb@sanger.ac.uk>
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+Institute nor the names of its contributors may be used to endorse or promote
+products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#ifndef HTSLIB_CRAM_STRUCTS_H
+#define HTSLIB_CRAM_STRUCTS_H
+
+/*
+ * Defines in-memory structs for the basic file-format objects in the
+ * CRAM format.
+ *
+ * The basic file format is:
+ *     File-def SAM-hdr Container Container ...
+ *
+ * Container:
+ *     Service-block data-block data-block ...
+ *
+ * Multiple blocks in a container are grouped together as slices,
+ * also sometimes referred to as landmarks in the spec.
+ */
+
+
+#include <pthread.h>
+#include <stdint.h>
+#include <sys/types.h>
+
+#include "../htslib/thread_pool.h"
+#include "../htslib/cram.h"
+#include "string_alloc.h"
+#include "mFILE.h"
+#include "../htslib/khash.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Generic hash-map integer -> integer
+KHASH_MAP_INIT_INT64(m_i2i, int)
+
+// Generic hash-set integer -> (existence)
+KHASH_SET_INIT_INT(s_i2i)
+
+// For brevity
+typedef unsigned char uc;
+
+/*
+ * A union for the preservation map. Required for khash.
+ */
+typedef union {
+    int i;
+    char *p;
+} pmap_t;
+
+// Generates static functions here which isn't ideal, but we have no way
+// currently to declare the kh_map_t structure here without also declaring a
+// duplicate in the .c files due to the nature of the KHASH macros.
+KHASH_MAP_INIT_STR(map, pmap_t)
+
+struct hFILE;
+
+#define SEQS_PER_SLICE 10000
+#define BASES_PER_SLICE (SEQS_PER_SLICE*500)
+#define SLICE_PER_CNT  1
+
+#define CRAM_SUBST_MATRIX "CGTNGTANCATNGCANACGT"
+
+#define MAX_STAT_VAL 1024
+//#define MAX_STAT_VAL 16
+typedef struct cram_stats {
+    int freqs[MAX_STAT_VAL];
+    khash_t(m_i2i) *h;
+    int nsamp; // total number of values added
+    int nvals; // total number of unique values added
+    int64_t min_val, max_val;
+} cram_stats;
+
+/* NB: matches java impl, not the spec */
+enum cram_encoding {
+    E_NULL               = 0,
+    E_EXTERNAL           = 1,  // Only for BYTE type in CRAM 4
+    E_GOLOMB             = 2,  // Not in CRAM 4
+    E_HUFFMAN            = 3,  // Not in CRAM 4
+    E_BYTE_ARRAY_LEN     = 4,
+    E_BYTE_ARRAY_STOP    = 5,
+    E_BETA               = 6,  // Not in CRAM 4
+    E_SUBEXP             = 7,  // Not in CRAM 4
+    E_GOLOMB_RICE        = 8,  // Not in CRAM 4
+    E_GAMMA              = 9,  // Not in CRAM 4
+
+    // CRAM 4 specific codecs
+    E_VARINT_UNSIGNED    = 41, // Specialisation of EXTERNAL
+    E_VARINT_SIGNED      = 42, // Specialisation of EXTERNAL
+    E_CONST_BYTE         = 43, // Alternative to HUFFMAN with 1 symbol
+    E_CONST_INT          = 44, // Alternative to HUFFMAN with 1 symbol
+
+    // More experimental ideas, not documented in spec yet
+    E_XHUFFMAN           = 50, // To external block
+    E_XPACK              = 51, // Transform to sub-codec
+    E_XRLE               = 52, // Transform to sub-codec
+    E_XDELTA             = 53, // Transform to sub-codec
+
+    // Total number of codecs, not a real one.
+    E_NUM_CODECS,
+};
+
+enum cram_external_type {
+    E_INT                = 1,
+    E_LONG               = 2,
+    E_BYTE               = 3,
+    E_BYTE_ARRAY         = 4,
+    E_BYTE_ARRAY_BLOCK   = 5,
+    E_SINT               = 6, // signed INT
+    E_SLONG              = 7, // signed LONG
+};
+
+/* External IDs used by this implementation (only assumed during writing) */
+enum cram_DS_ID {
+    DS_CORE   = 0,
+    DS_aux    = 1, // aux_blk
+    DS_aux_OQ = 2,
+    DS_aux_BQ = 3,
+    DS_aux_BD = 4,
+    DS_aux_BI = 5,
+    DS_aux_FZ = 6, // also ZM:B
+    DS_aux_oq = 7, // other qualities
+    DS_aux_os = 8, // other sequences
+    DS_aux_oz = 9, // other strings
+    DS_ref,
+    DS_RN, // name_blk
+    DS_QS, // qual_blk
+    DS_IN, // base_blk
+    DS_SC, // soft_blk
+
+    DS_BF, // start loop
+    DS_CF,
+    DS_AP,
+    DS_RG,
+    DS_MQ,
+    DS_NS,
+    DS_MF,
+    DS_TS,
+    DS_NP,
+    DS_NF,
+    DS_RL,
+    DS_FN,
+    DS_FC,
+    DS_FP,
+    DS_DL,
+    DS_BA,
+    DS_BS,
+    DS_TL,
+    DS_RI,
+    DS_RS,
+    DS_PD,
+    DS_HC,
+    DS_BB,
+    DS_QQ,
+
+    DS_TN, // end loop
+
+    DS_RN_len,
+    DS_SC_len,
+    DS_BB_len,
+    DS_QQ_len,
+
+    DS_TC, // CRAM v1.0 tags
+    DS_TM, // test
+    DS_TV, // test
+
+    DS_END,
+};
+
+/* "File Definition Structure" */
+struct cram_file_def {
+    char    magic[4];
+    uint8_t major_version;
+    uint8_t minor_version;
+    char    file_id[20] HTS_NONSTRING; // Filename or SHA1 checksum
+};
+
+#define CRAM_MAJOR_VERS(v) ((v) >> 8)
+#define CRAM_MINOR_VERS(v) ((v) & 0xff)
+
+struct cram_slice;
+
+// Internal version of htslib/cram.h enum.
+// Note these have to match the laout of methmap and methcost in
+// cram_io.c:cram_compress_block2
+enum cram_block_method_int {
+    // Public methods as defined in the CRAM spec.
+    BM_ERROR = -1,
+
+    // CRAM 2.x and 3.0
+    RAW      = 0,
+    GZIP     = 1,
+    BZIP2    = 2,
+    LZMA     = 3,
+    RANS     = 4, RANS0 = RANS,
+
+    // CRAM 3.1 onwards
+    RANSPR   = 5, RANS_PR0  = RANSPR,
+    ARITH    = 6, ARITH_PR0 = ARITH,
+    FQZ      = 7,
+    TOK3     = 8,
+    // BSC = 9, ZSTD = 10
+
+    // Methods not externalised, but used in metrics.
+    // Externally they become one of the above methods.
+    GZIP_RLE = 11,
+    GZIP_1,      // Z_DEFAULT_STRATEGY level 1, NB: not externalised in CRAM
+
+    FQZ_b, FQZ_c, FQZ_d, // Various preset FQZ methods
+
+  //RANS0,       // Order 0
+    RANS1,
+
+  //RANS_PR0,    // Order 0
+    RANS_PR1,    // Order 1
+    RANS_PR64,   // O0 + RLE
+    RANS_PR9,    // O1 + X4
+    RANS_PR128,  // O0 + Pack
+    RANS_PR129,  // O1 + Pack
+    RANS_PR192,  // O0 + RLE + pack
+    RANS_PR193,  // O1 + RLE + pack
+
+  //TOK3,   // tok+rans
+    TOKA,   // tok+arith
+
+  //ARITH_PR0,   // Order 0
+    ARITH_PR1,   // Order 1
+    ARITH_PR64,  // O0 + RLE
+    ARITH_PR9,   // O1 + X4
+    ARITH_PR128, // O0 + Pack
+    ARITH_PR129, // O1 + Pack
+    ARITH_PR192, // O0 + RLE + pack
+    ARITH_PR193, // O1 + RLE + pack
+
+    // NB: must end on no more than 31 unless we change to a
+    // 64-bit method type.
+};
+
+/* Now in htslib/cram.h
+enum cram_content_type {
+    CT_ERROR           = -1,
+    FILE_HEADER        = 0,
+    COMPRESSION_HEADER = 1,
+    MAPPED_SLICE       = 2,
+    UNMAPPED_SLICE     = 3, // CRAM V1.0 only
+    EXTERNAL           = 4,
+    CORE               = 5,
+};
+*/
+
+/* Maximum simultaneous codecs allowed, 1 per bit */
+#define CRAM_MAX_METHOD 32
+
+/* Compression metrics */
+struct cram_metrics {
+    // number of trials and time to next trial
+    int trial;
+    int next_trial;
+    int consistency;
+
+    // aggregate sizes during trials
+    int sz[CRAM_MAX_METHOD];
+    int input_avg_sz, input_avg_delta;
+
+    // resultant method from trials
+    int method, revised_method;
+    int strat;
+
+    // Revisions of method, to allow culling of continually failing ones.
+    int cnt[CRAM_MAX_METHOD];
+
+    double extra[CRAM_MAX_METHOD];
+
+    // Not amenable to rANS bit-packing techniques; cardinality > 16
+    int unpackable;
+};
+
+// Hash aux key (XX:i) to cram_metrics
+KHASH_MAP_INIT_INT(m_metrics, cram_metrics*)
+
+
+/* Block */
+struct cram_block {
+    enum cram_block_method_int  method, orig_method;
+    enum cram_content_type  content_type;
+    int32_t  content_id;
+    int32_t  comp_size;
+    int32_t  uncomp_size;
+    uint32_t crc32;
+    int32_t  idx; /* offset into data */
+    unsigned char    *data;
+
+    // For bit I/O
+    size_t alloc;
+    size_t byte;
+    int bit;
+
+    // To aid compression
+    cram_metrics *m; // used to track aux block compression only
+
+    int crc32_checked;
+    uint32_t crc_part;
+};
+
+struct cram_codec; /* defined in cram_codecs.h */
+struct cram_map;
+
+#define CRAM_MAP_HASH 32
+#define CRAM_MAP(a,b) (((a)*3+(b))&(CRAM_MAP_HASH-1))
+
+/* Compression header block */
+struct cram_block_compression_hdr {
+    int32_t ref_seq_id;
+    int64_t ref_seq_start;
+    int64_t ref_seq_span;
+    int32_t num_records;
+    int32_t num_landmarks;
+    int32_t *landmark;
+
+    /* Flags from preservation map */
+    int read_names_included;
+    int AP_delta;
+    // indexed by ref-base and subst. code
+    char substitution_matrix[5][4];
+    int no_ref;
+    int qs_seq_orient; // 1 => same as seq. 0 => original orientation
+
+    // TD Dictionary as a concatenated block
+    cram_block *TD_blk;          // Tag Dictionary
+    int nTL;                     // number of TL entries in TD
+    unsigned char **TL;          // array of size nTL, pointer into TD_blk.
+    khash_t(m_s2i) *TD_hash;     // Keyed on TD strings, map to TL[] indices
+    string_alloc_t *TD_keys;     // Pooled keys for TD hash.
+
+    khash_t(map) *preservation_map;
+    struct cram_map *rec_encoding_map[CRAM_MAP_HASH];
+    struct cram_map *tag_encoding_map[CRAM_MAP_HASH];
+
+    struct cram_codec *codecs[DS_END];
+
+    char *uncomp; // A single block of uncompressed data
+    size_t uncomp_size, uncomp_alloc;
+
+    // Total codec count, used for index to block_by_id for transforms
+    int ncodecs;
+};
+
+typedef struct cram_map {
+    int key;    /* 0xe0 + 3 bytes */
+    enum cram_encoding encoding;
+    int offset; /* Offset into a single block of memory */
+    int size;   /* Size */
+    struct cram_codec *codec;
+    struct cram_map *next; // for noddy internal hash
+} cram_map;
+
+typedef struct cram_tag_map {
+    struct cram_codec *codec;
+    cram_block *blk;
+    cram_block *blk2;
+    cram_metrics *m;
+} cram_tag_map;
+
+// Hash aux key (XX:i) to cram_tag_map
+KHASH_MAP_INIT_INT(m_tagmap, cram_tag_map*)
+
+/* Mapped or unmapped slice header block */
+struct cram_block_slice_hdr {
+    enum cram_content_type content_type;
+    int32_t ref_seq_id;     /* if content_type == MAPPED_SLICE */
+    int64_t ref_seq_start;  /* if content_type == MAPPED_SLICE */
+    int64_t ref_seq_span;   /* if content_type == MAPPED_SLICE */
+    int32_t num_records;
+    int64_t record_counter;
+    int32_t num_blocks;
+    int32_t num_content_ids;
+    int32_t *block_content_ids;
+    int32_t ref_base_id;    /* if content_type == MAPPED_SLICE */
+    unsigned char md5[16];
+};
+
+struct ref_entry;
+
+/*
+ * Container.
+ *
+ * Conceptually a container is split into slices, and slices into blocks.
+ * However on disk it's just a list of blocks and we need to query the
+ * block types to identify the start/end points of the slices.
+ *
+ * OR... are landmarks the start/end points of slices?
+ */
+struct cram_container {
+    int32_t  length;
+    int32_t  ref_seq_id;
+    int64_t  ref_seq_start;
+    int64_t  ref_seq_span;
+    int64_t  record_counter;
+    int64_t  num_bases;
+    int32_t  num_records;
+    int32_t  num_blocks;
+    int32_t  num_landmarks;
+    int32_t *landmark;
+
+    /* Size of container header above */
+    size_t   offset;
+
+    /* Compression header is always the first block? */
+    cram_block_compression_hdr *comp_hdr;
+    cram_block *comp_hdr_block;
+
+    /* For construction purposes */
+    int max_slice, curr_slice;   // maximum number of slices
+    int curr_slice_mt;           // Curr_slice when reading ahead (via threads)
+    int max_rec, curr_rec;       // current and max recs per slice
+    int max_c_rec, curr_c_rec;   // current and max recs per container
+    int slice_rec;               // rec no. for start of this slice
+    int curr_ref;                // current ref ID. -2 for no previous
+    int64_t last_pos;                // last record position
+    struct cram_slice **slices, *slice;
+    int pos_sorted;              // boolean, 1=>position sorted data
+    int64_t max_apos;                // maximum position, used if pos_sorted==0
+    int last_slice;              // number of reads in last slice (0 for 1st)
+    int multi_seq;               // true if packing multi seqs per cont/slice
+    int unsorted;                // true is AP_delta is 0.
+    int qs_seq_orient;           // 1 => same as seq. 0 => original orientation
+
+    /* Copied from fd before encoding, to allow multi-threading */
+    int ref_id;
+    hts_pos_t ref_start, first_base, last_base, ref_end;
+    char *ref;
+    int embed_ref;               // 1 if embedding ref, 2 if embedding cons
+    int no_ref;                  // true if referenceless
+    //struct ref_entry *ref;
+
+    /* For multi-threading */
+    bam_seq_t **bams;
+
+    /* Statistics for encoding */
+    cram_stats *stats[DS_END];
+
+    khash_t(m_tagmap) *tags_used; // set of tag types in use, for tag encoding map
+    int *refs_used;       // array of frequency of ref seq IDs
+
+    uint32_t crc32;       // CRC32
+
+    uint64_t s_num_bases; // number of bases in this slice
+    uint64_t s_aux_bytes; // number of bytes of aux in BAM
+
+    uint32_t n_mapped;    // Number of mapped reads
+    int ref_free;         // whether 'ref' is owned by us and must be freed.
+};
+
+/*
+ * A single cram record
+ */
+typedef struct cram_record {
+    struct cram_slice *s; // Filled out by cram_decode only
+
+    int32_t ref_id;       // fixed for all recs in slice?
+    int32_t flags;        // BF
+    int32_t cram_flags;   // CF
+    int32_t len;          // RL
+    int64_t apos;         // AP
+    int32_t rg;           // RG
+    int32_t name;         // RN; idx to s->names_blk
+    int32_t name_len;
+    int32_t mate_line;    // index to another cram_record
+    int32_t mate_ref_id;
+    int64_t mate_pos;     // NP
+    int64_t tlen;         // TS
+    int64_t explicit_tlen;// TS, but PNEXT/RNEXT still need auto-computing
+
+    // Auxiliary data
+    int32_t ntags;        // TC
+    uint32_t aux;         // idx to s->aux_blk
+    uint32_t aux_size;    // total size of packed ntags in aux_blk
+#ifndef TN_external
+    int32_t TN_idx;       // TN; idx to s->TN;
+#else
+    int32_t tn;           // idx to s->tn_blk
+#endif
+    int     TL;
+
+    uint32_t seq;         // idx to s->seqs_blk
+    uint32_t qual;        // idx to s->qual_blk
+    uint32_t cigar;       // idx to s->cigar
+    int32_t ncigar;
+    int64_t aend;         // alignment end
+    int32_t mqual;        // MQ
+
+    uint32_t feature;     // idx to s->feature
+    uint32_t nfeature;    // number of features
+    int32_t mate_flags;   // MF
+} cram_record;
+
+// Accessor macros as an analogue of the bam ones
+#define cram_qname(c)    (&(c)->s->name_blk->data[(c)->name])
+#define cram_seq(c)      (&(c)->s->seqs_blk->data[(c)->seq])
+#define cram_qual(c)     (&(c)->s->qual_blk->data[(c)->qual])
+#define cram_aux(c)      (&(c)->s->aux_blk->data[(c)->aux])
+#define cram_seqi(c,i)   (cram_seq((c))[(i)])
+#define cram_name_len(c) ((c)->name_len)
+#define cram_strand(c)   (((c)->flags & BAM_FREVERSE) != 0)
+#define cram_mstrand(c)  (((c)->flags & BAM_FMREVERSE) != 0)
+#define cram_cigar(c)    (&((cr)->s->cigar)[(c)->cigar])
+
+/*
+ * A feature is a base difference, used for the sequence reference encoding.
+ * (We generate these internally when writing CRAM.)
+ */
+typedef union cram_feature {
+    struct {
+        int pos;
+        int code;
+        int base;    // substitution code
+    } X;
+    struct {
+        int pos;
+        int code;
+        int base;    // actual base & qual
+        int qual;
+    } B;
+    struct {
+        int pos;
+        int code;
+        int seq_idx; // index to s->seqs_blk
+        int len;
+    } b;
+    struct {
+        int pos;
+        int code;
+        int qual;
+    } Q;
+    struct {
+        int pos;
+        int code;
+        int len;
+        int seq_idx; // soft-clip multiple bases
+    } S;
+    struct {
+        int pos;
+        int code;
+        int len;
+        int seq_idx; // insertion multiple bases
+    } I;
+    struct {
+        int pos;
+        int code;
+        int base; // insertion single base
+    } i;
+    struct {
+        int pos;
+        int code;
+        int len;
+    } D;
+    struct {
+        int pos;
+        int code;
+        int len;
+    } N;
+    struct {
+        int pos;
+        int code;
+        int len;
+    } P;
+    struct {
+        int pos;
+        int code;
+        int len;
+    } H;
+} cram_feature;
+
+/*
+ * A slice is really just a set of blocks, but it
+ * is the logical unit for decoding a number of
+ * sequences.
+ */
+struct cram_slice {
+    cram_block_slice_hdr *hdr;
+    cram_block *hdr_block;
+    cram_block **block;
+    cram_block **block_by_id;
+
+    /* State used during encoding/decoding */
+    int64_t last_apos, max_apos;
+
+    /* Array of decoded cram records */
+    cram_record *crecs;
+
+    /* An dynamically growing buffers for data pointed
+     * to by crecs[] array.
+     */
+    uint32_t  *cigar;
+    uint32_t   cigar_alloc;
+    uint32_t   ncigar;
+
+    cram_feature *features;
+    uint32_t      nfeatures;
+    uint32_t      afeatures; // allocated size of features
+
+#ifndef TN_external
+    // TN field (Tag Name)
+    uint32_t      *TN;
+    int           nTN, aTN;  // used and allocated size for TN[]
+#else
+    cram_block *tn_blk;
+    int tn_id;
+#endif
+
+    // For variable sized elements which are always external blocks.
+    cram_block *name_blk;
+    cram_block *seqs_blk;
+    cram_block *qual_blk;
+    cram_block *base_blk;
+    cram_block *soft_blk;
+    cram_block *aux_blk;       // BAM aux block, created while decoding CRAM
+
+    string_alloc_t *pair_keys; // Pooled keys for pair hash.
+    khash_t(m_s2i) *pair[2];   // for identifying read-pairs in this slice.
+
+    char *ref;                 // slice of current reference
+    hts_pos_t ref_start;       // start position of current reference;
+    hts_pos_t ref_end;         // end position of current reference;
+    int ref_id;
+
+    // For going from BAM to CRAM; an array of auxiliary blocks per type
+    int naux_block;
+    cram_block **aux_block;
+
+    unsigned int data_series; // See cram_fields enum
+    int decode_md;
+
+    int max_rec, curr_rec;       // current and max recs per slice
+    int slice_num;               // To be copied into c->curr_slice in decode
+};
+
+/*-----------------------------------------------------------------------------
+ * Consider moving reference handling to cram_refs.[ch]
+ */
+// from fa.fai / samtools faidx files
+typedef struct ref_entry {
+    char *name;
+    char *fn;
+    int64_t length;
+    int64_t offset;
+    int bases_per_line;
+    int line_length;
+    int64_t count;         // for shared references so we know to dealloc seq
+    char *seq;
+    mFILE *mf;
+    int is_md5;            // Reference comes from a raw seq found by MD5
+    int validated_md5;
+} ref_entry;
+
+KHASH_MAP_INIT_STR(refs, ref_entry*)
+
+// References structure.
+struct refs_t {
+    string_alloc_t *pool;  // String pool for holding filenames and SN vals
+
+    khash_t(refs) *h_meta; // ref_entry*, index by name
+    ref_entry **ref_id;    // ref_entry*, index by ID
+    int nref;              // number of ref_entry
+
+    char *fn;              // current file opened
+    BGZF *fp;              // and the hFILE* to go with it.
+
+    int count;             // how many cram_fd sharing this refs struct
+
+    pthread_mutex_t lock;  // Mutex for multi-threaded updating
+    ref_entry *last;       // Last queried sequence
+    int last_id;           // Used in cram_ref_decr_locked to delay free
+};
+
+/*-----------------------------------------------------------------------------
+ * CRAM index
+ *
+ * Detect format by number of entries per line.
+ * 5 => 1.0 (refid, start, nseq, C offset, slice)
+ * 6 => 1.1 (refid, start, span, C offset, S offset, S size)
+ *
+ * Indices are stored in a nested containment list, which is trivial to set
+ * up as the indices are on sorted data so we're appending to the nclist
+ * in sorted order. Basically if a slice entirely fits within a previous
+ * slice then we append to that slices list. This is done recursively.
+ *
+ * Lists are sorted on two dimensions: ref id + slice coords.
+ */
+typedef struct cram_index {
+    int nslice, nalloc;   // total number of slices
+    struct cram_index *e; // array of size nslice
+
+    int     refid;  // 1.0                 1.1
+    int     start;  // 1.0                 1.1
+    int     end;    //                     1.1
+    int     nseq;   // 1.0 - undocumented
+    int     slice;  // 1.0 landmark index, 1.1 landmark value
+    int     len;    //                     1.1 - size of slice in bytes
+    int64_t offset; // 1.0                 1.1
+
+    // Linked list of cram_index entries. Used to convert recursive
+    // NCList back to a linear list.
+    struct cram_index *e_next;
+} cram_index;
+
+typedef struct {
+    int refid;
+    int64_t start;
+    int64_t end;
+} cram_range;
+
+/*-----------------------------------------------------------------------------
+ */
+/* CRAM File handle */
+
+typedef struct spare_bams {
+    bam_seq_t **bams;
+    struct spare_bams *next;
+} spare_bams;
+
+struct cram_fd;
+typedef struct varint_vec {
+    // Returns number of bytes decoded from fd, 0 on error
+    int (*varint_decode32_crc)(struct cram_fd *fd, int32_t *val_p, uint32_t *crc);
+    int (*varint_decode32s_crc)(struct cram_fd *fd, int32_t *val_p, uint32_t *crc);
+    int (*varint_decode64_crc)(struct cram_fd *fd, int64_t *val_p, uint32_t *crc);
+
+    // Returns the value and increments *cp.  Sets err to 1 iff an error occurs.
+    // NOTE: Does not set err to 0 on success.
+    int64_t (*varint_get32) (char **cp, const char *endp, int *err);
+    int64_t (*varint_get32s)(char **cp, const char *endp, int *err);
+    int64_t (*varint_get64) (char **cp, const char *endp, int *err);
+    int64_t (*varint_get64s)(char **cp, const char *endp, int *err);
+
+    // Returns the number of bytes written, <= 0 on error.
+    int (*varint_put32) (char *cp, char *endp, int32_t val_p);
+    int (*varint_put32s)(char *cp, char *endp, int32_t val_p);
+    int (*varint_put64) (char *cp, char *endp, int64_t val_p);
+    int (*varint_put64s)(char *cp, char *endp, int64_t val_p);
+
+    // Returns the number of bytes written, <= 0 on error.
+    int (*varint_put32_blk) (cram_block *blk, int32_t val_p);
+    int (*varint_put32s_blk)(cram_block *blk, int32_t val_p);
+    int (*varint_put64_blk) (cram_block *blk, int64_t val_p);
+    int (*varint_put64s_blk)(cram_block *blk, int64_t val_p);
+
+    // Returns number of bytes needed to encode 'val'
+    int (*varint_size)(int64_t val);
+} varint_vec;
+
+struct cram_fd {
+    struct hFILE  *fp;
+    int            mode;     // 'r' or 'w'
+    int            version;
+    cram_file_def *file_def;
+    sam_hdr_t     *header;
+
+    char          *prefix;
+    int64_t        record_counter;
+    int            err;
+
+    // Most recent compression header decoded
+    //cram_block_compression_hdr *comp_hdr;
+    //cram_block_slice_hdr       *slice_hdr;
+
+    // Current container being processed
+    cram_container *ctr;
+
+    // Current container used for decoder threads
+    cram_container *ctr_mt;
+
+    // positions for encoding or decoding
+    int first_base, last_base; // copied to container
+
+    // cached reference portion
+    refs_t   *refs;                // ref meta-data structure
+    char     *ref, *ref_free;      // current portion held in memory
+    int       ref_id;              // copied to container
+    hts_pos_t ref_start;           // copied to container
+    hts_pos_t ref_end;             // copied to container
+    char     *ref_fn;              // reference fasta filename
+
+    // compression level and metrics
+    int level;
+    cram_metrics *m[DS_END];
+    khash_t(m_metrics) *tags_used; // cram_metrics[], per tag types in use.
+
+    // options
+    int decode_md; // Whether to export MD and NM tags
+    int seqs_per_slice;
+    int bases_per_slice;
+    int slices_per_container;
+    int embed_ref; // copied to container
+    int no_ref;    // copied to container
+    int no_ref_counter; // decide if permanent switch
+    int ignore_md5;
+    int use_bz2;
+    int use_rans;
+    int use_lzma;
+    int use_fqz;
+    int use_tok;
+    int use_arith;
+    int shared_ref;
+    unsigned int required_fields;
+    int store_md;
+    int store_nm;
+    cram_range range;
+
+    // lookup tables, stored here so we can be trivially multi-threaded
+    unsigned int bam_flag_swap[0x1000]; // cram -> bam flags
+    unsigned int cram_flag_swap[0x1000];// bam -> cram flags
+    unsigned char L1[256];              // ACGT{*} ->0123{4}
+    unsigned char L2[256];              // ACGTN{*}->01234{5}
+    char cram_sub_matrix[32][32];       // base substitution codes
+
+    int         index_sz;
+    cram_index *index;                  // array, sizeof index_sz
+    off_t first_container;
+    off_t curr_position;
+    int eof;
+    int last_slice;                     // number of recs encoded in last slice
+    int last_RI_count;                  // number of references encoded in last container
+    int multi_seq;                      // -1 is auto, 0 is one ref per container, 1 is multi...
+    int multi_seq_user;                 // Original user setting (CRAM_OPT_MULTI_SEQ_PER_SLICE)
+    int unsorted;
+    int last_mapped;                    // number of mapped reads in last container
+    int empty_container;                // Marker for EOF block
+
+    // thread pool
+    int own_pool;
+    hts_tpool *pool;
+    hts_tpool_process *rqueue;
+    pthread_mutex_t metrics_lock;
+    pthread_mutex_t ref_lock;
+    pthread_mutex_t range_lock;
+    spare_bams *bl;
+    pthread_mutex_t bam_list_lock;
+    void *job_pending;
+    int ooc;                            // out of containers.
+
+    int lossy_read_names;               // boolean
+    int tlen_approx;                    // max TLEN calculation offset.
+    int tlen_zero;                      // If true, permit tlen 0 (=> tlen calculated)
+
+    BGZF *idxfp;                        // File pointer for on-the-fly index creation
+
+    // variable integer decoding callbacks.
+    // This changed in CRAM4.0 to a data-size agnostic encoding.
+    varint_vec vv;
+
+    // Force AP delta even on non positional sorted data.
+    // This can be beneficial for pairs where pairs are nearby each other.
+    // We suffer with delta to unrelated things (previous pair), but gain
+    // in delta between them.  (Ideal would be a per read setting.)
+    int ap_delta;
+};
+
+// Translation of required fields to cram data series
+enum cram_fields {
+    CRAM_BF = 0x00000001,
+    CRAM_AP = 0x00000002,
+    CRAM_FP = 0x00000004,
+    CRAM_RL = 0x00000008,
+    CRAM_DL = 0x00000010,
+    CRAM_NF = 0x00000020,
+    CRAM_BA = 0x00000040,
+    CRAM_QS = 0x00000080,
+    CRAM_FC = 0x00000100,
+    CRAM_FN = 0x00000200,
+    CRAM_BS = 0x00000400,
+    CRAM_IN = 0x00000800,
+    CRAM_RG = 0x00001000,
+    CRAM_MQ = 0x00002000,
+    CRAM_TL = 0x00004000,
+    CRAM_RN = 0x00008000,
+    CRAM_NS = 0x00010000,
+    CRAM_NP = 0x00020000,
+    CRAM_TS = 0x00040000,
+    CRAM_MF = 0x00080000,
+    CRAM_CF = 0x00100000,
+    CRAM_RI = 0x00200000,
+    CRAM_RS = 0x00400000,
+    CRAM_PD = 0x00800000,
+    CRAM_HC = 0x01000000,
+    CRAM_SC = 0x02000000,
+    CRAM_BB = 0x04000000,
+    CRAM_BB_len = 0x08000000,
+    CRAM_QQ = 0x10000000,
+    CRAM_QQ_len = 0x20000000,
+    CRAM_aux= 0x40000000,
+    CRAM_ALL= 0x7fffffff,
+};
+
+// A CIGAR opcode, but not necessarily the implications of it. Eg FC/FP may
+// encode a base difference, but we don't need to know what it is for CIGAR.
+// If we have a soft-clip or insertion, we do need SC/IN though to know how
+// long that array is.
+#define CRAM_CIGAR (CRAM_FN | CRAM_FP | CRAM_FC | CRAM_DL | CRAM_IN | \
+                    CRAM_SC | CRAM_HC | CRAM_PD | CRAM_RS | CRAM_RL | CRAM_BF)
+
+#define CRAM_SEQ (CRAM_CIGAR | CRAM_BA | CRAM_BS | \
+                  CRAM_RL    | CRAM_AP | CRAM_BB)
+
+#define CRAM_QUAL (CRAM_CIGAR | CRAM_RL | CRAM_AP | CRAM_QS | CRAM_QQ)
+
+/* BF bitfields */
+/* Corrected in 1.1. Use bam_flag_swap[bf] and BAM_* macros for 1.0 & 1.1 */
+#define CRAM_FPAIRED      256
+#define CRAM_FPROPER_PAIR 128
+#define CRAM_FUNMAP        64
+#define CRAM_FREVERSE      32
+#define CRAM_FREAD1        16
+#define CRAM_FREAD2         8
+#define CRAM_FSECONDARY     4
+#define CRAM_FQCFAIL        2
+#define CRAM_FDUP           1
+
+#define DS_aux_S "\001"
+#define DS_aux_OQ_S "\002"
+#define DS_aux_BQ_S "\003"
+#define DS_aux_BD_S "\004"
+#define DS_aux_BI_S "\005"
+#define DS_aux_FZ_S "\006"
+#define DS_aux_oq_S "\007"
+#define DS_aux_os_S "\010"
+#define DS_aux_oz_S "\011"
+
+#define CRAM_M_REVERSE  1
+#define CRAM_M_UNMAP    2
+
+
+/* CF bitfields */
+#define CRAM_FLAG_PRESERVE_QUAL_SCORES (1<<0)
+#define CRAM_FLAG_DETACHED             (1<<1)
+#define CRAM_FLAG_MATE_DOWNSTREAM      (1<<2)
+#define CRAM_FLAG_NO_SEQ               (1<<3)
+#define CRAM_FLAG_EXPLICIT_TLEN        (1<<4)
+#define CRAM_FLAG_MASK                 ((1<<5)-1)
+
+/* Internal only */
+#define CRAM_FLAG_STATS_ADDED          (1<<30)
+#define CRAM_FLAG_DISCARD_NAME         (1U<<31)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* HTSLIB_CRAM_STRUCTS_H */
diff --git a/ext/htslib/cram/mFILE.c b/ext/htslib/cram/mFILE.c
new file mode 100644
index 0000000..3ecdca3
--- /dev/null
+++ b/ext/htslib/cram/mFILE.c
@@ -0,0 +1,668 @@
+/*
+Copyright (c) 2005-2006, 2008-2009, 2013, 2015, 2017-2019 Genome Research Ltd.
+Author: James Bonfield <jkb@sanger.ac.uk>
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+Institute nor the names of its contributors may be used to endorse or promote
+products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <stdarg.h>
+
+#include "../htslib/hts_log.h"
+#include "os.h"
+#include "mFILE.h"
+
+#ifdef HAVE_MMAP
+#include <sys/mman.h>
+#endif
+
+/*
+ * This file contains memory-based versions of the most commonly used
+ * (by io_lib) stdio functions.
+ *
+ * Actual file IO takes place either on opening or closing an mFILE.
+ *
+ * Coupled to this are a bunch of rather scary macros which can be obtained
+ * by including stdio_hack.h. It is recommended though that you use mFILE.h
+ * instead and replace fopen with mfopen (etc). This is more or less
+ * mandatory if you wish to use both FILE and mFILE structs in a single file.
+ */
+
+static mFILE *m_channel[3];  /* stdin, stdout and stderr fakes */
+
+/*
+ * Reads the entirety of fp into memory. If 'fn' exists it is the filename
+ * associated with fp. This will be used for more optimal reading (via a
+ * stat to identify the size and a single read). Otherwise we use successive
+ * reads until EOF.
+ *
+ * Returns a malloced buffer on success of length *size
+ *         NULL on failure
+ */
+static char *mfload(FILE *fp, const char *fn, size_t *size, int binary) {
+    struct stat sb;
+    char *data = NULL;
+    size_t allocated = 0, used = 0;
+    int bufsize = 8192;
+
+#ifdef _WIN32
+    if (binary)
+        _setmode(_fileno(fp), _O_BINARY);
+    else
+        _setmode(_fileno(fp), _O_TEXT);
+#endif
+
+    if (fn && -1 != stat(fn, &sb)) {
+        data = malloc(allocated = sb.st_size);
+        if (!data)
+            return NULL;
+        bufsize = sb.st_size;
+    } else {
+        fn = NULL;
+    }
+
+    do {
+        size_t len;
+        if (used + bufsize > allocated) {
+            allocated += bufsize;
+            char *datan = realloc(data, allocated);
+            if (datan) {
+                data = datan;
+            } else {
+                free(data);
+                return NULL;
+            }
+        }
+        len = fread(data + used, 1, allocated - used, fp);
+        if (len > 0)
+            used += len;
+    } while (!feof(fp) && (fn == NULL || used < sb.st_size));
+
+    *size = used;
+
+    return data;
+}
+
+
+#ifdef HAVE_MMAP
+/*
+ * mmaps in the file, but only for reading currently.
+ *
+ * Returns 0 on success
+ *        -1 on failure
+ */
+int mfmmap(mFILE *mf, FILE *fp, const char *fn) {
+    struct stat sb;
+
+    if (stat(fn, &sb) != 0)
+        return -1;
+
+    mf->size = sb.st_size;
+    mf->data = mmap(NULL, mf->size, PROT_READ, MAP_SHARED,
+                    fileno(fp), 0);
+
+    if (!mf->data || mf->data == (void *)-1)
+        return -1;
+
+    mf->alloced = 0;
+    return 0;
+}
+#endif
+
+
+/*
+ * Creates and returns m_channel[0].
+ * We initialise this on the first attempted read, which then slurps in
+ * all of stdin until EOF is met.
+ */
+mFILE *mstdin(void) {
+    if (m_channel[0])
+        return m_channel[0];
+
+    m_channel[0] = mfcreate(NULL, 0);
+    if (NULL == m_channel[0]) return NULL;
+    m_channel[0]->fp = stdin;
+    return m_channel[0];
+}
+
+static void init_mstdin(void) {
+    static int done_stdin = 0;
+    if (done_stdin)
+        return;
+
+    m_channel[0]->data = mfload(stdin, NULL, &m_channel[0]->size, 1);
+    m_channel[0]->mode = MF_READ;
+    done_stdin = 1;
+}
+
+/*
+ * Creates and returns m_channel[1]. This is the fake for stdout. It starts as
+ * an empty buffer which is physically written out only when mfflush or
+ * mfclose are called.
+ */
+mFILE *mstdout(void) {
+    if (m_channel[1])
+        return m_channel[1];
+
+    m_channel[1] = mfcreate(NULL, 0);
+    if (NULL == m_channel[1]) return NULL;
+    m_channel[1]->fp = stdout;
+    m_channel[1]->mode = MF_WRITE;
+    return m_channel[1];
+}
+
+/*
+ * Stderr as an mFILE.
+ * The code handles stderr by returning m_channel[2], but also checking
+ * for stderr in fprintf (the common usage of it) to auto-flush.
+ */
+mFILE *mstderr(void) {
+    if (m_channel[2])
+        return m_channel[2];
+
+    m_channel[2] = mfcreate(NULL, 0);
+    if (NULL == m_channel[2]) return NULL;
+    m_channel[2]->fp = stderr;
+    m_channel[2]->mode = MF_WRITE;
+    return m_channel[2];
+}
+
+
+/*
+ * For creating existing mFILE pointers directly from memory buffers.
+ */
+mFILE *mfcreate(char *data, int size) {
+    mFILE *mf = (mFILE *)malloc(sizeof(*mf));
+    if (NULL == mf) return NULL;
+    mf->fp = NULL;
+    mf->data = data;
+    mf->alloced = size;
+    mf->size = size;
+    mf->eof = 0;
+    mf->offset = 0;
+    mf->flush_pos = 0;
+    mf->mode = MF_READ | MF_WRITE;
+    return mf;
+}
+
+/*
+ * Recreate an existing mFILE to house new data/size.
+ * It also rewinds the file.
+ */
+void mfrecreate(mFILE *mf, char *data, int size) {
+    if (mf->data)
+        free(mf->data);
+    mf->data = data;
+    mf->size = size;
+    mf->alloced = size;
+    mf->eof = 0;
+    mf->offset = 0;
+    mf->flush_pos = 0;
+}
+
+
+/*
+ * Creates a new mFILE to contain the contents of the FILE pointer.
+ * This mFILE is purely for in-memory operations and has no links to the
+ * original FILE* it came from. It also doesn't close the FILE pointer.
+ * Consider using mfreopen() is you need different behaviour.
+ *
+ * Returns mFILE * on success
+ *         NULL on failure.
+ */
+mFILE *mfcreate_from(const char *path, const char *mode_str, FILE *fp) {
+    mFILE *mf;
+
+    /* Open using mfreopen() */
+    if (NULL == (mf = mfreopen(path, mode_str, fp)))
+        return NULL;
+
+    /* Disassociate from the input stream */
+    mf->fp = NULL;
+
+    return mf;
+}
+
+/*
+ * Converts a FILE * to an mFILE *.
+ * Use this for wrapper functions to turn external prototypes requiring
+ * FILE * as an argument into internal code using mFILE *.
+ */
+mFILE *mfreopen(const char *path, const char *mode_str, FILE *fp) {
+    mFILE *mf;
+    int r = 0, w = 0, a = 0, b = 0, x = 0, mode = 0;
+
+    /* Parse mode:
+     * r = read file contents (if truncated => don't read)
+     * w = write on close
+     * a = position at end of buffer
+     * x = position at same location as the original fp, don't seek on flush
+     * + = for update (read and write)
+     * m = mmap (read only)
+     */
+    if (strchr(mode_str, 'r'))
+        r = 1, mode |= MF_READ;
+    if (strchr(mode_str, 'w'))
+        w = 1, mode |= MF_WRITE | MF_TRUNC;
+    if (strchr(mode_str, 'a'))
+        w = a = 1, mode |= MF_WRITE | MF_APPEND;
+    if (strchr(mode_str, 'b'))
+        b = 1, mode |= MF_BINARY;
+    if (strchr(mode_str, 'x'))
+        x = 1;
+    if (strchr(mode_str, '+')) {
+        w = 1, mode |= MF_READ | MF_WRITE;
+        if (a)
+            r = 1;
+    }
+#ifdef HAVE_MMAP
+    if (strchr(mode_str, 'm'))
+        if (!w) mode |= MF_MMAP;
+#endif
+
+    if (r) {
+        mf = mfcreate(NULL, 0);
+        if (NULL == mf) return NULL;
+        if (!(mode & MF_TRUNC)) {
+#ifdef HAVE_MMAP
+            if (mode & MF_MMAP) {
+                if (mfmmap(mf, fp, path) == -1) {
+                    mf->data = NULL;
+                    mode &= ~MF_MMAP;
+                }
+            }
+#endif
+            if (!mf->data) {
+                mf->data = mfload(fp, path, &mf->size, b);
+                if (!mf->data) {
+                    free(mf);
+                    return NULL;
+                }
+                mf->alloced = mf->size;
+                if (!a)
+                    fseek(fp, 0, SEEK_SET);
+            }
+        }
+    } else if (w) {
+        /* Write - initialise the data structures */
+        mf = mfcreate(NULL, 0);
+        if (NULL == mf) return NULL;
+    } else {
+        hts_log_error("Must specify either r, w or a for mode");
+        return NULL;
+    }
+    mf->fp = fp;
+    mf->mode = mode;
+
+    if (x) {
+        mf->mode |= MF_MODEX;
+    }
+
+    if (a) {
+        mf->flush_pos = mf->size;
+        fseek(fp, 0, SEEK_END);
+    }
+
+    return mf;
+}
+
+/*
+ * Opens a file. If we have read access (r or a+) then it loads the entire
+ * file into memory. If We have write access then the pathname is stored.
+ * We do not actually write until an mfclose, which then checks this pathname.
+ */
+mFILE *mfopen(const char *path, const char *mode) {
+    FILE *fp;
+
+    if (NULL == (fp = fopen(path, mode)))
+        return NULL;
+    return mfreopen(path, mode, fp);
+}
+
+/*
+ * Closes an mFILE. If the filename is known (implying write access) then this
+ * also writes the data to disk.
+ *
+ * Stdout is handled by calling mfflush which writes to stdout if appropriate.
+ */
+int mfclose(mFILE *mf) {
+    if (!mf)
+        return -1;
+
+    mfflush(mf);
+
+#ifdef HAVE_MMAP
+    if ((mf->mode & MF_MMAP) && mf->data) {
+        /* Mmaped */
+        munmap(mf->data, mf->size);
+        mf->data = NULL;
+    }
+#endif
+
+    if (mf->fp)
+        fclose(mf->fp);
+
+    mfdestroy(mf);
+
+    return 0;
+}
+
+/*
+ * Closes the file pointer contained within the mFILE without destroying
+ * the in-memory data.
+ *
+ * Attempting to do this on an mmaped buffer is an error.
+ */
+int mfdetach(mFILE *mf) {
+    if (!mf)
+        return -1;
+
+    mfflush(mf);
+    if (mf->mode & MF_MMAP)
+        return -1;
+
+    if (mf->fp) {
+        fclose(mf->fp);
+        mf->fp = NULL;
+    }
+
+    return 0;
+}
+
+/*
+ * Destroys an mFILE structure but does not flush or close it
+ */
+int mfdestroy(mFILE *mf) {
+    if (!mf)
+        return -1;
+
+    if (mf->data)
+        free(mf->data);
+    free(mf);
+
+    return 0;
+}
+
+/*
+ * Steals that data out of an mFILE.  The mFILE itself will be closed.
+ * It is up to the caller to free the stolen buffer.  If size_out is
+ * not NULL, mf->size will be stored in it.
+ * This is more-or-less the opposite of mfcreate().
+ *
+ * Note, we cannot steal the allocated buffer from an mmaped mFILE.
+ */
+
+void *mfsteal(mFILE *mf, size_t *size_out) {
+    void *data;
+
+    if (!mf) return NULL;
+
+    data = mf->data;
+
+    if (NULL != size_out) *size_out = mf->size;
+
+    if (mfdetach(mf) != 0)
+        return NULL;
+
+    mf->data = NULL;
+    mfdestroy(mf);
+
+    return data;
+}
+
+/*
+ * Seek/tell functions. Nothing more than updating and reporting an
+ * in-memory index. NB we can seek on stdin or stdout even provided we
+ * haven't been flushing.
+ */
+int mfseek(mFILE *mf, long offset, int whence) {
+    switch (whence) {
+    case SEEK_SET:
+        mf->offset = offset;
+        break;
+    case SEEK_CUR:
+        mf->offset += offset;
+        break;
+    case SEEK_END:
+        mf->offset = mf->size + offset;
+        break;
+    default:
+        errno = EINVAL;
+        return -1;
+    }
+
+    mf->eof = 0;
+    return 0;
+}
+
+long mftell(mFILE *mf) {
+    return mf->offset;
+}
+
+void mrewind(mFILE *mf) {
+    mf->offset = 0;
+    mf->eof = 0;
+}
+
+/*
+ * mftruncate is not directly a translation of ftruncate as the latter
+ * takes a file descriptor instead of a FILE *. It performs the analogous
+ * role though.
+ *
+ * If offset is -1 then the file is truncated to be the current file
+ * offset.
+ */
+void mftruncate(mFILE *mf, long offset) {
+    mf->size = offset != -1 ? offset : mf->offset;
+    if (mf->offset > mf->size)
+        mf->offset = mf->size;
+}
+
+int mfeof(mFILE *mf) {
+    return mf->eof;
+}
+
+/*
+ * mFILE read/write functions. Basically these turn fread/fwrite syntax
+ * into memcpy statements, with appropriate memory handling for writing.
+ */
+size_t mfread(void *ptr, size_t size, size_t nmemb, mFILE *mf) {
+    size_t len;
+    char *cptr = (char *)ptr;
+
+    if (mf == m_channel[0]) init_mstdin();
+
+    if (mf->size <= mf->offset)
+        return 0;
+
+    len = size * nmemb <= mf->size - mf->offset
+        ? size * nmemb
+        : mf->size - mf->offset;
+    if (!size)
+        return 0;
+
+    memcpy(cptr, &mf->data[mf->offset], len);
+    mf->offset += len;
+
+    if (len != size * nmemb) {
+        mf->eof = 1;
+    }
+
+    return len / size;
+}
+
+size_t mfwrite(void *ptr, size_t size, size_t nmemb, mFILE *mf) {
+    if (!(mf->mode & MF_WRITE))
+        return 0;
+
+    /* Append mode => forced all writes to end of file */
+    if (mf->mode & MF_APPEND)
+        mf->offset = mf->size;
+
+    /* Make sure we have enough room */
+    while (size * nmemb + mf->offset > mf->alloced) {
+        size_t new_alloced = mf->alloced ? mf->alloced * 2 : 1024;
+        void * new_data = realloc(mf->data, new_alloced);
+        if (NULL == new_data) return 0;
+        mf->alloced = new_alloced;
+        mf->data    = new_data;
+    }
+
+    /* Record where we need to reflush from */
+    if (mf->offset < mf->flush_pos)
+        mf->flush_pos = mf->offset;
+
+    /* Copy the data over */
+    memcpy(&mf->data[mf->offset], ptr, size * nmemb);
+    mf->offset += size * nmemb;
+    if (mf->size < mf->offset)
+        mf->size = mf->offset;
+
+    return nmemb;
+}
+
+int mfgetc(mFILE *mf) {
+    if (mf == m_channel[0]) init_mstdin();
+    if (mf->offset < mf->size) {
+        return (unsigned char)mf->data[mf->offset++];
+    }
+
+    mf->eof = 1;
+    return -1;
+}
+
+int mungetc(int c, mFILE *mf) {
+    if (mf->offset > 0) {
+        mf->data[--mf->offset] = c;
+        return c;
+    }
+
+    mf->eof = 1;
+    return -1;
+}
+
+char *mfgets(char *s, int size, mFILE *mf) {
+    int i;
+
+    if (mf == m_channel[0]) init_mstdin();
+    *s = 0;
+    for (i = 0; i < size-1;) {
+        if (mf->offset < mf->size) {
+            s[i] = mf->data[mf->offset++];
+            if (s[i++] == '\n')
+                break;
+        } else {
+            mf->eof = 1;
+            break;
+        }
+    }
+
+    s[i] = 0;
+    return i ? s : NULL;
+}
+
+/*
+ * Flushes an mFILE. If this is a real open of a file in write mode then
+ * mFILE->fp will be set. We then write out any new data in mFILE since the
+ * last flush. We cannot tell what may have been modified as we don't keep
+ * track of that, so we typically rewrite out the entire file contents between
+ * the last flush_pos and the end of file.
+ *
+ * For stderr/stdout we also reset the offsets so we cannot modify things
+ * we've already output.
+ */
+int mfflush(mFILE *mf) {
+    if (!mf->fp)
+        return 0;
+
+    /* FIXME: only do this when opened in write mode */
+    if (mf == m_channel[1] || mf == m_channel[2]) {
+        if (mf->flush_pos < mf->size) {
+            size_t bytes = mf->size - mf->flush_pos;
+            if (fwrite(mf->data + mf->flush_pos, 1, bytes, mf->fp) < bytes)
+                return -1;
+            if (0 != fflush(mf->fp))
+                return -1;
+        }
+
+        /* Stdout & stderr are non-seekable streams so throw away the data */
+        mf->offset = mf->size = mf->flush_pos = 0;
+    }
+
+    /* only flush when opened in write mode */
+    if (mf->mode & MF_WRITE) {
+        if (mf->flush_pos < mf->size) {
+            size_t bytes = mf->size - mf->flush_pos;
+            if (!(mf->mode & MF_MODEX)) {
+                fseek(mf->fp, mf->flush_pos, SEEK_SET);
+            }
+            if (fwrite(mf->data + mf->flush_pos, 1, bytes, mf->fp) < bytes)
+                return -1;
+            if (0 != fflush(mf->fp))
+                return -1;
+        }
+        if (ftell(mf->fp) != -1 &&
+            ftruncate(fileno(mf->fp), ftell(mf->fp)) == -1)
+            return -1;
+        mf->flush_pos = mf->size;
+    }
+
+    return 0;
+}
+
+/*
+ * Converts an mFILE from binary to ascii mode by replacing all
+ * cr-nl with nl.
+ *
+ * Primarily used on windows when we've uncompressed a binary file which
+ * happens to be a text file (eg Experiment File). Previously we would have
+ * seeked back to the start and used _setmode(fileno(fp), _O_TEXT).
+ *
+ * Side effect: resets offset and flush_pos back to the start.
+ */
+void mfascii(mFILE *mf) {
+    size_t p1, p2;
+
+    for (p1 = p2 = 1; p1 < mf->size; p1++, p2++) {
+        if (mf->data[p1] == '\n' && mf->data[p1-1] == '\r') {
+            p2--; /* delete the \r */
+        }
+        mf->data[p2] = mf->data[p1];
+    }
+    mf->size = p2;
+
+    mf->offset = mf->flush_pos = 0;
+}
diff --git a/ext/htslib/cram/mFILE.h b/ext/htslib/cram/mFILE.h
new file mode 100644
index 0000000..ca7062c
--- /dev/null
+++ b/ext/htslib/cram/mFILE.h
@@ -0,0 +1,93 @@
+/*
+Copyright (c) 2005-2006, 2008-2009, 2013, 2018 Genome Research Ltd.
+Author: James Bonfield <jkb@sanger.ac.uk>
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+Institute nor the names of its contributors may be used to endorse or promote
+products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#ifndef CRAM_MFILE_H
+#define CRAM_MFILE_H
+
+#include <stdio.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+    FILE *fp;
+    char *data;
+    size_t alloced;
+    int eof;
+    int mode; /* open mode in MF_?? define bit pattern */
+    size_t size;
+    size_t offset;
+    size_t flush_pos;
+} mFILE;
+
+// Work around a clash with winuser.h
+#ifdef MF_APPEND
+#  undef MF_APPEND
+#endif
+
+#define MF_READ    1
+#define MF_WRITE   2
+#define MF_APPEND  4
+#define MF_BINARY  8
+#define MF_TRUNC  16
+#define MF_MODEX  32
+#define MF_MMAP   64
+
+mFILE *mfreopen(const char *path, const char *mode, FILE *fp);
+mFILE *mfopen(const char *path, const char *mode);
+int mfdetach(mFILE *mf);
+int mfclose(mFILE *mf);
+int mfdestroy(mFILE *mf);
+int mfseek(mFILE *mf, long offset, int whence);
+long mftell(mFILE *mf);
+void mrewind(mFILE *mf);
+void mftruncate(mFILE *mf, long offset);
+int mfeof(mFILE *mf);
+size_t mfread(void *ptr, size_t size, size_t nmemb, mFILE *mf);
+size_t mfwrite(void *ptr, size_t size, size_t nmemb, mFILE *mf);
+int mfgetc(mFILE *mf);
+int mungetc(int c, mFILE *mf);
+mFILE *mfcreate(char *data, int size);
+mFILE *mfcreate_from(const char *path, const char *mode_str, FILE *fp);
+void mfrecreate(mFILE *mf, char *data, int size);
+void *mfsteal(mFILE *mf, size_t *size_out);
+char *mfgets(char *s, int size, mFILE *mf);
+int mfflush(mFILE *mf);
+mFILE *mstdin(void);
+mFILE *mstdout(void);
+mFILE *mstderr(void);
+void mfascii(mFILE *mf);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* CRAM_MFILE_H */
diff --git a/ext/htslib/cram/misc.h b/ext/htslib/cram/misc.h
new file mode 100644
index 0000000..312dc7d
--- /dev/null
+++ b/ext/htslib/cram/misc.h
@@ -0,0 +1,77 @@
+/*
+Copyright (c) 1994-1997, 2001-2002 MEDICAL RESEARCH COUNCIL
+All rights reserved
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+   1 Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+   2 Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+   3 Neither the name of the MEDICAL RESEARCH COUNCIL, THE LABORATORY OF
+MOLECULAR BIOLOGY nor the names of its contributors may be used to endorse or
+promote products derived from this software without specific prior written
+permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+Copyright (c) 2003-2013, 2018-2019 Genome Research Ltd.
+
+Author: James Bonfield <jkb@sanger.ac.uk>
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+Institute nor the names of its contributors may be used to endorse or promote
+products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#ifndef CRAM_MISC_H
+#define CRAM_MISC_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define MIN(A,B) ( ( (A) < (B) ) ? (A) : (B) )
+#define MAX(A,B) ( ( (A) > (B) ) ? (A) : (B) )
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* CRAM_MISC_H */
diff --git a/ext/htslib/cram/open_trace_file.c b/ext/htslib/cram/open_trace_file.c
new file mode 100644
index 0000000..4d617b7
--- /dev/null
+++ b/ext/htslib/cram/open_trace_file.c
@@ -0,0 +1,438 @@
+/*
+Author: James Bonfield
+
+Copyright (c) 2000-2001 MEDICAL RESEARCH COUNCIL
+All rights reserved
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+   3. Neither the name of the MEDICAL RESEARCH COUNCIL, THE LABORATORY OF
+MOLECULAR BIOLOGY nor the names of its contributors may be used to endorse or
+promote products derived from this software without specific prior written
+permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+/*
+Copyright (c) 2008, 2009, 2013, 2014-2015, 2018-2020 Genome Research Ltd.
+Author: James Bonfield <jkb@sanger.ac.uk>
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+Institute nor the names of its contributors may be used to endorse or promote
+products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <limits.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+
+#include "os.h"
+#ifndef PATH_MAX
+#  define PATH_MAX 1024
+#endif
+
+#include "open_trace_file.h"
+#include "misc.h"
+#include "../htslib/hfile.h"
+#include "../htslib/hts_log.h"
+#include "../htslib/hts.h"
+
+/*
+ * Returns whether the path refers to a regular file.
+ */
+static int is_file(char *fn) {
+    struct stat buf;
+    if ( stat(fn,&buf) ) return 0;
+    return S_ISREG(buf.st_mode);
+}
+
+/*
+ * Tokenises the search path splitting on colons (unix) or semicolons
+ * (windows).
+ * We also  explicitly add a "./" to the end of the search path
+ *
+ * Returns: A new search path with items separated by nul chars. Two nul
+ *          chars in a row represent the end of the tokenised path.
+ * Returns NULL for a failure.
+ *
+ * The returned data has been malloced. It is up to the caller to free this
+ * memory.
+ */
+char *tokenise_search_path(const char *searchpath) {
+    char *newsearch;
+    unsigned int i, j;
+    size_t len;
+    char path_sep = HTS_PATH_SEPARATOR_CHAR;
+
+    if (!searchpath)
+        searchpath="";
+
+    newsearch = (char *)malloc((len = strlen(searchpath))+5);
+    if (!newsearch)
+        return NULL;
+
+    for (i = 0, j = 0; i < len; i++) {
+        /* "::" => ":". Used for escaping colons in http://foo */
+        if (i < len-1 && searchpath[i] == ':' && searchpath[i+1] == ':') {
+            newsearch[j++] = ':';
+            i++;
+            continue;
+        }
+
+        /* Handle http:// and ftp:// too without :: */
+        if (path_sep == ':') {
+            if ((i == 0 || (i > 0 && searchpath[i-1] == ':')) &&
+                (!strncmp(&searchpath[i], "http:",     5) ||
+                 !strncmp(&searchpath[i], "https:",    6) ||
+                 !strncmp(&searchpath[i], "ftp:",      4) ||
+                 !strncmp(&searchpath[i], "|http:",    6) ||
+                 !strncmp(&searchpath[i], "|https:",   7) ||
+                 !strncmp(&searchpath[i], "|ftp:",     5) ||
+                 !strncmp(&searchpath[i], "URL=http:", 9) ||
+                 !strncmp(&searchpath[i], "URL=https:",10)||
+                 !strncmp(&searchpath[i], "URL=ftp:",  8))) {
+                do {
+                    newsearch[j++] = searchpath[i];
+                } while (i<len && searchpath[i++] != ':');
+                if (searchpath[i] == ':')
+                    i++;
+                if (searchpath[i]=='/')
+                    newsearch[j++] = searchpath[i++];
+                if (searchpath[i]=='/')
+                    newsearch[j++] = searchpath[i++];
+                // Look for host:port
+                do {
+                    newsearch[j++] = searchpath[i++];
+                } while (i<len && searchpath[i] != ':' && searchpath[i] != '/');
+                newsearch[j++] = searchpath[i++];
+                if (searchpath[i] == ':')
+                    i++;
+            }
+        }
+
+        if (searchpath[i] == path_sep) {
+            /* Skip blank path components */
+            if (j && newsearch[j-1] != 0)
+                newsearch[j++] = 0;
+        } else {
+            newsearch[j++] = searchpath[i];
+        }
+    }
+
+    if (j)
+        newsearch[j++] = 0;
+    newsearch[j++] = '.';
+    newsearch[j++] = '/';
+    newsearch[j++] = 0;
+    newsearch[j++] = 0;
+
+    return newsearch;
+}
+
+static char *expand_path(const char *file, char *dirname, int max_s_digits);
+
+mFILE *find_file_url(const char *file, char *url) {
+    char *path = NULL, buf[8192];
+    mFILE *mf = NULL;
+    ssize_t len;
+    hFILE *hf = NULL;
+
+    /* Expand %s for the trace name.  Only one digit is allowed between
+       The % and s to avoid ambiguity with percent-encoded URLs */
+
+    path = expand_path(file, url, 1);
+    if (!path)
+        return NULL;
+
+    if (!(hf = hopen(path, "r"))) {
+        if (errno != ENOENT)
+            hts_log_warning("Failed to open reference \"%s\": %s", path, strerror(errno));
+        goto fail;
+    }
+
+    if (NULL == (mf = mfcreate(NULL, 0)))
+        goto fail;
+    while ((len = hread(hf, buf, sizeof(buf))) > 0) {
+        if (mfwrite(buf, len, 1, mf) <= 0) {
+            hclose_abruptly(hf);
+            goto fail;
+        }
+    }
+    if (hclose(hf) < 0 || len < 0) {
+        hts_log_warning("Failed to read reference \"%s\": %s", path, strerror(errno));
+        goto fail;
+    }
+
+    free(path);
+    mrewind(mf);
+    return mf;
+
+ fail:
+    mfdestroy(mf);
+    free(path);
+    return NULL;
+}
+
+/*
+ * Takes a dirname possibly including % rules and appends the filename
+ * to it.
+ *
+ * Returns expanded pathname or NULL for malloc failure.
+ */
+static char *expand_path(const char *file, char *dirname, int max_s_digits) {
+    size_t len = strlen(dirname);
+    size_t lenf = strlen(file);
+    char *cp, *path;
+
+    path = malloc(len+lenf+2); // worst expansion DIR/FILE
+    if (!path) {
+        hts_log_error("Out of memory");
+        return NULL;
+    }
+
+    if (dirname[len-1] == '/')
+        len--;
+
+    /* Special case for "./" or absolute filenames */
+    if (*file == '/' || (len==1 && *dirname == '.')) {
+        memcpy(path, file, lenf + 1);
+    } else {
+        /* Handle %[0-9]*s expansions, if required */
+        char *path_end = path;
+        *path = 0;
+        while ((cp = strchr(dirname, '%'))) {
+            char *endp;
+            long l = strtol(cp+1, &endp, 10);
+            if (*endp != 's' || endp - cp - 1 > max_s_digits) {
+                strncpy(path_end, dirname, (endp+1)-dirname);
+                path_end += (endp+1)-dirname;
+                dirname = endp+1;
+                continue;
+            }
+
+            strncpy(path_end, dirname, cp-dirname);
+            path_end += cp-dirname;
+            if (l) {
+                strncpy(path_end, file, l);
+                path_end += MIN(strlen(file), l);
+                file     += MIN(strlen(file), l);
+            } else {
+                strcpy(path_end, file);
+                path_end += strlen(file);
+                file     += strlen(file);
+            }
+            len -= (endp+1) - dirname;
+            dirname = endp+1;
+        }
+        strncpy(path_end, dirname, len);
+        path_end += MIN(strlen(dirname), len);
+        *path_end = 0;
+        if (*file) {
+            *path_end++ = '/';
+            strcpy(path_end, file);
+        }
+    }
+
+    //fprintf(stderr, "*PATH=\"%s\"\n", path);
+    return path;
+}
+
+/*
+ * Searches for file in the directory 'dirname'. If it finds it, it opens
+ * it. This also searches for compressed versions of the file in dirname
+ * too.
+ *
+ * Returns mFILE pointer if found
+ *         NULL if not
+ */
+static mFILE *find_file_dir(const char *file, char *dirname) {
+    char *path;
+    mFILE *mf = NULL;
+
+    path = expand_path(file, dirname, INT_MAX);
+    if (!path)
+        return NULL;
+
+    if (is_file(path))
+        mf = mfopen(path, "rbm");
+
+    free(path);
+    return mf;
+}
+
+/*
+ * ------------------------------------------------------------------------
+ * Public functions below.
+ */
+
+/*
+ * Opens a trace file named 'file'. This is initially looked for as a
+ * pathname relative to a file named "relative_to". This may (for
+ * example) be the name of an experiment file referencing the trace
+ * file. In this case by passing relative_to as the experiment file
+ * filename the trace file will be picked up in the same directory as
+ * the experiment file. Relative_to may be supplied as NULL.
+ *
+ * 'file' is looked for at relative_to, then the current directory, and then
+ * all of the locations listed in 'path' (which is a colon separated list).
+ * If 'path' is NULL it uses the RAWDATA environment variable instead.
+ *
+ * Returns a mFILE pointer when found.
+ *           NULL otherwise.
+ */
+mFILE *open_path_mfile(const char *file, char *path, char *relative_to) {
+    char *newsearch;
+    char *ele;
+    mFILE *fp;
+
+    /* Use path first */
+    if (!path)
+        path = getenv("RAWDATA");
+    if (NULL == (newsearch = tokenise_search_path(path)))
+        return NULL;
+
+    /*
+     * Step through the search path testing out each component.
+     * We now look through each path element treating some prefixes as
+     * special, otherwise we treat the element as a directory.
+     */
+    for (ele = newsearch; *ele; ele += strlen(ele)+1) {
+        char *ele2;
+
+        /*
+         * '|' prefixing a path component indicates that we do not
+         * wish to perform the compression extension searching in that
+         * location.
+         *
+         * NB: this has been removed from the htslib implementation.
+         */
+        if (*ele == '|') {
+            ele2 = ele+1;
+        } else {
+            ele2 = ele;
+        }
+
+        if (0 == strncmp(ele2, "URL=", 4)) {
+            if ((fp = find_file_url(file, ele2+4))) {
+                free(newsearch);
+                return fp;
+            }
+        } else if (!strncmp(ele2, "http:", 5) ||
+                   !strncmp(ele2, "https:", 6) ||
+                   !strncmp(ele2, "ftp:", 4)) {
+            if ((fp = find_file_url(file, ele2))) {
+                free(newsearch);
+                return fp;
+            }
+        } else if ((fp = find_file_dir(file, ele2))) {
+            free(newsearch);
+            return fp;
+        }
+    }
+
+    free(newsearch);
+
+    /* Look in the same location as the incoming 'relative_to' filename */
+    if (relative_to) {
+        char *cp;
+        char relative_path[PATH_MAX+1];
+        strcpy(relative_path, relative_to);
+        if ((cp = strrchr(relative_path, '/')))
+            *cp = 0;
+        if ((fp = find_file_dir(file, relative_path)))
+            return fp;
+    }
+
+    return NULL;
+}
+
+
+/*
+ * As per open_path_mfile, but searching only for local filenames.
+ * This is useful as we may avoid doing a full mfopen and loading
+ * the entire file into memory.
+ *
+ * Returns the expanded pathname if found.
+ *         NULL if not
+ */
+char *find_path(const char *file, const char *path) {
+    char *newsearch;
+    char *ele;
+    char *outpath = NULL;
+
+    /* Use path first */
+    if (!path)
+        path = getenv("RAWDATA");
+    if (NULL == (newsearch = tokenise_search_path(path)))
+        return NULL;
+
+    for (ele = newsearch; *ele; ele += strlen(ele)+1) {
+        char *ele2 = (*ele == '|') ? ele+1 : ele;
+
+        if (!strncmp(ele2, "URL=", 4) ||
+            !strncmp(ele2, "http:", 5) ||
+            !strncmp(ele2, "https:", 6) ||
+            !strncmp(ele2, "ftp:", 4)) {
+            continue;
+        } else {
+            outpath = expand_path(file, ele2, INT_MAX);
+            if (is_file(outpath)) {
+                free(newsearch);
+                return outpath;
+            } else {
+                free(outpath);
+            }
+        }
+    }
+
+    free(newsearch);
+
+    return NULL;
+}
diff --git a/ext/htslib/cram/open_trace_file.h b/ext/htslib/cram/open_trace_file.h
new file mode 100644
index 0000000..4586098
--- /dev/null
+++ b/ext/htslib/cram/open_trace_file.h
@@ -0,0 +1,125 @@
+/*
+Author: James Bonfield
+
+Copyright (c) 2000-2001 MEDICAL RESEARCH COUNCIL
+All rights reserved
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+   . Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+   . Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+   . Neither the name of the MEDICAL RESEARCH COUNCIL, THE LABORATORY OF
+MOLECULAR BIOLOGY nor the names of its contributors may be used to endorse or
+promote products derived from this software without specific prior written
+permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+/*
+Copyright (c) 2008, 2009, 2013, 2018 Genome Research Ltd.
+Author: James Bonfield <jkb@sanger.ac.uk>
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+Institute nor the names of its contributors may be used to endorse or promote
+products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#ifndef OPEN_TRACE_FILE_H
+#define OPEN_TRACE_FILE_H
+
+#include "mFILE.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Tokenises the search path splitting on colons (unix) or semicolons
+ * (windows).
+ * We also  explicitly add a "./" to the end of the search path
+ *
+ * Returns: A new search path with items separated by nul chars. Two nul
+ *          chars in a row represent the end of the tokenised path.
+ * Returns NULL for a failure.
+ *
+ * The returned data has been malloced. It is up to the caller to free this
+ * memory.
+ */
+char *tokenise_search_path(const char *searchpath);
+
+/*
+ * Opens a trace file named 'file'. This is initially looked for as a
+ * pathname relative to a file named "relative_to". This may (for
+ * example) be the name of an experiment file referencing the trace
+ * file. In this case by passing relative_to as the experiment file
+ * filename the trace file will be picked up in the same directory as
+ * the experiment file. Relative_to may be supplied as NULL.
+ *
+ * 'file' is looked for at relative_to, then the current directory, and then
+ * all of the locations listed in 'path' (which is a colon separated list).
+ * If 'path' is NULL it uses the RAWDATA environment variable instead.
+ *
+ * Returns a mFILE pointer when found.
+ *           NULL otherwise.
+ */
+mFILE *open_path_mfile(const char *file, char *path, char *relative_to);
+
+/*
+ * Returns a mFILE containing the entire contents of the url;
+ *         NULL on failure.
+ */
+mFILE *find_file_url(const char *file, char *url);
+
+
+/*
+ * As per open_path_mfile, but searching only for local filenames.
+ * This is useful as we may avoid doing a full mfopen and loading
+ * the entire file into memory.
+ *
+ * Returns the expanded pathname if found.
+ *         NULL if not
+ */
+char *find_path(const char *file, const char *path);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* OPEN_TRACE_FILE_H */
diff --git a/ext/htslib/cram/os.h b/ext/htslib/cram/os.h
new file mode 100644
index 0000000..1f39887
--- /dev/null
+++ b/ext/htslib/cram/os.h
@@ -0,0 +1,205 @@
+/*
+Copyright (c) 1993, 1995-2002 MEDICAL RESEARCH COUNCIL
+All rights reserved
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+   1 Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+   2 Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+   3 Neither the name of the MEDICAL RESEARCH COUNCIL, THE LABORATORY OF
+MOLECULAR BIOLOGY nor the names of its contributors may be used to endorse or
+promote products derived from this software without specific prior written
+permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+/*
+Copyright (c) 2004, 2006, 2009-2011, 2013, 2017-2018 Genome Research Ltd.
+Author: James Bonfield <jkb@sanger.ac.uk>
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+Institute nor the names of its contributors may be used to endorse or promote
+products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+/*
+ * File: os.h
+ *
+ * Author:
+ *         MRC Laboratory of Molecular Biology
+ *         Hills Road
+ *         Cambridge CB2 2QH
+ *         United Kingdom
+ *
+ * Description: operating system specific type definitions
+ *
+ */
+
+#ifndef CRAM_OS_H
+#define CRAM_OS_H
+
+#include <limits.h>
+#include <stdint.h>
+
+#include "../htslib/hts_endian.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+/*-----------------------------------------------------------------------------
+ * Byte swapping macros
+ */
+
+/*
+ * Our new swap runs at the same speed on Ultrix, but substantially faster
+ * (300% for swap_int4, ~50% for swap_int2) on an Alpha (due to the lack of
+ * decent 'char' support).
+ *
+ * They also have the ability to swap in situ (src == dst). Newer code now
+ * relies on this so don't change back!
+ */
+#define iswap_int8(x)                           \
+    (((x & 0x00000000000000ffLL) << 56) +       \
+     ((x & 0x000000000000ff00LL) << 40) +       \
+     ((x & 0x0000000000ff0000LL) << 24) +       \
+     ((x & 0x00000000ff000000LL) <<  8) +       \
+     ((x & 0x000000ff00000000LL) >>  8) +       \
+     ((x & 0x0000ff0000000000LL) >> 24) +       \
+     ((x & 0x00ff000000000000LL) >> 40) +       \
+     ((x & 0xff00000000000000LL) >> 56))
+
+#define iswap_int4(x)                           \
+    (((x & 0x000000ff) << 24) +                 \
+     ((x & 0x0000ff00) <<  8) +                 \
+     ((x & 0x00ff0000) >>  8) +                 \
+     ((x & 0xff000000) >> 24))
+
+#define iswap_int2(x)                           \
+    (((x & 0x00ff) << 8) +                      \
+     ((x & 0xff00) >> 8))
+
+/*
+ * Linux systems may use byteswap.h to get assembly versions of byte-swap
+ * on intel systems. This can be as trivial as the bswap opcode, which works
+ * out at over 2-times faster than iswap_int4 above.
+ */
+#if 0
+#if defined(__linux__)
+#    include <byteswap.h>
+#    undef iswap_int8
+#    undef iswap_int4
+#    undef iswap_int2
+#    define iswap_int8 bswap_64
+#    define iswap_int4 bswap_32
+#    define iswap_int2 bswap_16
+#endif
+#endif
+
+
+/*
+ * Macros to specify that data read in is of a particular endianness.
+ * The macros here swap to the appropriate order for the particular machine
+ * running the macro and return the new answer. These may also be used when
+ * writing to a file to specify that we wish to write in (eg) big endian
+ * format.
+ *
+ * This leads to efficient code as most of the time these macros are
+ * trivial.
+ */
+#if defined(HTS_BIG_ENDIAN)
+#define le_int4(x) iswap_int4((x))
+#define le_int2(x) iswap_int2((x))
+#elif defined(HTS_LITTLE_ENDIAN)
+#define le_int4(x) (x)
+#define le_int2(x) (x)
+#else
+static inline uint32_t le_int4(uint32_t x) {
+    return le_to_u32((uint8_t *) &x);
+}
+static inline uint16_t le_int2(uint16_t x) {
+    return le_to_u16((uint8_t *) &x);
+}
+#endif
+
+/*-----------------------------------------------------------------------------
+ * Operating system specifics.
+ * These ought to be done by autoconf, but are legacy code.
+ */
+/*
+ * SunOS 4.x
+ * Even though we use the ANSI gcc, we make use the the standard SunOS 4.x
+ * libraries and include files, which are non-ansi
+ */
+#if defined(__sun__) && !defined(__svr4__)
+#define SEEK_SET 0
+#define SEEK_CUR 1
+#define SEEK_END 2
+#endif
+
+/*
+ * Microsoft Visual C++
+ * Windows
+ */
+#if defined(_MSC_VER)
+#define popen _popen
+#define pclose _pclose
+#define ftruncate(fd,len) _chsize(fd,len)
+#endif
+
+
+/*
+ * Microsoft Windows running MinGW
+ */
+#if defined(__MINGW32__)
+#include <io.h>
+#define mkdir(filename,mode) mkdir((filename))
+#define sysconf(x) 512
+#ifndef ftruncate
+#  define ftruncate(fd,len) _chsize(fd,len)
+#endif
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* CRAM_OS_H */
diff --git a/ext/htslib/cram/pooled_alloc.c b/ext/htslib/cram/pooled_alloc.c
new file mode 100644
index 0000000..4601a7f
--- /dev/null
+++ b/ext/htslib/cram/pooled_alloc.c
@@ -0,0 +1,205 @@
+/*
+Copyright (c) 2009, 2013, 2015, 2018-2019 Genome Research Ltd.
+Author: Rob Davies <rmd@sanger.ac.uk>
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+Institute nor the names of its contributors may be used to endorse or promote
+products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <stdint.h>
+
+#include "pooled_alloc.h"
+#include "misc.h"
+
+//#define DISABLE_POOLED_ALLOC
+//#define TEST_MAIN
+
+#define PSIZE 1024*1024
+
+// credit to http://graphics.stanford.edu/~seander/bithacks.html
+static int next_power_2(unsigned int v) {
+    v--;
+    v |= v >> 1;
+    v |= v >> 2;
+    v |= v >> 4;
+    v |= v >> 8;
+    v |= v >> 16;
+    v++;
+
+    return v;
+}
+
+/*
+ * Creates a pool.
+ * Pool allocations are approx minimum of 1024*dsize or PSIZE.
+ * (Assumes we're not trying to use pools for >= 2Gb or more)
+ */
+pool_alloc_t *pool_create(size_t dsize) {
+    pool_alloc_t *p;
+
+    if (NULL == (p = (pool_alloc_t *)malloc(sizeof(*p))))
+        return NULL;
+
+    /* Minimum size is a pointer, for free list */
+    dsize = (dsize + sizeof(void *) - 1) & ~(sizeof(void *)-1);
+    if (dsize < sizeof(void *))
+        dsize = sizeof(void *);
+    p->dsize = dsize;
+    p->psize = MIN(PSIZE, next_power_2(p->dsize*1024));
+
+    p->npools = 0;
+    p->pools = NULL;
+    p->free  = NULL;
+
+    return p;
+}
+
+void pool_destroy(pool_alloc_t *p) {
+    size_t i;
+
+    for (i = 0; i < p->npools; i++) {
+        free(p->pools[i].pool);
+    }
+    free(p->pools);
+    free(p);
+}
+
+#ifndef DISABLE_POOLED_ALLOC
+
+static pool_t *new_pool(pool_alloc_t *p) {
+    size_t n = p->psize / p->dsize;
+    pool_t *pool;
+
+    pool = realloc(p->pools, (p->npools + 1) * sizeof(*p->pools));
+    if (NULL == pool) return NULL;
+    p->pools = pool;
+    pool = &p->pools[p->npools];
+
+    pool->pool = malloc(n * p->dsize);
+    if (NULL == pool->pool) return NULL;
+
+    pool->used = 0;
+
+    p->npools++;
+
+    return pool;
+}
+
+void *pool_alloc(pool_alloc_t *p) {
+    pool_t *pool;
+    void *ret;
+
+    /* Look on free list */
+    if (NULL != p->free) {
+        ret = p->free;
+        p->free = *((void **)p->free);
+        return ret;
+    }
+
+    /* Look for space in the last pool */
+    if (p->npools) {
+        pool = &p->pools[p->npools - 1];
+        if (pool->used + p->dsize < p->psize) {
+            ret = ((char *) pool->pool) + pool->used;
+            pool->used += p->dsize;
+            return ret;
+        }
+    }
+
+    /* Need a new pool */
+    pool = new_pool(p);
+    if (NULL == pool) return NULL;
+
+    pool->used = p->dsize;
+    return pool->pool;
+}
+
+void pool_free(pool_alloc_t *p, void *ptr) {
+    *(void **)ptr = p->free;
+    p->free = ptr;
+}
+
+#else
+
+void *pool_alloc(pool_alloc_t *p) {
+    return malloc(p->dsize);
+}
+
+void pool_free(pool_alloc_t *p, void *ptr) {
+    free(ptr);
+}
+
+#endif
+
+#ifdef TEST_MAIN
+typedef struct {
+    int x, y, z;
+} xyz;
+
+#define NP 10000
+int main(void) {
+    int i;
+    xyz *item;
+    xyz **items;
+    pool_alloc_t *p = pool_create(sizeof(xyz));
+
+    items = (xyz **)malloc(NP * sizeof(*items));
+
+    for (i = 0; i < NP; i++) {
+        item = pool_alloc(p);
+        item->x = i;
+        item->y = i+1;
+        item->z = i+2;
+        items[i] = item;
+    }
+
+    for (i = 0; i < NP; i++) {
+        item = items[i];
+        if (i % 3)
+            pool_free(p, item);
+    }
+
+    for (i = 0; i < NP; i++) {
+        item = pool_alloc(p);
+        item->x = 1000000+i;
+        item->y = 1000000+i+1;
+        item->z = 1000000+i+2;
+    }
+
+    for (i = 0; i < NP; i++) {
+        item = items[i];
+        printf("%d\t%d\t%d\t%d\n", i, item->x, item->y, item->z);
+        pool_free(p, item);
+    }
+
+    free(items);
+    return 0;
+}
+#endif
diff --git a/ext/htslib/cram/pooled_alloc.h b/ext/htslib/cram/pooled_alloc.h
new file mode 100644
index 0000000..bb49d11
--- /dev/null
+++ b/ext/htslib/cram/pooled_alloc.h
@@ -0,0 +1,66 @@
+/*
+Copyright (c) 2009, 2013, 2018 Genome Research Ltd.
+Author: Rob Davies <rmd@sanger.ac.uk>
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+Institute nor the names of its contributors may be used to endorse or promote
+products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#ifndef POOLED_ALLOC_H
+#define POOLED_ALLOC_H
+
+#include <stddef.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Implements a pooled block allocator where all items are the same size,
+ * but we need many of them.
+ */
+typedef struct {
+    void   *pool;
+    size_t  used;
+} pool_t;
+
+typedef struct {
+    size_t dsize;
+    size_t psize;
+    size_t npools;
+    pool_t *pools;
+    void *free;
+} pool_alloc_t;
+
+pool_alloc_t *pool_create(size_t dsize);
+void pool_destroy(pool_alloc_t *p);
+void *pool_alloc(pool_alloc_t *p);
+void pool_free(pool_alloc_t *p, void *ptr);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* POOLED_ALLOC_H */
diff --git a/ext/htslib/cram/string_alloc.c b/ext/htslib/cram/string_alloc.c
new file mode 100644
index 0000000..c339b10
--- /dev/null
+++ b/ext/htslib/cram/string_alloc.c
@@ -0,0 +1,162 @@
+/*
+Copyright (c) 2010, 2013, 2018-2019 Genome Research Ltd.
+Author: Andrew Whitwham <aw7@sanger.ac.uk>
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+Institute nor the names of its contributors may be used to endorse or promote
+products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+
+/*
+   A pooled string allocator intended to cut down on the
+   memory overhead of many small string allocations.
+
+   Andrew Whitwham, September 2010.
+*/
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+#include "string_alloc.h"
+
+#define MIN_STR_SIZE 1024
+
+
+/* creates the string pool. max_length is the initial size
+   a single string can be.  The max_length can grow as
+   needed */
+
+string_alloc_t *string_pool_create(size_t max_length) {
+    string_alloc_t *a_str;
+
+    if (NULL == (a_str = (string_alloc_t *)malloc(sizeof(*a_str)))) {
+        return NULL;
+    }
+
+    if (max_length < MIN_STR_SIZE) max_length = MIN_STR_SIZE;
+
+    a_str->nstrings    = 0;
+    a_str->max_strings = 0;
+    a_str->max_length  = max_length;
+    a_str->strings     = NULL;
+
+    return a_str;
+}
+
+
+/* internal function to do the actual memory allocation */
+
+static string_t *new_string_pool(string_alloc_t *a_str) {
+    string_t *str;
+
+    if (a_str->nstrings == a_str->max_strings) {
+        size_t new_max = (a_str->max_strings | (a_str->max_strings >> 2)) + 1;
+        str = realloc(a_str->strings, new_max * sizeof(*a_str->strings));
+
+        if (NULL == str) return NULL;
+
+        a_str->strings = str;
+        a_str->max_strings = new_max;
+    }
+
+    str = &a_str->strings[a_str->nstrings];
+
+    str->str = malloc(a_str->max_length);
+
+    if (NULL == str->str) return NULL;
+
+    str->used = 0;
+    a_str->nstrings++;
+
+    return str;
+}
+
+
+/* free allocated memory */
+
+void string_pool_destroy(string_alloc_t *a_str) {
+    size_t i;
+
+    for (i = 0; i < a_str->nstrings; i++) {
+        free(a_str->strings[i].str);
+    }
+
+    free(a_str->strings);
+    free(a_str);
+}
+
+
+/* allocate space for a string */
+
+char *string_alloc(string_alloc_t *a_str, size_t length) {
+    string_t *str;
+    char *ret;
+
+    if (length <= 0) return NULL;
+
+    // add to last string pool if we have space
+    if (a_str->nstrings) {
+        str = &a_str->strings[a_str->nstrings - 1];
+
+        if (str->used + length < a_str->max_length) {
+            ret = str->str + str->used;
+            str->used += length;
+            return ret;
+        }
+    }
+
+    // increase the max length if needs be
+    if (length > a_str->max_length) a_str->max_length = length;
+
+    // need a new string pool
+    str = new_string_pool(a_str);
+
+    if (NULL == str) return NULL;
+
+    str->used = length;
+    return str->str;
+}
+
+
+/* equivalent to strdup */
+
+char *string_dup(string_alloc_t *a_str, const char *instr) {
+    return string_ndup(a_str, instr, strlen(instr));
+}
+
+char *string_ndup(string_alloc_t *a_str, const char *instr, size_t len) {
+    char *str = string_alloc(a_str, len + 1);
+
+    if (NULL == str) return NULL;
+
+    memcpy(str, instr, len);
+    str[len] = 0;
+
+    return str;
+}
diff --git a/ext/htslib/cram/string_alloc.h b/ext/htslib/cram/string_alloc.h
new file mode 100644
index 0000000..42ebb0a
--- /dev/null
+++ b/ext/htslib/cram/string_alloc.h
@@ -0,0 +1,69 @@
+/*
+Copyright (c) 2010, 2013, 2018 Genome Research Ltd.
+Author: Andrew Whitwham <aw7@sanger.ac.uk>
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+Institute nor the names of its contributors may be used to endorse or promote
+products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#ifndef STRING_ALLOC_H
+#define STRING_ALLOC_H
+
+#include <stdlib.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * A pooled string allocator intended to cut down on the
+ * memory overhead of many small string allocations.
+ *
+ * Andrew Whitwham, September 2010.
+ */
+
+typedef struct {
+    char *str;
+    size_t used;
+} string_t;
+
+typedef struct {
+    size_t max_length;
+    size_t nstrings;
+    size_t max_strings;
+    string_t *strings;
+} string_alloc_t;
+
+string_alloc_t *string_pool_create(size_t max_length);
+void string_pool_destroy(string_alloc_t *a_str);
+char *string_alloc(string_alloc_t *a_str, size_t length);
+char *string_dup(string_alloc_t *a_str, const char *instr);
+char *string_ndup(string_alloc_t *a_str, const char *instr, size_t len);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/ext/htslib/errmod.c b/ext/htslib/errmod.c
new file mode 100644
index 0000000..df708e1
--- /dev/null
+++ b/ext/htslib/errmod.c
@@ -0,0 +1,208 @@
+/*  errmod.c -- revised MAQ error model.
+
+    Copyright (C) 2010 Broad Institute.
+    Copyright (C) 2012, 2013, 2016-2017, 2019 Genome Research Ltd.
+
+    Author: Heng Li <lh3@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+
+#include <math.h>
+#include "htslib/hts.h"
+#include "htslib/ksort.h"
+#include "htslib/hts_os.h" // for drand48
+
+KSORT_INIT_STATIC_GENERIC(uint16_t)
+
+struct errmod_t {
+    double depcorr;
+    /* table of constants generated for given depcorr and eta */
+    double *fk, *beta, *lhet;
+};
+
+typedef struct {
+    double fsum[16], bsum[16];
+    uint32_t c[16];
+} call_aux_t;
+
+/* \Gamma(n) = (n-1)! */
+#define lfact(n) lgamma(n+1)
+
+/* generates a success * trials table of bionomial probability densities (log transformed) */
+static double* logbinomial_table( const int n_size )
+{
+    /* prob distribution for binom var is p(k) = {n! \over k! (n-k)! } p^k (1-p)^{n-k} */
+    /* this calcs p(k) = {log(n!) - log(k!) - log((n-k)!) */
+    int k, n;
+    double *logbinom = (double*)calloc(n_size * n_size, sizeof(double));
+    if (!logbinom) return NULL;
+    for (n = 1; n < n_size; ++n) {
+        double lfn = lfact(n);
+        for (k = 1; k <= n; ++k)
+            logbinom[n<<8|k] = lfn - lfact(k) - lfact(n-k);
+    }
+    return logbinom;
+}
+
+static int cal_coef(errmod_t *em, double depcorr, double eta)
+{
+    int k, n, q;
+    double sum, sum1;
+    double *lC;
+
+    // initialize ->fk
+    em->fk = (double*)calloc(256, sizeof(double));
+    if (!em->fk) return -1;
+    em->fk[0] = 1.0;
+    for (n = 1; n < 256; ++n)
+        em->fk[n] = pow(1. - depcorr, n) * (1.0 - eta) + eta;
+
+    // initialize ->beta
+    em->beta = (double*)calloc(256 * 256 * 64, sizeof(double));
+    if (!em->beta) return -1;
+
+    lC = logbinomial_table( 256 );
+    if (!lC) return -1;
+
+    for (q = 1; q < 64; ++q) {
+        double e = pow(10.0, -q/10.0);
+        double le = log(e);
+        double le1 = log(1.0 - e);
+        for (n = 1; n <= 255; ++n) {
+            double *beta = em->beta + (q<<16|n<<8);
+            sum1 = lC[n<<8|n] + n*le;
+            beta[n] = HUGE_VAL;
+            for (k = n - 1; k >= 0; --k, sum1 = sum) {
+                sum = sum1 + log1p(exp(lC[n<<8|k] + k*le + (n-k)*le1 - sum1));
+                beta[k] = -10. / M_LN10 * (sum1 - sum);
+            }
+        }
+    }
+
+    // initialize ->lhet
+    em->lhet = (double*)calloc(256 * 256, sizeof(double));
+    if (!em->lhet) {
+        free(lC);
+        return -1;
+    }
+    for (n = 0; n < 256; ++n)
+        for (k = 0; k < 256; ++k)
+            em->lhet[n<<8|k] = lC[n<<8|k] - M_LN2 * n;
+    free(lC);
+    return 0;
+}
+
+/**
+ * Create errmod_t object with obj.depcorr set to depcorr and initialise
+ */
+errmod_t *errmod_init(double depcorr)
+{
+    errmod_t *em;
+    em = (errmod_t*)calloc(1, sizeof(errmod_t));
+    if (!em) return NULL;
+    em->depcorr = depcorr;
+    cal_coef(em, depcorr, 0.03);
+    return em;
+}
+
+/**
+ * Deallocate an errmod_t object
+ */
+void errmod_destroy(errmod_t *em)
+{
+    if (em == 0) return;
+    free(em->lhet); free(em->fk); free(em->beta);
+    free(em);
+}
+
+//
+// em: error model to fit to data
+// m: number of alleles across all samples
+// n: number of bases observed in sample
+// bases[i]: bases observed in pileup [6 bit quality|1 bit strand|4 bit base]
+// q[i*m+j]: (Output) phred-scaled likelihood of each genotype (i,j)
+int errmod_cal(const errmod_t *em, int n, int m, uint16_t *bases, float *q)
+{
+    // Aux
+    // aux.c is total count of each base observed (ignoring strand)
+    call_aux_t aux;
+    // Loop variables
+    int i, j, k;
+    // The total count of each base observed per strand
+    int w[32];
+
+    memset(q, 0, m * m * sizeof(float)); // initialise q to 0
+    if (n == 0) return 0;
+    // This section randomly downsamples to 255 depth so as not to go beyond our precalculated matrix
+    if (n > 255) { // if we exceed 255 bases observed then shuffle them to sample and only keep the first 255
+        ks_shuffle(uint16_t, n, bases);
+        n = 255;
+    }
+    ks_introsort(uint16_t, n, bases);
+    /* zero out w and aux */
+    memset(w, 0, 32 * sizeof(int));
+    memset(&aux, 0, sizeof(call_aux_t));
+
+    for (j = n - 1; j >= 0; --j) { // calculate esum and fsum
+        uint16_t b = bases[j];
+        /* extract quality and cap at 63 */
+        int qual = b>>5 < 4? 4 : b>>5;
+        if (qual > 63) qual = 63;
+        /* extract base ORed with strand */
+        int basestrand = b&0x1f;
+        /* extract base */
+        int base = b&0xf;
+        aux.fsum[base] += em->fk[w[basestrand]];
+        aux.bsum[base] += em->fk[w[basestrand]] * em->beta[qual<<16|n<<8|aux.c[base]];
+        ++aux.c[base];
+        ++w[basestrand];
+    }
+
+    // generate likelihood
+    for (j = 0; j < m; ++j) {
+        float tmp1, tmp3;
+        int tmp2;
+        // homozygous
+        for (k = 0, tmp1 = tmp3 = 0.0, tmp2 = 0; k < m; ++k) {
+            if (k == j) continue;
+            tmp1 += aux.bsum[k]; tmp2 += aux.c[k]; tmp3 += aux.fsum[k];
+        }
+        if (tmp2) {
+            q[j*m+j] = tmp1;
+        }
+        // heterozygous
+        for (k = j + 1; k < m; ++k) {
+            int cjk = aux.c[j] + aux.c[k];
+            for (i = 0, tmp2 = 0, tmp1 = tmp3 = 0.0; i < m; ++i) {
+                if (i == j || i == k) continue;
+                tmp1 += aux.bsum[i]; tmp2 += aux.c[i]; tmp3 += aux.fsum[i];
+            }
+            if (tmp2) {
+                q[j*m+k] = q[k*m+j] = -4.343 * em->lhet[cjk<<8|aux.c[k]] + tmp1;
+            } else q[j*m+k] = q[k*m+j] = -4.343 * em->lhet[cjk<<8|aux.c[k]]; // all the bases are either j or k
+        }
+        /* clamp to greater than 0 */
+        for (k = 0; k < m; ++k) if (q[j*m+k] < 0.0) q[j*m+k] = 0.0;
+    }
+
+    return 0;
+}
diff --git a/ext/htslib/faidx.5 b/ext/htslib/faidx.5
new file mode 100644
index 0000000..fb84fb2
--- /dev/null
+++ b/ext/htslib/faidx.5
@@ -0,0 +1,238 @@
+'\" t
+.TH faidx 5 "June 2018" "htslib" "Bioinformatics formats"
+.SH NAME
+faidx \- an index enabling random access to FASTA and FASTQ files
+.\"
+.\" Copyright (C) 2013, 2015, 2018 Genome Research Ltd.
+.\"
+.\" Author: John Marshall <jm18@sanger.ac.uk>
+.\"
+.\" Permission is hereby granted, free of charge, to any person obtaining a
+.\" copy of this software and associated documentation files (the "Software"),
+.\" to deal in the Software without restriction, including without limitation
+.\" the rights to use, copy, modify, merge, publish, distribute, sublicense,
+.\" and/or sell copies of the Software, and to permit persons to whom the
+.\" Software is furnished to do so, subject to the following conditions:
+.\"
+.\" The above copyright notice and this permission notice shall be included in
+.\" all copies or substantial portions of the Software.
+.\"
+.\" THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+.\" IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+.\" FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+.\" THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+.\" LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+.\" FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+.\" DEALINGS IN THE SOFTWARE.
+.\"
+.SH SYNOPSIS
+.IR file.fa .fai,
+.IR file.fasta .fai,
+.IR file.fq .fai,
+.IR file.fastq .fai
+.SH DESCRIPTION
+Using an \fBfai index\fP file in conjunction with a FASTA/FASTQ file containing
+reference sequences enables efficient access to arbitrary regions within
+those reference sequences.
+The index file typically has the same filename as the corresponding FASTA/FASTQ
+file, with \fB.fai\fP appended.
+.P
+An \fBfai index\fP file is a text file consisting of lines each with
+five TAB-delimited columns for a FASTA file and six for FASTQ:
+.TS
+lbl.
+NAME	Name of this reference sequence
+LENGTH	Total length of this reference sequence, in bases
+OFFSET	Offset in the FASTA/FASTQ file of this sequence's first base
+LINEBASES	The number of bases on each line
+LINEWIDTH	The number of bytes in each line, including the newline
+QUALOFFSET	Offset of sequence's first quality within the FASTQ file
+.TE
+.P
+The \fBNAME\fP and \fBLENGTH\fP columns contain the same
+data as would appear in the \fBSN\fP and \fBLN\fP fields of a
+SAM \fB@SQ\fP header for the same reference sequence.
+.P
+The \fBOFFSET\fP column contains the offset within the FASTA/FASTQ file, in
+bytes starting from zero, of the first base of this reference sequence, i.e., of
+the character following the newline at the end of the header line (the 
+"\fB>\fP" line in FASTA, "\fB@\fP" in FASTQ). Typically the lines of a
+\fBfai index\fP file appear in the order in which the reference sequences
+appear in the FASTA/FASTQ file, so \fB.fai\fP files are typically sorted
+according to this column.
+.P
+The \fBLINEBASES\fP column contains the number of bases in each of the sequence
+lines that form the body of this reference sequence, apart from the final line
+which may be shorter.
+The \fBLINEWIDTH\fP column contains the number of \fIbytes\fP in each of
+the sequence lines (except perhaps the final line), thus differing from
+\fBLINEBASES\fP in that it also counts the bytes forming the line terminator.
+.P
+The \fBQUALOFFSET\fP works the same way as \fBOFFSET\fP but for the first
+quality score of this reference sequence.  This would be the first character
+following the newline at the end of the "\fB+\fP" line.  For FASTQ files only.
+.SS FASTA Files
+In order to be indexed with \fBsamtools faidx\fP, a FASTA file must be a text
+file of the form
+.LP
+.RS
+.RI > name
+.RI [ description ...]
+.br
+ATGCATGCATGCATGCATGCATGCATGCAT
+.br
+GCATGCATGCATGCATGCATGCATGCATGC
+.br
+ATGCAT
+.br
+.RI > name
+.RI [ description ...]
+.br
+ATGCATGCATGCAT
+.br
+GCATGCATGCATGC
+.br
+[...]
+.RE
+.LP
+In particular, each reference sequence must be "well-formatted", i.e., all
+of its sequence lines must be the same length, apart from the final sequence
+line which may be shorter.
+(While this sequence line length must be the same within each sequence,
+it may vary between different reference sequences in the same FASTA file.)
+.P
+This also means that although the FASTA file may have Unix- or Windows-style
+or other line termination, the newline characters present must be consistent,
+at least within each reference sequence.
+.P
+The \fBsamtools\fP implementation uses the first word of the "\fB>\fP" header
+line text (i.e., up to the first whitespace character, having skipped any
+initial whitespace after the ">") as the \fBNAME\fP column.
+.SS FASTQ Files
+FASTQ files for indexing work in the same way as the FASTA files.
+.LP
+.RS
+.RI @ name
+.RI [ description...]
+.br
+ATGCATGCATGCATGCATGCATGCATGCAT
+.br
+GCATGCATGCATGCATGCATGCATGCATGC
+.br
+ATGCAT
+.br
+.RI +
+.br
+FFFA@@FFFFFFFFFFHHB:::@BFFFFGG
+.br
+HIHIIIIIIIIIIIIIIIIIIIIIIIFFFF
+.br
+8011<<
+.br
+.RI @ name
+.RI [ description...]
+.br
+ATGCATGCATGCAT
+.br
+GCATGCATGCATGC
+.br
+.RI +
+.br
+IIA94445EEII==
+.br
+=>IIIIIIIIICCC
+.br
+[...]
+.RE
+.LP
+Quality lines must be wrapped at the same length as the corresponding
+sequence lines.
+.SH EXAMPLE
+For example, given this FASTA file
+.LP
+.RS
+>one
+.br
+ATGCATGCATGCATGCATGCATGCATGCAT
+.br
+GCATGCATGCATGCATGCATGCATGCATGC
+.br
+ATGCAT
+.br
+>two another chromosome
+.br
+ATGCATGCATGCAT
+.br
+GCATGCATGCATGC
+.br
+.RE
+.LP
+formatted with Unix-style (LF) line termination, the corresponding fai index
+would be
+.RS
+.TS
+lnnnn.
+one	66	5	30	31
+two	28	98	14	15
+.TE
+.RE
+.LP
+If the FASTA file were formatted with Windows-style (CR-LF) line termination,
+the fai index would be
+.RS
+.TS
+lnnnn.
+one	66	6	30	32
+two	28	103	14	16
+.TE
+.RE
+.LP
+An example FASTQ file
+.LP
+.RS
+@fastq1
+.br
+ATGCATGCATGCATGCATGCATGCATGCAT
+.br
+GCATGCATGCATGCATGCATGCATGCATGC
+.br
+ATGCAT
+.br
++
+.br
+FFFA@@FFFFFFFFFFHHB:::@BFFFFGG
+.br
+HIHIIIIIIIIIIIIIIIIIIIIIIIFFFF
+.br
+8011<<
+.br
+@fastq2
+.br
+ATGCATGCATGCAT
+.br
+GCATGCATGCATGC
+.br
++
+.br
+IIA94445EEII==
+.br
+=>IIIIIIIIICCC
+.br
+.RE
+.LP
+Formatted with Unix-style line termination would give this fai index
+.RS
+.TS
+lnnnnn.
+fastq1	66	8	30	31	79
+fastq2	28	156	14	15	188
+.TE
+.RE
+.SH SEE ALSO
+.IR samtools (1)
+.TP
+https://en.wikipedia.org/wiki/FASTA_format
+.TP
+https://en.wikipedia.org/wiki/FASTQ_format
+
+Further description of the FASTA and FASTQ formats
diff --git a/ext/htslib/faidx.c b/ext/htslib/faidx.c
new file mode 100644
index 0000000..ed39c0c
--- /dev/null
+++ b/ext/htslib/faidx.c
@@ -0,0 +1,1066 @@
+/*  faidx.c -- FASTA and FASTQ random access.
+
+    Copyright (C) 2008, 2009, 2013-2020, 2022, 2024 Genome Research Ltd.
+    Portions copyright (C) 2011 Broad Institute.
+
+    Author: Heng Li <lh3@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+
+#include <ctype.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <inttypes.h>
+#include <errno.h>
+#include <limits.h>
+#include <unistd.h>
+#include <assert.h>
+
+#include "htslib/bgzf.h"
+#include "htslib/faidx.h"
+#include "htslib/hfile.h"
+#include "htslib/khash.h"
+#include "htslib/kstring.h"
+#include "hts_internal.h"
+
+// Faster isgraph; assumes ASCII
+static inline int isgraph_(unsigned char c) {
+    return c > ' ' && c <= '~';
+}
+
+#ifdef isgraph
+#  undef isgraph
+#endif
+#define isgraph isgraph_
+
+// An optimised bgzf_getc.
+// We could consider moving this to bgzf.h, but our own code uses it here only.
+static inline int bgzf_getc_(BGZF *fp) {
+    if (fp->block_offset+1 < fp->block_length) {
+        int c = ((unsigned char*)fp->uncompressed_block)[fp->block_offset++];
+        fp->uncompressed_address++;
+        return c;
+    }
+
+    return bgzf_getc(fp);
+}
+#define bgzf_getc bgzf_getc_
+
+typedef struct {
+    int id; // faidx_t->name[id] is for this struct.
+    uint32_t line_len, line_blen;
+    uint64_t len;
+    uint64_t seq_offset;
+    uint64_t qual_offset;
+} faidx1_t;
+KHASH_MAP_INIT_STR(s, faidx1_t)
+
+struct faidx_t {
+    BGZF *bgzf;
+    int n, m;
+    char **name;
+    khash_t(s) *hash;
+    enum fai_format_options format;
+};
+
+static int fai_name2id(void *v, const char *ref)
+{
+    faidx_t *fai = (faidx_t *)v;
+    khint_t k = kh_get(s, fai->hash, ref);
+    return k == kh_end(fai->hash) ? -1 : kh_val(fai->hash, k).id;
+}
+
+static inline int fai_insert_index(faidx_t *idx, const char *name, uint64_t len, uint32_t line_len, uint32_t line_blen, uint64_t seq_offset, uint64_t qual_offset)
+{
+    if (!name) {
+        hts_log_error("Malformed line");
+        return -1;
+    }
+
+    char *name_key = strdup(name);
+    int absent;
+    khint_t k = kh_put(s, idx->hash, name_key, &absent);
+    faidx1_t *v = &kh_value(idx->hash, k);
+
+    if (! absent) {
+        hts_log_warning("Ignoring duplicate sequence \"%s\" at byte offset %" PRIu64, name, seq_offset);
+        free(name_key);
+        return 0;
+    }
+
+    if (idx->n == idx->m) {
+        char **tmp;
+        idx->m = idx->m? idx->m<<1 : 16;
+        if (!(tmp = (char**)realloc(idx->name, sizeof(char*) * idx->m))) {
+            hts_log_error("Out of memory");
+            return -1;
+        }
+        idx->name = tmp;
+    }
+    v->id = idx->n;
+    idx->name[idx->n++] = name_key;
+    v->len = len;
+    v->line_len = line_len;
+    v->line_blen = line_blen;
+    v->seq_offset = seq_offset;
+    v->qual_offset = qual_offset;
+
+    return 0;
+}
+
+
+static faidx_t *fai_build_core(BGZF *bgzf) {
+    kstring_t name = { 0, 0, NULL };
+    int c, read_done, line_num;
+    faidx_t *idx;
+    uint64_t seq_offset, qual_offset;
+    uint64_t seq_len, qual_len;
+    uint64_t char_len, cl, line_len, ll;
+    enum read_state {OUT_READ, IN_NAME, IN_SEQ, SEQ_END, IN_QUAL} state;
+
+    idx = (faidx_t*)calloc(1, sizeof(faidx_t));
+    idx->hash = kh_init(s);
+    idx->format = FAI_NONE;
+
+    state = OUT_READ, read_done = 0, line_num = 1;
+    seq_offset = qual_offset = seq_len = qual_len = char_len = cl = line_len = ll = 0;
+
+    while ((c = bgzf_getc(bgzf)) >= 0) {
+        switch (state) {
+            case OUT_READ:
+                switch (c) {
+                    case '>':
+                        if (idx->format == FAI_FASTQ) {
+                            hts_log_error("Found '>' in a FASTQ file, error at line %d", line_num);
+                            goto fail;
+                        }
+
+                        idx->format = FAI_FASTA;
+                        state = IN_NAME;
+                    break;
+
+                    case '@':
+                        if (idx->format == FAI_FASTA) {
+                            hts_log_error("Found '@' in a FASTA file, error at line %d", line_num);
+                            goto fail;
+                        }
+
+                        idx->format = FAI_FASTQ;
+                        state = IN_NAME;
+                    break;
+
+                    case '\r':
+                        // Blank line with cr-lf ending?
+                        if ((c = bgzf_getc(bgzf)) == '\n') {
+                            line_num++;
+                        } else {
+                            hts_log_error("Format error, carriage return not followed by new line at line %d", line_num);
+                            goto fail;
+                        }
+                    break;
+
+                    case '\n':
+                        // just move onto the next line
+                        line_num++;
+                    break;
+
+                    default: {
+                        char s[4] = { '"', c, '"', '\0' };
+                        hts_log_error("Format error, unexpected %s at line %d", isprint(c) ? s : "character", line_num);
+                        goto fail;
+                    }
+                }
+            break;
+
+            case IN_NAME:
+                if (read_done) {
+                    if (fai_insert_index(idx, name.s, seq_len, line_len, char_len, seq_offset, qual_offset) != 0)
+                        goto fail;
+
+                    read_done = 0;
+                }
+
+                name.l = 0;
+
+                do {
+                    if (!isspace(c)) {
+                        kputc(c, &name);
+                    } else if (name.l > 0 || c == '\n') {
+                        break;
+                    }
+                } while ((c = bgzf_getc(bgzf)) >= 0);
+
+                kputsn("", 0, &name);
+
+                if (c < 0) {
+                    hts_log_error("The last entry '%s' has no sequence at line %d", name.s, line_num);
+                    goto fail;
+                }
+
+                // read the rest of the line if necessary
+                if (c != '\n') while ((c = bgzf_getc(bgzf)) >= 0 && c != '\n');
+
+                state = IN_SEQ; seq_len = qual_len = char_len = line_len = 0;
+                seq_offset = bgzf_utell(bgzf);
+                line_num++;
+            break;
+
+            case IN_SEQ:
+                if (idx->format == FAI_FASTA) {
+                    if (c == '\n') {
+                        state = OUT_READ;
+                        line_num++;
+                        continue;
+                    } else if (c == '>') {
+                        state = IN_NAME;
+                        continue;
+                    }
+                } else if (idx->format == FAI_FASTQ) {
+                    if (c == '+') {
+                        state = IN_QUAL;
+                        if (c != '\n') while ((c = bgzf_getc(bgzf)) >= 0 && c != '\n');
+                        qual_offset = bgzf_utell(bgzf);
+                        line_num++;
+                        continue;
+                    } else if (c == '\n') {
+                        hts_log_error("Inlined empty line is not allowed in sequence '%s' at line %d", name.s, line_num);
+                        goto fail;
+                    }
+                }
+
+                ll = cl = 0;
+
+                if (idx->format == FAI_FASTA) read_done = 1;
+
+                do {
+                    ll++;
+                    if (isgraph(c)) cl++;
+                } while ((c = bgzf_getc(bgzf)) >= 0 && c != '\n');
+
+                ll++; seq_len += cl;
+
+                if (line_len == 0) {
+                    line_len = ll;
+                    char_len = cl;
+                } else if (line_len > ll) {
+
+                    if (idx->format == FAI_FASTA)
+                        state = OUT_READ;
+                    else
+                        state = SEQ_END;
+
+                } else if (line_len < ll) {
+                    hts_log_error("Different line length in sequence '%s' at line %d", name.s, line_num);
+                    goto fail;
+                }
+
+                line_num++;
+            break;
+
+            case SEQ_END:
+                if (c == '+') {
+                    state = IN_QUAL;
+                    while ((c = bgzf_getc(bgzf)) >= 0 && c != '\n');
+                    qual_offset = bgzf_utell(bgzf);
+                    line_num++;
+                } else {
+                    hts_log_error("Format error, expecting '+', got '%c' at line %d", c, line_num);
+                    goto fail;
+                }
+            break;
+
+            case IN_QUAL:
+                if (c == '\n') {
+                    if (!read_done) {
+                        hts_log_error("Inlined empty line is not allowed in quality of sequence '%s' at line %d", name.s, line_num);
+                        goto fail;
+                    }
+
+                    state = OUT_READ;
+                    line_num++;
+                    continue;
+                } else if (c == '@' && read_done) {
+                    state = IN_NAME;
+                    continue;
+                }
+
+                ll = cl = 0;
+
+                do {
+                    ll++;
+                    if (isgraph(c)) cl++;
+                } while ((c = bgzf_getc(bgzf)) >= 0 && c != '\n');
+
+                ll++; qual_len += cl;
+
+                if (line_len < ll) {
+                    hts_log_error("Quality line length too long in '%s' at line %d", name.s, line_num);
+                    goto fail;
+                } else if (qual_len == seq_len) {
+                    read_done = 1;
+                } else if (qual_len > seq_len) {
+                    hts_log_error("Quality length longer than sequence in '%s' at line %d", name.s, line_num);
+                    goto fail;
+                } else if (line_len > ll) {
+                    hts_log_error("Quality line length too short in '%s' at line %d", name.s, line_num);
+                    goto fail;
+                }
+
+                line_num++;
+            break;
+        }
+    }
+
+    if (read_done) {
+        if (fai_insert_index(idx, name.s, seq_len, line_len, char_len, seq_offset, qual_offset) != 0)
+            goto fail;
+    } else {
+        hts_log_error("File truncated at line %d", line_num);
+        goto fail;
+    }
+
+    free(name.s);
+    return idx;
+
+fail:
+    free(name.s);
+    fai_destroy(idx);
+    return NULL;
+}
+
+
+static int fai_save(const faidx_t *fai, hFILE *fp) {
+    khint_t k;
+    int i;
+    char buf[96]; // Must be big enough for format below.
+
+    for (i = 0; i < fai->n; ++i) {
+        faidx1_t x;
+        k = kh_get(s, fai->hash, fai->name[i]);
+        assert(k < kh_end(fai->hash));
+        x = kh_value(fai->hash, k);
+
+        if (fai->format == FAI_FASTA) {
+            snprintf(buf, sizeof(buf),
+                 "\t%"PRIu64"\t%"PRIu64"\t%"PRIu32"\t%"PRIu32"\n",
+                 x.len, x.seq_offset, x.line_blen, x.line_len);
+        } else {
+            snprintf(buf, sizeof(buf),
+                 "\t%"PRIu64"\t%"PRIu64"\t%"PRIu32"\t%"PRIu32"\t%"PRIu64"\n",
+                 x.len, x.seq_offset, x.line_blen, x.line_len, x.qual_offset);
+        }
+
+        if (hputs(fai->name[i], fp) != 0) return -1;
+        if (hputs(buf, fp) != 0) return -1;
+    }
+    return 0;
+}
+
+
+static faidx_t *fai_read(hFILE *fp, const char *fname, int format)
+{
+    faidx_t *fai;
+    char *buf = NULL, *p;
+    ssize_t l, lnum = 1;
+
+    fai = (faidx_t*)calloc(1, sizeof(faidx_t));
+    if (!fai) return NULL;
+
+    fai->hash = kh_init(s);
+    if (!fai->hash) goto fail;
+
+    buf = (char*)calloc(0x10000, 1);
+    if (!buf) goto fail;
+
+    while ((l = hgetln(buf, 0x10000, fp)) > 0) {
+        uint32_t line_len, line_blen, n;
+        uint64_t len;
+        uint64_t seq_offset;
+        uint64_t qual_offset = 0;
+
+        for (p = buf; *p && !isspace_c(*p); ++p);
+
+        if (p - buf < l) {
+            *p = 0; ++p;
+        }
+
+        if (format == FAI_FASTA) {
+            n = sscanf(p, "%"SCNu64"%"SCNu64"%"SCNu32"%"SCNu32, &len, &seq_offset, &line_blen, &line_len);
+
+            if (n != 4) {
+                hts_log_error("Could not understand FASTA index %s line %zd", fname, lnum);
+                goto fail;
+            }
+        } else {
+            n = sscanf(p, "%"SCNu64"%"SCNu64"%"SCNu32"%"SCNu32"%"SCNu64, &len, &seq_offset, &line_blen, &line_len, &qual_offset);
+
+            if (n != 5) {
+                if (n == 4) {
+                    hts_log_error("Possibly this is a FASTA index, try using faidx.  Problem in %s line %zd", fname, lnum);
+                } else {
+                    hts_log_error("Could not understand FASTQ index %s line %zd", fname, lnum);
+                }
+
+                goto fail;
+            }
+        }
+
+        if (fai_insert_index(fai, buf, len, line_len, line_blen, seq_offset, qual_offset) != 0) {
+            goto fail;
+        }
+
+        if (buf[l - 1] == '\n') ++lnum;
+    }
+
+    if (l < 0) {
+        hts_log_error("Error while reading %s: %s", fname, strerror(errno));
+        goto fail;
+    }
+    free(buf);
+    return fai;
+
+ fail:
+    free(buf);
+    fai_destroy(fai);
+    return NULL;
+}
+
+void fai_destroy(faidx_t *fai)
+{
+    int i;
+    if (!fai) return;
+    for (i = 0; i < fai->n; ++i) free(fai->name[i]);
+    free(fai->name);
+    kh_destroy(s, fai->hash);
+    if (fai->bgzf) bgzf_close(fai->bgzf);
+    free(fai);
+}
+
+
+static int fai_build3_core(const char *fn, const char *fnfai, const char *fngzi)
+{
+    kstring_t fai_kstr = { 0, 0, NULL };
+    kstring_t gzi_kstr = { 0, 0, NULL };
+    BGZF *bgzf = NULL;
+    hFILE *fp = NULL;
+    faidx_t *fai = NULL;
+    int save_errno, res;
+    char *file_type;
+
+    bgzf = bgzf_open(fn, "r");
+
+    if ( !bgzf ) {
+        hts_log_error("Failed to open the file %s : %s", fn, strerror(errno));
+        goto fail;
+    }
+
+    if ( bgzf->is_compressed ) {
+        if (bgzf_index_build_init(bgzf) != 0) {
+            hts_log_error("Failed to allocate bgzf index");
+            goto fail;
+        }
+    }
+
+    fai = fai_build_core(bgzf);
+
+    if ( !fai ) {
+        if (bgzf->is_compressed && bgzf->is_gzip) {
+            hts_log_error("Cannot index files compressed with gzip, please use bgzip");
+        }
+        goto fail;
+    }
+
+    if (fai->format == FAI_FASTA) {
+        file_type   = "FASTA";
+    } else {
+        file_type   = "FASTQ";
+    }
+
+    if (!fnfai) {
+        if (ksprintf(&fai_kstr, "%s.fai", fn) < 0) goto fail;
+        fnfai = fai_kstr.s;
+    }
+
+    if (!fngzi) {
+        if (ksprintf(&gzi_kstr, "%s.gzi", fn) < 0) goto fail;
+        fngzi = gzi_kstr.s;
+    }
+
+    if ( bgzf->is_compressed ) {
+        if (bgzf_index_dump(bgzf, fngzi, NULL) < 0) {
+            hts_log_error("Failed to make bgzf index %s", fngzi);
+            goto fail;
+        }
+    }
+
+    res = bgzf_close(bgzf);
+    bgzf = NULL;
+
+    if (res < 0) {
+        hts_log_error("Error on closing %s : %s", fn, strerror(errno));
+        goto fail;
+    }
+
+    fp = hopen(fnfai, "wb");
+
+    if ( !fp ) {
+        hts_log_error("Failed to open %s index %s : %s", file_type, fnfai, strerror(errno));
+        goto fail;
+    }
+
+    if (fai_save(fai, fp) != 0) {
+        hts_log_error("Failed to write %s index %s : %s", file_type, fnfai, strerror(errno));
+        goto fail;
+    }
+
+    if (hclose(fp) != 0) {
+        hts_log_error("Failed on closing %s index %s : %s", file_type, fnfai, strerror(errno));
+        goto fail;
+    }
+
+    free(fai_kstr.s);
+    free(gzi_kstr.s);
+    fai_destroy(fai);
+    return 0;
+
+ fail:
+    save_errno = errno;
+    free(fai_kstr.s);
+    free(gzi_kstr.s);
+    bgzf_close(bgzf);
+    fai_destroy(fai);
+    errno = save_errno;
+    return -1;
+}
+
+
+int fai_build3(const char *fn, const char *fnfai, const char *fngzi) {
+    return fai_build3_core(fn, fnfai, fngzi);
+}
+
+
+int fai_build(const char *fn) {
+    return fai_build3(fn, NULL, NULL);
+}
+
+
+static faidx_t *fai_load3_core(const char *fn, const char *fnfai, const char *fngzi,
+                   int flags, int format)
+{
+    kstring_t fai_kstr = { 0, 0, NULL };
+    kstring_t gzi_kstr = { 0, 0, NULL };
+    hFILE *fp = NULL;
+    faidx_t *fai = NULL;
+    int res, gzi_index_needed = 0;
+    char *file_type;
+
+    if (format == FAI_FASTA) {
+        file_type   = "FASTA";
+    } else {
+        file_type   = "FASTQ";
+    }
+
+    if (fn == NULL)
+        return NULL;
+
+    if (fnfai == NULL) {
+        if (ksprintf(&fai_kstr, "%s.fai", fn) < 0) goto fail;
+        fnfai = fai_kstr.s;
+    }
+    if (fngzi == NULL) {
+        if (ksprintf(&gzi_kstr, "%s.gzi", fn) < 0) goto fail;
+        fngzi = gzi_kstr.s;
+    }
+
+    fp = hopen(fnfai, "rb");
+
+    if (fp) {
+        // index file present, check if a compressed index is needed
+        hFILE *gz = NULL;
+        BGZF *bgzf = bgzf_open(fn, "rb");
+
+        if (bgzf == 0) {
+            hts_log_error("Failed to open %s file %s", file_type, fn);
+            goto fail;
+        }
+
+        if (bgzf_compression(bgzf) == 2) { // BGZF compression
+            if ((gz = hopen(fngzi, "rb")) == 0) {
+
+                if (!(flags & FAI_CREATE) || errno != ENOENT) {
+                    hts_log_error("Failed to open %s index %s: %s", file_type, fngzi, strerror(errno));
+                    bgzf_close(bgzf);
+                    goto fail;
+                }
+
+                gzi_index_needed = 1;
+                res = hclose(fp); // closed as going to be re-indexed
+
+                if (res < 0) {
+                    hts_log_error("Failed on closing %s index %s : %s", file_type, fnfai, strerror(errno));
+                    goto fail;
+                }
+            } else {
+                res = hclose(gz);
+
+                if (res < 0) {
+                    hts_log_error("Failed on closing %s index %s : %s", file_type, fngzi, strerror(errno));
+                    goto fail;
+                }
+            }
+        }
+
+        bgzf_close(bgzf);
+    }
+
+    if (fp == 0 || gzi_index_needed) {
+        if (!(flags & FAI_CREATE) || errno != ENOENT) {
+            hts_log_error("Failed to open %s index %s: %s", file_type, fnfai, strerror(errno));
+            goto fail;
+        }
+
+        hts_log_info("Build %s index", file_type);
+
+        if (fai_build3_core(fn, fnfai, fngzi) < 0) {
+            goto fail;
+        }
+
+        fp = hopen(fnfai, "rb");
+        if (fp == 0) {
+            hts_log_error("Failed to open %s index %s: %s", file_type, fnfai, strerror(errno));
+            goto fail;
+        }
+    }
+
+    fai = fai_read(fp, fnfai, format);
+    if (fai == NULL) {
+        hts_log_error("Failed to read %s index %s", file_type, fnfai);
+        goto fail;
+    }
+
+    res = hclose(fp);
+    fp = NULL;
+    if (res < 0) {
+        hts_log_error("Failed on closing %s index %s : %s", file_type, fnfai, strerror(errno));
+        goto fail;
+    }
+
+    fai->bgzf = bgzf_open(fn, "rb");
+    if (fai->bgzf == 0) {
+        hts_log_error("Failed to open %s file %s", file_type, fn);
+        goto fail;
+    }
+
+    if ( fai->bgzf->is_compressed==1 ) {
+        if ( bgzf_index_load(fai->bgzf, fngzi, NULL) < 0 ) {
+            hts_log_error("Failed to load .gzi index: %s", fngzi);
+            goto fail;
+        }
+    }
+    free(fai_kstr.s);
+    free(gzi_kstr.s);
+    return fai;
+
+ fail:
+    if (fai) fai_destroy(fai);
+    if (fp) hclose_abruptly(fp);
+    free(fai_kstr.s);
+    free(gzi_kstr.s);
+    return NULL;
+}
+
+
+faidx_t *fai_load3(const char *fn, const char *fnfai, const char *fngzi,
+                   int flags) {
+    return fai_load3_core(fn, fnfai, fngzi, flags, FAI_FASTA);
+}
+
+
+faidx_t *fai_load(const char *fn)
+{
+    return fai_load3(fn, NULL, NULL, FAI_CREATE);
+}
+
+
+faidx_t *fai_load3_format(const char *fn, const char *fnfai, const char *fngzi,
+                   int flags, enum fai_format_options format) {
+    return fai_load3_core(fn, fnfai, fngzi, flags, format);
+}
+
+
+faidx_t *fai_load_format(const char *fn, enum fai_format_options format) {
+    return fai_load3_format(fn, NULL, NULL, FAI_CREATE, format);
+}
+
+
+static char *fai_retrieve(const faidx_t *fai, const faidx1_t *val,
+                          uint64_t offset, hts_pos_t beg, hts_pos_t end, hts_pos_t *len) {
+    char *buffer, *s;
+    ssize_t nread, remaining, firstline_len, firstline_blen;
+    int ret;
+
+    if ((uint64_t) end - (uint64_t) beg >= SIZE_MAX - 2) {
+        hts_log_error("Range %"PRId64"..%"PRId64" too big", beg, end);
+        *len = -1;
+        return NULL;
+    }
+
+    if (val->line_blen <= 0) {
+        hts_log_error("Invalid line length in index: %d", val->line_blen);
+        *len = -1;
+        return NULL;
+    }
+
+    ret = bgzf_useek(fai->bgzf,
+                     offset
+                     + beg / val->line_blen * val->line_len
+                     + beg % val->line_blen, SEEK_SET);
+
+    if (ret < 0) {
+        *len = -1;
+        hts_log_error("Failed to retrieve block. (Seeking in a compressed, .gzi unindexed, file?)");
+        return NULL;
+    }
+
+    // Over-allocate so there is extra space for one end-of-line sequence
+    buffer = (char*)malloc((size_t) end - beg + val->line_len - val->line_blen + 1);
+    if (!buffer) {
+        *len = -1;
+        return NULL;
+    }
+
+    remaining = *len = end - beg;
+    firstline_blen = val->line_blen - beg % val->line_blen;
+
+    // Special case when the entire interval requested is within a single FASTA/Q line
+    if (remaining <= firstline_blen) {
+        nread = bgzf_read_small(fai->bgzf, buffer, remaining);
+        if (nread < remaining) goto error;
+        buffer[nread] = '\0';
+        return buffer;
+    }
+
+    s = buffer;
+    firstline_len = val->line_len - beg % val->line_blen;
+
+    // Read the (partial) first line and its line terminator, but increment  s  past the
+    // line contents only, so the terminator characters will be overwritten by the next line.
+    nread = bgzf_read_small(fai->bgzf, s, firstline_len);
+    if (nread < firstline_len) goto error;
+    s += firstline_blen;
+    remaining -= firstline_blen;
+
+    // Similarly read complete lines and their line terminator characters, but overwrite the latter.
+    while (remaining > val->line_blen) {
+        nread = bgzf_read_small(fai->bgzf, s, val->line_len);
+        if (nread < (ssize_t) val->line_len) goto error;
+        s += val->line_blen;
+        remaining -= val->line_blen;
+    }
+
+    if (remaining > 0) {
+        nread = bgzf_read_small(fai->bgzf, s, remaining);
+        if (nread < remaining) goto error;
+        s += remaining;
+    }
+
+    *s = '\0';
+    return buffer;
+
+error:
+    hts_log_error("Failed to retrieve block: %s",
+                  (nread == 0)? "unexpected end of file" : "error reading file");
+    free(buffer);
+    *len = -1;
+    return NULL;
+}
+
+static int fai_get_val(const faidx_t *fai, const char *str,
+                       hts_pos_t *len, faidx1_t *val, hts_pos_t *fbeg, hts_pos_t *fend) {
+    khiter_t iter;
+    khash_t(s) *h;
+    int id;
+    hts_pos_t beg, end;
+
+    if (!fai_parse_region(fai, str, &id, &beg, &end, 0)) {
+        hts_log_warning("Reference %s not found in FASTA file, returning empty sequence", str);
+        *len = -2;
+        return 1;
+    }
+
+    h = fai->hash;
+    iter = kh_get(s, h, faidx_iseq(fai, id));
+    if (iter >= kh_end(h)) {
+        // should have already been caught above
+        abort();
+    }
+    *val = kh_value(h, iter);
+
+    if (beg >= val->len) beg = val->len;
+    if (end >= val->len) end = val->len;
+    if (beg > end) beg = end;
+
+    *fbeg = beg;
+    *fend = end;
+
+    return 0;
+}
+
+/*
+ *  The internal still has line_blen as uint32_t, but our references
+ *  can be longer, so for future proofing we use hts_pos_t.  We also needed
+ *  a signed value so we can return negatives as an error.
+ */
+hts_pos_t fai_line_length(const faidx_t *fai, const char *str)
+{
+    faidx1_t val;
+    int64_t beg, end;
+    hts_pos_t len;
+
+    if (fai_get_val(fai, str, &len, &val, &beg, &end))
+        return -1;
+    else
+        return val.line_blen;
+}
+
+char *fai_fetch64(const faidx_t *fai, const char *str, hts_pos_t *len)
+{
+    faidx1_t val;
+    int64_t beg, end;
+
+    if (fai_get_val(fai, str, len, &val, &beg, &end)) {
+        return NULL;
+    }
+
+    // now retrieve the sequence
+    return fai_retrieve(fai, &val, val.seq_offset, beg, end, len);
+}
+
+char *fai_fetch(const faidx_t *fai, const char *str, int *len)
+{
+    hts_pos_t len64;
+    char *ret = fai_fetch64(fai, str, &len64);
+    *len = len64 < INT_MAX ? len64 : INT_MAX; // trunc
+    return ret;
+}
+
+char *fai_fetchqual64(const faidx_t *fai, const char *str, hts_pos_t *len) {
+    faidx1_t val;
+    int64_t beg, end;
+
+    if (fai_get_val(fai, str, len, &val, &beg, &end)) {
+        return NULL;
+    }
+
+    // now retrieve the sequence
+    return fai_retrieve(fai, &val, val.qual_offset, beg, end, len);
+}
+
+char *fai_fetchqual(const faidx_t *fai, const char *str, int *len) {
+    hts_pos_t len64;
+    char *ret = fai_fetchqual64(fai, str, &len64);
+    *len = len64 < INT_MAX ? len64 : INT_MAX; // trunc
+    return ret;
+}
+
+int faidx_fetch_nseq(const faidx_t *fai)
+{
+    return fai->n;
+}
+
+int faidx_nseq(const faidx_t *fai)
+{
+    return fai->n;
+}
+
+const char *faidx_iseq(const faidx_t *fai, int i)
+{
+    return fai->name[i];
+}
+
+hts_pos_t faidx_seq_len64(const faidx_t *fai, const char *seq)
+{
+    khint_t k = kh_get(s, fai->hash, seq);
+    if ( k == kh_end(fai->hash) ) return -1;
+    return kh_val(fai->hash, k).len;
+}
+
+int faidx_seq_len(const faidx_t *fai, const char *seq)
+{
+    hts_pos_t len = faidx_seq_len64(fai, seq);
+    return len < INT_MAX ? len : INT_MAX;
+}
+
+static int faidx_adjust_position(const faidx_t *fai, int end_adjust,
+                                 faidx1_t *val_out, const char *c_name,
+                                 hts_pos_t *p_beg_i, hts_pos_t *p_end_i,
+                                 hts_pos_t *len) {
+    khiter_t iter;
+    faidx1_t *val;
+
+    // Adjust position
+    iter = kh_get(s, fai->hash, c_name);
+
+    if (iter == kh_end(fai->hash)) {
+        if (len)
+            *len = -2;
+        hts_log_error("The sequence \"%s\" was not found", c_name);
+        return 1;
+    }
+
+    val = &kh_value(fai->hash, iter);
+
+    if (val_out)
+        *val_out = *val;
+
+    if(*p_end_i < *p_beg_i)
+        *p_beg_i = *p_end_i;
+
+    if(*p_beg_i < 0)
+        *p_beg_i = 0;
+    else if(val->len <= *p_beg_i)
+        *p_beg_i = val->len;
+
+    if(*p_end_i < 0)
+        *p_end_i = 0;
+    else if(val->len <= *p_end_i)
+        *p_end_i = val->len - end_adjust;
+
+    return 0;
+}
+
+int fai_adjust_region(const faidx_t *fai, int tid,
+                      hts_pos_t *beg, hts_pos_t *end)
+{
+    hts_pos_t orig_beg, orig_end;
+
+    if (!fai || !beg || !end || tid < 0 || tid >= fai->n)
+        return -1;
+
+    orig_beg = *beg;
+    orig_end = *end;
+    if (faidx_adjust_position(fai, 0, NULL, fai->name[tid], beg, end, NULL) != 0) {
+        hts_log_error("Inconsistent faidx internal state - couldn't find \"%s\"",
+                      fai->name[tid]);
+        return -1;
+    }
+
+    return ((orig_beg != *beg ? 1 : 0) |
+            (orig_end != *end && orig_end < HTS_POS_MAX ? 2 : 0));
+}
+
+char *faidx_fetch_seq64(const faidx_t *fai, const char *c_name, hts_pos_t p_beg_i, hts_pos_t p_end_i, hts_pos_t *len)
+{
+    faidx1_t val;
+
+    // Adjust position
+    if (faidx_adjust_position(fai, 1, &val, c_name, &p_beg_i, &p_end_i, len)) {
+        return NULL;
+    }
+
+    // Now retrieve the sequence
+    return fai_retrieve(fai, &val, val.seq_offset, p_beg_i, p_end_i + 1, len);
+}
+
+char *faidx_fetch_seq(const faidx_t *fai, const char *c_name, int p_beg_i, int p_end_i, int *len)
+{
+    hts_pos_t len64;
+    char *ret = faidx_fetch_seq64(fai, c_name, p_beg_i, p_end_i, &len64);
+    *len = len64 < INT_MAX ? len64 : INT_MAX;  // trunc
+    return ret;
+}
+
+char *faidx_fetch_qual64(const faidx_t *fai, const char *c_name, hts_pos_t p_beg_i, hts_pos_t p_end_i, hts_pos_t *len)
+{
+    faidx1_t val;
+
+    // Adjust position
+    if (faidx_adjust_position(fai, 1, &val, c_name, &p_beg_i, &p_end_i, len)) {
+        return NULL;
+    }
+
+    // Now retrieve the sequence
+    return fai_retrieve(fai, &val, val.qual_offset, p_beg_i, p_end_i + 1, len);
+}
+
+char *faidx_fetch_qual(const faidx_t *fai, const char *c_name, int p_beg_i, int p_end_i, int *len)
+{
+    hts_pos_t len64;
+    char *ret = faidx_fetch_qual64(fai, c_name, p_beg_i, p_end_i, &len64);
+    *len = len64 < INT_MAX ? len64 : INT_MAX;  // trunc
+    return ret;
+}
+
+int faidx_has_seq(const faidx_t *fai, const char *seq)
+{
+    khiter_t iter = kh_get(s, fai->hash, seq);
+    if (iter == kh_end(fai->hash)) return 0;
+    return 1;
+}
+
+const char *fai_parse_region(const faidx_t *fai, const char *s,
+                             int *tid, hts_pos_t *beg, hts_pos_t *end,
+                             int flags)
+{
+    return hts_parse_region(s, tid, beg, end, (hts_name2id_f)fai_name2id, (void *)fai, flags);
+}
+
+void fai_set_cache_size(faidx_t *fai, int cache_size) {
+    bgzf_set_cache_size(fai->bgzf, cache_size);
+}
+
+// Adds a thread pool to the underlying BGZF layer.
+int fai_thread_pool(faidx_t *fai, struct hts_tpool *pool, int qsize) {
+    return bgzf_thread_pool(fai->bgzf, pool, qsize);
+}
+
+char *fai_path(const char *fa) {
+    char *fai = NULL;
+    if (!fa) {
+        hts_log_error("No reference file specified");
+    } else {
+        char *fai_tmp = strstr(fa, HTS_IDX_DELIM);
+        if (fai_tmp) {
+            fai_tmp += strlen(HTS_IDX_DELIM);
+            fai = strdup(fai_tmp);
+            if (!fai)
+                hts_log_error("Failed to allocate memory");
+        } else {
+            if (hisremote(fa)) {
+                fai = hts_idx_locatefn(fa, ".fai");       // get the remote fai file name, if any, but do not download the file
+                if (!fai)
+                    hts_log_error("Failed to locate index file for remote reference file '%s'", fa);
+            } else{
+                if (hts_idx_check_local(fa, HTS_FMT_FAI, &fai) == 0 && fai) {
+                    if (fai_build3(fa, fai, NULL) == -1) {      // create local fai file by indexing local fasta
+                        hts_log_error("Failed to build index file for reference file '%s'", fa);
+                        free(fai);
+                        fai = NULL;
+                    }
+                }
+            }
+        }
+    }
+
+    return fai;
+}
diff --git a/ext/htslib/fuzz_settings.h b/ext/htslib/fuzz_settings.h
new file mode 100644
index 0000000..8215819
--- /dev/null
+++ b/ext/htslib/fuzz_settings.h
@@ -0,0 +1,35 @@
+/*  fuzz_settings.h -- fuzz-tester specific definitions
+
+    Copyright (C) 2023 Genome Research Ltd.
+
+    Author: Rob Davies <rmd@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#ifndef HTSLIB_FUZZ_SETTINGS_H
+#define HTSLIB_FUZZ_SETTINGS_H
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+
+#ifndef FUZZ_ALLOC_LIMIT
+// By default libfuzzer reports out-of-memory on allocations > 2 Gbytes
+#define FUZZ_ALLOC_LIMIT 2000000000ULL
+#endif
+
+#endif
+#endif
diff --git a/ext/htslib/header.c b/ext/htslib/header.c
new file mode 100644
index 0000000..7f62074
--- /dev/null
+++ b/ext/htslib/header.c
@@ -0,0 +1,2803 @@
+/*
+Copyright (c) 2018-2020, 2023 Genome Research Ltd.
+Authors: James Bonfield <jkb@sanger.ac.uk>, Valeriu Ohan <vo2@sanger.ac.uk>
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+Institute nor the names of its contributors may be used to endorse or promote
+products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+
+#include <string.h>
+#include <assert.h>
+#include <errno.h>
+#include "textutils_internal.h"
+#include "header.h"
+
+// Hash table for removing multiple lines from the header
+KHASH_SET_INIT_STR(rm)
+// Used for long refs in SAM files
+KHASH_DECLARE(s2i, kh_cstr_t, int64_t)
+
+typedef khash_t(rm) rmhash_t;
+
+static int sam_hdr_link_pg(sam_hdr_t *bh);
+
+static int sam_hrecs_vupdate(sam_hrecs_t *hrecs, sam_hrec_type_t *type, va_list ap);
+static int sam_hrecs_update(sam_hrecs_t *hrecs, sam_hrec_type_t *type, ...);
+
+
+#define MAX_ERROR_QUOTE 320 // Prevent over-long error messages
+static void sam_hrecs_error(const char *msg, const char *line, size_t len, size_t lno) {
+    int j;
+
+    if (len > MAX_ERROR_QUOTE)
+        len = MAX_ERROR_QUOTE;
+    for (j = 0; j < len && line[j] != '\n'; j++)
+        ;
+    hts_log_error("%s at line %zd: \"%.*s\"", msg, lno, j, line);
+}
+
+/* ==== Static methods ==== */
+
+static int sam_hrecs_init_type_order(sam_hrecs_t *hrecs, char *type_list) {
+    if (!hrecs)
+        return -1;
+
+    if (!type_list) {
+        hrecs->type_count = 5;
+        hrecs->type_order = calloc(hrecs->type_count, 3);
+        if (!hrecs->type_order)
+            return -1;
+        memcpy(hrecs->type_order[0], "HD", 2);
+        memcpy(hrecs->type_order[1], "SQ", 2);
+        memcpy(hrecs->type_order[2], "RG", 2);
+        memcpy(hrecs->type_order[3], "PG", 2);
+        memcpy(hrecs->type_order[4], "CO", 2);
+    }
+
+    return 0;
+}
+
+static int sam_hrecs_add_ref_altnames(sam_hrecs_t *hrecs, int nref, const char *list) {
+    const char *token;
+    ks_tokaux_t aux;
+
+    if (!list)
+        return 0;
+
+    for (token = kstrtok(list, ",", &aux); token; token = kstrtok(NULL, NULL, &aux)) {
+        if (aux.p == token)
+            continue;
+
+        char *name = string_ndup(hrecs->str_pool, token, aux.p - token);
+        if (!name)
+            return -1;
+        int r;
+        khint_t k = kh_put(m_s2i, hrecs->ref_hash, name, &r);
+        if (r < 0) return -1;
+
+        if (r > 0)
+            kh_val(hrecs->ref_hash, k) = nref;
+        else if (kh_val(hrecs->ref_hash, k) != nref)
+            hts_log_warning("Duplicate entry AN:\"%s\" in sam header", name);
+    }
+
+    return 0;
+}
+
+static void sam_hrecs_remove_ref_altnames(sam_hrecs_t *hrecs, int expected, const char *list) {
+    const char *token, *sn;
+    ks_tokaux_t aux;
+    kstring_t str = KS_INITIALIZE;
+
+    if (expected < 0 || expected >= hrecs->nref)
+        return;
+    sn = hrecs->ref[expected].name;
+
+    for (token = kstrtok(list, ",", &aux); token; token = kstrtok(NULL, NULL, &aux)) {
+        kputsn(token, aux.p - token, ks_clear(&str));
+        khint_t k = kh_get(m_s2i, hrecs->ref_hash, str.s);
+        if (k != kh_end(hrecs->ref_hash)
+            && kh_val(hrecs->ref_hash, k) == expected
+            && strcmp(sn, str.s) != 0)
+            kh_del(m_s2i, hrecs->ref_hash, k);
+    }
+
+    free(str.s);
+}
+
+/* Updates the hash tables in the sam_hrecs_t structure.
+ *
+ * Returns 0 on success;
+ *        -1 on failure
+ */
+static int sam_hrecs_update_hashes(sam_hrecs_t *hrecs,
+                                   khint32_t type,
+                                   sam_hrec_type_t *h_type) {
+    /* Add to reference hash? */
+    if (type == TYPEKEY("SQ")) {
+        sam_hrec_tag_t *tag = h_type->tag;
+        int nref = hrecs->nref;
+        const char *name = NULL;
+        const char *altnames = NULL;
+        hts_pos_t len = -1;
+        int r;
+        khint_t k;
+
+        while (tag) {
+            if (tag->str[0] == 'S' && tag->str[1] == 'N') {
+                assert(tag->len >= 3);
+                name = tag->str+3;
+            } else if (tag->str[0] == 'L' && tag->str[1] == 'N') {
+                assert(tag->len >= 3);
+                len = strtoll(tag->str+3, NULL, 10);
+            } else if (tag->str[0] == 'A' && tag->str[1] == 'N') {
+                assert(tag->len >= 3);
+                altnames = tag->str+3;
+            }
+            tag = tag->next;
+        }
+
+        if (!name) {
+            hts_log_error("Header includes @SQ line with no SN: tag");
+            return -1; // SN should be present, according to spec.
+        }
+
+        if (len == -1) {
+            hts_log_error("Header includes @SQ line \"%s\" with no LN: tag",
+                          name);
+            return -1; // LN should be present, according to spec.
+        }
+
+        // Seen already?
+        k = kh_get(m_s2i, hrecs->ref_hash, name);
+        if (k < kh_end(hrecs->ref_hash)) {
+            nref = kh_val(hrecs->ref_hash, k);
+            int ref_changed_flag = 0;
+
+            // Check for hash entry added by sam_hrecs_refs_from_targets_array()
+            if (hrecs->ref[nref].ty == NULL) {
+                // Attach header line to existing stub entry.
+                hrecs->ref[nref].ty = h_type;
+                // Check lengths match; correct if not.
+                if (len != hrecs->ref[nref].len) {
+                    char tmp[32];
+                    snprintf(tmp, sizeof(tmp), "%" PRIhts_pos,
+                             hrecs->ref[nref].len);
+                    if (sam_hrecs_update(hrecs, h_type, "LN", tmp, NULL) < 0)
+                        return -1;
+                    ref_changed_flag = 1;
+                }
+                if (sam_hrecs_add_ref_altnames(hrecs, nref, altnames) < 0)
+                    return -1;
+
+                if (ref_changed_flag && (hrecs->refs_changed < 0 || hrecs->refs_changed > nref))
+                    hrecs->refs_changed = nref;
+                return 0;
+            }
+
+            // Check to see if an existing entry is being updated
+            if (hrecs->ref[nref].ty == h_type) {
+                if (hrecs->ref[nref].len != len) {
+                    hrecs->ref[nref].len = len;
+                    ref_changed_flag = 1;
+                }
+                if (!hrecs->ref[nref].name || strcmp(hrecs->ref[nref].name, name)) {
+                    hrecs->ref[nref].name = name;
+                    ref_changed_flag = 1;
+                }
+                if (sam_hrecs_add_ref_altnames(hrecs, nref, altnames) < 0)
+                    return -1;
+
+                if (ref_changed_flag && (hrecs->refs_changed < 0 || hrecs->refs_changed > nref))
+                    hrecs->refs_changed = nref;
+                return 0;
+            }
+
+            // If here, the name is a duplicate.
+            // Check to see if it matches the SN: tag from the earlier record.
+            if (strcmp(hrecs->ref[nref].name, name) == 0) {
+                hts_log_error("Duplicate entry \"%s\" in sam header",
+                                name);
+                return -1;
+            }
+
+            // Clash with an already-seen altname
+            // As SN: should be preferred to AN: add this as a new
+            // record and update the hash entry to point to it.
+            hts_log_warning("Ref name SN:\"%s\" is a duplicate of an existing AN key", name);
+            nref = hrecs->nref;
+        }
+
+        if (nref == hrecs->ref_sz) {
+            size_t new_sz = hrecs->ref_sz >= 4 ? hrecs->ref_sz + (hrecs->ref_sz / 4) : 32;
+            sam_hrec_sq_t *new_ref = realloc(hrecs->ref, sizeof(*hrecs->ref) * new_sz);
+            if (!new_ref)
+                return -1;
+            hrecs->ref = new_ref;
+            hrecs->ref_sz = new_sz;
+        }
+
+        hrecs->ref[nref].name = name;
+        hrecs->ref[nref].len  = len;
+        hrecs->ref[nref].ty = h_type;
+
+        k = kh_put(m_s2i, hrecs->ref_hash, hrecs->ref[nref].name, &r);
+        if (-1 == r) return -1;
+        kh_val(hrecs->ref_hash, k) = nref;
+
+        if (sam_hrecs_add_ref_altnames(hrecs, nref, altnames) < 0)
+            return -1;
+
+        if (hrecs->refs_changed < 0 || hrecs->refs_changed > hrecs->nref)
+            hrecs->refs_changed = hrecs->nref;
+        hrecs->nref++;
+    }
+
+    /* Add to read-group hash? */
+    if (type == TYPEKEY("RG")) {
+        sam_hrec_tag_t *tag = sam_hrecs_find_key(h_type, "ID", NULL);
+        int nrg = hrecs->nrg, r;
+        khint_t k;
+
+        if (!tag) {
+            hts_log_error("Header includes @RG line with no ID: tag");
+            return -1;  // ID should be present, according to spec.
+        }
+        assert(tag->str && tag->len >= 3);
+
+        // Seen already?
+        k = kh_get(m_s2i, hrecs->rg_hash, tag->str + 3);
+        if (k < kh_end(hrecs->rg_hash)) {
+            nrg = kh_val(hrecs->rg_hash, k);
+            assert(hrecs->rg[nrg].ty != NULL);
+            if (hrecs->rg[nrg].ty != h_type) {
+                hts_log_warning("Duplicate entry \"%s\" in sam header",
+                                tag->str + 3);
+            } else {
+                hrecs->rg[nrg].name = tag->str + 3;
+                hrecs->rg[nrg].name_len = tag->len - 3;
+            }
+            return 0;
+        }
+
+        if (nrg == hrecs->rg_sz) {
+            size_t new_sz = hrecs->rg_sz >= 4 ? hrecs->rg_sz + hrecs->rg_sz / 4 : 4;
+            sam_hrec_rg_t *new_rg = realloc(hrecs->rg, sizeof(*hrecs->rg) * new_sz);
+            if (!new_rg)
+                return -1;
+            hrecs->rg = new_rg;
+            hrecs->rg_sz = new_sz;
+        }
+
+        hrecs->rg[nrg].name = tag->str + 3;
+        hrecs->rg[nrg].name_len = tag->len - 3;
+        hrecs->rg[nrg].ty   = h_type;
+        hrecs->rg[nrg].id   = nrg;
+
+        k = kh_put(m_s2i, hrecs->rg_hash, hrecs->rg[nrg].name, &r);
+        if (-1 == r) return -1;
+        kh_val(hrecs->rg_hash, k) = nrg;
+
+        hrecs->nrg++;
+    }
+
+    /* Add to program hash? */
+    if (type == TYPEKEY("PG")) {
+        sam_hrec_tag_t *tag;
+        sam_hrec_pg_t *new_pg;
+        int npg = hrecs->npg;
+
+        if (npg == hrecs->pg_sz) {
+            size_t new_sz = hrecs->pg_sz >= 4 ? hrecs->pg_sz + hrecs->pg_sz / 4 : 4;
+            new_pg = realloc(hrecs->pg, sizeof(*hrecs->pg) * new_sz);
+            if (!new_pg)
+                return -1;
+            hrecs->pg = new_pg;
+            hrecs->pg_sz = new_sz;
+        }
+
+        tag = h_type->tag;
+        hrecs->pg[npg].name = NULL;
+        hrecs->pg[npg].name_len = 0;
+        hrecs->pg[npg].ty  = h_type;
+        hrecs->pg[npg].id   = npg;
+        hrecs->pg[npg].prev_id = -1;
+
+        while (tag) {
+            if (tag->str[0] == 'I' && tag->str[1] == 'D') {
+                /* Avoid duplicate ID tags coming from other applications */
+                if (!hrecs->pg[npg].name) {
+                    assert(tag->len >= 3);
+                    hrecs->pg[npg].name = tag->str + 3;
+                    hrecs->pg[npg].name_len = tag->len - 3;
+                } else {
+                    hts_log_warning("PG line with multiple ID tags. The first encountered was preferred - ID:%s", hrecs->pg[npg].name);
+                }
+            } else if (tag->str[0] == 'P' && tag->str[1] == 'P') {
+                // Resolve later if needed
+                khint_t k;
+                k = kh_get(m_s2i, hrecs->pg_hash, tag->str+3);
+
+                if (k != kh_end(hrecs->pg_hash)) {
+                    int p_id = kh_val(hrecs->pg_hash, k);
+                    hrecs->pg[npg].prev_id = hrecs->pg[p_id].id;
+
+                    /* Unmark previous entry as a PG termination */
+                    if (hrecs->npg_end > 0 &&
+                        hrecs->pg_end[hrecs->npg_end-1] == p_id) {
+                        hrecs->npg_end--;
+                    } else {
+                        int i;
+                        for (i = 0; i < hrecs->npg_end; i++) {
+                            if (hrecs->pg_end[i] == p_id) {
+                                memmove(&hrecs->pg_end[i], &hrecs->pg_end[i+1],
+                                        (hrecs->npg_end-i-1)*sizeof(*hrecs->pg_end));
+                                hrecs->npg_end--;
+                            }
+                        }
+                    }
+                } else {
+                    hrecs->pg[npg].prev_id = -1;
+                }
+            }
+            tag = tag->next;
+        }
+
+        if (hrecs->pg[npg].name) {
+            khint_t k;
+            int r;
+            k = kh_put(m_s2i, hrecs->pg_hash, hrecs->pg[npg].name, &r);
+            if (-1 == r) return -1;
+            kh_val(hrecs->pg_hash, k) = npg;
+        } else {
+            return -1; // ID should be present, according to spec.
+        }
+
+        /* Add to npg_end[] array. Remove later if we find a PP line */
+        if (hrecs->npg_end >= hrecs->npg_end_alloc) {
+            int *new_pg_end;
+            int  new_alloc = hrecs->npg_end_alloc ? hrecs->npg_end_alloc*2 : 4;
+
+            new_pg_end = realloc(hrecs->pg_end, new_alloc * sizeof(int));
+            if (!new_pg_end)
+                return -1;
+            hrecs->npg_end_alloc = new_alloc;
+            hrecs->pg_end = new_pg_end;
+        }
+        hrecs->pg_end[hrecs->npg_end++] = npg;
+
+        hrecs->npg++;
+    }
+
+    return 0;
+}
+
+static int sam_hrecs_remove_hash_entry(sam_hrecs_t *hrecs, khint32_t type, sam_hrec_type_t *h_type) {
+    if (!hrecs || !h_type)
+        return -1;
+
+    sam_hrec_tag_t *tag;
+    const char *key = NULL;
+    khint_t k;
+
+    /* Remove name and any alternative names from reference hash */
+    if (type == TYPEKEY("SQ")) {
+        const char *altnames = NULL;
+
+        tag = h_type->tag;
+
+        while (tag) {
+            if (tag->str[0] == 'S' && tag->str[1] == 'N') {
+                assert(tag->len >= 3);
+                key = tag->str + 3;
+            } else if (tag->str[0] == 'A' && tag->str[1] == 'N') {
+                assert(tag->len >= 3);
+                altnames = tag->str + 3;
+            }
+            tag = tag->next;
+        }
+
+        if (key) {
+            k = kh_get(m_s2i, hrecs->ref_hash, key);
+            if (k != kh_end(hrecs->ref_hash)) {
+                int idx = kh_val(hrecs->ref_hash, k);
+                if (idx + 1 < hrecs->nref)
+                    memmove(&hrecs->ref[idx], &hrecs->ref[idx+1],
+                            sizeof(sam_hrec_sq_t)*(hrecs->nref - idx - 1));
+                if (altnames)
+                    sam_hrecs_remove_ref_altnames(hrecs, idx, altnames);
+                kh_del(m_s2i, hrecs->ref_hash, k);
+                hrecs->nref--;
+                if (hrecs->refs_changed < 0 || hrecs->refs_changed > idx)
+                    hrecs->refs_changed = idx;
+                for (k = 0; k < kh_end(hrecs->ref_hash); k++) {
+                    if (kh_exist(hrecs->ref_hash, k)
+                        && kh_value(hrecs->ref_hash, k) > idx) {
+                        kh_value(hrecs->ref_hash, k)--;
+                    }
+                }
+            }
+        }
+    }
+
+    /* Remove from read-group hash */
+    if (type == TYPEKEY("RG")) {
+        tag = h_type->tag;
+
+        while (tag) {
+            if (tag->str[0] == 'I' && tag->str[1] == 'D') {
+                assert(tag->len >= 3);
+                key = tag->str + 3;
+                k = kh_get(m_s2i, hrecs->rg_hash, key);
+                if (k != kh_end(hrecs->rg_hash)) {
+                    int idx = kh_val(hrecs->rg_hash, k);
+                    if (idx + 1 < hrecs->nrg)
+                        memmove(&hrecs->rg[idx], &hrecs->rg[idx+1], sizeof(sam_hrec_rg_t)*(hrecs->nrg - idx - 1));
+                    kh_del(m_s2i, hrecs->rg_hash, k);
+                    hrecs->nrg--;
+                    for (k = 0; k < kh_end(hrecs->rg_hash); k++) {
+                        if (kh_exist(hrecs->rg_hash, k)
+                            && kh_value(hrecs->rg_hash, k) > idx) {
+                            kh_value(hrecs->rg_hash, k)--;
+                        }
+                    }
+                }
+                break;
+            }
+            tag = tag->next;
+        }
+    }
+
+    return 0;
+}
+
+/** Add a header record to the global line ordering
+ *
+ * If @p after is not NULL, the new record will be inserted after this one,
+ * otherwise it will go at the end.
+ *
+ * An exception is an HD record, which will always be put first unless
+ * one is already present.
+ */
+static void sam_hrecs_global_list_add(sam_hrecs_t *hrecs,
+                                      sam_hrec_type_t *h_type,
+                                      sam_hrec_type_t *after) {
+    const khint32_t hd_type = TYPEKEY("HD");
+    int update_first_line = 0;
+
+    // First line seen
+    if (!hrecs->first_line) {
+        hrecs->first_line = h_type->global_next = h_type->global_prev = h_type;
+        return;
+    }
+
+    // @HD goes at the top (unless there's one already)
+    if (h_type->type == hd_type && hrecs->first_line->type != hd_type) {
+        after = hrecs->first_line->global_prev;
+        update_first_line = 1;
+    }
+
+    // If no instructions given, put it at the end
+    if (!after)
+        after = hrecs->first_line->global_prev;
+
+    h_type->global_prev = after;
+    h_type->global_next = after->global_next;
+    h_type->global_prev->global_next = h_type;
+    h_type->global_next->global_prev = h_type;
+
+    if (update_first_line)
+        hrecs->first_line = h_type;
+}
+
+/*! Add header record with a va_list interface.
+ *
+ * Adds a single record to a SAM header.
+ *
+ * This takes a header record type, a va_list argument and one or more
+ * key,value pairs, ending with the NULL key.
+ *
+ * Eg. sam_hrecs_vadd(h, "SQ", args, "ID", "foo", "LN", "100", NULL).
+ *
+ * The purpose of the additional va_list parameter is to permit other
+ * varargs functions to call this while including their own additional
+ * parameters; an example is in sam_hdr_add_pg().
+ *
+ * Note: this function invokes va_arg at least once, making the value
+ * of ap indeterminate after the return. The caller should call
+ * va_start/va_end before/after calling this function or use va_copy.
+ *
+ * @return
+ * Returns >= 0 on success;
+ *        -1 on failure
+ */
+static int sam_hrecs_vadd(sam_hrecs_t *hrecs, const char *type, va_list ap, ...) {
+    va_list args;
+    sam_hrec_type_t *h_type;
+    sam_hrec_tag_t *h_tag, *last=NULL;
+    int new;
+    khint32_t type_i = TYPEKEY(type), k;
+
+    if (!strncmp(type, "HD", 2) && (h_type = sam_hrecs_find_type_id(hrecs, "HD", NULL, NULL)))
+        return sam_hrecs_vupdate(hrecs, h_type, ap);
+
+    if (!(h_type = pool_alloc(hrecs->type_pool)))
+        return -1;
+    k = kh_put(sam_hrecs_t, hrecs->h, type_i, &new);
+    if (new < 0)
+        return -1;
+
+    h_type->type = type_i;
+
+    // Form the ring, either with self or other lines of this type
+    if (!new) {
+        sam_hrec_type_t *t = kh_val(hrecs->h, k), *p;
+        p = t->prev;
+
+        assert(p->next == t);
+        p->next = h_type;
+        h_type->prev = p;
+
+        t->prev = h_type;
+        h_type->next = t;
+    } else {
+        kh_val(hrecs->h, k) = h_type;
+        h_type->prev = h_type->next = h_type;
+    }
+    h_type->tag = NULL;
+
+    // Add to global line ordering after any existing line of the same type,
+    // or at the end if no line of this type exists yet.
+    sam_hrecs_global_list_add(hrecs, h_type, !new ? h_type->prev : NULL);
+
+    // Check linked-list invariants
+    assert(h_type->prev->next == h_type);
+    assert(h_type->next->prev == h_type);
+    assert(h_type->global_prev->global_next == h_type);
+    assert(h_type->global_next->global_prev == h_type);
+
+    // Any ... varargs
+    va_start(args, ap);
+    for (;;) {
+        char *key, *val = NULL, *str;
+
+        if (!(key = (char *)va_arg(args, char *)))
+            break;
+        if (strncmp(type, "CO", 2) && !(val = (char *)va_arg(args, char *)))
+            break;
+        if (*val == '\0')
+            continue;
+
+        if (!(h_tag = pool_alloc(hrecs->tag_pool)))
+            return -1;
+
+        if (strncmp(type, "CO", 2)) {
+            h_tag->len = 3 + strlen(val);
+            str = string_alloc(hrecs->str_pool, h_tag->len+1);
+            if (!str || snprintf(str, h_tag->len+1, "%2.2s:%s", key, val) < 0)
+                return -1;
+            h_tag->str = str;
+        } else {
+            h_tag->len = strlen(key);
+            h_tag->str = string_ndup(hrecs->str_pool, key, h_tag->len);
+            if (!h_tag->str)
+                return -1;
+        }
+
+        h_tag->next = NULL;
+        if (last)
+            last->next = h_tag;
+        else
+            h_type->tag = h_tag;
+
+        last = h_tag;
+    }
+    va_end(args);
+
+    // Plus the specified va_list params
+    for (;;) {
+        char *key, *val = NULL, *str;
+
+        if (!(key = (char *)va_arg(ap, char *)))
+            break;
+        if (strncmp(type, "CO", 2) && !(val = (char *)va_arg(ap, char *)))
+            break;
+
+        if (!(h_tag = pool_alloc(hrecs->tag_pool)))
+            return -1;
+
+        if (strncmp(type, "CO", 2)) {
+            h_tag->len = 3 + strlen(val);
+            str = string_alloc(hrecs->str_pool, h_tag->len+1);
+            if (!str || snprintf(str, h_tag->len+1, "%2.2s:%s", key, val) < 0)
+                return -1;
+            h_tag->str = str;
+        } else {
+            h_tag->len = strlen(key);
+            h_tag->str = string_ndup(hrecs->str_pool, key, h_tag->len);
+            if (!h_tag->str)
+                return -1;
+        }
+
+        h_tag->next = NULL;
+        if (last)
+            last->next = h_tag;
+        else
+            h_type->tag = h_tag;
+
+        last = h_tag;
+    }
+
+    if (-1 == sam_hrecs_update_hashes(hrecs, TYPEKEY(type), h_type))
+        return -1;
+
+    if (!strncmp(type, "PG", 2))
+        hrecs->pgs_changed = 1;
+
+    hrecs->dirty = 1;
+
+    return 0;
+}
+
+// As sam_hrecs_vadd(), but without the extra va_list parameter
+static int sam_hrecs_add(sam_hrecs_t *hrecs, const char *type, ...) {
+    va_list args;
+    int res;
+    va_start(args, type);
+    res = sam_hrecs_vadd(hrecs, type, args, NULL);
+    va_end(args);
+    return res;
+}
+
+/*
+ * Function for deallocating a list of tags
+ */
+
+static void sam_hrecs_free_tags(sam_hrecs_t *hrecs, sam_hrec_tag_t *tag) {
+    if (!hrecs || !tag)
+        return;
+    if (tag->next)
+        sam_hrecs_free_tags(hrecs, tag->next);
+
+    pool_free(hrecs->tag_pool, tag);
+}
+
+static int sam_hrecs_remove_line(sam_hrecs_t *hrecs, const char *type_name, sam_hrec_type_t *type_found, int remove_hash) {
+    if (!hrecs || !type_name || !type_found)
+        return -1;
+
+    khint32_t itype = TYPEKEY(type_name);
+    khint_t k = kh_get(sam_hrecs_t, hrecs->h, itype);
+    if (k == kh_end(hrecs->h))
+        return -1;
+
+    // Remove from global list (remembering it could be the only line)
+    if (hrecs->first_line == type_found) {
+        hrecs->first_line = (type_found->global_next != type_found
+                             ? type_found->global_next : NULL);
+    }
+    type_found->global_next->global_prev = type_found->global_prev;
+    type_found->global_prev->global_next = type_found->global_next;
+
+    /* single element in the list */
+    if (type_found->prev == type_found || type_found->next == type_found) {
+        kh_del(sam_hrecs_t, hrecs->h, k);
+    } else {
+        type_found->prev->next = type_found->next;
+        type_found->next->prev = type_found->prev;
+        if (kh_val(hrecs->h, k) == type_found) { //first element
+            kh_val(hrecs->h, k) = type_found->next;
+        }
+    }
+
+    if (remove_hash && (!strncmp(type_name, "SQ", 2) || !strncmp(type_name, "RG", 2)))
+        sam_hrecs_remove_hash_entry(hrecs, itype, type_found);
+
+    sam_hrecs_free_tags(hrecs, type_found->tag);
+    pool_free(hrecs->type_pool, type_found);
+
+    hrecs->dirty = 1;
+
+    return 0;
+}
+
+// Paste together a line from the parsed data structures
+static int build_header_line(const sam_hrec_type_t *ty, kstring_t *ks) {
+    sam_hrec_tag_t *tag;
+    int r = 0;
+    char c[2]= { ty->type >> 8, ty->type & 0xff };
+
+    r |= (kputc_('@', ks) == EOF);
+    r |= (kputsn(c, 2, ks) == EOF);
+    for (tag = ty->tag; tag; tag = tag->next) {
+        r |= (kputc_('\t', ks) == EOF);
+        r |= (kputsn(tag->str, tag->len, ks) == EOF);
+    }
+
+    return r;
+}
+
+static int sam_hrecs_rebuild_lines(const sam_hrecs_t *hrecs, kstring_t *ks) {
+    const sam_hrec_type_t *t1, *t2;
+
+    if (!hrecs->first_line)
+        return kputsn("", 0, ks) >= 0 ? 0 : -1;
+
+    t1 = t2 = hrecs->first_line;
+    do {
+        if (build_header_line(t1, ks) != 0)
+            return -1;
+        if (kputc('\n', ks) < 0)
+            return -1;
+
+        t1 = t1->global_next;
+    } while (t1 != t2);
+
+    return 0;
+}
+
+static int sam_hrecs_parse_lines(sam_hrecs_t *hrecs, const char *hdr, size_t len) {
+    size_t i, lno;
+
+    if (!hrecs || len > SSIZE_MAX)
+        return -1;
+
+    if (!len)
+        len = strlen(hdr);
+
+    if (len < 3) {
+        if (len == 0 || *hdr == '\0') return 0;
+        sam_hrecs_error("Header line too short", hdr, len, 1);
+        return -1;
+    }
+
+    for (i = 0, lno = 1; i < len - 3 && hdr[i] != '\0'; i++, lno++) {
+        khint32_t type;
+        khint_t k;
+
+        int l_start = i, new;
+        sam_hrec_type_t *h_type;
+        sam_hrec_tag_t *h_tag, *last;
+
+        if (hdr[i] != '@') {
+            sam_hrecs_error("Header line does not start with '@'",
+                          &hdr[l_start], len - l_start, lno);
+            return -1;
+        }
+
+        if (!isalpha_c(hdr[i+1]) || !isalpha_c(hdr[i+2])) {
+            sam_hrecs_error("Header line does not have a two character key",
+                          &hdr[l_start], len - l_start, lno);
+            return -1;
+        }
+        type = TYPEKEY(&hdr[i+1]);
+
+        i += 3;
+        if (i == len || hdr[i] == '\n')
+            continue;
+
+        // Add the header line type
+        if (!(h_type = pool_alloc(hrecs->type_pool)))
+            return -1;
+        k = kh_put(sam_hrecs_t, hrecs->h, type, &new);
+        if (new < 0)
+            return -1;
+
+        h_type->type = type;
+
+        // Add to end of global list
+        sam_hrecs_global_list_add(hrecs, h_type, NULL);
+
+        // Form the ring, either with self or other lines of this type
+        if (!new) {
+            sam_hrec_type_t *t = kh_val(hrecs->h, k), *p;
+            p = t->prev;
+
+            assert(p->next == t);
+            p->next = h_type;
+            h_type->prev = p;
+
+            t->prev = h_type;
+            h_type->next = t;
+        } else {
+            kh_val(hrecs->h, k) = h_type;
+            h_type->prev = h_type->next = h_type;
+        }
+
+        // Parse the tags on this line
+        last = NULL;
+        if (type == TYPEKEY("CO")) {
+            size_t j;
+
+            if (i == len || hdr[i] != '\t') {
+                sam_hrecs_error("Missing tab",
+                              &hdr[l_start], len - l_start, lno);
+                return -1;
+            }
+
+            for (j = ++i; j < len && hdr[j] != '\0' && hdr[j] != '\n'; j++)
+                ;
+
+            if (!(h_type->tag = h_tag = pool_alloc(hrecs->tag_pool)))
+                return -1;
+            h_tag->str = string_ndup(hrecs->str_pool, &hdr[i], j-i);
+            h_tag->len = j-i;
+            h_tag->next = NULL;
+            if (!h_tag->str)
+                return -1;
+
+            i = j;
+
+        } else {
+            do {
+                size_t j;
+
+                if (i == len || hdr[i] != '\t') {
+                    sam_hrecs_error("Missing tab",
+                                  &hdr[l_start], len - l_start, lno);
+                    return -1;
+                }
+
+                for (j = ++i; j < len && hdr[j] != '\0' && hdr[j] != '\n' && hdr[j] != '\t'; j++)
+                    ;
+
+                if (j - i < 3 || hdr[i + 2] != ':') {
+                    sam_hrecs_error("Malformed key:value pair",
+                                   &hdr[l_start], len - l_start, lno);
+                    return -1;
+                }
+
+                if (!(h_tag = pool_alloc(hrecs->tag_pool)))
+                    return -1;
+                h_tag->str = string_ndup(hrecs->str_pool, &hdr[i], j-i);
+                h_tag->len = j-i;
+                h_tag->next = NULL;
+                if (!h_tag->str)
+                    return -1;
+
+                if (last)
+                    last->next = h_tag;
+                else
+                    h_type->tag = h_tag;
+
+                last = h_tag;
+                i = j;
+            } while (i < len && hdr[i] != '\0' && hdr[i] != '\n');
+        }
+
+        /* Update RG/SQ hashes */
+        if (-1 == sam_hrecs_update_hashes(hrecs, type, h_type))
+            return -1;
+    }
+
+    return 0;
+}
+
+/*! Update sam_hdr_t target_name and target_len arrays
+ *
+ *  @return 0 on success; -1 on failure
+ */
+int sam_hdr_update_target_arrays(sam_hdr_t *bh, const sam_hrecs_t *hrecs,
+                                 int refs_changed) {
+    if (!bh || !hrecs)
+        return -1;
+
+    if (refs_changed < 0)
+        return 0;
+
+    // Grow arrays if necessary
+    if (bh->n_targets < hrecs->nref) {
+        char **new_names = realloc(bh->target_name,
+                                   hrecs->nref * sizeof(*new_names));
+        if (!new_names)
+            return -1;
+        bh->target_name = new_names;
+        uint32_t *new_lens = realloc(bh->target_len,
+                                     hrecs->nref * sizeof(*new_lens));
+        if (!new_lens)
+            return -1;
+        bh->target_len = new_lens;
+    }
+
+    // Update names and lengths where changed
+    // hrecs->refs_changed is the first ref that has been updated, so ones
+    // before that can be skipped.
+    int i;
+    khint_t k;
+    khash_t(s2i) *long_refs = (khash_t(s2i) *) bh->sdict;
+    for (i = refs_changed; i < hrecs->nref; i++) {
+        if (i >= bh->n_targets
+            || strcmp(bh->target_name[i], hrecs->ref[i].name) != 0) {
+            if (i < bh->n_targets)
+                free(bh->target_name[i]);
+            bh->target_name[i] = strdup(hrecs->ref[i].name);
+            if (!bh->target_name[i])
+                return -1;
+        }
+        if (hrecs->ref[i].len < UINT32_MAX) {
+            bh->target_len[i] = hrecs->ref[i].len;
+
+            if (!long_refs)
+                continue;
+
+            // Check if we have an old length, if so remove it.
+            k = kh_get(s2i, long_refs, bh->target_name[i]);
+            if (k < kh_end(long_refs))
+                kh_del(s2i, long_refs, k);
+        } else {
+            bh->target_len[i] = UINT32_MAX;
+            if (bh->hrecs != hrecs) {
+                // Called from sam_hdr_dup; need to add sdict entries
+                if (!long_refs) {
+                    if (!(bh->sdict = long_refs = kh_init(s2i)))
+                        return -1;
+                }
+
+                // Add / update length
+                int absent;
+                k = kh_put(s2i, long_refs, bh->target_name[i], &absent);
+                if (absent < 0)
+                    return -1;
+                kh_val(long_refs, k) = hrecs->ref[i].len;
+            }
+        }
+    }
+
+    // Free up any names that have been removed
+    for (; i < bh->n_targets; i++) {
+        if (long_refs) {
+            k = kh_get(s2i, long_refs, bh->target_name[i]);
+            if (k < kh_end(long_refs))
+                kh_del(s2i, long_refs, k);
+        }
+        free(bh->target_name[i]);
+    }
+
+    bh->n_targets = hrecs->nref;
+    return 0;
+}
+
+static int rebuild_target_arrays(sam_hdr_t *bh) {
+    if (!bh || !bh->hrecs)
+        return -1;
+
+    sam_hrecs_t *hrecs = bh->hrecs;
+    if (hrecs->refs_changed < 0)
+        return 0;
+
+    if (sam_hdr_update_target_arrays(bh, hrecs, hrecs->refs_changed) != 0)
+        return -1;
+
+    hrecs->refs_changed = -1;
+    return 0;
+}
+
+/// Populate hrecs refs array from header target_name, target_len arrays
+/**
+ * @return 0 on success; -1 on failure
+ *
+ * Pre-fills the refs hash from the target arrays.  For BAM files this
+ * will ensure that they are in the correct order as the target arrays
+ * are the canonical source for converting target ids to names and lengths.
+ *
+ * The added entries do not link to a header line. sam_hrecs_update_hashes()
+ * will add the links later for lines found in the text header.
+ *
+ * This should be called before the text header is parsed.
+ */
+static int sam_hrecs_refs_from_targets_array(sam_hrecs_t *hrecs,
+                                             const sam_hdr_t *bh) {
+    int32_t tid = 0;
+
+    if (!hrecs || !bh)
+        return -1;
+
+    // This should always be called before parsing the text header
+    // so the ref array should start off empty, and we don't have to try
+    // to reconcile any existing data.
+    if (hrecs->nref > 0) {
+        hts_log_error("Called with non-empty ref array");
+        return -1;
+    }
+
+    if (hrecs->ref_sz < bh->n_targets) {
+        sam_hrec_sq_t *new_ref = realloc(hrecs->ref,
+                                         bh->n_targets * sizeof(*new_ref));
+        if (!new_ref)
+            return -1;
+
+        hrecs->ref = new_ref;
+        hrecs->ref_sz = bh->n_targets;
+    }
+
+    for (tid = 0; tid < bh->n_targets; tid++) {
+        khint_t k;
+        int r;
+        hrecs->ref[tid].name = string_dup(hrecs->str_pool, bh->target_name[tid]);
+        if (!hrecs->ref[tid].name) goto fail;
+        if (bh->target_len[tid] < UINT32_MAX || !bh->sdict) {
+            hrecs->ref[tid].len  = bh->target_len[tid];
+        } else {
+            khash_t(s2i) *long_refs = (khash_t(s2i) *) bh->sdict;
+            k = kh_get(s2i, long_refs, hrecs->ref[tid].name);
+            if (k < kh_end(long_refs)) {
+                hrecs->ref[tid].len = kh_val(long_refs, k);
+            } else {
+                hrecs->ref[tid].len = UINT32_MAX;
+            }
+        }
+        hrecs->ref[tid].ty   = NULL;
+        k = kh_put(m_s2i, hrecs->ref_hash, hrecs->ref[tid].name, &r);
+        if (r < 0) goto fail;
+        if (r == 0) {
+            hts_log_error("Duplicate entry \"%s\" in target list",
+                            hrecs->ref[tid].name);
+            return -1;
+        } else {
+            kh_val(hrecs->ref_hash, k) = tid;
+        }
+    }
+    hrecs->nref = bh->n_targets;
+    return 0;
+
+ fail: {
+        int32_t i;
+        hts_log_error("%s", strerror(errno));
+        for (i = 0; i < tid; i++) {
+            khint_t k;
+            if (!hrecs->ref[i].name) continue;
+            k = kh_get(m_s2i, hrecs->ref_hash, hrecs->ref[tid].name);
+            if (k < kh_end(hrecs->ref_hash)) kh_del(m_s2i, hrecs->ref_hash, k);
+        }
+        hrecs->nref = 0;
+        return -1;
+    }
+}
+
+/*
+ * Add SQ header records for any references in the hrecs->ref array that
+ * were added by sam_hrecs_refs_from_targets_array() but have not
+ * been linked to an @SQ line by sam_hrecs_update_hashes() yet.
+ *
+ * This may be needed either because:
+ *
+ *   - A bam file was read that had entries in its refs list with no
+ *     corresponding @SQ line.
+ *
+ *   - A program constructed a sam_hdr_t which has target_name and target_len
+ *     array entries with no corresponding @SQ line in text.
+ */
+static int add_stub_ref_sq_lines(sam_hrecs_t *hrecs) {
+    int tid;
+    char len[32];
+
+    for (tid = 0; tid < hrecs->nref; tid++) {
+        if (hrecs->ref[tid].ty == NULL) {
+            snprintf(len, sizeof(len), "%"PRIhts_pos, hrecs->ref[tid].len);
+            if (sam_hrecs_add(hrecs, "SQ",
+                              "SN", hrecs->ref[tid].name,
+                              "LN", len, NULL) != 0)
+                return -1;
+
+            // Check that the stub has actually been filled
+            if(hrecs->ref[tid].ty == NULL) {
+                hts_log_error("Reference stub with tid=%d, name=\"%s\", len=%"PRIhts_pos" could not be filled",
+                        tid, hrecs->ref[tid].name, hrecs->ref[tid].len);
+                return -1;
+            }
+        }
+    }
+    return 0;
+}
+
+int sam_hdr_fill_hrecs(sam_hdr_t *bh) {
+    sam_hrecs_t *hrecs = sam_hrecs_new();
+
+    if (!hrecs)
+        return -1;
+
+    if (bh->target_name && bh->target_len && bh->n_targets > 0) {
+        if (sam_hrecs_refs_from_targets_array(hrecs, bh) != 0) {
+            sam_hrecs_free(hrecs);
+            return -1;
+        }
+    }
+
+    // Parse existing header text
+    if (bh->text && bh->l_text > 0) {
+        if (sam_hrecs_parse_lines(hrecs, bh->text, bh->l_text) != 0) {
+            sam_hrecs_free(hrecs);
+            return -1;
+        }
+    }
+
+    if (add_stub_ref_sq_lines(hrecs) < 0) {
+        sam_hrecs_free(hrecs);
+        return -1;
+    }
+
+    bh->hrecs = hrecs;
+
+    if (hrecs->refs_changed >= 0 && rebuild_target_arrays(bh) != 0)
+        return -1;
+
+    return 0;
+}
+
+/** Remove outdated header text
+
+    @param bh     BAM header
+
+    This is called when API functions have changed the header so that the
+    text version is no longer valid.
+ */
+static void redact_header_text(sam_hdr_t *bh) {
+    assert(bh->hrecs && bh->hrecs->dirty);
+    bh->l_text = 0;
+    free(bh->text);
+    bh->text = NULL;
+}
+
+/** Find nth header record of a given type
+
+    @param type   Header type (SQ, RG etc.)
+    @param idx    0-based index
+
+    @return sam_hrec_type_t pointer to the record on success
+            NULL if no record exists with the given type and index
+ */
+
+static sam_hrec_type_t *sam_hrecs_find_type_pos(sam_hrecs_t *hrecs,
+                                                const char *type, int idx) {
+    sam_hrec_type_t *first, *itr;
+
+    if (idx < 0)
+        return NULL;
+
+    if (type[0] == 'S' && type[1] == 'Q')
+        return idx < hrecs->nref ? hrecs->ref[idx].ty : NULL;
+
+    if (type[0] == 'R' && type[1] == 'G')
+        return idx < hrecs->nrg ? hrecs->rg[idx].ty : NULL;
+
+    if (type[0] == 'P' && type[1] == 'G')
+        return idx < hrecs->npg ? hrecs->pg[idx].ty : NULL;
+
+    first = itr = sam_hrecs_find_type_id(hrecs, type, NULL, NULL);
+    if (!first)
+        return NULL;
+
+    while (idx > 0) {
+        itr = itr->next;
+        if (itr == first)
+            break;
+        --idx;
+    }
+
+    return idx == 0 ? itr : NULL;
+}
+
+/* ==== Public methods ==== */
+
+size_t sam_hdr_length(sam_hdr_t *bh) {
+    if (!bh || -1 == sam_hdr_rebuild(bh))
+        return SIZE_MAX;
+
+    return bh->l_text;
+}
+
+const char *sam_hdr_str(sam_hdr_t *bh) {
+    if (!bh || -1 == sam_hdr_rebuild(bh))
+        return NULL;
+
+    return bh->text;
+}
+
+int sam_hdr_nref(const sam_hdr_t *bh) {
+    if (!bh)
+        return -1;
+
+    return bh->hrecs ? bh->hrecs->nref : bh->n_targets;
+}
+
+/*
+ * Reconstructs the text representation from the header hash table.
+ * Returns 0 on success
+ *        -1 on failure
+ */
+int sam_hdr_rebuild(sam_hdr_t *bh) {
+    sam_hrecs_t *hrecs;
+    if (!bh)
+        return -1;
+
+    if (!(hrecs = bh->hrecs))
+        return bh->text ? 0 : -1;
+
+    if (hrecs->refs_changed >= 0) {
+        if (rebuild_target_arrays(bh) < 0) {
+            hts_log_error("Header target array rebuild has failed");
+            return -1;
+        }
+    }
+
+    /* If header text wasn't changed or header is empty, don't rebuild it. */
+    if (!hrecs->dirty)
+        return 0;
+
+    if (hrecs->pgs_changed && sam_hdr_link_pg(bh) < 0) {
+        hts_log_error("Linking @PG lines has failed");
+        return -1;
+    }
+
+    kstring_t ks = KS_INITIALIZE;
+    if (sam_hrecs_rebuild_text(hrecs, &ks) != 0) {
+        ks_free(&ks);
+        hts_log_error("Header text rebuild has failed");
+        return -1;
+    }
+
+    hrecs->dirty = 0;
+
+    /* Sync */
+    free(bh->text);
+    bh->l_text = ks_len(&ks);
+    bh->text = ks_release(&ks);
+
+    return 0;
+}
+
+/*
+ * Appends a formatted line to an existing SAM header.
+ * Line is a full SAM header record, eg "@SQ\tSN:foo\tLN:100", with
+ * optional new-line. If it contains more than 1 line then multiple lines
+ * will be added in order.
+ *
+ * Input text is of maximum length len or as terminated earlier by a NUL.
+ * len may be 0 if unknown, in which case lines must be NUL-terminated.
+ *
+ * Returns 0 on success
+ *        -1 on failure
+ */
+int sam_hdr_add_lines(sam_hdr_t *bh, const char *lines, size_t len) {
+    sam_hrecs_t *hrecs;
+
+    if (!bh || !lines)
+        return -1;
+
+    if (len == 0 && *lines == '\0')
+        return 0;
+
+    if (!(hrecs = bh->hrecs)) {
+        if (sam_hdr_fill_hrecs(bh) != 0)
+            return -1;
+        hrecs = bh->hrecs;
+    }
+
+    if (sam_hrecs_parse_lines(hrecs, lines, len) != 0)
+        return -1;
+
+    if (hrecs->refs_changed >= 0 && rebuild_target_arrays(bh) != 0)
+        return -1;
+
+    hrecs->dirty = 1;
+    redact_header_text(bh);
+
+    return 0;
+}
+
+/*
+ * Adds a single line to a SAM header.
+ * Specify type and one or more key,value pairs, ending with the NULL key.
+ * Eg. sam_hdr_add_line(h, "SQ", "ID", "foo", "LN", "100", NULL).
+ *
+ * Returns 0 on success
+ *        -1 on failure
+ */
+int sam_hdr_add_line(sam_hdr_t *bh, const char *type, ...) {
+    va_list args;
+    sam_hrecs_t *hrecs;
+
+    if (!bh || !type)
+        return -1;
+
+    if (!(hrecs = bh->hrecs)) {
+        if (sam_hdr_fill_hrecs(bh) != 0)
+            return -1;
+        hrecs = bh->hrecs;
+    }
+
+    va_start(args, type);
+    int ret = sam_hrecs_vadd(hrecs, type, args, NULL);
+    va_end(args);
+
+    if (ret == 0) {
+        if (hrecs->refs_changed >= 0 && rebuild_target_arrays(bh) != 0)
+            return -1;
+
+        if (hrecs->dirty)
+            redact_header_text(bh);
+    }
+
+    return ret;
+}
+
+/*
+ * Returns a complete line of formatted text for a specific head type/ID
+ * combination. If ID_key is NULL then it returns the first line of the specified
+ * type.
+ */
+int sam_hdr_find_line_id(sam_hdr_t *bh, const char *type,
+                      const char *ID_key, const char *ID_val, kstring_t *ks) {
+    sam_hrecs_t *hrecs;
+    if (!bh || !type)
+        return -2;
+
+    if (!(hrecs = bh->hrecs)) {
+        if (sam_hdr_fill_hrecs(bh) != 0)
+            return -2;
+        hrecs = bh->hrecs;
+    }
+
+    sam_hrec_type_t *ty = sam_hrecs_find_type_id(hrecs, type, ID_key, ID_val);
+    if (!ty)
+        return -1;
+
+    ks->l = 0;
+    if (build_header_line(ty, ks) < 0) {
+        return -2;
+    }
+
+    return 0;
+}
+
+int sam_hdr_find_line_pos(sam_hdr_t *bh, const char *type,
+                          int pos, kstring_t *ks) {
+    sam_hrecs_t *hrecs;
+    if (!bh || !type)
+        return -2;
+
+    if (!(hrecs = bh->hrecs)) {
+        if (sam_hdr_fill_hrecs(bh) != 0)
+            return -2;
+        hrecs = bh->hrecs;
+    }
+
+    sam_hrec_type_t *ty = sam_hrecs_find_type_pos(hrecs, type, pos);
+    if (!ty)
+        return -1;
+
+    ks->l = 0;
+    if (build_header_line(ty, ks) < 0) {
+        return -2;
+    }
+
+    return 0;
+}
+
+/*
+ * Remove a line from the header by specifying a tag:value that uniquely
+ * identifies a line, i.e. the @SQ line containing "SN:ref1".
+ * @SQ line is uniquely identified by SN tag.
+ * @RG line is uniquely identified by ID tag.
+ * @PG line is uniquely identified by ID tag.
+ *
+ * Returns 0 on success and -1 on error
+ */
+
+int sam_hdr_remove_line_id(sam_hdr_t *bh, const char *type, const char *ID_key, const char *ID_value) {
+    sam_hrecs_t *hrecs;
+    if (!bh || !type)
+        return -1;
+
+    if (!(hrecs = bh->hrecs)) {
+        if (sam_hdr_fill_hrecs(bh) != 0)
+            return -1;
+        hrecs = bh->hrecs;
+    }
+
+    if (!strncmp(type, "PG", 2)) {
+        hts_log_warning("Removing PG lines is not supported!");
+        return -1;
+    }
+
+    sam_hrec_type_t *type_found = sam_hrecs_find_type_id(hrecs, type, ID_key, ID_value);
+    if (!type_found)
+        return 0;
+
+    int ret = sam_hrecs_remove_line(hrecs, type, type_found, 1);
+    if (ret == 0) {
+        if (hrecs->refs_changed >= 0 && rebuild_target_arrays(bh) != 0)
+            return -1;
+
+        if (hrecs->dirty)
+            redact_header_text(bh);
+    }
+
+    return ret;
+}
+
+/*
+ * Remove a line from the header by specifying the position in the type
+ * group, i.e. 3rd @SQ line.
+ *
+ * Returns 0 on success and -1 on error
+ */
+
+int sam_hdr_remove_line_pos(sam_hdr_t *bh, const char *type, int position) {
+    sam_hrecs_t *hrecs;
+    if (!bh || !type || position <= 0)
+        return -1;
+
+    if (!(hrecs = bh->hrecs)) {
+        if (sam_hdr_fill_hrecs(bh) != 0)
+            return -1;
+        hrecs = bh->hrecs;
+    }
+
+    if (!strncmp(type, "PG", 2)) {
+        hts_log_warning("Removing PG lines is not supported!");
+        return -1;
+    }
+
+    sam_hrec_type_t *type_found = sam_hrecs_find_type_pos(hrecs, type,
+                                                          position);
+    if (!type_found)
+        return -1;
+
+    int ret = sam_hrecs_remove_line(hrecs, type, type_found, 1);
+    if (ret == 0) {
+        if (hrecs->refs_changed >= 0 && rebuild_target_arrays(bh) != 0)
+            return -1;
+
+        if (hrecs->dirty)
+            redact_header_text(bh);
+    }
+
+    return ret;
+}
+
+/*
+ * Check if sam_hdr_update_line() is being used to change the name of
+ * a record, and if the new name is going to clash with an existing one.
+ *
+ * If ap includes repeated keys, we go with the last one as sam_hrecs_vupdate()
+ * will go through them all and leave the final one in place.
+ *
+ * Returns 0 if the name does not change
+ *         1 if the name changes but does not clash
+ *        -1 if the name changes and the new one is already in use
+ */
+static int check_for_name_update(sam_hrecs_t *hrecs, sam_hrec_type_t *rec,
+                                 va_list ap, const char **old_name,
+                                 const char **new_name,
+                                 char id_tag_out[3],
+                                 khash_t(m_s2i) **hash_out) {
+    char *key, *val;
+    const char *id_tag;
+    sam_hrec_tag_t *tag, *prev;
+    khash_t(m_s2i) *hash;
+    khint_t k;
+    int ret = 0;
+
+    if        (rec->type == TYPEKEY("SQ")) {
+        id_tag = "SN"; hash = hrecs->ref_hash;
+    } else if (rec->type == TYPEKEY("RG")) {
+        id_tag = "ID"; hash = hrecs->rg_hash;
+    } else if (rec->type == TYPEKEY("PG")) {
+        id_tag = "ID"; hash = hrecs->pg_hash;
+    } else {
+        return 0;
+    }
+
+    memcpy(id_tag_out, id_tag, 3);
+    *hash_out = hash;
+
+    tag = sam_hrecs_find_key(rec, id_tag, &prev);
+    if (!tag)
+        return 0;
+    assert(tag->len >= 3);
+    *old_name = tag->str + 3;
+
+    while ((key = va_arg(ap, char *)) != NULL) {
+        val = va_arg(ap, char *);
+        if (!val) val = "";
+        if (strcmp(key, id_tag) != 0) continue;
+        if (strcmp(val, tag->str + 3) == 0) { ret = 0; continue; }
+        k = kh_get(m_s2i, hash, val);
+        ret = k < kh_end(hash) ? -1 : 1;
+        *new_name = val;
+    }
+    return ret;
+}
+
+int sam_hdr_update_line(sam_hdr_t *bh, const char *type,
+        const char *ID_key, const char *ID_value, ...) {
+    sam_hrecs_t *hrecs;
+    if (!bh)
+        return -1;
+
+    if (!(hrecs = bh->hrecs)) {
+        if (sam_hdr_fill_hrecs(bh) != 0)
+            return -1;
+        hrecs = bh->hrecs;
+    }
+
+    int ret, rename;
+    sam_hrec_type_t *ty = sam_hrecs_find_type_id(hrecs, type, ID_key, ID_value);
+    if (!ty)
+        return -1;
+
+    va_list args;
+    const char *old_name = "?", *new_name = "?";
+    char id_tag[3];
+    khash_t(m_s2i) *hash = NULL;
+    va_start(args, ID_value);
+    rename = check_for_name_update(hrecs, ty, args,
+                                   &old_name, &new_name, id_tag, &hash);
+    va_end(args);
+    if (rename < 0) {
+        hts_log_error("Cannot rename @%s \"%s\" to \"%s\" : already exists",
+                      type, old_name, new_name);
+        return -1;
+    }
+    if (rename > 0 && TYPEKEY(type) == TYPEKEY("PG")) {
+        // This is just too complicated
+        hts_log_error("Renaming @PG records is not supported");
+        return -1;
+    }
+    va_start(args, ID_value);
+    ret = sam_hrecs_vupdate(hrecs, ty, args);
+    va_end(args);
+
+    if (ret)
+        return ret;
+
+    // TODO Account for @SQ-AN altnames
+
+    if (rename) {
+        // Adjust the hash table to point to the new name
+        // sam_hrecs_update_hashes() should sort out everything else
+        khint_t k = kh_get(m_s2i, hash, old_name);
+        sam_hrec_tag_t *new_tag = sam_hrecs_find_key(ty, id_tag, NULL);
+        int r, pos;
+        assert(k < kh_end(hash));        // Or we wouldn't have found it earlier
+        assert(new_tag && new_tag->str); // id_tag should exist
+        assert(new_tag->len > 3);
+        pos = kh_val(hash, k);
+        kh_del(m_s2i, hash, k);
+        k = kh_put(m_s2i, hash, new_tag->str + 3, &r);
+        if (r < 1) {
+            hts_log_error("Failed to rename item in hash table");
+            return -1;
+        }
+        kh_val(hash, k) = pos;
+    }
+
+    ret = sam_hrecs_update_hashes(hrecs, TYPEKEY(type), ty);
+
+    if (!ret && hrecs->refs_changed >= 0)
+        ret = rebuild_target_arrays(bh);
+
+    if (!ret && hrecs->dirty)
+        redact_header_text(bh);
+
+    return ret;
+}
+
+static int rebuild_hash(sam_hrecs_t *hrecs, const char *type) {
+    sam_hrec_type_t *head, *step;
+    khiter_t k;
+
+    if (strncmp(type, "SQ", 2) == 0) {
+        hrecs->nref = 0;
+        kh_clear(m_s2i, hrecs->ref_hash);
+    } else if (strncmp(type, "RG", 2) == 0) {
+        hrecs->nrg = 0;
+        kh_clear(m_s2i, hrecs->rg_hash);
+    }
+
+    k = kh_get(sam_hrecs_t, hrecs->h, TYPEKEY(type));
+
+    if (k != kh_end(hrecs->h)) { // something to rebuild
+        head = kh_val(hrecs->h, k);
+        step = head;
+
+        do {
+            if (sam_hrecs_update_hashes(hrecs, TYPEKEY(type), step) == -1) {
+                hts_log_error("Unable to rebuild hashes");
+                return -1;
+            }
+
+            step = step->next;
+        } while (step != head);
+    }
+
+    return 0;
+}
+
+int sam_hdr_remove_except(sam_hdr_t *bh, const char *type, const char *ID_key, const char *ID_value) {
+    sam_hrecs_t *hrecs;
+    if (!bh || !type)
+        return -1;
+
+    if (!(hrecs = bh->hrecs)) {
+        if (sam_hdr_fill_hrecs(bh) != 0)
+            return -1;
+        hrecs = bh->hrecs;
+    }
+
+    sam_hrec_type_t *step;
+    int ret = 1, remove_all = (ID_key == NULL);
+
+    if (!strncmp(type, "PG", 2) || !strncmp(type, "CO", 2)) {
+        hts_log_warning("Removing PG or CO lines is not supported!");
+        return -1;
+    }
+
+    sam_hrec_type_t *type_found = sam_hrecs_find_type_id(hrecs, type, ID_key, ID_value);
+    if (!type_found) { // remove all line of this type
+        khint_t k = kh_get(sam_hrecs_t, hrecs->h, TYPEKEY(type));
+        if (k == kh_end(hrecs->h))
+            return 0;
+        type_found =  kh_val(hrecs->h, k);
+        if (!type_found)
+            return 0;
+        remove_all = 1;
+    }
+
+    step = type_found->next;
+    while (step != type_found) {
+        sam_hrec_type_t *to_remove = step;
+        step = step->next;
+        ret &= sam_hrecs_remove_line(hrecs, type, to_remove, 0);
+    }
+
+    if (remove_all)
+        ret &= sam_hrecs_remove_line(hrecs, type, type_found, 0);
+
+    /* if RG or SQ, delete then rebuild the hashes (as it is faster
+       to rebuild than delete one by one).
+    */
+
+    if ((strncmp(type, "SQ", 2) == 0) || (strncmp(type, "RG", 2) == 0)) {
+        if (rebuild_hash(hrecs, type)) {
+            return -1;
+        }
+    }
+
+    if (!ret && hrecs->dirty)
+        redact_header_text(bh);
+
+    return 0;
+}
+
+int sam_hdr_remove_lines(sam_hdr_t *bh, const char *type, const char *id, void *vrh) {
+    sam_hrecs_t *hrecs;
+    rmhash_t *rh = (rmhash_t *)vrh;
+
+    if (!bh || !type)
+        return -1;
+    if (!rh) // remove all lines
+        return sam_hdr_remove_except(bh, type, NULL, NULL);
+    if (!id)
+        return -1;
+
+    if (!(hrecs = bh->hrecs)) {
+        if (sam_hdr_fill_hrecs(bh) != 0)
+            return -1;
+        hrecs = bh->hrecs;
+    }
+
+    khint_t k = kh_get(sam_hrecs_t, hrecs->h, TYPEKEY(type));
+    if (k == kh_end(hrecs->h)) // nothing to remove from
+        return 0;
+
+    sam_hrec_type_t *head = kh_val(hrecs->h, k);
+    if (!head) {
+        hts_log_error("Header inconsistency");
+        return -1;
+    }
+
+    int ret = 0;
+    sam_hrec_type_t *step = head->next;
+    while (step != head) {
+        sam_hrec_tag_t *tag = sam_hrecs_find_key(step, id, NULL);
+        if (tag && tag->str && tag->len >= 3) {
+           k = kh_get(rm, rh, tag->str+3);
+           if (k == kh_end(rh)) { // value is not in the hash table, so remove
+               sam_hrec_type_t *to_remove = step;
+               step = step->next;
+               ret |= sam_hrecs_remove_line(hrecs, type, to_remove, 0);
+           } else {
+               step = step->next;
+           }
+        } else { // tag is not on the line, so skip to next line
+            step = step->next;
+        }
+    }
+
+    // process the first line
+    sam_hrec_tag_t * tag = sam_hrecs_find_key(head, id, NULL);
+    if (tag && tag->str && tag->len >= 3) {
+       k = kh_get(rm, rh, tag->str+3);
+       if (k == kh_end(rh)) { // value is not in the hash table, so remove
+           sam_hrec_type_t *to_remove = head;
+           head = head->next;
+           ret |= sam_hrecs_remove_line(hrecs, type, to_remove, 0);
+       }
+    }
+
+    /* if RG or SQ, delete then rebuild the hashes (as it is faster
+       to rebuild than delete one by one).
+    */
+
+    if ((strncmp(type, "SQ", 2) == 0) || (strncmp(type, "RG", 2) == 0)) {
+        if (rebuild_hash(hrecs, type)) {
+            return -1;
+        }
+    }
+
+    if (!ret && hrecs->dirty)
+        redact_header_text(bh);
+
+    return ret;
+}
+
+int sam_hdr_count_lines(sam_hdr_t *bh, const char *type) {
+    int count;
+    sam_hrec_type_t *first_ty, *itr_ty;
+
+    if (!bh || !type)
+        return -1;
+
+    if (!bh->hrecs) {
+        if (sam_hdr_fill_hrecs(bh) != 0)
+            return -1;
+    }
+
+    // Deal with types that have counts
+    switch (type[0]) {
+    case 'S':
+        if (type[1] == 'Q')
+            return bh->hrecs->nref;
+        break;
+    case 'R':
+        if (type[1] == 'G')
+            return bh->hrecs->nrg;
+        break;
+    case 'P':
+        if (type[1] == 'G')
+            return bh->hrecs->npg;
+        break;
+    default:
+        break;
+    }
+
+    first_ty = sam_hrecs_find_type_id(bh->hrecs, type, NULL, NULL);
+    if (!first_ty)
+        return 0;
+
+    count = 1;
+    for (itr_ty = first_ty->next;
+         itr_ty && itr_ty != first_ty; itr_ty = itr_ty->next) {
+        count++;
+    }
+
+    return count;
+}
+
+int sam_hdr_line_index(sam_hdr_t *bh,
+                       const char *type,
+                       const char *key) {
+    sam_hrecs_t *hrecs;
+    if (!bh || !type || !key)
+        return -2;
+
+    if (!(hrecs = bh->hrecs)) {
+        if (sam_hdr_fill_hrecs(bh) != 0)
+            return -2;
+        hrecs = bh->hrecs;
+    }
+
+    khint_t k;
+    int idx = -1;
+    switch (type[0]) {
+    case 'S':
+        if (type[1] == 'Q') {
+            k = kh_get(m_s2i, hrecs->ref_hash, key);
+            if (k != kh_end(hrecs->ref_hash))
+                idx = kh_val(hrecs->ref_hash, k);
+        } else {
+            hts_log_warning("Type '%s' not supported. Only @SQ, @RG and @PG lines are indexed", type);
+        }
+        break;
+    case 'R':
+        if (type[1] == 'G') {
+            k = kh_get(m_s2i, hrecs->rg_hash, key);
+            if (k != kh_end(hrecs->rg_hash))
+                idx = kh_val(hrecs->rg_hash, k);
+        } else {
+            hts_log_warning("Type '%s' not supported. Only @SQ, @RG and @PG lines are indexed", type);
+        }
+        break;
+    case 'P':
+        if (type[1] == 'G') {
+            k = kh_get(m_s2i, hrecs->pg_hash, key);
+            if (k != kh_end(hrecs->pg_hash))
+                idx = kh_val(hrecs->pg_hash, k);
+        } else {
+            hts_log_warning("Type '%s' not supported. Only @SQ, @RG and @PG lines are indexed", type);
+        }
+        break;
+    default:
+        hts_log_warning("Type '%s' not supported. Only @SQ, @RG and @PG lines are indexed", type);
+    }
+
+    return idx;
+}
+
+const char *sam_hdr_line_name(sam_hdr_t *bh,
+                              const char *type,
+                              int pos) {
+    sam_hrecs_t *hrecs;
+    if (!bh || !type || pos < 0)
+        return NULL;
+
+    if (!(hrecs = bh->hrecs)) {
+        if (sam_hdr_fill_hrecs(bh) != 0)
+            return NULL;
+        hrecs = bh->hrecs;
+    }
+
+    switch (type[0]) {
+    case 'S':
+        if (type[1] == 'Q') {
+            if (pos < hrecs->nref)
+                return hrecs->ref[pos].name;
+        } else {
+            hts_log_warning("Type '%s' not supported. Only @SQ, @RG and @PG lines are indexed", type);
+        }
+        break;
+    case 'R':
+        if (type[1] == 'G') {
+            if (pos < hrecs->nrg)
+                return hrecs->rg[pos].name;
+        } else {
+            hts_log_warning("Type '%s' not supported. Only @SQ, @RG and @PG lines are indexed", type);
+        }
+        break;
+    case 'P':
+        if (type[1] == 'G') {
+            if (pos < hrecs->npg)
+                return hrecs->pg[pos].name;
+        } else {
+            hts_log_warning("Type '%s' not supported. Only @SQ, @RG and @PG lines are indexed", type);
+        }
+        break;
+    default:
+        hts_log_warning("Type '%s' not supported. Only @SQ, @RG and @PG lines are indexed", type);
+    }
+
+    return NULL;
+}
+
+/* ==== Key:val level methods ==== */
+
+int sam_hdr_find_tag_id(sam_hdr_t *bh,
+                     const char *type,
+                     const char *ID_key,
+                     const char *ID_value,
+                     const char *key,
+                     kstring_t *ks) {
+    sam_hrecs_t *hrecs;
+    if (!bh || !type || !key)
+        return -2;
+
+    if (!(hrecs = bh->hrecs)) {
+        if (sam_hdr_fill_hrecs(bh) != 0)
+            return -2;
+        hrecs = bh->hrecs;
+    }
+
+    sam_hrec_type_t *ty = sam_hrecs_find_type_id(hrecs, type, ID_key, ID_value);
+    if (!ty)
+        return -1;
+
+    sam_hrec_tag_t *tag = sam_hrecs_find_key(ty, key, NULL);
+    if (!tag || !tag->str || tag->len < 4)
+        return -1;
+
+    ks->l = 0;
+    if (kputsn(tag->str+3, tag->len-3, ks) == EOF) {
+        return -2;
+    }
+
+    return 0;
+}
+
+int sam_hdr_find_tag_pos(sam_hdr_t *bh,
+                     const char *type,
+                     int pos,
+                     const char *key,
+                     kstring_t *ks) {
+    sam_hrecs_t *hrecs;
+    if (!bh || !type || !key)
+        return -2;
+
+    if (!(hrecs = bh->hrecs)) {
+        if (sam_hdr_fill_hrecs(bh) != 0)
+            return -2;
+        hrecs = bh->hrecs;
+    }
+
+    sam_hrec_type_t *ty = sam_hrecs_find_type_pos(hrecs, type, pos);
+    if (!ty)
+        return -1;
+
+    sam_hrec_tag_t *tag = sam_hrecs_find_key(ty, key, NULL);
+    if (!tag || !tag->str || tag->len < 4)
+        return -1;
+
+    ks->l = 0;
+    if (kputsn(tag->str+3, tag->len-3, ks) == EOF) {
+        return -2;
+    }
+
+    return 0;
+}
+
+int sam_hdr_remove_tag_id(sam_hdr_t *bh,
+        const char *type,
+        const char *ID_key,
+        const char *ID_value,
+        const char *key) {
+    sam_hrecs_t *hrecs;
+    if (!bh || !type || !key)
+        return -1;
+
+    if (!(hrecs = bh->hrecs)) {
+        if (sam_hdr_fill_hrecs(bh) != 0)
+            return -1;
+        hrecs = bh->hrecs;
+    }
+
+    sam_hrec_type_t *ty = sam_hrecs_find_type_id(hrecs, type, ID_key, ID_value);
+    if (!ty)
+        return -1;
+
+    int ret = sam_hrecs_remove_key(hrecs, ty, key);
+    if (!ret && hrecs->dirty)
+        redact_header_text(bh);
+
+    return ret;
+}
+
+/*
+ * Reconstructs a kstring from the header hash table.
+ * Returns 0 on success
+ *        -1 on failure
+ */
+int sam_hrecs_rebuild_text(const sam_hrecs_t *hrecs, kstring_t *ks) {
+    ks->l = 0;
+
+    if (!hrecs->h || !hrecs->h->size) {
+        return kputsn("", 0, ks) >= 0 ? 0 : -1;
+    }
+    if (sam_hrecs_rebuild_lines(hrecs, ks) != 0)
+        return -1;
+
+    return 0;
+}
+
+/*
+ * Looks up a reference sequence by name and returns the numerical ID.
+ * Returns -1 if unknown reference; -2 if header could not be parsed.
+ */
+int sam_hdr_name2tid(sam_hdr_t *bh, const char *ref) {
+    sam_hrecs_t *hrecs;
+    khint_t k;
+
+    if (!bh)
+        return -1;
+
+    if (!(hrecs = bh->hrecs)) {
+        if (sam_hdr_fill_hrecs(bh) != 0)
+            return -2;
+        hrecs = bh->hrecs;
+    }
+
+    if (!hrecs->ref_hash)
+        return -1;
+
+    k = kh_get(m_s2i, hrecs->ref_hash, ref);
+    return k == kh_end(hrecs->ref_hash) ? -1 : kh_val(hrecs->ref_hash, k);
+}
+
+const char *sam_hdr_tid2name(const sam_hdr_t *h, int tid) {
+    sam_hrecs_t *hrecs;
+
+    if (!h || tid < 0)
+        return NULL;
+
+    if ((hrecs = h->hrecs) != NULL && tid < hrecs->nref) {
+        return hrecs->ref[tid].name;
+    } else {
+        if (tid < h->n_targets)
+            return h->target_name[tid];
+    }
+
+    return NULL;
+}
+
+hts_pos_t sam_hdr_tid2len(const sam_hdr_t *h, int tid) {
+    sam_hrecs_t *hrecs;
+
+    if (!h || tid < 0)
+        return 0;
+
+    if ((hrecs = h->hrecs) != NULL && tid < hrecs->nref) {
+        return hrecs->ref[tid].len;
+    } else {
+        if (tid < h->n_targets) {
+            if (h->target_len[tid] < UINT32_MAX || !h->sdict) {
+                return h->target_len[tid];
+            } else {
+                khash_t(s2i) *long_refs = (khash_t(s2i) *) h->sdict;
+                khint_t k = kh_get(s2i, long_refs, h->target_name[tid]);
+                if (k < kh_end(long_refs)) {
+                    return kh_val(long_refs, k);
+                } else {
+                    return UINT32_MAX;
+                }
+            }
+        }
+    }
+
+    return 0;
+}
+
+/*
+ * Fixes any PP links in @PG headers.
+ * If the entries are in order then this doesn't need doing, but in case
+ * our header is out of order this goes through the hrecs->pg[] array
+ * setting the prev_id field.
+ *
+ * Note we can have multiple complete chains. This code should identify the
+ * tails of these chains as these are the entries we have to link to in
+ * subsequent PP records.
+ *
+ * Returns 0 on success
+ *        -1 on failure (indicating broken PG/PP records)
+ */
+static int sam_hdr_link_pg(sam_hdr_t *bh) {
+    sam_hrecs_t *hrecs;
+    int i, j, ret = 0, *new_pg_end;
+
+    if (!bh)
+        return -1;
+
+    if (!(hrecs = bh->hrecs)) {
+        if (sam_hdr_fill_hrecs(bh) != 0)
+            return -1;
+        hrecs = bh->hrecs;
+    }
+
+    if (!hrecs->pgs_changed || !hrecs->npg)
+        return 0;
+
+    hrecs->npg_end_alloc = hrecs->npg;
+    new_pg_end = realloc(hrecs->pg_end, hrecs->npg * sizeof(*new_pg_end));
+    if (!new_pg_end)
+        return -1;
+    hrecs->pg_end = new_pg_end;
+    int *chain_size = calloc(hrecs->npg, sizeof(int));
+    if (!chain_size)
+        return -1;
+
+    for (i = 0; i < hrecs->npg; i++)
+        hrecs->pg_end[i] = i;
+
+    for (i = 0; i < hrecs->npg; i++) {
+        khint_t k;
+        sam_hrec_tag_t *tag;
+
+        assert(hrecs->pg[i].ty != NULL);
+        for (tag = hrecs->pg[i].ty->tag; tag; tag = tag->next) {
+            if (tag->str[0] == 'P' && tag->str[1] == 'P')
+                break;
+        }
+        if (!tag) {
+            /* Chain start points */
+            continue;
+        }
+
+        k = kh_get(m_s2i, hrecs->pg_hash, tag->str+3);
+
+        if (k == kh_end(hrecs->pg_hash)) {
+            hts_log_warning("PG line with ID:%s has a PP link to missing program '%s'",
+                    hrecs->pg[i].name, tag->str+3);
+            continue;
+        }
+
+        int pp_idx = kh_val(hrecs->pg_hash, k);
+        if (pp_idx == i) {
+            hts_log_warning("PG line with ID:%s has a PP link to itself",
+                            hrecs->pg[i].name);
+            continue;
+        }
+
+        hrecs->pg[i].prev_id = hrecs->pg[pp_idx].id;
+        hrecs->pg_end[pp_idx] = -1;
+        chain_size[i] = chain_size[pp_idx]+1;
+    }
+
+    int last_end = -1;
+    for (i = j = 0; i < hrecs->npg; i++) {
+        if (hrecs->pg_end[i] != -1) {
+            last_end = hrecs->pg_end[i];
+            if (chain_size[i] > 0)
+                hrecs->pg_end[j++] = hrecs->pg_end[i];
+        }
+    }
+    /* Only leafs? Choose the last one! */
+    if (!j && hrecs->npg_end > 0 && last_end >= 0) {
+        hrecs->pg_end[0] = last_end;
+        j = 1;
+    }
+
+    hrecs->npg_end = j;
+    hrecs->pgs_changed = 0;
+
+    /* mark as dirty or empty for rebuild */
+    hrecs->dirty = 1;
+    redact_header_text(bh);
+    free(chain_size);
+
+    return ret;
+}
+
+/*
+ * Returns a unique ID from a base name.
+ *
+ * The value returned is valid until the next call to
+ * this function.
+ */
+const char *sam_hdr_pg_id(sam_hdr_t *bh, const char *name) {
+    sam_hrecs_t *hrecs;
+    size_t name_len;
+    const size_t name_extra = 17;
+    if (!bh || !name)
+        return NULL;
+
+    if (!(hrecs = bh->hrecs)) {
+        if (sam_hdr_fill_hrecs(bh) != 0)
+            return NULL;
+        hrecs = bh->hrecs;
+    }
+
+    khint_t k = kh_get(m_s2i, hrecs->pg_hash, name);
+    if (k == kh_end(hrecs->pg_hash))
+        return name;
+
+    name_len = strlen(name);
+    if (name_len > 1000) name_len = 1000;
+    if (hrecs->ID_buf_sz < name_len + name_extra) {
+        char *new_ID_buf = realloc(hrecs->ID_buf, name_len + name_extra);
+        if (new_ID_buf == NULL)
+            return NULL;
+        hrecs->ID_buf = new_ID_buf;
+        hrecs->ID_buf_sz = name_len + name_extra;
+    }
+
+    do {
+        snprintf(hrecs->ID_buf, hrecs->ID_buf_sz, "%.1000s.%d", name, hrecs->ID_cnt++);
+        k = kh_get(m_s2i, hrecs->pg_hash, hrecs->ID_buf);
+    } while (k != kh_end(hrecs->pg_hash));
+
+    return hrecs->ID_buf;
+}
+
+/*
+ * Add an @PG line.
+ *
+ * If we wish complete control over this use sam_hdr_add_line() directly. This
+ * function uses that, but attempts to do a lot of tedious house work for
+ * you too.
+ *
+ * - It will generate a suitable ID if the supplied one clashes.
+ * - It will generate multiple @PG records if we have multiple PG chains.
+ *
+ * Call it as per sam_hdr_add_line() with a series of key,value pairs ending
+ * in NULL.
+ *
+ * Returns 0 on success
+ *        -1 on failure
+ */
+int sam_hdr_add_pg(sam_hdr_t *bh, const char *name, ...) {
+    sam_hrecs_t *hrecs;
+    const char *specified_id = NULL, *specified_pn = NULL, *specified_pp = NULL;
+    const char *key, *val;
+    if (!bh)
+        return -1;
+
+    if (!(hrecs = bh->hrecs)) {
+        if (sam_hdr_fill_hrecs(bh) != 0)
+            return -1;
+        hrecs = bh->hrecs;
+    }
+
+    bh->hrecs->pgs_changed = 1;
+    if (sam_hdr_link_pg(bh) < 0) {
+        hts_log_error("Error linking @PG lines");
+        return -1;
+    }
+
+    va_list args;
+    // Check for ID / PN / PP tags in varargs list
+    va_start(args, name);
+    while ((key = va_arg(args, const char *)) != NULL) {
+        val = va_arg(args, const char *);
+        if (!val) break;
+        if (strcmp(key, "PN") == 0 && *val != '\0')
+            specified_pn = val;
+        else if (strcmp(key, "PP") == 0 && *val != '\0')
+            specified_pp = val;
+        else if (strcmp(key, "ID") == 0 && *val != '\0')
+            specified_id = val;
+    }
+    va_end(args);
+
+    if (specified_id && hrecs->pg_hash) {
+        khint_t k = kh_get(m_s2i, hrecs->pg_hash, specified_id);
+        if (k != kh_end(hrecs->pg_hash)) {
+            hts_log_error("Header @PG ID:%s already present", specified_id);
+            return -1;
+        }
+    }
+
+    if (specified_pp && hrecs->pg_hash) {
+        khint_t k = kh_get(m_s2i, hrecs->pg_hash, specified_pp);
+        if (k == kh_end(hrecs->pg_hash)) {
+            hts_log_error("Header @PG ID:%s referred to by PP tag not present",
+                          specified_pp);
+            return -1;
+        }
+    }
+
+    if (!specified_pp && hrecs->npg_end) {
+        /* Copy ends array to avoid us looping while modifying it */
+        int *end = malloc(hrecs->npg_end * sizeof(int));
+        int i, nends = hrecs->npg_end;
+
+        if (!end)
+            return -1;
+
+        memcpy(end, hrecs->pg_end, nends * sizeof(*end));
+
+        for (i = 0; i < nends; i++) {
+            const char *id = !specified_id ? sam_hdr_pg_id(bh, name) : "";
+            if (!id) {
+                free(end);
+                return -1;
+            }
+            assert(end[i] >= 0 && end[i] < hrecs->npg);
+            va_start(args, name);
+            if (-1 == sam_hrecs_vadd(hrecs, "PG", args,
+                                     "ID", id,
+                                     "PN", !specified_pn ? name : "",
+                                     "PP", hrecs->pg[end[i]].name,
+                                     NULL)) {
+                free(end);
+                return  -1;
+            }
+            va_end(args);
+        }
+
+        free(end);
+    } else {
+        const char *id = !specified_id ? sam_hdr_pg_id(bh, name) : "";
+        if (!id)
+            return -1;
+        va_start(args, name);
+        if (-1 == sam_hrecs_vadd(hrecs, "PG", args,
+                                 "ID", id,
+                                 "PN", !specified_pn ? name : "",
+                                 NULL))
+            return -1;
+        va_end(args);
+    }
+
+    hrecs->dirty = 1;
+    redact_header_text(bh);
+
+    return 0;
+}
+
+/*! Increments a reference count on bh.
+ *
+ * This permits multiple files to share the same header, all calling
+ * sam_hdr_destroy when done, without causing errors for other open files.
+ */
+void sam_hdr_incr_ref(sam_hdr_t *bh) {
+    if (!bh)
+        return;
+    bh->ref_count++;
+}
+
+/* ==== Internal methods ==== */
+
+/*
+ * Creates an empty SAM header.  Allocates space for the SAM header
+ * structures (hash tables) ready to be populated.
+ *
+ * Returns a sam_hrecs_t struct on success (free with sam_hrecs_free())
+ *         NULL on failure
+ */
+sam_hrecs_t *sam_hrecs_new(void) {
+    sam_hrecs_t *hrecs = calloc(1, sizeof(*hrecs));
+
+    if (!hrecs)
+        return NULL;
+
+    hrecs->h = kh_init(sam_hrecs_t);
+    if (!hrecs->h)
+        goto err;
+
+    hrecs->ID_cnt = 1;
+
+    hrecs->nref = 0;
+    hrecs->ref_sz = 0;
+    hrecs->ref  = NULL;
+    if (!(hrecs->ref_hash = kh_init(m_s2i)))
+        goto err;
+    hrecs->refs_changed = -1;
+
+    hrecs->nrg = 0;
+    hrecs->rg_sz = 0;
+    hrecs->rg  = NULL;
+    if (!(hrecs->rg_hash = kh_init(m_s2i)))
+        goto err;
+
+    hrecs->npg = 0;
+    hrecs->pg_sz = 0;
+    hrecs->pg  = NULL;
+    hrecs->npg_end = hrecs->npg_end_alloc = 0;
+    hrecs->pg_end = NULL;
+    if (!(hrecs->pg_hash = kh_init(m_s2i)))
+        goto err;
+
+    if (!(hrecs->tag_pool = pool_create(sizeof(sam_hrec_tag_t))))
+        goto err;
+
+    if (!(hrecs->type_pool = pool_create(sizeof(sam_hrec_type_t))))
+        goto err;
+
+    if (!(hrecs->str_pool = string_pool_create(65536)))
+        goto err;
+
+    if (sam_hrecs_init_type_order(hrecs, NULL))
+        goto err;
+
+    return hrecs;
+
+err:
+    if (hrecs->h)
+        kh_destroy(sam_hrecs_t, hrecs->h);
+
+    if (hrecs->tag_pool)
+        pool_destroy(hrecs->tag_pool);
+
+    if (hrecs->type_pool)
+        pool_destroy(hrecs->type_pool);
+
+    if (hrecs->str_pool)
+        string_pool_destroy(hrecs->str_pool);
+
+    free(hrecs);
+
+    return NULL;
+}
+#if 0
+/*
+ * Produces a duplicate copy of source and returns it.
+ * Returns NULL on failure
+ */
+sam_hrecs_t *sam_hrecs_dup(sam_hrecs_t *source) {
+        return NULL;
+}
+#endif
+/*! Deallocates all storage used by a sam_hrecs_t struct.
+ *
+ * This also decrements the header reference count. If after decrementing
+ * it is still non-zero then the header is assumed to be in use by another
+ * caller and the free is not done.
+ *
+ */
+void sam_hrecs_free(sam_hrecs_t *hrecs) {
+    if (!hrecs)
+        return;
+
+    if (hrecs->h)
+        kh_destroy(sam_hrecs_t, hrecs->h);
+
+    if (hrecs->ref_hash)
+        kh_destroy(m_s2i, hrecs->ref_hash);
+
+    if (hrecs->ref)
+        free(hrecs->ref);
+
+    if (hrecs->rg_hash)
+        kh_destroy(m_s2i, hrecs->rg_hash);
+
+    if (hrecs->rg)
+        free(hrecs->rg);
+
+    if (hrecs->pg_hash)
+        kh_destroy(m_s2i, hrecs->pg_hash);
+
+    if (hrecs->pg)
+        free(hrecs->pg);
+
+    if (hrecs->pg_end)
+        free(hrecs->pg_end);
+
+    if (hrecs->type_pool)
+        pool_destroy(hrecs->type_pool);
+
+    if (hrecs->tag_pool)
+        pool_destroy(hrecs->tag_pool);
+
+    if (hrecs->str_pool)
+        string_pool_destroy(hrecs->str_pool);
+
+    if (hrecs->type_order)
+        free(hrecs->type_order);
+
+    if (hrecs->ID_buf)
+        free(hrecs->ID_buf);
+
+    free(hrecs);
+}
+
+/*
+ * Internal method already used by the CRAM code
+ * Returns the first header item matching 'type'. If ID is non-NULL it checks
+ * for the tag ID: and compares against the specified ID.
+ *
+ * Returns NULL if no type/ID is found
+ */
+sam_hrec_type_t *sam_hrecs_find_type_id(sam_hrecs_t *hrecs, const char *type,
+                                     const char *ID_key, const char *ID_value) {
+    if (!hrecs || !type)
+        return NULL;
+    sam_hrec_type_t *t1, *t2;
+    khint_t k;
+
+    /* Special case for types we have prebuilt hashes on */
+    if (ID_key) {
+        if (!ID_value)
+            return NULL;
+
+        if (type[0]   == 'S' && type[1]   == 'Q' &&
+            ID_key[0] == 'S' && ID_key[1] == 'N') {
+            k = kh_get(m_s2i, hrecs->ref_hash, ID_value);
+            return k != kh_end(hrecs->ref_hash)
+                ? hrecs->ref[kh_val(hrecs->ref_hash, k)].ty
+                : NULL;
+        }
+
+        if (type[0]   == 'R' && type[1]   == 'G' &&
+            ID_key[0] == 'I' && ID_key[1] == 'D') {
+            k = kh_get(m_s2i, hrecs->rg_hash, ID_value);
+            return k != kh_end(hrecs->rg_hash)
+                ? hrecs->rg[kh_val(hrecs->rg_hash, k)].ty
+                : NULL;
+        }
+
+        if (type[0]   == 'P' && type[1]   == 'G' &&
+            ID_key[0] == 'I' && ID_key[1] == 'D') {
+            k = kh_get(m_s2i, hrecs->pg_hash, ID_value);
+            return k != kh_end(hrecs->pg_hash)
+                ? hrecs->pg[kh_val(hrecs->pg_hash, k)].ty
+                : NULL;
+        }
+    }
+
+    k = kh_get(sam_hrecs_t, hrecs->h, TYPEKEY(type));
+    if (k == kh_end(hrecs->h))
+        return NULL;
+
+    if (!ID_key)
+        return kh_val(hrecs->h, k);
+
+    t1 = t2 = kh_val(hrecs->h, k);
+    do {
+        sam_hrec_tag_t *tag;
+        for (tag = t1->tag; tag; tag = tag->next) {
+            if (tag->str[0] == ID_key[0] && tag->str[1] == ID_key[1]) {
+                const char *cp1 = tag->str+3;
+                const char *cp2 = ID_value;
+                while (*cp1 && *cp1 == *cp2)
+                    cp1++, cp2++;
+                if (*cp2 || *cp1)
+                    continue;
+                return t1;
+            }
+        }
+        t1 = t1->next;
+    } while (t1 != t2);
+
+    return NULL;
+}
+
+/*
+ * Adds or updates tag key,value pairs in a header line.
+ * Eg for adding M5 tags to @SQ lines or updating sort order for the
+ * @HD line.
+ *
+ * va_list contains multiple key,value pairs ending in NULL.
+ *
+ * Returns 0 on success
+ *        -1 on failure
+ */
+int sam_hrecs_vupdate(sam_hrecs_t *hrecs, sam_hrec_type_t *type, va_list ap) {
+    if (!hrecs)
+        return -1;
+
+    for (;;) {
+        char *k, *v, *str;
+        sam_hrec_tag_t *tag, *prev = NULL;
+
+        if (!(k = (char *)va_arg(ap, char *)))
+            break;
+        if (!(v = va_arg(ap, char *)))
+            v = "";
+
+        tag = sam_hrecs_find_key(type, k, &prev);
+        if (!tag) {
+            if (!(tag = pool_alloc(hrecs->tag_pool)))
+                return -1;
+            if (prev)
+                prev->next = tag;
+            else
+                type->tag = tag;
+
+            tag->next = NULL;
+        }
+
+        tag->len = 3 + strlen(v);
+        str = string_alloc(hrecs->str_pool, tag->len+1);
+        if (!str)
+            return -1;
+
+        if (snprintf(str, tag->len+1, "%2.2s:%s", k, v) < 0)
+            return -1;
+
+        tag->str = str;
+    }
+
+    hrecs->dirty = 1; //mark text as dirty and force a rebuild
+
+    return 0;
+}
+
+/*
+ * Adds or updates tag key,value pairs in a header line.
+ * Eg for adding M5 tags to @SQ lines or updating sort order for the
+ * @HD line.
+ *
+ * Specify multiple key,value pairs ending in NULL.
+ *
+ * Returns 0 on success
+ *        -1 on failure
+ */
+static int sam_hrecs_update(sam_hrecs_t *hrecs, sam_hrec_type_t *type, ...) {
+    va_list args;
+    int res;
+    va_start(args, type);
+    res = sam_hrecs_vupdate(hrecs, type, args);
+    va_end(args);
+    return res;
+}
+
+/*
+ * Looks for a specific key in a single sam header line identified by *type.
+ * If prev is non-NULL it also fills this out with the previous tag, to
+ * permit use in key removal. *prev is set to NULL when the tag is the first
+ * key in the list. When a tag isn't found, prev (if non NULL) will be the last
+ * tag in the existing list.
+ *
+ * Returns the tag pointer on success
+ *         NULL on failure
+ */
+sam_hrec_tag_t *sam_hrecs_find_key(sam_hrec_type_t *type,
+                                   const char *key,
+                                   sam_hrec_tag_t **prev) {
+    sam_hrec_tag_t *tag, *p = NULL;
+    if (!type)
+        return NULL;
+
+    for (tag = type->tag; tag; p = tag, tag = tag->next) {
+        if (tag->str[0] == key[0] && tag->str[1] == key[1]) {
+            if (prev)
+                *prev = p;
+            return tag;
+        }
+    }
+
+    if (prev)
+        *prev = p;
+
+    return NULL;
+}
+
+int sam_hrecs_remove_key(sam_hrecs_t *hrecs,
+                         sam_hrec_type_t *type,
+                         const char *key) {
+    sam_hrec_tag_t *tag, *prev;
+    if (!hrecs)
+        return -1;
+    tag = sam_hrecs_find_key(type, key, &prev);
+    if (!tag)
+        return 0; // Not there anyway
+
+    if (type->type == TYPEKEY("SQ") && tag->str[0] == 'A' && tag->str[1] == 'N') {
+        assert(tag->len >= 3);
+        sam_hrec_tag_t *sn_tag = sam_hrecs_find_key(type, "SN", NULL);
+        if (sn_tag) {
+            assert(sn_tag->len >= 3);
+            khint_t k = kh_get(m_s2i, hrecs->ref_hash, sn_tag->str + 3);
+            if (k != kh_end(hrecs->ref_hash))
+                sam_hrecs_remove_ref_altnames(hrecs, kh_val(hrecs->ref_hash, k), tag->str + 3);
+        }
+    }
+
+    if (!prev) { //first tag
+        type->tag = tag->next;
+    } else {
+        prev->next = tag->next;
+    }
+    pool_free(hrecs->tag_pool, tag);
+    hrecs->dirty = 1; //mark text as dirty and force a rebuild
+
+    return 1;
+}
+
+/*
+ * Looks up a read-group by name and returns a pointer to the start of the
+ * associated tag list.
+ *
+ * Returns NULL on failure
+ */
+sam_hrec_rg_t *sam_hrecs_find_rg(sam_hrecs_t *hrecs, const char *rg) {
+    khint_t k = kh_get(m_s2i, hrecs->rg_hash, rg);
+    return k == kh_end(hrecs->rg_hash)
+        ? NULL
+        : &hrecs->rg[kh_val(hrecs->rg_hash, k)];
+}
+
+#if DEBUG_HEADER
+void sam_hrecs_dump(sam_hrecs_t *hrecs) {
+    khint_t k;
+    int i;
+
+    printf("===DUMP===\n");
+    for (k = kh_begin(hrecs->h); k != kh_end(hrecs->h); k++) {
+        sam_hrec_type_t *t1, *t2;
+        char c[2];
+        int idx = 0;
+
+        if (!kh_exist(hrecs->h, k))
+            continue;
+
+        t1 = t2 = kh_val(hrecs->h, k);
+        c[0] = kh_key(hrecs->h, k)>>8;
+        c[1] = kh_key(hrecs->h, k)&0xff;
+        printf("Type %.2s\n", c);
+
+        do {
+            sam_hrec_tag_t *tag;
+            printf(">>>%d ", idx++);
+            for (tag = t1->tag; tag; tag=tag->next) {
+                if (strncmp(c, "CO", 2))
+                    printf("\"%.2s\":\"%.*s\"\t", tag->str, tag->len-3, tag->str+3);
+                else
+                    printf("%s", tag->str);
+            }
+            putchar('\n');
+            t1 = t1->next;
+        } while (t1 != t2);
+    }
+
+    /* Dump out PG chains */
+    printf("\n@PG chains:\n");
+    for (i = 0; i < hrecs->npg_end; i++) {
+        int j;
+        printf("  %d:", i);
+        for (j = hrecs->pg_end[i]; j != -1; j = hrecs->pg[j].prev_id) {
+            printf("%s%d(%.*s)",
+                   j == hrecs->pg_end[i] ? " " : "->",
+                   j, hrecs->pg[j].name_len, hrecs->pg[j].name);
+        }
+        printf("\n");
+    }
+
+    puts("===END DUMP===");
+}
+#endif
+
+/*
+ * Returns the sort order:
+ */
+enum sam_sort_order sam_hrecs_sort_order(sam_hrecs_t *hrecs) {
+    khint_t k;
+    enum sam_sort_order so;
+
+    so = ORDER_UNKNOWN;
+    k = kh_get(sam_hrecs_t, hrecs->h, TYPEKEY("HD"));
+    if (k != kh_end(hrecs->h)) {
+        sam_hrec_type_t *ty = kh_val(hrecs->h, k);
+        sam_hrec_tag_t *tag;
+        for (tag = ty->tag; tag; tag = tag->next) {
+            if (tag->str[0] == 'S' && tag->str[1] == 'O') {
+                if (strcmp(tag->str+3, "unsorted") == 0)
+                    so = ORDER_UNSORTED;
+                else if (strcmp(tag->str+3, "queryname") == 0)
+                    so = ORDER_NAME;
+                else if (strcmp(tag->str+3, "coordinate") == 0)
+                    so = ORDER_COORD;
+                else if (strcmp(tag->str+3, "unknown") != 0)
+                    hts_log_error("Unknown sort order field: %s", tag->str+3);
+            }
+        }
+    }
+
+    return so;
+}
+
+enum sam_group_order sam_hrecs_group_order(sam_hrecs_t *hrecs) {
+    khint_t k;
+    enum sam_group_order go;
+
+    go = ORDER_NONE;
+    k = kh_get(sam_hrecs_t, hrecs->h, TYPEKEY("HD"));
+    if (k != kh_end(hrecs->h)) {
+        sam_hrec_type_t *ty = kh_val(hrecs->h, k);
+        sam_hrec_tag_t *tag;
+        for (tag = ty->tag; tag; tag = tag->next) {
+            if (tag->str[0] == 'G' && tag->str[1] == 'O') {
+                if (strcmp(tag->str+3, "query") == 0)
+                    go = ORDER_QUERY;
+                else if (strcmp(tag->str+3, "reference") == 0)
+                    go = ORDER_REFERENCE;
+            }
+        }
+    }
+
+    return go;
+}
diff --git a/ext/htslib/header.h b/ext/htslib/header.h
new file mode 100644
index 0000000..a98d306
--- /dev/null
+++ b/ext/htslib/header.h
@@ -0,0 +1,319 @@
+/*
+Copyright (c) 2013-2019 Genome Research Ltd.
+Authors: James Bonfield <jkb@sanger.ac.uk>, Valeriu Ohan <vo2@sanger.ac.uk>
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+   3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+Institute nor the names of its contributors may be used to endorse or promote
+products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH LTD OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+/*! \file
+ * SAM header parsing.
+ *
+ * These functions can be shared between SAM, BAM and CRAM file
+ * formats as all three internally use the same string encoding for
+ * header fields.
+ */
+
+
+#ifndef HEADER_H_
+#define HEADER_H_
+
+#include <stdarg.h>
+
+#include "cram/string_alloc.h"
+#include "cram/pooled_alloc.h"
+
+#include "htslib/khash.h"
+#include "htslib/kstring.h"
+#include "htslib/sam.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*! Make a single integer out of a two-letter type code */
+static inline khint32_t TYPEKEY(const char *type) {
+    unsigned int u0 = (unsigned char) type[0];
+    unsigned int u1 = (unsigned char) type[1];
+    return (u0 << 8) | u1;
+}
+
+/*
+ * Proposed new SAM header parsing
+
+1 @SQ ID:foo LN:100
+2 @SQ ID:bar LN:200
+3 @SQ ID:ram LN:300 UR:xyz
+4 @RG ID:r ...
+5 @RG ID:s ...
+
+Hash table for 2-char @keys without dup entries.
+If dup lines, we form a circular linked list. Ie hash keys = {RG, SQ}.
+
+HASH("SQ")--\
+            |
+    (3) <-> 1 <-> 2 <-> 3 <-> (1)
+
+HASH("RG")--\
+            |
+    (5) <-> 4 <-> 5 <-> (4)
+
+Items stored in the hash values also form their own linked lists:
+Ie SQ->ID(foo)->LN(100)
+   SQ->ID(bar)->LN(200)
+   SQ->ID(ram)->LN(300)->UR(xyz)
+   RG->ID(r)
+ */
+
+/*! A single key:value pair on a header line
+ *
+ * These form a linked list and hold strings. The strings are
+ * allocated from a string_alloc_t pool referenced in the master
+ * sam_hrecs_t structure. Do not attempt to free, malloc or manipulate
+ * these strings directly.
+ */
+typedef struct sam_hrec_tag_s {
+    struct sam_hrec_tag_s *next;
+    const char *str;
+    int   len;
+} sam_hrec_tag_t;
+
+/*! The parsed version of the SAM header string.
+ *
+ * Each header type (SQ, RG, HD, etc) points to its own sam_hdr_type
+ * struct via the main hash table h in the sam_hrecs_t struct.
+ *
+ * These in turn consist of circular bi-directional linked lists (ie
+ * rings) to hold the multiple instances of the same header type
+ * code. For example if we have 5 \@SQ lines the primary hash table
+ * will key on \@SQ pointing to the first sam_hdr_type and that in turn
+ * will be part of a ring of 5 elements.
+ *
+ * For each sam_hdr_type structure we also point to a sam_hdr_tag
+ * structure which holds the tokenised attributes; the tab separated
+ * key:value pairs per line.
+ */
+typedef struct sam_hrec_type_s {
+    struct sam_hrec_type_s *next; // circular list of this type
+    struct sam_hrec_type_s *prev; // circular list of this type
+    struct sam_hrec_type_s *global_next; // circular list of all lines
+    struct sam_hrec_type_s *global_prev; // circular list of all lines
+    sam_hrec_tag_t *tag;          // first tag
+    khint32_t type;               // Two-letter type code as an int
+} sam_hrec_type_t;
+
+/*! Parsed \@SQ lines */
+typedef struct {
+    const char *name;
+    hts_pos_t len;
+    sam_hrec_type_t *ty;
+} sam_hrec_sq_t;
+
+/*! Parsed \@RG lines */
+typedef struct {
+    const char *name;
+    sam_hrec_type_t *ty;
+    int name_len;
+    int id;           // numerical ID
+} sam_hrec_rg_t;
+
+/*! Parsed \@PG lines */
+typedef struct {
+    const char *name;
+    sam_hrec_type_t *ty;
+    int name_len;
+    int id;           // numerical ID
+    int prev_id;      // -1 if none
+} sam_hrec_pg_t;
+
+
+/*! Sort order parsed from @HD line */
+enum sam_sort_order {
+    ORDER_UNKNOWN  =-1,
+    ORDER_UNSORTED = 0,
+    ORDER_NAME     = 1,
+    ORDER_COORD    = 2
+  //ORDER_COLLATE  = 3 // maybe one day!
+};
+
+enum sam_group_order {
+    ORDER_NONE      =-1,
+    ORDER_QUERY     = 0,
+    ORDER_REFERENCE = 1
+};
+
+KHASH_MAP_INIT_INT(sam_hrecs_t, sam_hrec_type_t*)
+KHASH_MAP_INIT_STR(m_s2i, int)
+
+/*! Primary structure for header manipulation
+ *
+ * The initial header text is held in the text kstring_t, but is also
+ * parsed out into SQ, RG and PG arrays. These have a hash table
+ * associated with each to allow lookup by ID or SN fields instead of
+ * their numeric array indices. Additionally PG has an array to hold
+ * the linked list start points (the last in a PP chain).
+ *
+ * Use the appropriate sam_hdr_* functions to edit the header, and
+ * call sam_hdr_rebuild() any time the textual form needs to be
+ * updated again.
+ */
+struct sam_hrecs_t {
+    khash_t(sam_hrecs_t) *h;
+    sam_hrec_type_t *first_line; //!< First line (usually @HD)
+    string_alloc_t *str_pool; //!< Pool of sam_hdr_tag->str strings
+    pool_alloc_t   *type_pool;//!< Pool of sam_hdr_type structs
+    pool_alloc_t   *tag_pool; //!< Pool of sam_hdr_tag structs
+
+    // @SQ lines / references
+    int nref;                  //!< Number of \@SQ lines
+    int ref_sz;                //!< Number of entries available in ref[]
+    sam_hrec_sq_t *ref;        //!< Array of parsed \@SQ lines
+    khash_t(m_s2i) *ref_hash;  //!< Maps SQ SN field to ref[] index
+
+    // @RG lines / read-groups
+    int nrg;                   //!< Number of \@RG lines
+    int rg_sz;                 //!< number of entries available in rg[]
+    sam_hrec_rg_t *rg;         //!< Array of parsed \@RG lines
+    khash_t(m_s2i) *rg_hash;   //!< Maps RG ID field to rg[] index
+
+    // @PG lines / programs
+    int npg;                   //!< Number of \@PG lines
+    int pg_sz;                //!< Number of entries available in pg[]
+    int npg_end;               //!< Number of terminating \@PG lines
+    int npg_end_alloc;         //!< Size of pg_end field
+    sam_hrec_pg_t *pg;         //!< Array of parsed \@PG lines
+    khash_t(m_s2i) *pg_hash;   //!< Maps PG ID field to pg[] index
+    int *pg_end;               //!< \@PG chain termination IDs
+
+    // @cond internal
+    char *ID_buf;             // temporary buffer for sam_hdr_pg_id
+    uint32_t ID_buf_sz;
+    int ID_cnt;
+    // @endcond
+
+    int dirty;                // marks the header as modified, so it can be rebuilt
+    int refs_changed;         // Index of first changed ref (-1 if unchanged)
+    int pgs_changed;          // New PG line added
+    int type_count;
+    char (*type_order)[3];
+};
+
+/*!
+ * Method for parsing the header text and populating the
+ * internal hash tables. After calling this method, the
+ * parsed representation becomes the single source of truth.
+ *
+ * @param bh    Header structure, previously initialised by a
+ *              sam_hdr_init call
+ * @return      0 on success, -1 on failure
+ */
+int sam_hdr_fill_hrecs(sam_hdr_t *bh);
+
+/*!
+ * Reconstructs the text representation of the header from
+ * the hash table data after a change has been performed on
+ * the header.
+ *
+ * @return  0 on success, -1 on failure
+ */
+int sam_hdr_rebuild(sam_hdr_t *bh);
+
+/*! Creates an empty SAM header, ready to be populated.
+ *
+ * @return
+ * Returns a sam_hrecs_t struct on success (free with sam_hrecs_free())
+ *         NULL on failure
+ */
+sam_hrecs_t *sam_hrecs_new(void);
+
+/*! Produces a duplicate copy of hrecs and returns it.
+ * @return
+ * Returns NULL on failure
+ */
+sam_hrecs_t *sam_hrecs_dup(sam_hrecs_t *hrecs);
+
+/*! Update sam_hdr_t target_name and target_len arrays
+ *
+ *  sam_hdr_t and sam_hrecs_t are specified separately so that sam_hdr_dup
+ *  can use it to construct target arrays from the source header.
+ *
+ *  @return 0 on success; -1 on failure
+ */
+int sam_hdr_update_target_arrays(sam_hdr_t *bh, const sam_hrecs_t *hrecs,
+                                 int refs_changed);
+
+/*! Reconstructs a kstring from the header hash table.
+ *
+ * @return
+ * Returns 0 on success
+ *        -1 on failure
+ */
+int sam_hrecs_rebuild_text(const sam_hrecs_t *hrecs, kstring_t *ks);
+
+/*! Deallocates all storage used by a sam_hrecs_t struct.
+ *
+ * This also decrements the header reference count. If after decrementing
+ * it is still non-zero then the header is assumed to be in use by another
+ * caller and the free is not done.
+ */
+void sam_hrecs_free(sam_hrecs_t *hrecs);
+
+/*!
+ * @return
+ * Returns the first header item matching 'type'. If ID is non-NULL it checks
+ * for the tag ID: and compares against the specified ID.
+ *
+ * Returns NULL if no type/ID is found
+ */
+sam_hrec_type_t *sam_hrecs_find_type_id(sam_hrecs_t *hrecs, const char *type,
+                                     const char *ID_key, const char *ID_value);
+
+sam_hrec_tag_t *sam_hrecs_find_key(sam_hrec_type_t *type,
+                                   const char *key,
+                                   sam_hrec_tag_t **prev);
+
+int sam_hrecs_remove_key(sam_hrecs_t *hrecs,
+                         sam_hrec_type_t *type,
+                         const char *key);
+
+/*! Looks up a read-group by name and returns a pointer to the start of the
+ * associated tag list.
+ *
+ * @return
+ * Returns NULL on failure
+ */
+sam_hrec_rg_t *sam_hrecs_find_rg(sam_hrecs_t *hrecs, const char *rg);
+
+/*! Returns the sort order from the @HD SO: field */
+enum sam_sort_order sam_hrecs_sort_order(sam_hrecs_t *hrecs);
+
+/*! Returns the group order from the @HD SO: field */
+enum sam_group_order sam_hrecs_group_order(sam_hrecs_t *hrecs);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* HEADER_H_ */
diff --git a/ext/htslib/hfile.c b/ext/htslib/hfile.c
new file mode 100644
index 0000000..552b717
--- /dev/null
+++ b/ext/htslib/hfile.c
@@ -0,0 +1,1440 @@
+/*  hfile.c -- buffered low-level input/output streams.
+
+    Copyright (C) 2013-2021, 2023-2024 Genome Research Ltd.
+
+    Author: John Marshall <jm18@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <stddef.h>
+#include <string.h>
+#include <errno.h>
+#include <limits.h>
+
+#include <pthread.h>
+
+#ifdef ENABLE_PLUGINS
+#if defined(_WIN32) || defined(__CYGWIN__) || defined(__MSYS__)
+#define USING_WINDOWS_PLUGIN_DLLS
+#include <dlfcn.h>
+#endif
+#endif
+
+#include "htslib/hfile.h"
+#include "hfile_internal.h"
+#include "htslib/kstring.h"
+
+#ifndef ENOTSUP
+#define ENOTSUP EINVAL
+#endif
+#ifndef EOVERFLOW
+#define EOVERFLOW ERANGE
+#endif
+#ifndef EPROTONOSUPPORT
+#define EPROTONOSUPPORT ENOSYS
+#endif
+
+#ifndef SSIZE_MAX /* SSIZE_MAX is POSIX 1 */
+#define SSIZE_MAX LONG_MAX
+#endif
+
+/* hFILE fields are used as follows:
+
+   char *buffer;     // Pointer to the start of the I/O buffer
+   char *begin;      // First not-yet-read character / unused position
+   char *end;        // First unfilled/unfillable position
+   char *limit;      // Pointer to the first position past the buffer
+
+   const hFILE_backend *backend;  // Methods to refill/flush I/O buffer
+
+   off_t offset;     // Offset within the stream of buffer position 0
+   unsigned at_eof:1;// For reading, whether EOF has been seen
+   unsigned mobile:1;// Buffer is a mobile window or fixed full contents
+   unsigned readonly:1;// Whether opened as "r" rather than "r+"/"w"/"a"
+   int has_errno;    // Error number from the last failure on this stream
+
+For reading, begin is the first unread character in the buffer and end is the
+first unfilled position:
+
+   -----------ABCDEFGHIJKLMNO---------------
+   ^buffer    ^begin         ^end           ^limit
+
+For writing, begin is the first unused position and end is unused so remains
+equal to buffer:
+
+   ABCDEFGHIJKLMNOPQRSTUVWXYZ---------------
+   ^buffer                   ^begin         ^limit
+   ^end
+
+Thus if begin > end then there is a non-empty write buffer, if begin < end
+then there is a non-empty read buffer, and if begin == end then both buffers
+are empty.  In all cases, the stream's file position indicator corresponds
+to the position pointed to by begin.
+
+The above is the normal scenario of a mobile window.  For in-memory
+streams (eg via hfile_init_fixed) the buffer can be used as the full
+contents without any separate backend behind it.  These always have at_eof
+set, offset set to 0, need no read() method, and should just return EINVAL
+for seek():
+
+   abcdefghijkLMNOPQRSTUVWXYZ------
+   ^buffer    ^begin         ^end  ^limit
+*/
+HTSLIB_EXPORT
+hFILE *hfile_init(size_t struct_size, const char *mode, size_t capacity)
+{
+    hFILE *fp = (hFILE *) malloc(struct_size);
+    if (fp == NULL) goto error;
+
+    if (capacity == 0) capacity = 32768;
+    // FIXME For now, clamp input buffer sizes so mpileup doesn't eat memory
+    if (strchr(mode, 'r') && capacity > 32768) capacity = 32768;
+
+    fp->buffer = (char *) malloc(capacity);
+    if (fp->buffer == NULL) goto error;
+
+    fp->begin = fp->end = fp->buffer;
+    fp->limit = &fp->buffer[capacity];
+
+    fp->offset = 0;
+    fp->at_eof = 0;
+    fp->mobile = 1;
+    fp->readonly = (strchr(mode, 'r') && ! strchr(mode, '+'));
+    fp->preserve = 0;
+    fp->has_errno = 0;
+    return fp;
+
+error:
+    hfile_destroy(fp);
+    return NULL;
+}
+
+hFILE *hfile_init_fixed(size_t struct_size, const char *mode,
+                        char *buffer, size_t buf_filled, size_t buf_size)
+{
+    hFILE *fp = (hFILE *) malloc(struct_size);
+    if (fp == NULL) return NULL;
+
+    fp->buffer = fp->begin = buffer;
+    fp->end = &fp->buffer[buf_filled];
+    fp->limit = &fp->buffer[buf_size];
+
+    fp->offset = 0;
+    fp->at_eof = 1;
+    fp->mobile = 0;
+    fp->readonly = (strchr(mode, 'r') && ! strchr(mode, '+'));
+    fp->preserve = 0;
+    fp->has_errno = 0;
+    return fp;
+}
+
+static const struct hFILE_backend mem_backend;
+
+HTSLIB_EXPORT
+void hfile_destroy(hFILE *fp)
+{
+    int save = errno;
+    if (fp) free(fp->buffer);
+    free(fp);
+    errno = save;
+}
+
+static inline int writebuffer_is_nonempty(hFILE *fp)
+{
+    return fp->begin > fp->end;
+}
+
+/* Refills the read buffer from the backend (once, so may only partially
+   fill the buffer), returning the number of additional characters read
+   (which might be 0), or negative when an error occurred.  */
+static ssize_t refill_buffer(hFILE *fp)
+{
+    ssize_t n;
+
+    // Move any unread characters to the start of the buffer
+    if (fp->mobile && fp->begin > fp->buffer) {
+        fp->offset += fp->begin - fp->buffer;
+        memmove(fp->buffer, fp->begin, fp->end - fp->begin);
+        fp->end = &fp->buffer[fp->end - fp->begin];
+        fp->begin = fp->buffer;
+    }
+
+    // Read into the available buffer space at fp->[end,limit)
+    if (fp->at_eof || fp->end == fp->limit) n = 0;
+    else {
+        n = fp->backend->read(fp, fp->end, fp->limit - fp->end);
+        if (n < 0) { fp->has_errno = errno; return n; }
+        else if (n == 0) fp->at_eof = 1;
+    }
+
+    fp->end += n;
+    return n;
+}
+
+/*
+ * Changes the buffer size for an hFILE.  Ideally this is done
+ * immediately after opening.  If performed later, this function may
+ * fail if we are reducing the buffer size and the current offset into
+ * the buffer is beyond the new capacity.
+ *
+ * Returns 0 on success;
+ *        -1 on failure.
+ */
+HTSLIB_EXPORT
+int hfile_set_blksize(hFILE *fp, size_t bufsiz) {
+    char *buffer;
+    ptrdiff_t curr_used;
+    if (!fp) return -1;
+    curr_used = (fp->begin > fp->end ? fp->begin : fp->end) - fp->buffer;
+    if (bufsiz == 0) bufsiz = 32768;
+
+    // Ensure buffer resize will not erase live data
+    if (bufsiz < curr_used)
+        return -1;
+
+    if (!(buffer = (char *) realloc(fp->buffer, bufsiz))) return -1;
+
+    fp->begin  = buffer + (fp->begin - fp->buffer);
+    fp->end    = buffer + (fp->end   - fp->buffer);
+    fp->buffer = buffer;
+    fp->limit  = &fp->buffer[bufsiz];
+
+    return 0;
+}
+
+/* Called only from hgetc(), when our buffer is empty.  */
+HTSLIB_EXPORT
+int hgetc2(hFILE *fp)
+{
+    return (refill_buffer(fp) > 0)? (unsigned char) *(fp->begin++) : EOF;
+}
+
+ssize_t hgetdelim(char *buffer, size_t size, int delim, hFILE *fp)
+{
+    char *found;
+    size_t n, copied = 0;
+    ssize_t got;
+
+    if (size < 1 || size > SSIZE_MAX) {
+        fp->has_errno = errno = EINVAL;
+        return -1;
+    }
+    if (writebuffer_is_nonempty(fp)) {
+        fp->has_errno = errno = EBADF;
+        return -1;
+    }
+
+    --size; /* to allow space for the NUL terminator */
+
+    do {
+        n = fp->end - fp->begin;
+        if (n > size - copied) n = size - copied;
+
+        /* Look in the hFILE buffer for the delimiter */
+        found = memchr(fp->begin, delim, n);
+        if (found != NULL) {
+            n = found - fp->begin + 1;
+            memcpy(buffer + copied, fp->begin, n);
+            buffer[n + copied] = '\0';
+            fp->begin += n;
+            return n + copied;
+        }
+
+        /* No delimiter yet, copy as much as we can and refill if necessary */
+        memcpy(buffer + copied, fp->begin, n);
+        fp->begin += n;
+        copied += n;
+
+        if (copied == size) { /* Output buffer full */
+            buffer[copied] = '\0';
+            return copied;
+        }
+
+        got = refill_buffer(fp);
+    } while (got > 0);
+
+    if (got < 0) return -1; /* Error on refill. */
+
+    buffer[copied] = '\0';  /* EOF, return anything that was copied. */
+    return copied;
+}
+
+char *hgets(char *buffer, int size, hFILE *fp)
+{
+    if (size < 1) {
+        fp->has_errno = errno = EINVAL;
+        return NULL;
+    }
+    return hgetln(buffer, size, fp) > 0 ? buffer : NULL;
+}
+
+ssize_t hpeek(hFILE *fp, void *buffer, size_t nbytes)
+{
+    size_t n = fp->end - fp->begin;
+    while (n < nbytes) {
+        ssize_t ret = refill_buffer(fp);
+        if (ret < 0) return ret;
+        else if (ret == 0) break;
+        else n += ret;
+    }
+
+    if (n > nbytes) n = nbytes;
+    memcpy(buffer, fp->begin, n);
+    return n;
+}
+
+/* Called only from hread(); when called, our buffer is empty and nread bytes
+   have already been placed in the destination buffer.  */
+HTSLIB_EXPORT
+ssize_t hread2(hFILE *fp, void *destv, size_t nbytes, size_t nread)
+{
+    const size_t capacity = fp->limit - fp->buffer;
+    int buffer_invalidated = 0;
+    char *dest = (char *) destv;
+    dest += nread, nbytes -= nread;
+
+    // Read large requests directly into the destination buffer
+    while (nbytes * 2 >= capacity && !fp->at_eof) {
+        ssize_t n = fp->backend->read(fp, dest, nbytes);
+        if (n < 0) { fp->has_errno = errno; return n; }
+        else if (n == 0) fp->at_eof = 1;
+        else buffer_invalidated = 1;
+        fp->offset += n;
+        dest += n, nbytes -= n;
+        nread += n;
+    }
+
+    if (buffer_invalidated) {
+        // Our unread buffer is empty, so begin == end, but our already-read
+        // buffer [buffer,begin) is likely non-empty and is no longer valid as
+        // its contents are no longer adjacent to the file position indicator.
+        // Discard it so that hseek() can't try to take advantage of it.
+        fp->offset += fp->begin - fp->buffer;
+        fp->begin = fp->end = fp->buffer;
+    }
+
+    while (nbytes > 0 && !fp->at_eof) {
+        size_t n;
+        ssize_t ret = refill_buffer(fp);
+        if (ret < 0) return ret;
+
+        n = fp->end - fp->begin;
+        if (n > nbytes) n = nbytes;
+        memcpy(dest, fp->begin, n);
+        fp->begin += n;
+        dest += n, nbytes -= n;
+        nread += n;
+    }
+
+    return nread;
+}
+
+/* Flushes the write buffer, fp->[buffer,begin), out through the backend
+   returning 0 on success or negative if an error occurred.  */
+static ssize_t flush_buffer(hFILE *fp)
+{
+    const char *buffer = fp->buffer;
+    while (buffer < fp->begin) {
+        ssize_t n = fp->backend->write(fp, buffer, fp->begin - buffer);
+        if (n < 0) { fp->has_errno = errno; return n; }
+        buffer += n;
+        fp->offset += n;
+    }
+
+    fp->begin = fp->buffer;  // Leave the buffer empty
+    return 0;
+}
+
+int hflush(hFILE *fp)
+{
+    if (flush_buffer(fp) < 0) return EOF;
+    if (fp->backend->flush) {
+        if (fp->backend->flush(fp) < 0) { fp->has_errno = errno; return EOF; }
+    }
+    return 0;
+}
+
+/* Called only from hputc(), when our buffer is already full.  */
+HTSLIB_EXPORT
+int hputc2(int c, hFILE *fp)
+{
+    if (flush_buffer(fp) < 0) return EOF;
+    *(fp->begin++) = c;
+    return c;
+}
+
+/* Called only from hwrite() and hputs2(); when called, our buffer is either
+   full and ncopied bytes from the source have already been copied to our
+   buffer; or completely empty, ncopied is zero and totalbytes is greater than
+   the buffer size.  */
+HTSLIB_EXPORT
+ssize_t hwrite2(hFILE *fp, const void *srcv, size_t totalbytes, size_t ncopied)
+{
+    const char *src = (const char *) srcv;
+    ssize_t ret;
+    const size_t capacity = fp->limit - fp->buffer;
+    size_t remaining = totalbytes - ncopied;
+    src += ncopied;
+
+    ret = flush_buffer(fp);
+    if (ret < 0) return ret;
+
+    // Write large blocks out directly from the source buffer
+    while (remaining * 2 >= capacity) {
+        ssize_t n = fp->backend->write(fp, src, remaining);
+        if (n < 0) { fp->has_errno = errno; return n; }
+        fp->offset += n;
+        src += n, remaining -= n;
+    }
+
+    // Just buffer any remaining characters
+    memcpy(fp->begin, src, remaining);
+    fp->begin += remaining;
+
+    return totalbytes;
+}
+
+/* Called only from hputs(), when our buffer is already full.  */
+HTSLIB_EXPORT
+int hputs2(const char *text, size_t totalbytes, size_t ncopied, hFILE *fp)
+{
+    return (hwrite2(fp, text, totalbytes, ncopied) >= 0)? 0 : EOF;
+}
+
+off_t hseek(hFILE *fp, off_t offset, int whence)
+{
+    off_t curpos, pos;
+
+    if (writebuffer_is_nonempty(fp) && fp->mobile) {
+        int ret = flush_buffer(fp);
+        if (ret < 0) return ret;
+    }
+
+    curpos = htell(fp);
+
+    // Relative offsets are given relative to the hFILE's stream position,
+    // which may differ from the backend's physical position due to buffering
+    // read-ahead.  Correct for this by converting to an absolute position.
+    if (whence == SEEK_CUR) {
+        if (curpos + offset < 0) {
+            // Either a negative offset resulted in a position before the
+            // start of the file, or we overflowed when given a positive offset
+            fp->has_errno = errno = (offset < 0)? EINVAL : EOVERFLOW;
+            return -1;
+        }
+
+        whence = SEEK_SET;
+        offset = curpos + offset;
+    }
+    // For fixed immobile buffers, convert everything else to SEEK_SET too
+    // so that seeking can be avoided for all (within range) requests.
+    else if (! fp->mobile && whence == SEEK_END) {
+        size_t length = fp->end - fp->buffer;
+        if (offset > 0 || -offset > length) {
+            fp->has_errno = errno = EINVAL;
+            return -1;
+        }
+
+        whence = SEEK_SET;
+        offset = length + offset;
+    }
+
+    // Avoid seeking if the desired position is within our read buffer.
+    // (But not when the next operation may be a write on a mobile buffer.)
+    if (whence == SEEK_SET && (! fp->mobile || fp->readonly) &&
+        offset >= fp->offset && offset - fp->offset <= fp->end - fp->buffer) {
+        fp->begin = &fp->buffer[offset - fp->offset];
+        return offset;
+    }
+
+    pos = fp->backend->seek(fp, offset, whence);
+    if (pos < 0) { fp->has_errno = errno; return pos; }
+
+    // Seeking succeeded, so discard any non-empty read buffer
+    fp->begin = fp->end = fp->buffer;
+    fp->at_eof = 0;
+
+    fp->offset = pos;
+    return pos;
+}
+
+int hclose(hFILE *fp)
+{
+    int err = fp->has_errno;
+
+    if (writebuffer_is_nonempty(fp) && hflush(fp) < 0) err = fp->has_errno;
+    if (!fp->preserve) {
+        if (fp->backend->close(fp) < 0) err = errno;
+        hfile_destroy(fp);
+    }
+
+    if (err) {
+        errno = err;
+        return EOF;
+    }
+    else return 0;
+}
+
+void hclose_abruptly(hFILE *fp)
+{
+    int save = errno;
+    if (fp->preserve)
+        return;
+    if (fp->backend->close(fp) < 0) { /* Ignore subsequent errors */ }
+    hfile_destroy(fp);
+    errno = save;
+}
+
+
+/***************************
+ * File descriptor backend *
+ ***************************/
+
+#ifndef _WIN32
+#include <sys/socket.h>
+#include <sys/stat.h>
+#define HAVE_STRUCT_STAT_ST_BLKSIZE
+#else
+#include <winsock2.h>
+#define HAVE_CLOSESOCKET
+#define HAVE_SETMODE
+#endif
+#include <fcntl.h>
+#include <unistd.h>
+
+/* For Unix, it doesn't matter whether a file descriptor is a socket.
+   However Windows insists on send()/recv() and its own closesocket()
+   being used when fd happens to be a socket.  */
+
+typedef struct {
+    hFILE base;
+    int fd;
+    unsigned is_socket:1, is_shared:1;
+} hFILE_fd;
+
+static ssize_t fd_read(hFILE *fpv, void *buffer, size_t nbytes)
+{
+    hFILE_fd *fp = (hFILE_fd *) fpv;
+    ssize_t n;
+    do {
+        n = fp->is_socket? recv(fp->fd, buffer, nbytes, 0)
+                         : read(fp->fd, buffer, nbytes);
+    } while (n < 0 && errno == EINTR);
+    return n;
+}
+
+static ssize_t fd_write(hFILE *fpv, const void *buffer, size_t nbytes)
+{
+    hFILE_fd *fp = (hFILE_fd *) fpv;
+    ssize_t n;
+    do {
+        n = fp->is_socket?  send(fp->fd, buffer, nbytes, 0)
+                         : write(fp->fd, buffer, nbytes);
+    } while (n < 0 && errno == EINTR);
+#ifdef _WIN32
+        // On windows we have no SIGPIPE.  Instead write returns
+        // EINVAL.  We check for this and our fd being a pipe.
+        // If so, we raise SIGTERM instead of SIGPIPE.  It's not
+        // ideal, but I think the only alternative is extra checking
+        // in every single piece of code.
+        if (n < 0 && errno == EINVAL &&
+            GetLastError() == ERROR_NO_DATA &&
+            GetFileType((HANDLE)_get_osfhandle(fp->fd)) == FILE_TYPE_PIPE) {
+            raise(SIGTERM);
+        }
+#endif
+    return n;
+}
+
+static off_t fd_seek(hFILE *fpv, off_t offset, int whence)
+{
+    hFILE_fd *fp = (hFILE_fd *) fpv;
+#ifdef _WIN32
+    // On windows lseek can return non-zero values even on a pipe.  Instead
+    // it's likely to seek somewhere within the pipe memory buffer.
+    // This breaks bgzf_check_EOF among other things.
+    if (GetFileType((HANDLE)_get_osfhandle(fp->fd)) == FILE_TYPE_PIPE) {
+        errno = ESPIPE;
+        return -1;
+    }
+#endif
+
+    return lseek(fp->fd, offset, whence);
+}
+
+static int fd_flush(hFILE *fpv)
+{
+    int ret = 0;
+    do {
+#ifdef HAVE_FDATASYNC
+        hFILE_fd *fp = (hFILE_fd *) fpv;
+        ret = fdatasync(fp->fd);
+#elif defined(HAVE_FSYNC)
+        hFILE_fd *fp = (hFILE_fd *) fpv;
+        ret = fsync(fp->fd);
+#endif
+        // Ignore invalid-for-fsync(2) errors due to being, e.g., a pipe,
+        // and operation-not-supported errors (Mac OS X)
+        if (ret < 0 && (errno == EINVAL || errno == ENOTSUP)) ret = 0;
+    } while (ret < 0 && errno == EINTR);
+    return ret;
+}
+
+static int fd_close(hFILE *fpv)
+{
+    hFILE_fd *fp = (hFILE_fd *) fpv;
+    int ret;
+
+    // If we don't own the fd, return successfully without actually closing it
+    if (fp->is_shared) return 0;
+
+    do {
+#ifdef HAVE_CLOSESOCKET
+        ret = fp->is_socket? closesocket(fp->fd) : close(fp->fd);
+#else
+        ret = close(fp->fd);
+#endif
+    } while (ret < 0 && errno == EINTR);
+    return ret;
+}
+
+static const struct hFILE_backend fd_backend =
+{
+    fd_read, fd_write, fd_seek, fd_flush, fd_close
+};
+
+static size_t blksize(int fd)
+{
+#ifdef HAVE_STRUCT_STAT_ST_BLKSIZE
+    struct stat sbuf;
+    if (fstat(fd, &sbuf) != 0) return 0;
+    return sbuf.st_blksize;
+#else
+    return 0;
+#endif
+}
+
+static hFILE *hopen_fd(const char *filename, const char *mode)
+{
+    hFILE_fd *fp = NULL;
+    int fd = open(filename, hfile_oflags(mode), 0666);
+    if (fd < 0) goto error;
+
+    fp = (hFILE_fd *) hfile_init(sizeof (hFILE_fd), mode, blksize(fd));
+    if (fp == NULL) goto error;
+
+    fp->fd = fd;
+    fp->is_socket = 0;
+    fp->is_shared = 0;
+    fp->base.backend = &fd_backend;
+    return &fp->base;
+
+error:
+    if (fd >= 0) { int save = errno; (void) close(fd); errno = save; }
+    hfile_destroy((hFILE *) fp);
+    return NULL;
+}
+
+// Loads the contents of filename to produced a read-only, in memory,
+// immobile hfile.  fp is the already opened file.  We always close this
+// input fp, irrespective of whether we error or whether we return a new
+// immobile hfile.
+static hFILE *hpreload(hFILE *fp) {
+    hFILE *mem_fp;
+    char *buf = NULL;
+    off_t buf_sz = 0, buf_a = 0, buf_inc = 8192, len;
+
+    for (;;) {
+        if (buf_a - buf_sz < 5000) {
+            buf_a += buf_inc;
+            char *t = realloc(buf, buf_a);
+            if (!t) goto err;
+            buf = t;
+            if (buf_inc < 1000000) buf_inc *= 1.3;
+        }
+        len = hread(fp, buf+buf_sz, buf_a-buf_sz);
+        if (len > 0)
+            buf_sz += len;
+        else
+            break;
+    }
+
+    if (len < 0) goto err;
+    mem_fp = hfile_init_fixed(sizeof(hFILE), "r", buf, buf_sz, buf_a);
+    if (!mem_fp) goto err;
+    mem_fp->backend = &mem_backend;
+
+    if (hclose(fp) < 0) {
+        hclose_abruptly(mem_fp);
+        goto err;
+    }
+    return mem_fp;
+
+ err:
+    free(buf);
+    hclose_abruptly(fp);
+    return NULL;
+}
+
+static int is_preload_url_remote(const char *url){
+    return hisremote(url + 8); // len("preload:") = 8
+}
+
+static hFILE *hopen_preload(const char *url, const char *mode){
+    hFILE* fp = hopen(url + 8, mode);
+    return fp ? hpreload(fp) : NULL;
+}
+
+hFILE *hdopen(int fd, const char *mode)
+{
+    hFILE_fd *fp = (hFILE_fd*) hfile_init(sizeof (hFILE_fd), mode, blksize(fd));
+    if (fp == NULL) return NULL;
+
+    fp->fd = fd;
+    fp->is_socket = (strchr(mode, 's') != NULL);
+    fp->is_shared = (strchr(mode, 'S') != NULL);
+    fp->base.backend = &fd_backend;
+    return &fp->base;
+}
+
+static hFILE *hopen_fd_fileuri(const char *url, const char *mode)
+{
+    if (strncmp(url, "file://localhost/", 17) == 0) url += 16;
+    else if (strncmp(url, "file:///", 8) == 0) url += 7;
+    else { errno = EPROTONOSUPPORT; return NULL; }
+
+#if defined(_WIN32) || defined(__MSYS__)
+    // For cases like C:/foo
+    if (url[0] == '/' && url[1] && url[2] == ':' && url[3] == '/') url++;
+#endif
+
+    return hopen_fd(url, mode);
+}
+
+static hFILE *hopen_fd_stdinout(const char *mode)
+{
+    int fd = (strchr(mode, 'r') != NULL)? STDIN_FILENO : STDOUT_FILENO;
+    char mode_shared[101];
+    snprintf(mode_shared, sizeof mode_shared, "S%s", mode);
+#if defined HAVE_SETMODE && defined O_BINARY
+    if (setmode(fd, O_BINARY) < 0) return NULL;
+#endif
+    return hdopen(fd, mode_shared);
+}
+
+HTSLIB_EXPORT
+int hfile_oflags(const char *mode)
+{
+    int rdwr = 0, flags = 0;
+    const char *s;
+    for (s = mode; *s; s++)
+        switch (*s) {
+        case 'r': rdwr = O_RDONLY;  break;
+        case 'w': rdwr = O_WRONLY; flags |= O_CREAT | O_TRUNC;  break;
+        case 'a': rdwr = O_WRONLY; flags |= O_CREAT | O_APPEND;  break;
+        case '+': rdwr = O_RDWR;  break;
+#ifdef O_CLOEXEC
+        case 'e': flags |= O_CLOEXEC;  break;
+#endif
+#ifdef O_EXCL
+        case 'x': flags |= O_EXCL;  break;
+#endif
+        default:  break;
+        }
+
+#ifdef O_BINARY
+    flags |= O_BINARY;
+#endif
+
+    return rdwr | flags;
+}
+
+
+/*********************
+ * In-memory backend *
+ *********************/
+
+#include "hts_internal.h"
+
+typedef struct {
+    hFILE base;
+} hFILE_mem;
+
+static off_t mem_seek(hFILE *fpv, off_t offset, int whence)
+{
+    errno = EINVAL;
+    return -1;
+}
+
+static int mem_close(hFILE *fpv)
+{
+    return 0;
+}
+
+static const struct hFILE_backend mem_backend =
+{
+    NULL, NULL, mem_seek, NULL, mem_close
+};
+
+static int cmp_prefix(const char *key, const char *s)
+{
+    while (*key)
+        if (tolower_c(*s) != *key) return +1;
+        else s++, key++;
+
+    return 0;
+}
+
+static hFILE *create_hfile_mem(char* buffer, const char* mode, size_t buf_filled, size_t buf_size)
+{
+    hFILE_mem *fp = (hFILE_mem *) hfile_init_fixed(sizeof(hFILE_mem), mode, buffer, buf_filled, buf_size);
+    if (fp == NULL)
+        return NULL;
+
+    fp->base.backend = &mem_backend;
+    return &fp->base;
+}
+
+static hFILE *hopen_mem(const char *url, const char *mode)
+{
+    size_t length, size;
+    char *buffer;
+    const char *data, *comma = strchr(url, ',');
+    if (comma == NULL) { errno = EINVAL; return NULL; }
+    data = comma+1;
+
+    // TODO Implement write modes
+    if (strchr(mode, 'r') == NULL) { errno = EROFS; return NULL; }
+
+    if (comma - url >= 7 && cmp_prefix(";base64", &comma[-7]) == 0) {
+        size = hts_base64_decoded_length(strlen(data));
+        buffer = malloc(size);
+        if (buffer == NULL) return NULL;
+        hts_decode_base64(buffer, &length, data);
+    }
+    else {
+        size = strlen(data) + 1;
+        buffer = malloc(size);
+        if (buffer == NULL) return NULL;
+        hts_decode_percent(buffer, &length, data);
+    }
+    hFILE* hf;
+
+    if(!(hf = create_hfile_mem(buffer, mode, length, size))){
+        free(buffer);
+        return NULL;
+    }
+
+    return hf;
+}
+
+static hFILE *hopenv_mem(const char *filename, const char *mode, va_list args)
+{
+    char* buffer = va_arg(args, char*);
+    size_t sz = va_arg(args, size_t);
+    va_end(args);
+
+    hFILE* hf;
+
+    if(!(hf = create_hfile_mem(buffer, mode, sz, sz))){
+        free(buffer);
+        return NULL;
+    }
+
+    return hf;
+}
+
+char *hfile_mem_get_buffer(hFILE *file, size_t *length) {
+    if (file->backend != &mem_backend) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    if (length)
+        *length = file->buffer - file->limit;
+
+    return file->buffer;
+}
+
+char *hfile_mem_steal_buffer(hFILE *file, size_t *length) {
+    char *buf = hfile_mem_get_buffer(file, length);
+    if (buf)
+        file->buffer = NULL;
+    return buf;
+}
+
+// open() stub for mem: which only works with the vopen() interface
+// Use 'data:,' for data encoded in the URL
+static hFILE *hopen_not_supported(const char *fname, const char *mode) {
+    errno = EINVAL;
+    return NULL;
+}
+
+int hfile_plugin_init_mem(struct hFILE_plugin *self)
+{
+    // mem files are declared remote so they work with a tabix index
+    static const struct hFILE_scheme_handler handler =
+            {hopen_not_supported, hfile_always_remote, "mem", 2000 + 50, hopenv_mem};
+    self->name = "mem";
+    hfile_add_scheme_handler("mem", &handler);
+    return 0;
+}
+
+/**********************************************************************
+ * Dummy crypt4gh plug-in.  Does nothing apart from advise how to get *
+ * the real one.  It will be overridden by the actual plug-in.        *
+ **********************************************************************/
+
+static hFILE *crypt4gh_needed(const char *url, const char *mode)
+{
+    const char *u = strncmp(url, "crypt4gh:", 9) == 0 ? url + 9 : url;
+#if defined(ENABLE_PLUGINS)
+    const char *enable_plugins = "";
+#else
+    const char *enable_plugins = "You also need to rebuild HTSlib with plug-ins enabled.\n";
+#endif
+
+    hts_log_error("Accessing \"%s\" needs the crypt4gh plug-in.\n"
+                  "It can be found at "
+                  "https://github.com/samtools/htslib-crypt4gh\n"
+                  "%s"
+                  "If you have the plug-in, please ensure it can be "
+                  "found on your HTS_PATH.",
+                  u, enable_plugins);
+
+    errno = EPROTONOSUPPORT;
+    return NULL;
+}
+
+int hfile_plugin_init_crypt4gh_needed(struct hFILE_plugin *self)
+{
+    static const struct hFILE_scheme_handler handler =
+        { crypt4gh_needed, hfile_always_local, "crypt4gh-needed", 0, NULL };
+    self->name = "crypt4gh-needed";
+    hfile_add_scheme_handler("crypt4gh", &handler);
+    return 0;
+}
+
+
+/*****************************************
+ * Plugin and hopen() backend dispatcher *
+ *****************************************/
+
+#include "htslib/khash.h"
+
+KHASH_MAP_INIT_STR(scheme_string, const struct hFILE_scheme_handler *)
+static khash_t(scheme_string) *schemes = NULL;
+
+struct hFILE_plugin_list {
+    struct hFILE_plugin plugin;
+    struct hFILE_plugin_list *next;
+};
+
+static struct hFILE_plugin_list *plugins = NULL;
+static pthread_mutex_t plugins_lock = PTHREAD_MUTEX_INITIALIZER;
+
+void hfile_shutdown(int do_close_plugin)
+{
+    pthread_mutex_lock(&plugins_lock);
+
+    if (schemes) {
+        kh_destroy(scheme_string, schemes);
+        schemes = NULL;
+    }
+
+    while (plugins != NULL) {
+        struct hFILE_plugin_list *p = plugins;
+        if (p->plugin.destroy) p->plugin.destroy();
+#ifdef ENABLE_PLUGINS
+        if (p->plugin.obj && do_close_plugin) close_plugin(p->plugin.obj);
+#endif
+        plugins = p->next;
+        free(p);
+    }
+
+    pthread_mutex_unlock(&plugins_lock);
+}
+
+static void hfile_exit(void)
+{
+    hfile_shutdown(0);
+    pthread_mutex_destroy(&plugins_lock);
+}
+
+static inline int priority(const struct hFILE_scheme_handler *handler)
+{
+    return handler->priority % 1000;
+}
+
+#ifdef USING_WINDOWS_PLUGIN_DLLS
+/*
+ * Work-around for Windows plug-in dlls where the plug-in could be
+ * using a different HTSlib library to the executable (for example
+ * because the latter was build against a static libhts.a).  When this
+ * happens, the plug-in can call the wrong copy of hfile_add_scheme_handler().
+ * If this is detected, it calls this function which attempts to fix the
+ * problem by redirecting to the hfile_add_scheme_handler() in the main
+ * executable.
+ */
+static int try_exe_add_scheme_handler(const char *scheme,
+                                      const struct hFILE_scheme_handler *handler)
+{
+    static void (*add_scheme_handler)(const char *scheme,
+                                      const struct hFILE_scheme_handler *handler);
+    if (!add_scheme_handler) {
+        // dlopen the main executable and resolve hfile_add_scheme_handler
+        void *exe_handle = dlopen(NULL, RTLD_LAZY);
+        if (!exe_handle) return -1;
+        *(void **) (&add_scheme_handler) = dlsym(exe_handle, "hfile_add_scheme_handler");
+        dlclose(exe_handle);
+    }
+    // Check that the symbol was obtained and isn't the one in this copy
+    // of the library (to avoid infinite recursion)
+    if (!add_scheme_handler || add_scheme_handler == hfile_add_scheme_handler)
+        return -1;
+    add_scheme_handler(scheme, handler);
+    return 0;
+}
+#else
+static int try_exe_add_scheme_handler(const char *scheme,
+                                      const struct hFILE_scheme_handler *handler)
+{
+    return -1;
+}
+#endif
+
+HTSLIB_EXPORT
+void hfile_add_scheme_handler(const char *scheme,
+                              const struct hFILE_scheme_handler *handler)
+{
+    int absent;
+    if (handler->open == NULL || handler->isremote == NULL) {
+        hts_log_warning("Couldn't register scheme handler for %s: missing method", scheme);
+        return;
+    }
+    if (!schemes) {
+        if (try_exe_add_scheme_handler(scheme, handler) != 0) {
+            hts_log_warning("Couldn't register scheme handler for %s", scheme);
+        }
+        return;
+    }
+    khint_t k = kh_put(scheme_string, schemes, scheme, &absent);
+    if (absent < 0) {
+        hts_log_warning("Couldn't register scheme handler for %s : %s",
+                        scheme, strerror(errno));
+        return;
+    }
+    if (absent || priority(handler) > priority(kh_value(schemes, k))) {
+        kh_value(schemes, k) = handler;
+    }
+}
+
+static int init_add_plugin(void *obj, int (*init)(struct hFILE_plugin *),
+                           const char *pluginname)
+{
+    struct hFILE_plugin_list *p = malloc (sizeof (struct hFILE_plugin_list));
+    if (p == NULL) {
+        hts_log_debug("Failed to allocate memory for plugin \"%s\"", pluginname);
+        return -1;
+    }
+
+    p->plugin.api_version = 1;
+    p->plugin.obj = obj;
+    p->plugin.name = NULL;
+    p->plugin.destroy = NULL;
+
+    int ret = (*init)(&p->plugin);
+
+    if (ret != 0) {
+        hts_log_debug("Initialisation failed for plugin \"%s\": %d", pluginname, ret);
+        free(p);
+        return ret;
+    }
+
+    hts_log_debug("Loaded \"%s\"", pluginname);
+
+    p->next = plugins, plugins = p;
+    return 0;
+}
+
+/*
+ * Returns 0 on success,
+ *        <0 on failure
+ */
+static int load_hfile_plugins(void)
+{
+    static const struct hFILE_scheme_handler
+        data = { hopen_mem, hfile_always_local, "built-in", 80 },
+        file = { hopen_fd_fileuri, hfile_always_local, "built-in", 80 },
+        preload = { hopen_preload, is_preload_url_remote, "built-in", 80 };
+
+    schemes = kh_init(scheme_string);
+    if (schemes == NULL)
+        return -1;
+
+    hfile_add_scheme_handler("data", &data);
+    hfile_add_scheme_handler("file", &file);
+    hfile_add_scheme_handler("preload", &preload);
+    init_add_plugin(NULL, hfile_plugin_init_mem, "mem");
+    init_add_plugin(NULL, hfile_plugin_init_crypt4gh_needed, "crypt4gh-needed");
+
+#ifdef ENABLE_PLUGINS
+    struct hts_path_itr path;
+    const char *pluginname;
+    hts_path_itr_setup(&path, NULL, NULL, "hfile_", 6, NULL, 0);
+    while ((pluginname = hts_path_itr_next(&path)) != NULL) {
+        void *obj;
+        int (*init)(struct hFILE_plugin *) = (int (*)(struct hFILE_plugin *))
+            load_plugin(&obj, pluginname, "hfile_plugin_init");
+
+        if (init) {
+            if (init_add_plugin(obj, init, pluginname) != 0)
+                close_plugin(obj);
+        }
+    }
+#else
+
+#ifdef HAVE_LIBCURL
+    init_add_plugin(NULL, hfile_plugin_init_libcurl, "libcurl");
+#endif
+#ifdef ENABLE_GCS
+    init_add_plugin(NULL, hfile_plugin_init_gcs, "gcs");
+#endif
+#ifdef ENABLE_S3
+    init_add_plugin(NULL, hfile_plugin_init_s3, "s3");
+    init_add_plugin(NULL, hfile_plugin_init_s3_write, "s3w");
+#endif
+
+#endif
+
+    // In the unlikely event atexit() fails, it's better to succeed here and
+    // carry on; then eventually when the program exits, we'll merely close
+    // down the plugins uncleanly, as if we had aborted.
+    (void) atexit(hfile_exit);
+
+    return 0;
+}
+
+/* A filename like "foo:bar" in which we don't recognise the scheme is
+   either an ordinary file or an indication of a missing or broken plugin.
+   Try to open it as an ordinary file; but if there's no such file, set
+   errno distinctively to make the plugin issue apparent.  */
+static hFILE *hopen_unknown_scheme(const char *fname, const char *mode)
+{
+    hFILE *fp = hopen_fd(fname, mode);
+    if (fp == NULL && errno == ENOENT) errno = EPROTONOSUPPORT;
+    return fp;
+}
+
+/* Returns the appropriate handler, or NULL if the string isn't an URL.  */
+static const struct hFILE_scheme_handler *find_scheme_handler(const char *s)
+{
+    static const struct hFILE_scheme_handler unknown_scheme =
+        { hopen_unknown_scheme, hfile_always_local, "built-in", 0 };
+
+    char scheme[12];
+    int i;
+
+    for (i = 0; i < sizeof scheme; i++)
+        if (isalnum_c(s[i]) || s[i] == '+' || s[i] == '-' || s[i] == '.')
+            scheme[i] = tolower_c(s[i]);
+        else if (s[i] == ':') break;
+        else return NULL;
+
+    // 1 byte schemes are likely windows C:/foo pathnames
+    if (i <= 1 || i >= sizeof scheme) return NULL;
+    scheme[i] = '\0';
+
+    pthread_mutex_lock(&plugins_lock);
+    if (!schemes && load_hfile_plugins() < 0) {
+        pthread_mutex_unlock(&plugins_lock);
+        return NULL;
+    }
+    pthread_mutex_unlock(&plugins_lock);
+
+    khint_t k = kh_get(scheme_string, schemes, scheme);
+    return (k != kh_end(schemes))? kh_value(schemes, k) : &unknown_scheme;
+}
+
+
+/***************************
+ * Library introspection functions
+ ***************************/
+
+/*
+ * Fills out sc_list[] with the list of known URL schemes.
+ * This can be restricted to just ones from a specific plugin,
+ * or all (plugin == NULL).
+ *
+ * Returns number of schemes found on success;
+ *        -1 on failure.
+ */
+HTSLIB_EXPORT
+int hfile_list_schemes(const char *plugin, const char *sc_list[], int *nschemes)
+{
+    pthread_mutex_lock(&plugins_lock);
+    if (!schemes && load_hfile_plugins() < 0) {
+        pthread_mutex_unlock(&plugins_lock);
+        return -1;
+    }
+    pthread_mutex_unlock(&plugins_lock);
+
+    khiter_t k;
+    int ns = 0;
+
+    for (k = kh_begin(schemes); k != kh_end(schemes); k++) {
+        if (!kh_exist(schemes, k))
+            continue;
+
+        const struct hFILE_scheme_handler *s = kh_value(schemes, k);
+        if (plugin && strcmp(s->provider, plugin) != 0)
+            continue;
+
+        if (ns < *nschemes)
+            sc_list[ns] = kh_key(schemes, k);
+        ns++;
+    }
+
+    if (*nschemes > ns)
+        *nschemes = ns;
+
+    return ns;
+}
+
+
+/*
+ * Fills out plist[] with the list of known hFILE plugins.
+ *
+ * Returns number of schemes found on success;
+ *        -1 on failure
+ */
+HTSLIB_EXPORT
+int hfile_list_plugins(const char *plist[], int *nplugins)
+{
+    pthread_mutex_lock(&plugins_lock);
+    if (!schemes && load_hfile_plugins() < 0) {
+        pthread_mutex_unlock(&plugins_lock);
+        return -1;
+    }
+    pthread_mutex_unlock(&plugins_lock);
+
+    int np = 0;
+    if (*nplugins)
+        plist[np++] = "built-in";
+
+    struct hFILE_plugin_list *p = plugins;
+    while (p) {
+        if (np < *nplugins)
+            plist[np] = p->plugin.name;
+
+        p = p->next;
+        np++;
+    }
+
+    if (*nplugins > np)
+        *nplugins = np;
+
+    return np;
+}
+
+
+/*
+ * Tests for the presence of a specific hFILE plugin.
+ *
+ * Returns 1 if true
+ *         0 otherwise
+ */
+HTSLIB_EXPORT
+int hfile_has_plugin(const char *name)
+{
+    pthread_mutex_lock(&plugins_lock);
+    if (!schemes && load_hfile_plugins() < 0) {
+        pthread_mutex_unlock(&plugins_lock);
+        return -1;
+    }
+    pthread_mutex_unlock(&plugins_lock);
+
+    struct hFILE_plugin_list *p = plugins;
+    while (p) {
+        if (strcmp(p->plugin.name, name) == 0)
+            return 1;
+        p = p->next;
+    }
+
+    return 0;
+}
+
+/***************************
+ * hFILE interface proper
+ ***************************/
+
+hFILE *hopen(const char *fname, const char *mode, ...)
+{
+    const struct hFILE_scheme_handler *handler = find_scheme_handler(fname);
+    if (handler) {
+        if (strchr(mode, ':') == NULL
+            || handler->priority < 2000
+            || handler->vopen == NULL) {
+            return handler->open(fname, mode);
+        }
+        else {
+            hFILE *fp;
+            va_list arg;
+            va_start(arg, mode);
+            fp = handler->vopen(fname, mode, arg);
+            va_end(arg);
+            return fp;
+        }
+    }
+    else if (strcmp(fname, "-") == 0) return hopen_fd_stdinout(mode);
+    else return hopen_fd(fname, mode);
+}
+
+HTSLIB_EXPORT
+int hfile_always_local (const char *fname) { return 0; }
+
+HTSLIB_EXPORT
+int hfile_always_remote(const char *fname) { return 1; }
+
+int hisremote(const char *fname)
+{
+    const struct hFILE_scheme_handler *handler = find_scheme_handler(fname);
+    return handler? handler->isremote(fname) : 0;
+}
+
+// Remove an extension, if any, from the basename part of [start,limit).
+// Note: Doesn't notice percent-encoded '.' and '/' characters. Don't do that.
+static const char *strip_extension(const char *start, const char *limit)
+{
+    const char *s = limit;
+    while (s > start) {
+        --s;
+        if (*s == '.') return s;
+        else if (*s == '/') break;
+    }
+    return limit;
+}
+
+char *haddextension(struct kstring_t *buffer, const char *filename,
+                    int replace, const char *new_extension)
+{
+    const char *trailing, *end;
+
+    if (find_scheme_handler(filename)) {
+        // URL, so alter extensions before any trailing query or fragment parts
+        // Allow # symbols in s3 URLs
+        trailing = filename + ((strncmp(filename, "s3://", 5) && strncmp(filename, "s3+http://", 10) && strncmp(filename, "s3+https://", 11))  ? strcspn(filename, "?#") : strcspn(filename, "?"));
+    }
+    else {
+        // Local path, so alter extensions at the end of the filename
+        trailing = strchr(filename, '\0');
+    }
+
+    end = replace? strip_extension(filename, trailing) : trailing;
+
+    buffer->l = 0;
+    if (kputsn(filename, end - filename, buffer) >= 0 &&
+        kputs(new_extension, buffer) >= 0 &&
+        kputs(trailing, buffer) >= 0) return buffer->s;
+    else return NULL;
+}
+
+
+/*
+ * ----------------------------------------------------------------------
+ * Minimal stub functions for knet, added after the removal of
+ * hfile_net.c and knetfile.c.
+ *
+ * They exist purely for ABI compatibility, but are simply wrappers to
+ * hFILE.  API should be compatible except knet_fileno (unused?).
+ *
+ * CULL THESE and knetfile.h at the next .so version bump.
+ */
+typedef struct knetFile_s {
+    // As per htslib/knetfile.h.  Duplicated here as we don't wish to
+    // have any dependence on the deprecated knetfile.h interface, plus
+    // it's hopefully only temporary.
+    int type, fd;
+    int64_t offset;
+    char *host, *port;
+    int ctrl_fd, pasv_ip[4], pasv_port, max_response, no_reconnect, is_ready;
+    char *response, *retr, *size_cmd;
+    int64_t seek_offset;
+    int64_t file_size;
+    char *path, *http_host;
+
+    // Our local addition
+    hFILE *hf;
+} knetFile;
+
+HTSLIB_EXPORT
+knetFile *knet_open(const char *fn, const char *mode) {
+    knetFile *fp = calloc(1, sizeof(*fp));
+    if (!fp) return NULL;
+    if (!(fp->hf = hopen(fn, mode))) {
+        free(fp);
+        return NULL;
+    }
+
+    // FD backend is the only one implementing knet_fileno
+    fp->fd = fp->hf->backend == &fd_backend
+        ? ((hFILE_fd *)fp->hf)->fd
+        : -1;
+
+    return fp;
+}
+
+HTSLIB_EXPORT
+knetFile *knet_dopen(int fd, const char *mode) {
+    knetFile *fp = calloc(1, sizeof(*fp));
+    if (!fp) return NULL;
+    if (!(fp->hf = hdopen(fd, mode))) {
+        free(fp);
+        return NULL;
+    }
+    fp->fd = fd;
+    return fp;
+}
+
+HTSLIB_EXPORT
+ssize_t knet_read(knetFile *fp, void *buf, size_t len) {
+    ssize_t r = hread(fp->hf, buf, len);
+    fp->offset += r>0?r:0;
+    return r;
+}
+
+HTSLIB_EXPORT
+off_t knet_seek(knetFile *fp, off_t off, int whence) {
+    off_t r = hseek(fp->hf, off, whence);
+    if (r >= 0)
+        fp->offset = r;
+    return r;
+}
+
+HTSLIB_EXPORT
+int knet_close(knetFile *fp) {
+    int r = hclose(fp->hf);
+    free(fp);
+    return r;
+}
diff --git a/ext/htslib/hfile_gcs.c b/ext/htslib/hfile_gcs.c
new file mode 100644
index 0000000..2f01a20
--- /dev/null
+++ b/ext/htslib/hfile_gcs.c
@@ -0,0 +1,160 @@
+/*  hfile_gcs.c -- Google Cloud Storage backend for low-level file streams.
+
+    Copyright (C) 2016, 2021 Genome Research Ltd.
+
+    Author: John Marshall <jm18@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+
+#include "htslib/hts.h"
+#include "htslib/kstring.h"
+#include "hfile_internal.h"
+#ifdef ENABLE_PLUGINS
+#include "version.h"
+#endif
+
+static hFILE *
+gcs_rewrite(const char *gsurl, const char *mode, int mode_has_colon,
+            va_list *argsp)
+{
+    const char *bucket, *path, *access_token, *requester_pays_project;
+    kstring_t mode_colon = { 0, 0, NULL };
+    kstring_t url = { 0, 0, NULL };
+    kstring_t auth_hdr = { 0, 0, NULL };
+    kstring_t requester_pays_hdr = { 0, 0, NULL };
+    hFILE *fp = NULL;
+
+    // GCS URL format is gs[+SCHEME]://BUCKET/PATH
+
+    if (gsurl[2] == '+') {
+        bucket = strchr(gsurl, ':') + 1;
+        kputsn(&gsurl[3], bucket - &gsurl[3], &url);
+    }
+    else {
+        kputs("https:", &url);
+        bucket = &gsurl[3];
+    }
+    while (*bucket == '/') kputc(*bucket++, &url);
+
+    path = bucket + strcspn(bucket, "/?#");
+
+    kputsn(bucket, path - bucket, &url);
+    if (strchr(mode, 'r')) kputs(".storage-download", &url);
+    else if (strchr(mode, 'w')) kputs(".storage-upload", &url);
+    else kputs(".storage", &url);
+    kputs(".googleapis.com", &url);
+
+    kputs(path, &url);
+
+    if (hts_verbose >= 8)
+        fprintf(stderr, "[M::gcs_open] rewrote URL as %s\n", url.s);
+
+    // TODO Find the access token in a more standard way
+    access_token = getenv("GCS_OAUTH_TOKEN");
+
+    if (access_token) {
+        kputs("Authorization: Bearer ", &auth_hdr);
+        kputs(access_token, &auth_hdr);
+    }
+
+    requester_pays_project = getenv("GCS_REQUESTER_PAYS_PROJECT");
+
+    if (requester_pays_project) {
+        kputs("X-Goog-User-Project: ", &requester_pays_hdr);
+        kputs(requester_pays_project, &requester_pays_hdr);
+    }
+
+    if (argsp || mode_has_colon || auth_hdr.l > 0 || requester_pays_hdr.l > 0) {
+        if (! mode_has_colon) {
+            kputs(mode, &mode_colon);
+            kputc(':', &mode_colon);
+            mode = mode_colon.s;
+        }
+
+        if (auth_hdr.l > 0 && requester_pays_hdr.l > 0) {
+            fp = hopen(
+                url.s, mode, "va_list", argsp,
+                   "httphdr:l",
+                   auth_hdr.s,
+                   requester_pays_hdr.s,
+                   NULL,
+                   NULL
+            );
+
+        }
+        else {
+            fp = hopen(url.s, mode, "va_list", argsp,
+                       "httphdr", (auth_hdr.l > 0)? auth_hdr.s : NULL, NULL);
+        }
+    }
+    else
+        fp = hopen(url.s, mode);
+
+    free(mode_colon.s);
+    free(url.s);
+    free(auth_hdr.s);
+    free(requester_pays_hdr.s);
+    return fp;
+}
+
+static hFILE *gcs_open(const char *url, const char *mode)
+{
+    return gcs_rewrite(url, mode, 0, NULL);
+}
+
+static hFILE *gcs_vopen(const char *url, const char *mode_colon, va_list args0)
+{
+    // Need to use va_copy() as we can only take the address of an actual
+    // va_list object, not that of a parameter as its type may have decayed.
+    va_list args;
+    va_copy(args, args0);
+    hFILE *fp = gcs_rewrite(url, mode_colon, 1, &args);
+    va_end(args);
+    return fp;
+}
+
+int PLUGIN_GLOBAL(hfile_plugin_init,_gcs)(struct hFILE_plugin *self)
+{
+    static const struct hFILE_scheme_handler handler =
+        { gcs_open, hfile_always_remote, "Google Cloud Storage",
+          2000 + 50, gcs_vopen
+        };
+
+#ifdef ENABLE_PLUGINS
+    // Embed version string for examination via strings(1) or what(1)
+    static const char id[] = "@(#)hfile_gcs plugin (htslib)\t" HTS_VERSION_TEXT;
+    if (hts_verbose >= 9)
+        fprintf(stderr, "[M::hfile_gcs.init] version %s\n", strchr(id, '\t')+1);
+#endif
+
+    self->name = "Google Cloud Storage";
+    hfile_add_scheme_handler("gs", &handler);
+    hfile_add_scheme_handler("gs+http", &handler);
+    hfile_add_scheme_handler("gs+https", &handler);
+    return 0;
+}
diff --git a/ext/htslib/hfile_internal.h b/ext/htslib/hfile_internal.h
new file mode 100644
index 0000000..2e365ae
--- /dev/null
+++ b/ext/htslib/hfile_internal.h
@@ -0,0 +1,209 @@
+/*  hfile_internal.h -- internal parts of low-level input/output streams.
+
+    Copyright (C) 2013-2016, 2019 Genome Research Ltd.
+
+    Author: John Marshall <jm18@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#ifndef HFILE_INTERNAL_H
+#define HFILE_INTERNAL_H
+
+#include <stdarg.h>
+
+#include "htslib/hts_defs.h"
+#include "htslib/hfile.h"
+
+#include "textutils_internal.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*!
+  @abstract  Resizes the buffer within an hFILE.
+
+  @notes  Changes the buffer size for an hFILE.  Ideally this is done
+  immediately after opening.  If performed later, this function may
+  fail if we are reducing the buffer size and the current offset into
+  the buffer is beyond the new capacity.
+
+  @param fp        The file stream
+  @param bufsiz    The size of the new buffer
+
+  @return Returns 0 on success, -1 on failure.
+ */
+int hfile_set_blksize(hFILE *fp, size_t bufsiz);
+
+struct BGZF;
+/*!
+  @abstract Return the hFILE connected to a BGZF
+ */
+struct hFILE *bgzf_hfile(struct BGZF *fp);
+
+/*!
+  @abstract Closes all hFILE plugins that have been loaded
+*/
+void hfile_shutdown(int do_close_plugin);
+
+struct hFILE_backend {
+    /* As per read(2), returning the number of bytes read (possibly 0) or
+       negative (and setting errno) on errors.  Front-end code will call this
+       repeatedly if necessary to attempt to get the desired byte count.  */
+    ssize_t (*read)(hFILE *fp, void *buffer, size_t nbytes) HTS_RESULT_USED;
+
+    /* As per write(2), returning the number of bytes written or negative (and
+       setting errno) on errors.  Front-end code will call this repeatedly if
+       necessary until the desired block is written or an error occurs.  */
+    ssize_t (*write)(hFILE *fp, const void *buffer, size_t nbytes)
+        HTS_RESULT_USED;
+
+    /* As per lseek(2), returning the resulting offset within the stream or
+       negative (and setting errno) on errors.  */
+    off_t (*seek)(hFILE *fp, off_t offset, int whence) HTS_RESULT_USED;
+
+    /* Performs low-level flushing, if any, e.g., fsync(2); for writing streams
+       only.  Returns 0 for success or negative (and sets errno) on errors. */
+    int (*flush)(hFILE *fp) HTS_RESULT_USED;
+
+    /* Closes the underlying stream (for output streams, the buffer will
+       already have been flushed), returning 0 for success or negative (and
+       setting errno) on errors, as per close(2).  */
+    int (*close)(hFILE *fp) HTS_RESULT_USED;
+};
+
+/* May be called by hopen_*() functions to decode a fopen()-style mode into
+   open(2)-style flags.  */
+HTSLIB_EXPORT
+int hfile_oflags(const char *mode);
+
+/* Must be called by hopen_*() functions to allocate the hFILE struct and set
+   up its base.  Capacity is a suggested buffer size (e.g., via fstat(2))
+   or 0 for a default-sized buffer.  */
+HTSLIB_EXPORT
+hFILE *hfile_init(size_t struct_size, const char *mode, size_t capacity);
+
+/* Alternative to hfile_init() for in-memory backends for which the base
+   buffer is the only storage.  Buffer is already allocated via malloc(2)
+   of size buf_size and with buf_filled bytes already filled.  Ownership
+   of the buffer is transferred to the resulting hFILE.  */
+hFILE *hfile_init_fixed(size_t struct_size, const char *mode,
+                        char *buffer, size_t buf_filled, size_t buf_size);
+
+/* May be called by hopen_*() functions to undo the effects of hfile_init()
+   in the event opening the stream subsequently fails.  (This is safe to use
+   even if fp is NULL.  This takes care to preserve errno.)  */
+HTSLIB_EXPORT
+void hfile_destroy(hFILE *fp);
+
+
+struct hFILE_scheme_handler {
+    /* Opens a stream when dispatched by hopen(); should call hfile_init()
+       to malloc a struct "derived" from hFILE and initialise it appropriately,
+       including setting base.backend to its own backend vector.  */
+    hFILE *(*open)(const char *filename, const char *mode) HTS_RESULT_USED;
+
+    /* Returns whether the URL denotes remote storage when dispatched by
+       hisremote().  For simple cases, use one of hfile_always_*() below.  */
+    int (*isremote)(const char *filename) HTS_RESULT_USED;
+
+    /* The name of the plugin or other code providing this handler.  */
+    const char *provider;
+
+    /* If multiple handlers are registered for the same scheme, the one with
+       the highest priority is used; range is 0 (lowest) to 100 (highest).
+       This field is used modulo 1000 as a priority; thousands indicate
+       later revisions to this structure, as noted below.  */
+    int priority;
+
+    /* Fields below are present when priority >= 2000.  */
+
+    /* Same as the open() method, used when extra arguments have been given
+       to hopen().  */
+    hFILE *(*vopen)(const char *filename, const char *mode, va_list args)
+        HTS_RESULT_USED;
+};
+
+/* May be used as an isremote() function in simple cases.  */
+HTSLIB_EXPORT
+extern int hfile_always_local (const char *fname);
+HTSLIB_EXPORT
+extern int hfile_always_remote(const char *fname);
+
+/* Should be called by plugins for each URL scheme they wish to handle.  */
+HTSLIB_EXPORT
+void hfile_add_scheme_handler(const char *scheme,
+                              const struct hFILE_scheme_handler *handler);
+
+struct hFILE_plugin {
+    /* On entry, HTSlib's plugin API version (currently 1).  */
+    int api_version;
+
+    /* On entry, the plugin's handle as returned by dlopen() etc.  */
+    void *obj;
+
+    /* The plugin should fill this in with its (human-readable) name.  */
+    const char *name;
+
+    /* The plugin may wish to fill in a function to be called on closing.  */
+    void (*destroy)(void);
+};
+
+#ifdef ENABLE_PLUGINS
+#define PLUGIN_GLOBAL(identifier,suffix) identifier
+
+/* Plugins must define an entry point with this signature.  */
+HTSLIB_EXPORT
+extern int hfile_plugin_init(struct hFILE_plugin *self);
+
+#else
+#define PLUGIN_GLOBAL(identifier,suffix) identifier##suffix
+
+/* Only plugins distributed within the HTSlib source that might be built
+   even with --disable-plugins need to use PLUGIN_GLOBAL and be listed here;
+   others can simply define hfile_plugin_init().  */
+
+extern int hfile_plugin_init_gcs(struct hFILE_plugin *self);
+extern int hfile_plugin_init_libcurl(struct hFILE_plugin *self);
+extern int hfile_plugin_init_s3(struct hFILE_plugin *self);
+extern int hfile_plugin_init_s3_write(struct hFILE_plugin *self);
+#endif
+
+// Callback to allow headers to be set in http connections.  Currently used
+// to allow s3 to renew tokens when seeking.  Kept internal for now,
+// although we may consider exposing it in the API later.
+typedef int (* hts_httphdr_callback) (void *cb_data, char ***hdrs);
+
+/** Callback for handling 3xx redirect responses from http connections.
+
+    @param data       is passed to the callback
+    @param response   http response code (e.g. 301)
+    @param headers    http response headers
+    @param new_url    the callback should write the url to switch to in here
+
+    Currently used by s3 to handle switching region endpoints.
+*/
+typedef int (*redirect_callback) (void *data, long response,
+                                  kstring_t *headers, kstring_t *new_url);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/ext/htslib/hfile_libcurl.c b/ext/htslib/hfile_libcurl.c
new file mode 100644
index 0000000..3463acf
--- /dev/null
+++ b/ext/htslib/hfile_libcurl.c
@@ -0,0 +1,1563 @@
+/*  hfile_libcurl.c -- libcurl backend for low-level file streams.
+
+    Copyright (C) 2015-2017, 2019-2020 Genome Research Ltd.
+
+    Author: John Marshall <jm18@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+
+#include <stdarg.h>
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+#include <errno.h>
+#include <pthread.h>
+#ifndef _WIN32
+# include <sys/select.h>
+#endif
+#include <assert.h>
+
+#include "hfile_internal.h"
+#ifdef ENABLE_PLUGINS
+#include "version.h"
+#endif
+#include "htslib/hts.h"  // for hts_version() and hts_verbose
+#include "htslib/kstring.h"
+#include "htslib/khash.h"
+
+#include <curl/curl.h>
+
+// Number of seconds to take off auth_token expiry, to allow for clock skew
+// and slow servers
+#define AUTH_REFRESH_EARLY_SECS 60
+
+// Minimum number of bytes to skip when seeking forward.  Seeks less than
+// this will just read the data and throw it away.  The optimal value
+// depends on how long it takes to make a new connection compared
+// to how fast the data arrives.
+#define MIN_SEEK_FORWARD 1000000
+
+typedef struct {
+    char *path;
+    char *token;
+    time_t expiry;
+    int failed;
+    pthread_mutex_t lock;
+} auth_token;
+
+// For the authorization header cache
+KHASH_MAP_INIT_STR(auth_map, auth_token *)
+
+// Curl-compatible header linked list
+typedef struct {
+    struct curl_slist *list;
+    unsigned int num;
+    unsigned int size;
+} hdrlist;
+
+typedef struct {
+    hdrlist fixed;                   // List of headers supplied at hopen()
+    hdrlist extra;                   // List of headers from callback
+    hts_httphdr_callback callback;   // Callback to get more headers
+    void *callback_data;             // Data to pass to httphdr callback
+    auth_token *auth;                // Authentication token
+    int auth_hdr_num;                // Location of auth_token in hdrlist extra
+                                     // If -1, Authorization header is in fixed
+                                     //    -2, it came from the callback
+                                     //    -3, "auth_token_enabled", "false"
+                                     //        passed to hopen()
+    redirect_callback redirect;      // Callback to handle 3xx redirects
+    void *redirect_data;             // Data to pass to redirect_callback
+    long *http_response_ptr;         // Location to store http response code.
+    int fail_on_error;               // Open fails on >400 response code
+                                     //    (default true)
+} http_headers;
+
+typedef struct {
+    hFILE base;
+    CURL *easy;
+    CURLM *multi;
+    off_t file_size;
+    struct {
+        union { char *rd; const char *wr; } ptr;
+        size_t len;
+    } buffer;
+    CURLcode final_result;  // easy result code for finished transfers
+    // Flags for communicating with libcurl callbacks:
+    unsigned paused : 1;    // callback tells us that it has paused transfer
+    unsigned closing : 1;   // informs callback that hclose() has been invoked
+    unsigned finished : 1;  // wait_perform() tells us transfer is complete
+    unsigned perform_again : 1;
+    unsigned is_read : 1;   // Opened in read mode
+    unsigned can_seek : 1;  // Can (attempt to) seek on this handle
+    unsigned is_recursive:1; // Opened by hfile_libcurl itself
+    unsigned tried_seek : 1; // At least one seek has been attempted
+    int nrunning;
+    http_headers headers;
+
+    off_t delayed_seek;      // Location to seek to before reading
+    off_t last_offset;       // Location we're seeking from
+    char *preserved;         // Preserved buffer content on seek
+    size_t preserved_bytes;  // Number of preserved bytes
+    size_t preserved_size;   // Size of preserved buffer
+} hFILE_libcurl;
+
+static off_t libcurl_seek(hFILE *fpv, off_t offset, int whence);
+static int restart_from_position(hFILE_libcurl *fp, off_t pos);
+
+static int http_status_errno(int status)
+{
+    if (status >= 500)
+        switch (status) {
+        case 501: return ENOSYS;
+        case 503: return EBUSY;
+        case 504: return ETIMEDOUT;
+        default:  return EIO;
+        }
+    else if (status >= 400)
+        switch (status) {
+        case 401: return EPERM;
+        case 403: return EACCES;
+        case 404: return ENOENT;
+        case 405: return EROFS;
+        case 407: return EPERM;
+        case 408: return ETIMEDOUT;
+        case 410: return ENOENT;
+        default:  return EINVAL;
+        }
+    else return 0;
+}
+
+static int easy_errno(CURL *easy, CURLcode err)
+{
+    long lval;
+
+    switch (err) {
+    case CURLE_OK:
+        return 0;
+
+    case CURLE_UNSUPPORTED_PROTOCOL:
+    case CURLE_URL_MALFORMAT:
+        return EINVAL;
+
+#if LIBCURL_VERSION_NUM >= 0x071505
+    case CURLE_NOT_BUILT_IN:
+        return ENOSYS;
+#endif
+
+    case CURLE_COULDNT_RESOLVE_PROXY:
+    case CURLE_COULDNT_RESOLVE_HOST:
+    case CURLE_FTP_CANT_GET_HOST:
+        return EDESTADDRREQ; // Lookup failure
+
+    case CURLE_COULDNT_CONNECT:
+    case CURLE_SEND_ERROR:
+    case CURLE_RECV_ERROR:
+        if (curl_easy_getinfo(easy, CURLINFO_OS_ERRNO, &lval) == CURLE_OK)
+            return lval;
+        else
+            return ECONNABORTED;
+
+    case CURLE_REMOTE_ACCESS_DENIED:
+    case CURLE_LOGIN_DENIED:
+    case CURLE_TFTP_PERM:
+        return EACCES;
+
+    case CURLE_PARTIAL_FILE:
+        return EPIPE;
+
+    case CURLE_HTTP_RETURNED_ERROR:
+        if (curl_easy_getinfo(easy, CURLINFO_RESPONSE_CODE, &lval) == CURLE_OK)
+            return http_status_errno(lval);
+        else
+            return EIO;
+
+    case CURLE_OUT_OF_MEMORY:
+        return ENOMEM;
+
+    case CURLE_OPERATION_TIMEDOUT:
+        return ETIMEDOUT;
+
+    case CURLE_RANGE_ERROR:
+        return ESPIPE;
+
+    case CURLE_SSL_CONNECT_ERROR:
+        // TODO return SSL error buffer messages
+        return ECONNABORTED;
+
+    case CURLE_FILE_COULDNT_READ_FILE:
+    case CURLE_TFTP_NOTFOUND:
+        return ENOENT;
+
+    case CURLE_TOO_MANY_REDIRECTS:
+        return ELOOP;
+
+    case CURLE_FILESIZE_EXCEEDED:
+        return EFBIG;
+
+    case CURLE_REMOTE_DISK_FULL:
+        return ENOSPC;
+
+    case CURLE_REMOTE_FILE_EXISTS:
+        return EEXIST;
+
+    default:
+        hts_log_error("Libcurl reported error %d (%s)", (int) err,
+                      curl_easy_strerror(err));
+        return EIO;
+    }
+}
+
+static int multi_errno(CURLMcode errm)
+{
+    switch (errm) {
+    case CURLM_CALL_MULTI_PERFORM:
+    case CURLM_OK:
+        return 0;
+
+    case CURLM_BAD_HANDLE:
+    case CURLM_BAD_EASY_HANDLE:
+    case CURLM_BAD_SOCKET:
+        return EBADF;
+
+    case CURLM_OUT_OF_MEMORY:
+        return ENOMEM;
+
+    default:
+        hts_log_error("Libcurl reported error %d (%s)", (int) errm,
+                      curl_multi_strerror(errm));
+        return EIO;
+    }
+}
+
+static struct {
+    kstring_t useragent;
+    CURLSH *share;
+    char *auth_path;
+    khash_t(auth_map) *auth_map;
+    int allow_unencrypted_auth_header;
+    pthread_mutex_t auth_lock;
+    pthread_mutex_t share_lock;
+} curl = { { 0, 0, NULL }, NULL, NULL, NULL, 0, PTHREAD_MUTEX_INITIALIZER,
+           PTHREAD_MUTEX_INITIALIZER };
+
+static void share_lock(CURL *handle, curl_lock_data data,
+                       curl_lock_access access, void *userptr) {
+    pthread_mutex_lock(&curl.share_lock);
+}
+
+static void share_unlock(CURL *handle, curl_lock_data data, void *userptr) {
+    pthread_mutex_unlock(&curl.share_lock);
+}
+
+static void free_auth(auth_token *tok) {
+    if (!tok) return;
+    if (pthread_mutex_destroy(&tok->lock)) abort();
+    free(tok->path);
+    free(tok->token);
+    free(tok);
+}
+
+static void libcurl_exit(void)
+{
+    if (curl_share_cleanup(curl.share) == CURLSHE_OK)
+        curl.share = NULL;
+
+    free(curl.useragent.s);
+    curl.useragent.l = curl.useragent.m = 0; curl.useragent.s = NULL;
+
+    free(curl.auth_path);
+    curl.auth_path = NULL;
+
+    if (curl.auth_map) {
+        khiter_t i;
+        for (i = kh_begin(curl.auth_map); i != kh_end(curl.auth_map); ++i) {
+            if (kh_exist(curl.auth_map, i)) {
+                free_auth(kh_value(curl.auth_map, i));
+                kh_key(curl.auth_map, i) = NULL;
+                kh_value(curl.auth_map, i) = NULL;
+            }
+        }
+        kh_destroy(auth_map, curl.auth_map);
+        curl.auth_map = NULL;
+    }
+
+    curl_global_cleanup();
+}
+
+static int append_header(hdrlist *hdrs, const char *data, int dup) {
+    if (hdrs->num == hdrs->size) {
+        unsigned int new_sz = hdrs->size ? hdrs->size * 2 : 4, i;
+        struct curl_slist *new_list = realloc(hdrs->list,
+                                              new_sz * sizeof(*new_list));
+        if (!new_list) return -1;
+        hdrs->size = new_sz;
+        hdrs->list = new_list;
+        for (i = 1; i < hdrs->num; i++) hdrs->list[i-1].next = &hdrs->list[i];
+    }
+    // Annoyingly, libcurl doesn't declare the char * as const...
+    hdrs->list[hdrs->num].data = dup ? strdup(data) : (char *) data;
+    if (!hdrs->list[hdrs->num].data) return -1;
+    if (hdrs->num > 0) hdrs->list[hdrs->num - 1].next = &hdrs->list[hdrs->num];
+    hdrs->list[hdrs->num].next = NULL;
+    hdrs->num++;
+    return 0;
+}
+
+static void free_headers(hdrlist *hdrs, int completely) {
+    unsigned int i;
+    for (i = 0; i < hdrs->num; i++) {
+        free(hdrs->list[i].data);
+        hdrs->list[i].data = NULL;
+        hdrs->list[i].next = NULL;
+    }
+    hdrs->num = 0;
+    if (completely) {
+        free(hdrs->list);
+        hdrs->size = 0;
+        hdrs->list = NULL;
+    }
+}
+
+static struct curl_slist * get_header_list(hFILE_libcurl *fp) {
+    if (fp->headers.fixed.num > 0)
+        return &fp->headers.fixed.list[0];
+    if (fp->headers.extra.num > 0)
+        return &fp->headers.extra.list[0];
+    return 0;
+}
+
+static inline int is_authorization(const char *hdr) {
+    return (strncasecmp("authorization:", hdr, 14) == 0);
+}
+
+static int add_callback_headers(hFILE_libcurl *fp) {
+    char **hdrs = NULL, **hdr;
+
+    if (!fp->headers.callback)
+        return 0;
+
+    // Get the headers from the callback
+    if (fp->headers.callback(fp->headers.callback_data, &hdrs) != 0) {
+        return -1;
+    }
+
+    if (!hdrs) // No change
+        return 0;
+
+    // Remove any old callback headers
+    if (fp->headers.fixed.num > 0) {
+        // Unlink lists
+        fp->headers.fixed.list[fp->headers.fixed.num - 1].next = NULL;
+    }
+    free_headers(&fp->headers.extra, 0);
+
+    if (fp->headers.auth_hdr_num > 0 || fp->headers.auth_hdr_num == -2)
+        fp->headers.auth_hdr_num = 0; // Just removed it...
+
+    // Convert to libcurl-suitable form
+    for (hdr = hdrs; *hdr; hdr++) {
+        if (append_header(&fp->headers.extra, *hdr, 0) < 0) {
+            goto cleanup;
+        }
+        if (is_authorization(*hdr) && !fp->headers.auth_hdr_num)
+            fp->headers.auth_hdr_num = -2;
+    }
+    for (hdr = hdrs; *hdr; hdr++) *hdr = NULL;
+
+    if (fp->headers.fixed.num > 0 && fp->headers.extra.num > 0) {
+        // Relink lists
+        fp->headers.fixed.list[fp->headers.fixed.num - 1].next
+            = &fp->headers.extra.list[0];
+    }
+    return 0;
+
+ cleanup:
+    while (hdr && *hdr) {
+        free(*hdr);
+        *hdr = NULL;
+    }
+    return -1;
+}
+
+/*
+ * Read an OAUTH2-style Bearer access token (see
+ * https://tools.ietf.org/html/rfc6750#section-4).
+ * Returns 'v' for valid; 'i' for invalid (token missing or wrong sort);
+ * '?' for a JSON parse error; 'm' if it runs out of memory.
+ */
+static int read_auth_json(auth_token *tok, hFILE *auth_fp) {
+    hts_json_token *t = hts_json_alloc_token();
+    kstring_t str = {0, 0, NULL};
+    char *token = NULL, *type = NULL, *expiry = NULL;
+    int ret = 'i';
+
+    if (!t) goto error;
+
+    if ((ret = hts_json_fnext(auth_fp, t, &str)) != '{') goto error;
+    while (hts_json_fnext(auth_fp, t, &str) != '}') {
+        char *key;
+        if (hts_json_token_type(t) != 's') {
+            ret = '?';
+            goto error;
+        }
+        key = hts_json_token_str(t);
+        if (!key) goto error;
+        if (strcmp(key, "access_token") == 0) {
+            if ((ret = hts_json_fnext(auth_fp, t, &str)) != 's') goto error;
+            token = ks_release(&str);
+        } else if (strcmp(key, "token_type") == 0) {
+            if ((ret = hts_json_fnext(auth_fp, t, &str)) != 's') goto error;
+            type = ks_release(&str);
+        } else if (strcmp(key, "expires_in") == 0) {
+            if ((ret = hts_json_fnext(auth_fp, t, &str)) != 'n') goto error;
+            expiry = ks_release(&str);
+        } else if (hts_json_fskip_value(auth_fp, '\0') != 'v') {
+            ret = '?';
+            goto error;
+        }
+    }
+
+    if (!token || (type && strcmp(type, "Bearer") != 0)) {
+        ret = 'i';
+        goto error;
+    }
+
+    ret = 'm';
+    str.l = 0;
+    if (kputs("Authorization: Bearer ", &str) < 0) goto error;
+    if (kputs(token, &str) < 0) goto error;
+    free(tok->token);
+    tok->token = ks_release(&str);
+    if (expiry) {
+        long exp = strtol(expiry, NULL, 10);
+        if (exp < 0) exp = 0;
+        tok->expiry = time(NULL) + exp;
+    } else {
+        tok->expiry = 0;
+    }
+    ret = 'v';
+
+ error:
+    free(token);
+    free(type);
+    free(expiry);
+    free(str.s);
+    hts_json_free_token(t);
+    return ret;
+}
+
+static int read_auth_plain(auth_token *tok, hFILE *auth_fp) {
+    kstring_t line = {0, 0, NULL};
+    kstring_t token = {0, 0, NULL};
+    const char *start, *end;
+
+    if (kgetline(&line, (char * (*)(char *, int, void *)) hgets, auth_fp) < 0) goto error;
+    if (kputc('\0', &line) < 0) goto error;
+
+    for (start = line.s; *start && isspace_c(*start); start++) {}
+    for (end = start; *end && !isspace_c(*end); end++) {}
+
+    if (end > start) {
+        if (kputs("Authorization: Bearer ", &token) < 0) goto error;
+        if (kputsn(start, end - start, &token) < 0) goto error;
+    }
+
+    free(tok->token);
+    tok->token = ks_release(&token);
+    tok->expiry = 0;
+    free(line.s);
+    return 0;
+
+ error:
+    free(line.s);
+    free(token.s);
+    return -1;
+}
+
+static int renew_auth_token(auth_token *tok, int *changed) {
+    hFILE *auth_fp = NULL;
+    char buffer[16];
+    ssize_t len;
+
+    *changed = 0;
+    if (tok->expiry == 0 || time(NULL) + AUTH_REFRESH_EARLY_SECS < tok->expiry)
+        return 0; // Still valid
+
+    if (tok->failed)
+        return -1;
+
+    *changed = 1;
+    auth_fp = hopen(tok->path, "rR");
+    if (!auth_fp) {
+        // Not worried about missing files; other errors are bad.
+        if (errno != ENOENT)
+            goto fail;
+
+        tok->expiry = 0; // Prevent retry
+        free(tok->token); // Just in case it was set
+        return 0;
+    }
+
+    len = hpeek(auth_fp, buffer, sizeof(buffer));
+    if (len < 0)
+        goto fail;
+
+    if (memchr(buffer, '{', len) != NULL) {
+        if (read_auth_json(tok, auth_fp) != 'v')
+            goto fail;
+    } else {
+        if (read_auth_plain(tok, auth_fp) < 0)
+            goto fail;
+    }
+
+    return hclose(auth_fp) < 0 ? -1 : 0;
+
+ fail:
+    tok->failed = 1;
+    if (auth_fp) hclose_abruptly(auth_fp);
+    return -1;
+}
+
+static int add_auth_header(hFILE_libcurl *fp) {
+    int changed = 0;
+
+    if (fp->headers.auth_hdr_num < 0)
+        return 0; // Have an Authorization header from open or header callback
+
+    if (!fp->headers.auth)
+        return 0; // Nothing to add
+
+    pthread_mutex_lock(&fp->headers.auth->lock);
+    if (renew_auth_token(fp->headers.auth, &changed) < 0)
+        goto unlock_fail;
+
+    if (!changed && fp->headers.auth_hdr_num > 0) {
+        pthread_mutex_unlock(&fp->headers.auth->lock);
+        return 0;
+    }
+
+    if (fp->headers.auth_hdr_num > 0) {
+        // Had a previous header, so swap in the new one
+        char *header = fp->headers.auth->token;
+        char *header_copy = header ? strdup(header) : NULL;
+        int idx = fp->headers.auth_hdr_num - 1;
+        if (header && !header_copy)
+            goto unlock_fail;
+
+        if (header_copy) {
+            free(fp->headers.extra.list[idx].data);
+            fp->headers.extra.list[idx].data = header_copy;
+        } else {
+            unsigned int j;
+            // More complicated case - need to get rid of the old header
+            // and tidy up linked lists
+            free(fp->headers.extra.list[idx].data);
+            for (j = idx + 1; j < fp->headers.extra.num; j++) {
+                fp->headers.extra.list[j - 1] = fp->headers.extra.list[j];
+                fp->headers.extra.list[j - 1].next = &fp->headers.extra.list[j];
+            }
+            fp->headers.extra.num--;
+            if (fp->headers.extra.num > 0) {
+                fp->headers.extra.list[fp->headers.extra.num-1].next = NULL;
+            } else if (fp->headers.fixed.num > 0) {
+                fp->headers.fixed.list[fp->headers.fixed.num - 1].next = NULL;
+            }
+            fp->headers.auth_hdr_num = 0;
+        }
+    } else if (fp->headers.auth->token) {
+        // Add new header and remember where it is
+        if (append_header(&fp->headers.extra,
+                          fp->headers.auth->token, 1) < 0) {
+            goto unlock_fail;
+        }
+        fp->headers.auth_hdr_num = fp->headers.extra.num;
+    }
+
+    pthread_mutex_unlock(&fp->headers.auth->lock);
+    return 0;
+
+ unlock_fail:
+    pthread_mutex_unlock(&fp->headers.auth->lock);
+    return -1;
+}
+
+static int get_auth_token(hFILE_libcurl *fp, const char *url) {
+    const char *host = NULL, *p, *q;
+    kstring_t name = {0, 0, NULL};
+    size_t host_len = 0;
+    khiter_t idx;
+    auth_token *tok = NULL;
+
+    // Nothing to do if:
+    //   curl.auth_path has not been set
+    //   fp was made by hfile_libcurl (e.g. auth_path is a http:// url)
+    //   we already have an Authorization header
+    if (!curl.auth_path || fp->is_recursive || fp->headers.auth_hdr_num != 0)
+        return 0;
+
+    // Insist on having a secure connection unless the user insists harder
+    if (!curl.allow_unencrypted_auth_header && strncmp(url, "https://", 8) != 0)
+        return 0;
+
+    host = strstr(url, "://");
+    if (host) {
+        host += 3;
+        host_len = strcspn(host, "/");
+    }
+
+    p = curl.auth_path;
+    while ((q = strstr(p, "%h")) != NULL) {
+        if (q - p > INT_MAX || host_len > INT_MAX) goto error;
+        if (kputsn_(p, q - p, &name) < 0) goto error;
+        if (kputsn_(host, host_len, &name) < 0) goto error;
+        p = q + 2;
+    }
+    if (kputs(p, &name) < 0) goto error;
+
+    pthread_mutex_lock(&curl.auth_lock);
+    idx = kh_get(auth_map, curl.auth_map, name.s);
+    if (idx < kh_end(curl.auth_map)) {
+        tok = kh_value(curl.auth_map, idx);
+    } else {
+        tok = calloc(1, sizeof(*tok));
+        if (tok && pthread_mutex_init(&tok->lock, NULL) != 0) {
+            free(tok);
+            tok = NULL;
+        }
+        if (tok) {
+            int ret = -1;
+            tok->path = ks_release(&name);
+            tok->token = NULL;
+            tok->expiry = 1; // Force refresh
+            idx = kh_put(auth_map, curl.auth_map, tok->path, &ret);
+            if (ret < 0) {
+                free_auth(tok);
+                tok = NULL;
+            }
+            kh_value(curl.auth_map, idx) = tok;
+        }
+    }
+    pthread_mutex_unlock(&curl.auth_lock);
+
+    fp->headers.auth = tok;
+    free(name.s);
+
+    return add_auth_header(fp);
+
+ error:
+    free(name.s);
+    return -1;
+}
+
+static void process_messages(hFILE_libcurl *fp)
+{
+    CURLMsg *msg;
+    int remaining;
+
+    while ((msg = curl_multi_info_read(fp->multi, &remaining)) != NULL) {
+        switch (msg->msg) {
+        case CURLMSG_DONE:
+            fp->finished = 1;
+            fp->final_result = msg->data.result;
+            break;
+
+        default:
+            break;
+        }
+    }
+}
+
+static int wait_perform(hFILE_libcurl *fp)
+{
+    fd_set rd, wr, ex;
+    int maxfd, nrunning;
+    long timeout;
+    CURLMcode errm;
+
+    if (!fp->perform_again) {
+        FD_ZERO(&rd);
+        FD_ZERO(&wr);
+        FD_ZERO(&ex);
+        if (curl_multi_fdset(fp->multi, &rd, &wr, &ex, &maxfd) != CURLM_OK)
+            maxfd = -1, timeout = 1000;
+        else {
+            if (curl_multi_timeout(fp->multi, &timeout) != CURLM_OK)
+                timeout = 1000;
+            else if (timeout < 0) {
+                timeout = 10000;  // as recommended by curl_multi_timeout(3)
+            }
+        }
+        if (maxfd < 0) {
+            if (timeout > 100)
+                timeout = 100; // as recommended by curl_multi_fdset(3)
+#ifdef _WIN32
+            /* Windows ignores the first argument of select, so calling select
+             * with maxfd=-1 does not give the expected result of sleeping for
+             * timeout milliseconds in the conditional block below.
+             * So sleep here and skip the next block.
+             */
+            Sleep(timeout);
+            timeout = 0;
+#endif
+        }
+
+        if (timeout > 0) {
+            struct timeval tval;
+            tval.tv_sec  = (timeout / 1000);
+            tval.tv_usec = (timeout % 1000) * 1000;
+
+            if (select(maxfd + 1, &rd, &wr, &ex, &tval) < 0) return -1;
+        }
+    }
+
+    errm = curl_multi_perform(fp->multi, &nrunning);
+    fp->perform_again = 0;
+    if (errm == CURLM_CALL_MULTI_PERFORM) fp->perform_again = 1;
+    else if (errm != CURLM_OK) { errno = multi_errno(errm); return -1; }
+
+    if (nrunning < fp->nrunning) process_messages(fp);
+    return 0;
+}
+
+
+static size_t recv_callback(char *ptr, size_t size, size_t nmemb, void *fpv)
+{
+    hFILE_libcurl *fp = (hFILE_libcurl *) fpv;
+    size_t n = size * nmemb;
+
+    if (n > fp->buffer.len) {
+        fp->paused = 1;
+        return CURL_WRITEFUNC_PAUSE;
+    }
+    else if (n == 0) return 0;
+
+    memcpy(fp->buffer.ptr.rd, ptr, n);
+    fp->buffer.ptr.rd += n;
+    fp->buffer.len -= n;
+    return n;
+}
+
+
+static size_t header_callback(void *contents, size_t size, size_t nmemb,
+                              void *userp)
+{
+    size_t realsize = size * nmemb;
+    kstring_t *resp = (kstring_t *)userp;
+
+    if (kputsn((const char *)contents, realsize, resp) == EOF) {
+        return 0;
+    }
+
+    return realsize;
+}
+
+
+static ssize_t libcurl_read(hFILE *fpv, void *bufferv, size_t nbytes)
+{
+    hFILE_libcurl *fp = (hFILE_libcurl *) fpv;
+    char *buffer = (char *) bufferv;
+    off_t to_skip = -1;
+    ssize_t got = 0;
+    CURLcode err;
+
+    if (fp->delayed_seek >= 0) {
+        assert(fp->base.offset == fp->delayed_seek);
+
+        if (fp->preserved
+            && fp->last_offset > fp->delayed_seek
+            && fp->last_offset - fp->preserved_bytes <= fp->delayed_seek) {
+            // Can use buffer contents copied when seeking started, to
+            // avoid having to re-read data discarded by hseek().
+            // Note fp->last_offset is the offset of the *end* of the
+            // preserved buffer.
+            size_t n = fp->last_offset - fp->delayed_seek;
+            char *start = fp->preserved + (fp->preserved_bytes - n);
+            size_t bytes = n <= nbytes ? n : nbytes;
+            memcpy(buffer, start, bytes);
+            if (bytes < n) { // Part of the preserved buffer still left
+                fp->delayed_seek += bytes;
+            } else {
+                fp->last_offset = fp->delayed_seek = -1;
+            }
+            return bytes;
+        }
+
+        if (fp->last_offset >= 0
+            && fp->delayed_seek > fp->last_offset
+            && fp->delayed_seek - fp->last_offset < MIN_SEEK_FORWARD) {
+            // If not seeking far, just read the data and throw it away.  This
+            // is likely to be quicker than opening a new stream
+            to_skip = fp->delayed_seek - fp->last_offset;
+        } else {
+            if (restart_from_position(fp, fp->delayed_seek) < 0) {
+                return -1;
+            }
+        }
+        fp->delayed_seek = -1;
+        fp->last_offset = -1;
+        fp->preserved_bytes = 0;
+    }
+
+    do {
+        fp->buffer.ptr.rd = buffer;
+        fp->buffer.len = nbytes;
+        fp->paused = 0;
+        if (!fp->finished) {
+            err = curl_easy_pause(fp->easy, CURLPAUSE_CONT);
+            if (err != CURLE_OK) {
+                errno = easy_errno(fp->easy, err);
+                return -1;
+            }
+        }
+
+        while (! fp->paused && ! fp->finished) {
+            if (wait_perform(fp) < 0) return -1;
+        }
+
+        got = fp->buffer.ptr.rd - buffer;
+
+        if (to_skip >= 0) { // Skipping over a small seek
+            if (got <= to_skip) { // Need to skip more data
+                to_skip -= got;
+            } else {
+                got -= to_skip;
+                if (got > 0) {  // If enough was skipped, return the rest
+                    memmove(buffer, buffer + to_skip, got);
+                    to_skip = -1;
+                }
+            }
+        }
+    } while (to_skip >= 0 && ! fp->finished);
+    fp->buffer.ptr.rd = NULL;
+    fp->buffer.len = 0;
+
+    if (fp->finished && fp->final_result != CURLE_OK) {
+        errno = easy_errno(fp->easy, fp->final_result);
+        return -1;
+    }
+
+    return got;
+}
+
+static size_t send_callback(char *ptr, size_t size, size_t nmemb, void *fpv)
+{
+    hFILE_libcurl *fp = (hFILE_libcurl *) fpv;
+    size_t n = size * nmemb;
+
+    if (fp->buffer.len == 0) {
+        // Send buffer is empty; normally pause, or signal EOF if we're closing
+        if (fp->closing) return 0;
+        else { fp->paused = 1; return CURL_READFUNC_PAUSE; }
+    }
+
+    if (n > fp->buffer.len) n = fp->buffer.len;
+    memcpy(ptr, fp->buffer.ptr.wr, n);
+    fp->buffer.ptr.wr += n;
+    fp->buffer.len -= n;
+    return n;
+}
+
+static ssize_t libcurl_write(hFILE *fpv, const void *bufferv, size_t nbytes)
+{
+    hFILE_libcurl *fp = (hFILE_libcurl *) fpv;
+    const char *buffer = (const char *) bufferv;
+    CURLcode err;
+
+    fp->buffer.ptr.wr = buffer;
+    fp->buffer.len = nbytes;
+    fp->paused = 0;
+    err = curl_easy_pause(fp->easy, CURLPAUSE_CONT);
+    if (err != CURLE_OK) { errno = easy_errno(fp->easy, err); return -1; }
+
+    while (! fp->paused && ! fp->finished)
+        if (wait_perform(fp) < 0) return -1;
+
+    nbytes = fp->buffer.ptr.wr - buffer;
+    fp->buffer.ptr.wr = NULL;
+    fp->buffer.len = 0;
+
+    if (fp->finished && fp->final_result != CURLE_OK) {
+        errno = easy_errno(fp->easy, fp->final_result);
+        return -1;
+    }
+
+    return nbytes;
+}
+
+static void preserve_buffer_content(hFILE_libcurl *fp)
+{
+    if (fp->base.begin == fp->base.end) {
+        fp->preserved_bytes = 0;
+        return;
+    }
+    if (!fp->preserved
+        || fp->preserved_size < fp->base.limit - fp->base.buffer) {
+        fp->preserved = malloc(fp->base.limit - fp->base.buffer);
+        if (!fp->preserved) return;
+        fp->preserved_size = fp->base.limit - fp->base.buffer;
+    }
+
+    assert(fp->base.end - fp->base.begin <= fp->preserved_size);
+
+    memcpy(fp->preserved, fp->base.begin, fp->base.end - fp->base.begin);
+    fp->preserved_bytes = fp->base.end - fp->base.begin;
+    return;
+}
+
+static off_t libcurl_seek(hFILE *fpv, off_t offset, int whence)
+{
+    hFILE_libcurl *fp = (hFILE_libcurl *) fpv;
+    off_t origin, pos;
+
+    if (!fp->is_read || !fp->can_seek) {
+        // Cowardly refuse to seek when writing or a previous seek failed.
+        errno = ESPIPE;
+        return -1;
+    }
+
+    switch (whence) {
+    case SEEK_SET:
+        origin = 0;
+        break;
+    case SEEK_CUR:
+        errno = ENOSYS;
+        return -1;
+    case SEEK_END:
+        if (fp->file_size < 0) { errno = ESPIPE; return -1; }
+        origin = fp->file_size;
+        break;
+    default:
+        errno = EINVAL;
+        return -1;
+    }
+
+    // Check 0 <= origin+offset < fp->file_size carefully, avoiding overflow
+    if ((offset < 0)? origin + offset < 0
+                : (fp->file_size >= 0 && offset > fp->file_size - origin)) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    pos = origin + offset;
+
+    if (fp->tried_seek) {
+        /* Seeking has worked at least once, so now we can delay doing
+           the actual work until the next read.  This avoids lots of pointless
+           http or ftp reconnections if the caller does lots of seeks
+           without any intervening reads. */
+        if (fp->delayed_seek < 0) {
+            fp->last_offset = fp->base.offset + (fp->base.end - fp->base.buffer);
+            // Stash the current hFILE buffer content in case it's useful later
+            preserve_buffer_content(fp);
+        }
+        fp->delayed_seek = pos;
+        return pos;
+    }
+
+    if (restart_from_position(fp, pos) < 0) {
+        /* This value for errno may not be entirely true, but the caller may be
+           able to carry on with the existing handle. */
+        errno = ESPIPE;
+        return -1;
+    }
+
+    fp->tried_seek = 1;
+    return pos;
+}
+
+static int restart_from_position(hFILE_libcurl *fp, off_t pos) {
+    hFILE_libcurl temp_fp;
+    CURLcode err;
+    CURLMcode errm;
+    int update_headers = 0;
+    int save_errno = 0;
+
+    // TODO If we seem to be doing random access, use CURLOPT_RANGE to do
+    // limited reads (e.g. about a BAM block!) so seeking can reuse the
+    // existing connection more often.
+
+    // Get new headers from the callback (if defined).  This changes the
+    // headers in fp before it gets duplicated, but they should be have been
+    // sent by now.
+
+    if (fp->headers.callback) {
+        if (add_callback_headers(fp) != 0)
+            return -1;
+        update_headers = 1;
+    }
+    if (fp->headers.auth_hdr_num > 0 && fp->headers.auth) {
+        if (add_auth_header(fp) != 0)
+            return -1;
+        update_headers = 1;
+    }
+    if (update_headers) {
+        struct curl_slist *list = get_header_list(fp);
+        if (list) {
+            err = curl_easy_setopt(fp->easy, CURLOPT_HTTPHEADER, list);
+            if (err != CURLE_OK) {
+                errno = easy_errno(fp->easy,err);
+                return -1;
+            }
+        }
+    }
+
+    /*
+      Duplicate the easy handle, and use CURLOPT_RESUME_FROM_LARGE to open
+      a new request to the server, reading from the location that we want
+      to seek to.  If the new request works and returns the correct data,
+      the original easy handle in *fp is closed and replaced with the new
+      one.  If not, we close the new handle and leave *fp unchanged.
+     */
+
+    memcpy(&temp_fp, fp, sizeof(temp_fp));
+    temp_fp.buffer.len = 0;
+    temp_fp.buffer.ptr.rd = NULL;
+    temp_fp.easy = curl_easy_duphandle(fp->easy);
+    if (!temp_fp.easy)
+        goto early_error;
+
+    err = curl_easy_setopt(temp_fp.easy, CURLOPT_RESUME_FROM_LARGE,(curl_off_t)pos);
+    err |= curl_easy_setopt(temp_fp.easy, CURLOPT_PRIVATE, &temp_fp);
+    err |= curl_easy_setopt(temp_fp.easy, CURLOPT_WRITEDATA, &temp_fp);
+    if (err != CURLE_OK) {
+        save_errno = easy_errno(temp_fp.easy, err);
+        goto error;
+    }
+
+    temp_fp.buffer.len = 0;  // Ensures we only read the response headers
+    temp_fp.paused = temp_fp.finished = 0;
+
+    // fp->multi and temp_fp.multi are the same.
+    errm = curl_multi_add_handle(fp->multi, temp_fp.easy);
+    if (errm != CURLM_OK) {
+        save_errno = multi_errno(errm);
+        goto error;
+    }
+    temp_fp.nrunning = ++fp->nrunning;
+
+    while (! temp_fp.paused && ! temp_fp.finished)
+        if (wait_perform(&temp_fp) < 0) {
+            save_errno = errno;
+            goto error_remove;
+        }
+
+    if (temp_fp.finished && temp_fp.final_result != CURLE_OK) {
+        save_errno = easy_errno(temp_fp.easy, temp_fp.final_result);
+        goto error_remove;
+    }
+
+    // We've got a good response, close the original connection and
+    // replace it with the new one.
+
+    errm = curl_multi_remove_handle(fp->multi, fp->easy);
+    if (errm != CURLM_OK) {
+        // Clean up as much as possible
+        curl_easy_reset(temp_fp.easy);
+        if (curl_multi_remove_handle(fp->multi, temp_fp.easy) == CURLM_OK) {
+            fp->nrunning--;
+            curl_easy_cleanup(temp_fp.easy);
+        }
+        save_errno = multi_errno(errm);
+        goto early_error;
+    }
+    fp->nrunning--;
+
+    curl_easy_cleanup(fp->easy);
+    fp->easy = temp_fp.easy;
+    err = curl_easy_setopt(fp->easy, CURLOPT_WRITEDATA, fp);
+    err |= curl_easy_setopt(fp->easy, CURLOPT_PRIVATE, fp);
+    if (err != CURLE_OK) {
+        save_errno = easy_errno(fp->easy, err);
+        curl_easy_reset(fp->easy);
+        errno = save_errno;
+        return -1;
+    }
+    fp->buffer.len = 0;
+    fp->paused = temp_fp.paused;
+    fp->finished = temp_fp.finished;
+    fp->perform_again = temp_fp.perform_again;
+    fp->final_result = temp_fp.final_result;
+
+    return 0;
+
+ error_remove:
+    curl_easy_reset(temp_fp.easy); // Ensure no pointers to on-stack temp_fp
+    errm = curl_multi_remove_handle(fp->multi, temp_fp.easy);
+    if (errm != CURLM_OK) {
+        errno = multi_errno(errm);
+        return -1;
+    }
+    fp->nrunning--;
+ error:
+    curl_easy_cleanup(temp_fp.easy);
+ early_error:
+    fp->can_seek = 0;  // Don't try to seek again
+    if (save_errno)
+        errno = save_errno;
+    return -1;
+}
+
+static int libcurl_close(hFILE *fpv)
+{
+    hFILE_libcurl *fp = (hFILE_libcurl *) fpv;
+    CURLcode err;
+    CURLMcode errm;
+    int save_errno = 0;
+
+    // Before closing the file, unpause it and perform on it so that uploads
+    // have the opportunity to signal EOF to the server -- see send_callback().
+
+    fp->buffer.len = 0;
+    fp->closing = 1;
+    fp->paused = 0;
+    if (!fp->finished) {
+        err = curl_easy_pause(fp->easy, CURLPAUSE_CONT);
+        if (err != CURLE_OK) save_errno = easy_errno(fp->easy, err);
+    }
+
+    while (save_errno == 0 && ! fp->paused && ! fp->finished)
+        if (wait_perform(fp) < 0) save_errno = errno;
+
+    if (fp->finished && fp->final_result != CURLE_OK)
+        save_errno = easy_errno(fp->easy, fp->final_result);
+
+    errm = curl_multi_remove_handle(fp->multi, fp->easy);
+    if (errm != CURLM_OK && save_errno == 0) save_errno = multi_errno(errm);
+    fp->nrunning--;
+
+    curl_easy_cleanup(fp->easy);
+    curl_multi_cleanup(fp->multi);
+
+    if (fp->headers.callback) // Tell callback to free any data it needs to
+        fp->headers.callback(fp->headers.callback_data, NULL);
+    free_headers(&fp->headers.fixed, 1);
+    free_headers(&fp->headers.extra, 1);
+
+    free(fp->preserved);
+
+    if (save_errno) { errno = save_errno; return -1; }
+    else return 0;
+}
+
+static const struct hFILE_backend libcurl_backend =
+{
+    libcurl_read, libcurl_write, libcurl_seek, NULL, libcurl_close
+};
+
+static hFILE *
+libcurl_open(const char *url, const char *modes, http_headers *headers)
+{
+    hFILE_libcurl *fp;
+    struct curl_slist *list;
+    char mode;
+    const char *s;
+    CURLcode err;
+    CURLMcode errm;
+    int save, is_recursive;
+    kstring_t in_header = {0, 0, NULL};
+    long response;
+
+    is_recursive = strchr(modes, 'R') != NULL;
+
+    if ((s = strpbrk(modes, "rwa+")) != NULL) {
+        mode = *s;
+        if (strpbrk(&s[1], "rwa+")) mode = 'e';
+    }
+    else mode = '\0';
+
+    if (mode != 'r' && mode != 'w') { errno = EINVAL; goto early_error; }
+
+    fp = (hFILE_libcurl *) hfile_init(sizeof (hFILE_libcurl), modes, 0);
+    if (fp == NULL) goto early_error;
+
+    if (headers) {
+        fp->headers = *headers;
+    } else {
+        memset(&fp->headers, 0, sizeof(fp->headers));
+        fp->headers.fail_on_error = 1;
+    }
+
+    fp->file_size = -1;
+    fp->buffer.ptr.rd = NULL;
+    fp->buffer.len = 0;
+    fp->final_result = (CURLcode) -1;
+    fp->paused = fp->closing = fp->finished = fp->perform_again = 0;
+    fp->can_seek = 1;
+    fp->tried_seek = 0;
+    fp->delayed_seek = fp->last_offset = -1;
+    fp->preserved = NULL;
+    fp->preserved_bytes = fp->preserved_size = 0;
+    fp->is_recursive = is_recursive;
+    fp->nrunning = 0;
+    fp->easy = NULL;
+
+    fp->multi = curl_multi_init();
+    if (fp->multi == NULL) { errno = ENOMEM; goto error; }
+
+    fp->easy = curl_easy_init();
+    if (fp->easy == NULL) { errno = ENOMEM; goto error; }
+
+    // Make a route to the hFILE_libcurl* given just a CURL* easy handle
+    err = curl_easy_setopt(fp->easy, CURLOPT_PRIVATE, fp);
+
+    // Avoid many repeated CWD calls with FTP, instead requesting the filename
+    // by full path (but not strictly compliant with RFC1738).
+    err |= curl_easy_setopt(fp->easy, CURLOPT_FTP_FILEMETHOD, CURLFTPMETHOD_NOCWD);
+
+    if (mode == 'r') {
+        err |= curl_easy_setopt(fp->easy, CURLOPT_WRITEFUNCTION, recv_callback);
+        err |= curl_easy_setopt(fp->easy, CURLOPT_WRITEDATA, fp);
+        fp->is_read = 1;
+    }
+    else {
+        err |= curl_easy_setopt(fp->easy, CURLOPT_READFUNCTION, send_callback);
+        err |= curl_easy_setopt(fp->easy, CURLOPT_READDATA, fp);
+        err |= curl_easy_setopt(fp->easy, CURLOPT_UPLOAD, 1L);
+        if (append_header(&fp->headers.fixed,
+                          "Transfer-Encoding: chunked", 1) < 0)
+            goto error;
+        fp->is_read = 0;
+    }
+
+    err |= curl_easy_setopt(fp->easy, CURLOPT_SHARE, curl.share);
+    err |= curl_easy_setopt(fp->easy, CURLOPT_URL, url);
+    {
+        char* env_curl_ca_bundle = getenv("CURL_CA_BUNDLE");
+        if (env_curl_ca_bundle) {
+            err |= curl_easy_setopt(fp->easy, CURLOPT_CAINFO, env_curl_ca_bundle);
+        }
+    }
+    err |= curl_easy_setopt(fp->easy, CURLOPT_USERAGENT, curl.useragent.s);
+    if (fp->headers.callback) {
+        if (add_callback_headers(fp) != 0) goto error;
+    }
+    if (get_auth_token(fp, url) < 0)
+        goto error;
+    if ((list = get_header_list(fp)) != NULL)
+        err |= curl_easy_setopt(fp->easy, CURLOPT_HTTPHEADER, list);
+
+    if (hts_verbose <= 8 && fp->headers.fail_on_error)
+        err |= curl_easy_setopt(fp->easy, CURLOPT_FAILONERROR, 1L);
+    if (hts_verbose >= 8)
+        err |= curl_easy_setopt(fp->easy, CURLOPT_VERBOSE, 1L);
+
+    if (fp->headers.redirect) {
+        err |= curl_easy_setopt(fp->easy, CURLOPT_HEADERFUNCTION, header_callback);
+        err |= curl_easy_setopt(fp->easy, CURLOPT_HEADERDATA, (void *)&in_header);
+    } else {
+        err |= curl_easy_setopt(fp->easy, CURLOPT_FOLLOWLOCATION, 1L);
+    }
+
+    if (err != 0) { errno = ENOSYS; goto error; }
+
+    errm = curl_multi_add_handle(fp->multi, fp->easy);
+    if (errm != CURLM_OK) { errno = multi_errno(errm); goto error; }
+    fp->nrunning++;
+
+    while (! fp->paused && ! fp->finished) {
+        if (wait_perform(fp) < 0) goto error_remove;
+    }
+
+    curl_easy_getinfo(fp->easy, CURLINFO_RESPONSE_CODE, &response);
+    if (fp->headers.http_response_ptr) {
+        *fp->headers.http_response_ptr = response;
+    }
+
+    if (fp->finished && fp->final_result != CURLE_OK) {
+        errno = easy_errno(fp->easy, fp->final_result);
+        goto error_remove;
+    }
+
+    if (fp->headers.redirect) {
+        if (response >= 300 && response < 400) { // redirection
+            kstring_t new_url = {0, 0, NULL};
+
+            if (fp->headers.redirect(fp->headers.redirect_data, response,
+                                     &in_header, &new_url)) {
+                errno = ENOSYS;
+                goto error;
+            }
+
+            err |= curl_easy_setopt(fp->easy, CURLOPT_URL, new_url.s);
+            err |= curl_easy_setopt(fp->easy, CURLOPT_HEADERFUNCTION, NULL);
+            err |= curl_easy_setopt(fp->easy, CURLOPT_HEADERDATA, NULL);
+            free(ks_release(&in_header));
+
+            if (err != 0) { errno = ENOSYS; goto error; }
+            free(ks_release(&new_url));
+
+            if (restart_from_position(fp, 0) < 0) {
+                goto error_remove;
+            }
+
+            if (fp->headers.http_response_ptr) {
+                curl_easy_getinfo(fp->easy, CURLINFO_RESPONSE_CODE,
+                                  fp->headers.http_response_ptr);
+            }
+
+            if (fp->finished && fp->final_result != CURLE_OK) {
+                errno = easy_errno(fp->easy, fp->final_result);
+                goto error_remove;
+            }
+        } else {
+            // we no longer need to look at the headers
+            err |= curl_easy_setopt(fp->easy, CURLOPT_HEADERFUNCTION, NULL);
+            err |= curl_easy_setopt(fp->easy, CURLOPT_HEADERDATA, NULL);
+            free(ks_release(&in_header));
+
+            if (err != 0) { errno = ENOSYS; goto error; }
+        }
+    }
+
+    if (mode == 'r') {
+#if LIBCURL_VERSION_NUM >= 0x073700 // 7.55.0
+        curl_off_t offset;
+
+        if (curl_easy_getinfo(fp->easy, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T,
+                              &offset) == CURLE_OK && offset > 0)
+            fp->file_size = (off_t) offset;
+#else
+        double dval;
+
+        if (curl_easy_getinfo(fp->easy, CURLINFO_CONTENT_LENGTH_DOWNLOAD,
+                              &dval) == CURLE_OK && dval >= 0.0)
+            fp->file_size = (off_t) (dval + 0.1);
+#endif
+    }
+    fp->base.backend = &libcurl_backend;
+    return &fp->base;
+
+error_remove:
+    save = errno;
+    (void) curl_multi_remove_handle(fp->multi, fp->easy);
+    fp->nrunning--;
+    errno = save;
+
+error:
+    if (fp->headers.redirect) free(in_header.s);
+    save = errno;
+    if (fp->easy) curl_easy_cleanup(fp->easy);
+    if (fp->multi) curl_multi_cleanup(fp->multi);
+    free_headers(&fp->headers.extra, 1);
+    hfile_destroy((hFILE *) fp);
+    errno = save;
+    return NULL;
+
+early_error:
+    return NULL;
+}
+
+static hFILE *hopen_libcurl(const char *url, const char *modes)
+{
+    return libcurl_open(url, modes, NULL);
+}
+
+static int parse_va_list(http_headers *headers, va_list args)
+{
+    const char *argtype;
+
+    while ((argtype = va_arg(args, const char *)) != NULL)
+        if (strcmp(argtype, "httphdr:v") == 0) {
+            const char **hdr;
+            for (hdr = va_arg(args, const char **); *hdr; hdr++) {
+                if (append_header(&headers->fixed, *hdr, 1) < 0)
+                    return -1;
+                if (is_authorization(*hdr))
+                    headers->auth_hdr_num = -1;
+            }
+        }
+        else if (strcmp(argtype, "httphdr:l") == 0) {
+            const char *hdr;
+            while ((hdr = va_arg(args, const char *)) != NULL) {
+                if (append_header(&headers->fixed, hdr, 1) < 0)
+                    return -1;
+                if (is_authorization(hdr))
+                    headers->auth_hdr_num = -1;
+            }
+        }
+        else if (strcmp(argtype, "httphdr") == 0) {
+            const char *hdr = va_arg(args, const char *);
+            if (hdr) {
+                if (append_header(&headers->fixed, hdr, 1) < 0)
+                    return -1;
+                if (is_authorization(hdr))
+                    headers->auth_hdr_num = -1;
+            }
+        }
+        else if (strcmp(argtype, "httphdr_callback") == 0) {
+            headers->callback = va_arg(args, const hts_httphdr_callback);
+        }
+        else if (strcmp(argtype, "httphdr_callback_data") == 0) {
+            headers->callback_data = va_arg(args, void *);
+        }
+        else if (strcmp(argtype, "va_list") == 0) {
+            va_list *args2 = va_arg(args, va_list *);
+            if (args2) {
+                if (parse_va_list(headers, *args2) < 0) return -1;
+            }
+        }
+        else if (strcmp(argtype, "auth_token_enabled") == 0) {
+            const char *flag = va_arg(args, const char *);
+            if (strcmp(flag, "false") == 0)
+                headers->auth_hdr_num = -3;
+        }
+        else if (strcmp(argtype, "redirect_callback") == 0) {
+            headers->redirect = va_arg(args, const redirect_callback);
+        }
+        else if (strcmp(argtype, "redirect_callback_data") == 0) {
+            headers->redirect_data = va_arg(args, void *);
+        }
+        else if (strcmp(argtype, "http_response_ptr") == 0) {
+            headers->http_response_ptr = va_arg(args, long *);
+        }
+        else if (strcmp(argtype, "fail_on_error") == 0) {
+            headers->fail_on_error = va_arg(args, int);
+        }
+        else { errno = EINVAL; return -1; }
+
+    return 0;
+}
+
+/*
+  HTTP headers to be added to the request can be passed in as extra
+  arguments to hopen().  The headers can be specified as follows:
+
+  * Single header:
+    hopen(url, mode, "httphdr", "X-Hdr-1: text", NULL);
+
+  * Multiple headers in the argument list:
+    hopen(url, mode, "httphdr:l", "X-Hdr-1: text", "X-Hdr-2: text", NULL, NULL);
+
+  * Multiple headers in a char* array:
+    hopen(url, mode, "httphdr:v", hdrs, NULL);
+    where `hdrs` is a char **.  The list ends with a NULL pointer.
+
+  * A callback function
+    hopen(url, mode, "httphdr_callback", func,
+                     "httphdr_callback_data", arg, NULL);
+    `func` has type
+         int (* hts_httphdr_callback) (void *cb_data, char ***hdrs);
+    `arg` is passed to the callback as a void *.
+
+    The function is called at file open, and when attempting to seek (which
+    opens a new HTTP request).  This allows, for example, access tokens
+    that may have gone stale to be regenerated.  The function is also
+    called (with `hdrs` == NULL) on file close so that the callback can
+    free any memory that it needs to.
+
+    The callback should return 0 on success, non-zero on failure.  It should
+    return in *hdrs a list of strings containing the new headers (terminated
+    with a NULL pointer).  These will replace any headers previously supplied
+    by the callback.  If no changes are necessary, it can return NULL
+    in *hdrs, in which case the previous headers will be left unchanged.
+
+    Ownership of the strings in the header list passes to hfile_libcurl,
+    so the callback should not attempt to use or free them itself.  The memory
+    containing the array belongs to the callback and will not be freed by
+    hfile_libcurl.
+
+    Headers supplied by the callback are appended after any specified
+    using the "httphdr", "httphdr:l" or "httphdr:v" methods.  No attempt
+    is made to replace these headers (even if a key is repeated) so anything
+    that is expected to vary needs to come from the callback.
+ */
+
+static hFILE *vhopen_libcurl(const char *url, const char *modes, va_list args)
+{
+    hFILE *fp = NULL;
+    http_headers headers = { .fail_on_error = 1 };
+
+    if (parse_va_list(&headers, args) == 0) {
+        fp = libcurl_open(url, modes, &headers);
+    }
+
+    if (!fp) {
+        free_headers(&headers.fixed, 1);
+    }
+    return fp;
+}
+
+int PLUGIN_GLOBAL(hfile_plugin_init,_libcurl)(struct hFILE_plugin *self)
+{
+    static const struct hFILE_scheme_handler handler =
+        { hopen_libcurl, hfile_always_remote, "libcurl",
+          2000 + 50,
+          vhopen_libcurl };
+
+#ifdef ENABLE_PLUGINS
+    // Embed version string for examination via strings(1) or what(1)
+    static const char id[] =
+        "@(#)hfile_libcurl plugin (htslib)\t" HTS_VERSION_TEXT;
+    const char *version = strchr(id, '\t')+1;
+#else
+    const char *version = hts_version();
+#endif
+    const curl_version_info_data *info;
+    const char * const *protocol;
+    const char *auth;
+    CURLcode err;
+    CURLSHcode errsh;
+
+    err = curl_global_init(CURL_GLOBAL_ALL);
+    if (err != CURLE_OK) { errno = easy_errno(NULL, err); return -1; }
+
+    curl.share = curl_share_init();
+    if (curl.share == NULL) { curl_global_cleanup(); errno = EIO; return -1; }
+    errsh = curl_share_setopt(curl.share, CURLSHOPT_LOCKFUNC, share_lock);
+    errsh |= curl_share_setopt(curl.share, CURLSHOPT_UNLOCKFUNC, share_unlock);
+    errsh |= curl_share_setopt(curl.share, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS);
+    if (errsh != 0) {
+        curl_share_cleanup(curl.share);
+        curl_global_cleanup();
+        errno = EIO;
+        return -1;
+    }
+
+    if ((auth = getenv("HTS_AUTH_LOCATION")) != NULL) {
+        curl.auth_path = strdup(auth);
+        curl.auth_map = kh_init(auth_map);
+        if (!curl.auth_path || !curl.auth_map) {
+            int save_errno = errno;
+            free(curl.auth_path);
+            kh_destroy(auth_map, curl.auth_map);
+            curl_share_cleanup(curl.share);
+            curl_global_cleanup();
+            errno = save_errno;
+            return -1;
+        }
+    }
+    if ((auth = getenv("HTS_ALLOW_UNENCRYPTED_AUTHORIZATION_HEADER")) != NULL
+        && strcmp(auth, "I understand the risks") == 0) {
+        curl.allow_unencrypted_auth_header = 1;
+    }
+
+    info = curl_version_info(CURLVERSION_NOW);
+    ksprintf(&curl.useragent, "htslib/%s libcurl/%s", version, info->version);
+
+    self->name = "libcurl";
+    self->destroy = libcurl_exit;
+
+    for (protocol = info->protocols; *protocol; protocol++)
+        hfile_add_scheme_handler(*protocol, &handler);
+    return 0;
+}
diff --git a/ext/htslib/hfile_s3.c b/ext/htslib/hfile_s3.c
new file mode 100644
index 0000000..c7c52e6
--- /dev/null
+++ b/ext/htslib/hfile_s3.c
@@ -0,0 +1,1442 @@
+/*  hfile_s3.c -- Amazon S3 backend for low-level file streams.
+
+    Copyright (C) 2015-2017, 2019-2024 Genome Research Ltd.
+
+    Author: John Marshall <jm18@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+#include <time.h>
+
+#include <errno.h>
+
+#include "hfile_internal.h"
+#ifdef ENABLE_PLUGINS
+#include "version.h"
+#endif
+#include "htslib/hts.h"  // for hts_version() and hts_verbose
+#include "htslib/kstring.h"
+#include "hts_time_funcs.h"
+
+typedef struct s3_auth_data {
+    kstring_t id;
+    kstring_t token;
+    kstring_t secret;
+    kstring_t region;
+    kstring_t canonical_query_string;
+    kstring_t user_query_string;
+    kstring_t host;
+    kstring_t profile;
+    enum {s3_auto, s3_virtual, s3_path} url_style;
+    time_t creds_expiry_time;
+    char *bucket;
+    kstring_t auth_hdr;
+    time_t auth_time;
+    char date[40];
+    char date_long[17];
+    char date_short[9];
+    kstring_t date_html;
+    char mode;
+    char *headers[5];
+    int refcount;
+} s3_auth_data;
+
+#define AUTH_LIFETIME 60  // Regenerate auth headers if older than this
+#define CREDENTIAL_LIFETIME 60 // Seconds before expiry to reread credentials
+
+#if defined HAVE_COMMONCRYPTO
+
+#include <CommonCrypto/CommonHMAC.h>
+
+#define DIGEST_BUFSIZ CC_SHA1_DIGEST_LENGTH
+#define SHA256_DIGEST_BUFSIZE CC_SHA256_DIGEST_LENGTH
+#define HASH_LENGTH_SHA256 (SHA256_DIGEST_BUFSIZE * 2) + 1
+
+static size_t
+s3_sign(unsigned char *digest, kstring_t *key, kstring_t *message)
+{
+    CCHmac(kCCHmacAlgSHA1, key->s, key->l, message->s, message->l, digest);
+    return CC_SHA1_DIGEST_LENGTH;
+}
+
+
+static void s3_sha256(const unsigned char *in, size_t length, unsigned char *out) {
+    CC_SHA256(in, length, out);
+}
+
+
+static void s3_sign_sha256(const void *key, int key_len, const unsigned char *d, int n, unsigned char *md, unsigned int *md_len) {
+    CCHmac(kCCHmacAlgSHA256, key, key_len, d, n, md);
+    *md_len = CC_SHA256_DIGEST_LENGTH;
+}
+
+
+#elif defined HAVE_HMAC
+
+#include <openssl/hmac.h>
+#include <openssl/sha.h>
+
+#define DIGEST_BUFSIZ EVP_MAX_MD_SIZE
+#define SHA256_DIGEST_BUFSIZE SHA256_DIGEST_LENGTH
+#define HASH_LENGTH_SHA256 (SHA256_DIGEST_BUFSIZE * 2) + 1
+
+static size_t
+s3_sign(unsigned char *digest, kstring_t *key, kstring_t *message)
+{
+    unsigned int len;
+    HMAC(EVP_sha1(), key->s, key->l,
+         (unsigned char *) message->s, message->l, digest, &len);
+    return len;
+}
+
+
+static void s3_sha256(const unsigned char *in, size_t length, unsigned char *out) {
+    SHA256(in, length, out);
+}
+
+
+static void s3_sign_sha256(const void *key, int key_len, const unsigned char *d, int n, unsigned char *md, unsigned int *md_len) {
+    HMAC(EVP_sha256(), key, key_len, d, n, md, md_len);
+}
+
+#else
+#error No HMAC() routine found by configure
+#endif
+
+static void
+urldecode_kput(const char *s, int len, kstring_t *str)
+{
+    char buf[3];
+    int i = 0;
+
+    while (i < len)
+        if (s[i] == '%' && i+2 < len) {
+            buf[0] = s[i+1], buf[1] = s[i+2], buf[2] = '\0';
+            kputc(strtol(buf, NULL, 16), str);
+            i += 3;
+        }
+        else kputc(s[i++], str);
+}
+
+static void base64_kput(const unsigned char *data, size_t len, kstring_t *str)
+{
+    static const char base64[] =
+        "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+
+    size_t i = 0;
+    unsigned x = 0;
+    int bits = 0, pad = 0;
+
+    while (bits || i < len) {
+        if (bits < 6) {
+            x <<= 8, bits += 8;
+            if (i < len) x |= data[i++];
+            else pad++;
+        }
+
+        bits -= 6;
+        kputc(base64[(x >> bits) & 63], str);
+    }
+
+    str->l -= pad;
+    kputsn("==", pad, str);
+}
+
+static int is_dns_compliant(const char *s0, const char *slim, int is_https)
+{
+    int has_nondigit = 0, len = 0;
+    const char *s;
+
+    for (s = s0; s < slim; len++, s++)
+        if (islower_c(*s))
+            has_nondigit = 1;
+        else if (*s == '-') {
+            has_nondigit = 1;
+            if (s == s0 || s+1 == slim) return 0;
+        }
+        else if (isdigit_c(*s))
+            ;
+        else if (*s == '.') {
+            if (is_https) return 0;
+            if (s == s0 || ! isalnum_c(s[-1])) return 0;
+            if (s+1 == slim || ! isalnum_c(s[1])) return 0;
+        }
+        else return 0;
+
+    return has_nondigit && len >= 3 && len <= 63;
+}
+
+static FILE *expand_tilde_open(const char *fname, const char *mode)
+{
+    FILE *fp;
+
+    if (strncmp(fname, "~/", 2) == 0) {
+        kstring_t full_fname = { 0, 0, NULL };
+        const char *home = getenv("HOME");
+        if (! home) return NULL;
+
+        kputs(home, &full_fname);
+        kputs(&fname[1], &full_fname);
+
+        fp = fopen(full_fname.s, mode);
+        free(full_fname.s);
+    }
+    else
+        fp = fopen(fname, mode);
+
+    return fp;
+}
+
+static void parse_ini(const char *fname, const char *section, ...)
+{
+    kstring_t line = { 0, 0, NULL };
+    int active = 1;  // Start active, so global properties are accepted
+    char *s;
+
+    FILE *fp = expand_tilde_open(fname, "r");
+    if (fp == NULL) return;
+
+    while (line.l = 0, kgetline(&line, (kgets_func *) fgets, fp) >= 0)
+        if (line.s[0] == '[' && (s = strchr(line.s, ']')) != NULL) {
+            *s = '\0';
+            active = (strcmp(&line.s[1], section) == 0);
+        }
+        else if (active && (s = strpbrk(line.s, ":=")) != NULL) {
+            const char *key = line.s, *value = &s[1], *akey;
+            va_list args;
+
+            while (isspace_c(*key)) key++;
+            while (s > key && isspace_c(s[-1])) s--;
+            *s = '\0';
+
+            while (isspace_c(*value)) value++;
+            while (line.l > 0 && isspace_c(line.s[line.l-1]))
+                line.s[--line.l] = '\0';
+
+            va_start(args, section);
+            while ((akey = va_arg(args, const char *)) != NULL) {
+                kstring_t *avar = va_arg(args, kstring_t *);
+                if (strcmp(key, akey) == 0) {
+                    avar->l = 0;
+                    kputs(value, avar);
+                    break; }
+            }
+            va_end(args);
+        }
+
+    fclose(fp);
+    free(line.s);
+}
+
+static void parse_simple(const char *fname, kstring_t *id, kstring_t *secret)
+{
+    kstring_t text = { 0, 0, NULL };
+    char *s;
+    size_t len;
+
+    FILE *fp = expand_tilde_open(fname, "r");
+    if (fp == NULL) return;
+
+    while (kgetline(&text, (kgets_func *) fgets, fp) >= 0)
+        kputc(' ', &text);
+    fclose(fp);
+
+    s = text.s;
+    while (isspace_c(*s)) s++;
+    kputsn(s, len = strcspn(s, " \t"), id);
+
+    s += len;
+    while (isspace_c(*s)) s++;
+    kputsn(s, strcspn(s, " \t"), secret);
+
+    free(text.s);
+}
+
+static int copy_auth_headers(s3_auth_data *ad, char ***hdrs) {
+    char **hdr = &ad->headers[0];
+    int idx = 0;
+    *hdrs = hdr;
+
+    hdr[idx] = strdup(ad->date);
+    if (!hdr[idx]) return -1;
+    idx++;
+
+    if (ad->token.l) {
+        kstring_t token_hdr = KS_INITIALIZE;
+        kputs("X-Amz-Security-Token: ", &token_hdr);
+        kputs(ad->token.s, &token_hdr);
+        if (token_hdr.s) {
+            hdr[idx++] = token_hdr.s;
+        } else {
+            goto fail;
+        }
+    }
+
+    if (ad->auth_hdr.l) {
+        hdr[idx] = strdup(ad->auth_hdr.s);
+        if (!hdr[idx]) goto fail;
+        idx++;
+    }
+
+    hdr[idx] = NULL;
+    return 0;
+
+ fail:
+    for (--idx; idx >= 0; --idx)
+        free(hdr[idx]);
+    return -1;
+}
+
+static void free_auth_data(s3_auth_data *ad) {
+    if (ad->refcount > 0) {
+        --ad->refcount;
+        return;
+    }
+    free(ad->profile.s);
+    free(ad->id.s);
+    free(ad->token.s);
+    free(ad->secret.s);
+    free(ad->region.s);
+    free(ad->canonical_query_string.s);
+    free(ad->user_query_string.s);
+    free(ad->host.s);
+    free(ad->bucket);
+    free(ad->auth_hdr.s);
+    free(ad->date_html.s);
+    free(ad);
+}
+
+static time_t parse_rfc3339_date(kstring_t *datetime)
+{
+    int offset = 0;
+    time_t when;
+    int num;
+    char should_be_t = '\0', timezone[10] = { '\0' };
+    unsigned int year, mon, day, hour, min, sec;
+
+    if (!datetime->s)
+        return 0;
+
+    // It should be possible to do this with strptime(), but it seems
+    // to not get on with our feature definitions.
+    num = sscanf(datetime->s, "%4u-%2u-%2u%c%2u:%2u:%2u%9s",
+                 &year, &mon, &day, &should_be_t, &hour, &min, &sec, timezone);
+    if (num < 8)
+        return 0;
+    if (should_be_t != 'T' && should_be_t != 't' && should_be_t != ' ')
+        return 0;
+    struct tm parsed = { sec, min, hour, day, mon - 1, year - 1900, 0, 0, 0 };
+
+    switch (timezone[0]) {
+      case 'Z':
+      case 'z':
+      case '\0':
+          break;
+      case '+':
+      case '-': {
+          unsigned hr_off, min_off;
+          if (sscanf(timezone + 1, "%2u:%2u", &hr_off, &min_off)) {
+              if (hr_off < 24 && min_off <= 60) {
+                  offset = ((hr_off * 60 + min_off)
+                            * (timezone[0] == '+' ? -60 : 60));
+              }
+          }
+          break;
+      }
+      default:
+          return 0;
+    }
+
+    when = hts_time_gm(&parsed);
+    return when >= 0 ? when + offset : 0;
+}
+
+static void refresh_auth_data(s3_auth_data *ad) {
+    // Basically a copy of the AWS_SHARED_CREDENTIALS_FILE part of
+    // setup_auth_data(), but this only reads the authorisation parts.
+    const char *v = getenv("AWS_SHARED_CREDENTIALS_FILE");
+    kstring_t expiry_time = KS_INITIALIZE;
+    parse_ini(v? v : "~/.aws/credentials", ad->profile.s,
+              "aws_access_key_id", &ad->id,
+              "aws_secret_access_key", &ad->secret,
+              "aws_session_token", &ad->token,
+              "expiry_time", &expiry_time);
+    if (expiry_time.l) {
+        ad->creds_expiry_time = parse_rfc3339_date(&expiry_time);
+    }
+    ks_free(&expiry_time);
+}
+
+static int auth_header_callback(void *ctx, char ***hdrs) {
+    s3_auth_data *ad = (s3_auth_data *) ctx;
+
+    time_t now = time(NULL);
+#ifdef HAVE_GMTIME_R
+    struct tm tm_buffer;
+    struct tm *tm = gmtime_r(&now, &tm_buffer);
+#else
+    struct tm *tm = gmtime(&now);
+#endif
+    kstring_t message = { 0, 0, NULL };
+    unsigned char digest[DIGEST_BUFSIZ];
+    size_t digest_len;
+
+    if (!hdrs) { // Closing connection
+        free_auth_data(ad);
+        return 0;
+    }
+
+    if (ad->creds_expiry_time > 0
+        && ad->creds_expiry_time - now < CREDENTIAL_LIFETIME) {
+        refresh_auth_data(ad);
+    } else if (now - ad->auth_time < AUTH_LIFETIME) {
+        // Last auth string should still be valid
+        *hdrs = NULL;
+        return 0;
+    }
+
+    strftime(ad->date, sizeof(ad->date), "Date: %a, %d %b %Y %H:%M:%S GMT", tm);
+    if (!ad->id.l || !ad->secret.l) {
+        ad->auth_time = now;
+        return copy_auth_headers(ad, hdrs);
+    }
+
+    if (ksprintf(&message, "%s\n\n\n%s\n%s%s%s%s",
+                 ad->mode == 'r' ? "GET" : "PUT", ad->date + 6,
+                 ad->token.l ? "x-amz-security-token:" : "",
+                 ad->token.l ? ad->token.s : "",
+                 ad->token.l ? "\n" : "",
+                 ad->bucket) < 0) {
+        return -1;
+    }
+
+    digest_len = s3_sign(digest, &ad->secret, &message);
+    ad->auth_hdr.l = 0;
+    if (ksprintf(&ad->auth_hdr, "Authorization: AWS %s:", ad->id.s) < 0)
+        goto fail;
+    base64_kput(digest, digest_len, &ad->auth_hdr);
+
+    free(message.s);
+    ad->auth_time = now;
+    return copy_auth_headers(ad, hdrs);
+
+ fail:
+    free(message.s);
+    return -1;
+}
+
+
+/* like a escape path but for query strings '=' and '&' are untouched */
+static char *escape_query(const char *qs) {
+    size_t i, j = 0, length, alloced;
+    char *escaped;
+
+    length = strlen(qs);
+    alloced = length * 3 + 1;
+    if ((escaped = malloc(alloced)) == NULL) {
+        return NULL;
+    }
+
+    for (i = 0; i < length; i++) {
+        int c = qs[i];
+
+        if ((c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') ||
+             c == '_' || c == '-' || c == '~' || c == '.' || c == '/' || c == '=' || c == '&') {
+            escaped[j++] = c;
+        } else {
+            snprintf(escaped + j, alloced - j, "%%%02X", c);
+            j += 3;
+        }
+    }
+
+    escaped[j] = '\0';
+
+    return escaped;
+}
+
+
+static char *escape_path(const char *path) {
+    size_t i, j = 0, length, alloced;
+    char *escaped;
+
+    length = strlen(path);
+    alloced = length * 3 + 1;
+
+    if ((escaped = malloc(alloced)) == NULL) {
+        return NULL;
+    }
+
+    for (i = 0; i < length; i++) {
+        int c = path[i];
+
+        if (c == '?') break; // don't escape ? or beyond
+
+        if ((c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') ||
+             c == '_' || c == '-' || c == '~' || c == '.' || c == '/') {
+            escaped[j++] = c;
+        } else {
+            snprintf(escaped + j, alloced - j, "%%%02X", c);
+            j += 3;
+        }
+    }
+
+    if (i != length) {
+        // in the case of a '?' copy the rest of the path across unchanged
+        strcpy(escaped + j, path + i);
+    } else {
+        escaped[j] = '\0';
+    }
+
+    return escaped;
+}
+
+
+static int is_escaped(const char *str) {
+    const char *c = str;
+    int escaped = 0;
+    int needs_escape = 0;
+
+    while (*c != '\0') {
+        if (*c == '%' && c[1] != '\0' && c[2] != '\0') {
+            if (isxdigit_c(c[1]) && isxdigit_c(c[2])) {
+                escaped = 1;
+                c += 3;
+                continue;
+            } else {
+                // only escaped if all % signs are escaped
+                escaped = 0;
+            }
+        }
+        if (!((*c >= '0' && *c <= '9') || (*c >= 'A' && *c <= 'Z')
+              || (*c >= 'a' && *c <= 'z') ||
+              *c == '_' || *c == '-' || *c == '~' || *c == '.' || *c == '/')) {
+            needs_escape = 1;
+        }
+        c++;
+    }
+
+    return escaped || !needs_escape;
+}
+
+static int redirect_endpoint_callback(void *auth, long response,
+                                      kstring_t *header, kstring_t *url) {
+    s3_auth_data *ad = (s3_auth_data *)auth;
+    char *new_region;
+    char *end;
+    int ret = -1;
+
+    // get the new region from the reply header
+    if ((new_region = strstr(header->s, "x-amz-bucket-region: "))) {
+
+        new_region += strlen("x-amz-bucket-region: ");
+        end = new_region;
+
+        while (isalnum_c(*end) || ispunct_c(*end)) end++;
+
+        *end = 0;
+
+        if (strstr(ad->host.s, "amazonaws.com")) {
+            ad->region.l = 0;
+            kputs(new_region, &ad->region);
+
+            ad->host.l = 0;
+
+            if (ad->url_style == s3_path) {
+                // Path style https://s3.{region-code}.amazonaws.com/{bucket-name}/{key-name}
+                ksprintf(&ad->host, "s3.%s.amazonaws.com", new_region);
+            } else {
+                // Virtual https://{bucket-name}.s3.{region-code}.amazonaws.com/{key-name}
+                // Extract the {bucket-name} from {ad->host} to include in subdomain
+                kstring_t url_prefix = KS_INITIALIZE;
+                kputsn(ad->host.s, strcspn(ad->host.s, "."), &url_prefix);
+
+                ksprintf(&ad->host, "%s.s3.%s.amazonaws.com", url_prefix.s, new_region);
+                free(url_prefix.s);
+            }
+            if (ad->region.l && ad->host.l) {
+               int e = 0;
+               url->l = 0;
+               e |= kputs("https://", url) < 0;
+               e |= kputs(ad->host.s, url) < 0;
+               e |= kputsn(ad->bucket, strlen(ad->bucket), url) < 0;
+
+               if (!e)
+                   ret = 0;
+            }
+            if (ad->user_query_string.l) {
+                kputc('?', url);
+                kputsn(ad->user_query_string.s, ad->user_query_string.l, url);
+            }
+        }
+    }
+
+    return ret;
+}
+
+static s3_auth_data * setup_auth_data(const char *s3url, const char *mode,
+                                      int sigver, kstring_t *url)
+{
+    s3_auth_data *ad = calloc(1, sizeof(*ad));
+    const char *bucket, *path;
+    char *escaped = NULL;
+    size_t url_path_pos;
+    ptrdiff_t bucket_len;
+    int is_https = 1, dns_compliant;
+    char *query_start;
+
+    if (!ad)
+        return NULL;
+    ad->mode = strchr(mode, 'r') ? 'r' : 'w';
+    ad->url_style = s3_auto;
+
+    // Our S3 URL format is s3[+SCHEME]://[ID[:SECRET[:TOKEN]]@]BUCKET/PATH
+
+    if (s3url[2] == '+') {
+        bucket = strchr(s3url, ':') + 1;
+        if (bucket == NULL) {
+            free(ad);
+            return NULL;
+        }
+        kputsn(&s3url[3], bucket - &s3url[3], url);
+        is_https = strncmp(url->s, "https:", 6) == 0;
+    }
+    else {
+        kputs("https:", url);
+        bucket = &s3url[3];
+    }
+    while (*bucket == '/') kputc(*bucket++, url);
+
+    path = bucket + strcspn(bucket, "/?#@");
+
+    if (*path == '@') {
+        const char *colon = strpbrk(bucket, ":@");
+        if (*colon != ':') {
+            urldecode_kput(bucket, colon - bucket, &ad->profile);
+        }
+        else {
+            const char *colon2 = strpbrk(&colon[1], ":@");
+            urldecode_kput(bucket, colon - bucket, &ad->id);
+            urldecode_kput(&colon[1], colon2 - &colon[1], &ad->secret);
+            if (*colon2 == ':')
+                urldecode_kput(&colon2[1], path - &colon2[1], &ad->token);
+        }
+
+        bucket = &path[1];
+        path = bucket + strcspn(bucket, "/?#");
+    }
+    else {
+        // If the URL has no ID[:SECRET]@, consider environment variables.
+        const char *v;
+        if ((v = getenv("AWS_ACCESS_KEY_ID")) != NULL) kputs(v, &ad->id);
+        if ((v = getenv("AWS_SECRET_ACCESS_KEY")) != NULL) kputs(v, &ad->secret);
+        if ((v = getenv("AWS_SESSION_TOKEN")) != NULL) kputs(v, &ad->token);
+        if ((v = getenv("AWS_DEFAULT_REGION")) != NULL) kputs(v, &ad->region);
+        if ((v = getenv("HTS_S3_HOST")) != NULL) kputs(v, &ad->host);
+
+        if ((v = getenv("AWS_DEFAULT_PROFILE")) != NULL) kputs(v, &ad->profile);
+        else if ((v = getenv("AWS_PROFILE")) != NULL) kputs(v, &ad->profile);
+        else kputs("default", &ad->profile);
+
+        if ((v = getenv("HTS_S3_ADDRESS_STYLE")) != NULL) {
+            if (strcasecmp(v, "virtual") == 0) {
+                ad->url_style = s3_virtual;
+            } else if (strcasecmp(v, "path") == 0) {
+                ad->url_style = s3_path;
+            }
+        }
+    }
+
+    if (ad->id.l == 0) {
+        kstring_t url_style = KS_INITIALIZE;
+        kstring_t expiry_time = KS_INITIALIZE;
+        const char *v = getenv("AWS_SHARED_CREDENTIALS_FILE");
+        parse_ini(v? v : "~/.aws/credentials", ad->profile.s,
+                  "aws_access_key_id", &ad->id,
+                  "aws_secret_access_key", &ad->secret,
+                  "aws_session_token", &ad->token,
+                  "region", &ad->region,
+                  "addressing_style", &url_style,
+                  "expiry_time", &expiry_time,
+                  NULL);
+
+        if (url_style.l) {
+            if (strcmp(url_style.s, "virtual") == 0) {
+                ad->url_style = s3_virtual;
+            } else if (strcmp(url_style.s, "path") == 0) {
+                ad->url_style = s3_path;
+            } else {
+                ad->url_style = s3_auto;
+            }
+        }
+        if (expiry_time.l) {
+            // Not a real part of the AWS configuration file, but it allows
+            // support for short-term credentials like those for the IAM
+            // service.  The botocore library uses the key "expiry_time"
+            // internally for this purpose.
+            // See https://github.com/boto/botocore/blob/develop/botocore/credentials.py
+            ad->creds_expiry_time = parse_rfc3339_date(&expiry_time);
+        }
+
+        ks_free(&url_style);
+        ks_free(&expiry_time);
+    }
+
+    if (ad->id.l == 0) {
+        kstring_t url_style = KS_INITIALIZE;
+        const char *v = getenv("HTS_S3_S3CFG");
+        parse_ini(v? v : "~/.s3cfg", ad->profile.s, "access_key", &ad->id,
+                  "secret_key", &ad->secret, "access_token", &ad->token,
+                  "host_base", &ad->host,
+                  "bucket_location", &ad->region,
+                  "host_bucket", &url_style,
+                  NULL);
+
+        if (url_style.l) {
+            // Conforming to s3cmd's GitHub PR#416, host_bucket without the "%(bucket)s" string
+            // indicates use of path style adressing.
+            if (strstr(url_style.s, "%(bucket)s") == NULL) {
+                ad->url_style = s3_path;
+            } else {
+                ad->url_style = s3_auto;
+            }
+        }
+
+        ks_free(&url_style);
+    }
+
+    if (ad->id.l == 0)
+        parse_simple("~/.awssecret", &ad->id, &ad->secret);
+
+
+    // if address_style is set, force the dns_compliant setting
+    if (ad->url_style == s3_virtual) {
+        dns_compliant = 1;
+    } else if (ad->url_style == s3_path) {
+        dns_compliant = 0;
+    } else {
+        dns_compliant = is_dns_compliant(bucket, path, is_https);
+    }
+
+    if (ad->host.l == 0)
+        kputs("s3.amazonaws.com", &ad->host);
+
+    if (!dns_compliant && ad->region.l > 0
+        && strcmp(ad->host.s, "s3.amazonaws.com") == 0) {
+        // Can avoid a redirection by including the region in the host name
+        // (assuming the right one has been specified)
+        ad->host.l = 0;
+        ksprintf(&ad->host, "s3.%s.amazonaws.com", ad->region.s);
+    }
+
+    if (ad->region.l == 0)
+        kputs("us-east-1", &ad->region);
+
+    if (!is_escaped(path)) {
+        escaped = escape_path(path);
+        if (escaped == NULL) {
+            goto error;
+        }
+    }
+
+    bucket_len = path - bucket;
+
+    // Use virtual hosted-style access if possible, otherwise path-style.
+    if (dns_compliant) {
+        size_t url_host_pos = url->l;
+        // Append "bucket.host" to url
+        kputsn_(bucket, bucket_len, url);
+        kputc('.', url);
+        kputsn(ad->host.s, ad->host.l, url);
+        url_path_pos = url->l;
+
+        if (sigver == 4) {
+            // Copy back to ad->host to use when making the signature
+            ad->host.l = 0;
+            kputsn(url->s + url_host_pos, url->l - url_host_pos, &ad->host);
+        }
+    }
+    else {
+        // Append "host/bucket" to url
+        kputsn(ad->host.s, ad->host.l, url);
+        url_path_pos = url->l;
+        kputc('/', url);
+        kputsn(bucket, bucket_len, url);
+    }
+
+    kputs(escaped == NULL ? path : escaped, url);
+
+    if (sigver == 4 || !dns_compliant) {
+        ad->bucket = malloc(url->l - url_path_pos + 1);
+        if (ad->bucket == NULL) {
+            goto error;
+        }
+        memcpy(ad->bucket, url->s + url_path_pos, url->l - url_path_pos + 1);
+    }
+    else {
+        ad->bucket = malloc(url->l - url_path_pos + bucket_len + 2);
+        if (ad->bucket == NULL) {
+            goto error;
+        }
+        ad->bucket[0] = '/';
+        memcpy(ad->bucket + 1, bucket, bucket_len);
+        memcpy(ad->bucket + bucket_len + 1,
+               url->s + url_path_pos, url->l - url_path_pos + 1);
+    }
+
+    // write any query strings to its own place to use later
+    if ((query_start = strchr(ad->bucket, '?'))) {
+        kputs(query_start + 1, &ad->user_query_string);
+        *query_start = 0;
+    }
+
+    free(escaped);
+
+    return ad;
+
+ error:
+    free(escaped);
+    free_auth_data(ad);
+    return NULL;
+}
+
+static hFILE * s3_rewrite(const char *s3url, const char *mode, va_list *argsp)
+{
+    kstring_t url = { 0, 0, NULL };
+    s3_auth_data *ad = setup_auth_data(s3url, mode, 2, &url);
+
+    if (!ad)
+        return NULL;
+
+    hFILE *fp = hopen(url.s, mode, "va_list", argsp,
+                      "httphdr_callback", auth_header_callback,
+                      "httphdr_callback_data", ad,
+                      "redirect_callback", redirect_endpoint_callback,
+                      "redirect_callback_data", ad,
+                      NULL);
+    if (!fp) goto fail;
+
+    free(url.s);
+    return fp;
+
+ fail:
+    free(url.s);
+    free_auth_data(ad);
+    return NULL;
+}
+
+/***************************************************************
+
+AWS S3 sig version 4 writing code
+
+****************************************************************/
+
+static void hash_string(char *in, size_t length, char *out, size_t out_len) {
+    unsigned char hashed[SHA256_DIGEST_BUFSIZE];
+    int i, j;
+
+    s3_sha256((const unsigned char *)in, length, hashed);
+
+    for (i = 0, j = 0; i < SHA256_DIGEST_BUFSIZE; i++, j+= 2) {
+        snprintf(out + j, out_len - j, "%02x", hashed[i]);
+    }
+}
+
+static void ksinit(kstring_t *s) {
+    s->l = 0;
+    s->m = 0;
+    s->s = NULL;
+}
+
+
+static void ksfree(kstring_t *s) {
+    free(s->s);
+    ksinit(s);
+}
+
+
+static int make_signature(s3_auth_data *ad, kstring_t *string_to_sign, char *signature_string, size_t sig_string_len) {
+    unsigned char date_key[SHA256_DIGEST_BUFSIZE];
+    unsigned char date_region_key[SHA256_DIGEST_BUFSIZE];
+    unsigned char date_region_service_key[SHA256_DIGEST_BUFSIZE];
+    unsigned char signing_key[SHA256_DIGEST_BUFSIZE];
+    unsigned char signature[SHA256_DIGEST_BUFSIZE];
+
+    const unsigned char service[] = "s3";
+    const unsigned char request[] = "aws4_request";
+
+    kstring_t secret_access_key = KS_INITIALIZE;
+    unsigned int len;
+    unsigned int i, j;
+
+    ksprintf(&secret_access_key, "AWS4%s", ad->secret.s);
+
+    if (secret_access_key.l == 0) {
+        return -1;
+    }
+
+    s3_sign_sha256(secret_access_key.s, secret_access_key.l, (const unsigned char *)ad->date_short, strlen(ad->date_short), date_key, &len);
+    s3_sign_sha256(date_key, len, (const unsigned char *)ad->region.s, ad->region.l, date_region_key, &len);
+    s3_sign_sha256(date_region_key, len, service, 2, date_region_service_key, &len);
+    s3_sign_sha256(date_region_service_key, len, request, 12, signing_key, &len);
+    s3_sign_sha256(signing_key, len, (const unsigned char *)string_to_sign->s, string_to_sign->l, signature, &len);
+
+    for (i = 0, j = 0; i < len; i++, j+= 2) {
+        snprintf(signature_string + j, sig_string_len - j, "%02x", signature[i]);
+    }
+
+    ksfree(&secret_access_key);
+
+    return 0;
+}
+
+
+static int make_authorisation(s3_auth_data *ad, char *http_request, char *content, kstring_t *auth) {
+    kstring_t signed_headers = KS_INITIALIZE;
+    kstring_t canonical_headers = KS_INITIALIZE;
+    kstring_t canonical_request = KS_INITIALIZE;
+    kstring_t scope = KS_INITIALIZE;
+    kstring_t string_to_sign = KS_INITIALIZE;
+    char cr_hash[HASH_LENGTH_SHA256];
+    char signature_string[HASH_LENGTH_SHA256];
+    int ret = -1;
+
+
+    if (!ad->token.l) {
+        kputs("host;x-amz-content-sha256;x-amz-date", &signed_headers);
+    } else {
+        kputs("host;x-amz-content-sha256;x-amz-date;x-amz-security-token", &signed_headers);
+    }
+
+    if (signed_headers.l == 0) {
+        return -1;
+    }
+
+
+    if (!ad->token.l) {
+        ksprintf(&canonical_headers, "host:%s\nx-amz-content-sha256:%s\nx-amz-date:%s\n",
+        ad->host.s, content, ad->date_long);
+    } else {
+        ksprintf(&canonical_headers, "host:%s\nx-amz-content-sha256:%s\nx-amz-date:%s\nx-amz-security-token:%s\n",
+        ad->host.s, content, ad->date_long, ad->token.s);
+    }
+
+    if (canonical_headers.l == 0) {
+        goto cleanup;
+    }
+
+    // bucket == canonical_uri
+    ksprintf(&canonical_request, "%s\n%s\n%s\n%s\n%s\n%s",
+        http_request, ad->bucket, ad->canonical_query_string.s,
+        canonical_headers.s, signed_headers.s, content);
+
+    if (canonical_request.l == 0) {
+        goto cleanup;
+    }
+
+    hash_string(canonical_request.s, canonical_request.l, cr_hash, sizeof(cr_hash));
+
+    ksprintf(&scope, "%s/%s/s3/aws4_request", ad->date_short, ad->region.s);
+
+    if (scope.l == 0) {
+        goto cleanup;
+    }
+
+    ksprintf(&string_to_sign, "AWS4-HMAC-SHA256\n%s\n%s\n%s", ad->date_long, scope.s, cr_hash);
+
+    if (string_to_sign.l == 0) {
+        goto cleanup;
+    }
+
+    if (make_signature(ad, &string_to_sign, signature_string, sizeof(signature_string))) {
+        goto cleanup;
+    }
+
+    ksprintf(auth, "Authorization: AWS4-HMAC-SHA256 Credential=%s/%s/%s/s3/aws4_request,SignedHeaders=%s,Signature=%s",
+                ad->id.s, ad->date_short, ad->region.s, signed_headers.s, signature_string);
+
+    if (auth->l == 0) {
+        goto cleanup;
+    }
+
+    ret = 0;
+
+ cleanup:
+    ksfree(&signed_headers);
+    ksfree(&canonical_headers);
+    ksfree(&canonical_request);
+    ksfree(&scope);
+    ksfree(&string_to_sign);
+
+    return ret;
+}
+
+
+static int update_time(s3_auth_data *ad, time_t now) {
+    int ret = -1;
+#ifdef HAVE_GMTIME_R
+    struct tm tm_buffer;
+    struct tm *tm = gmtime_r(&now, &tm_buffer);
+#else
+    struct tm *tm = gmtime(&now);
+#endif
+
+    if (now - ad->auth_time > AUTH_LIFETIME) {
+        // update timestamp
+        ad->auth_time = now;
+
+        if (strftime(ad->date_long, 17, "%Y%m%dT%H%M%SZ", tm) != 16) {
+            return -1;
+        }
+
+        if (strftime(ad->date_short, 9, "%Y%m%d", tm) != 8) {
+            return -1;;
+        }
+
+        ad->date_html.l = 0;
+        ksprintf(&ad->date_html, "x-amz-date: %s", ad->date_long);
+    }
+
+    if (ad->date_html.l) ret = 0;
+
+    return ret;
+}
+
+
+static int query_cmp(const void *p1, const void *p2) {
+    char **q1 = (char **)p1;
+    char **q2 = (char **)p2;
+
+    return strcmp(*q1, *q2);
+}
+
+
+/* Query strings must be in alphabetical order for authorisation */
+
+static int order_query_string(kstring_t *qs) {
+    int *query_offset = NULL;
+    int num_queries, i;
+    char **queries = NULL;
+    kstring_t ordered = KS_INITIALIZE;
+    char *escaped = NULL;
+    int ret = -1;
+
+    if ((query_offset = ksplit(qs, '&', &num_queries)) == NULL) {
+        return -1;
+    }
+
+    if ((queries = malloc(num_queries * sizeof(char*))) == NULL)
+        goto err;
+
+    for (i = 0; i < num_queries; i++) {
+        queries[i] = qs->s + query_offset[i];
+    }
+
+    qsort(queries, num_queries, sizeof(char *), query_cmp);
+
+    for (i = 0; i < num_queries; i++) {
+        if (i) {
+            kputs("&", &ordered);
+        }
+
+        kputs(queries[i], &ordered);
+    }
+
+    if ((escaped = escape_query(ordered.s)) == NULL)
+        goto err;
+
+    qs->l = 0;
+    kputs(escaped, qs);
+
+    ret = 0;
+ err:
+    free(ordered.s);
+    free(queries);
+    free(query_offset);
+    free(escaped);
+
+    return ret;
+}
+
+
+static int write_authorisation_callback(void *auth, char *request, kstring_t *content, char *cqs,
+                                        kstring_t *hash, kstring_t *auth_str, kstring_t *date,
+                                        kstring_t *token, int uqs) {
+    s3_auth_data *ad = (s3_auth_data *)auth;
+    char content_hash[HASH_LENGTH_SHA256];
+    time_t now;
+
+    if (request == NULL) {
+        // signal to free auth data
+        free_auth_data(ad);
+        return 0;
+    }
+
+    now = time(NULL);
+
+    if (update_time(ad, now)) {
+        return -1;
+    }
+    if (ad->creds_expiry_time > 0
+        && ad->creds_expiry_time - now < CREDENTIAL_LIFETIME) {
+        refresh_auth_data(ad);
+    }
+
+    if (content) {
+        hash_string(content->s, content->l, content_hash, sizeof(content_hash));
+    } else {
+        // empty hash
+        hash_string("", 0, content_hash, sizeof(content_hash));
+    }
+
+    ad->canonical_query_string.l = 0;
+    kputs(cqs, &ad->canonical_query_string);
+
+    if (ad->canonical_query_string.l == 0) {
+        return -1;
+    }
+
+    /* add a user provided query string, normally only useful on upload initiation */
+    if (uqs) {
+        kputs("&", &ad->canonical_query_string);
+        kputs(ad->user_query_string.s, &ad->canonical_query_string);
+
+        if (order_query_string(&ad->canonical_query_string)) {
+            return -1;
+        }
+    }
+
+    if (make_authorisation(ad, request, content_hash, auth_str)) {
+        return -1;
+    }
+
+    kputs(ad->date_html.s, date);
+    kputsn(content_hash, HASH_LENGTH_SHA256, hash);
+
+    if (date->l == 0 || hash->l == 0) {
+        return -1;
+    }
+
+    if (ad->token.l) {
+        ksprintf(token, "x-amz-security-token: %s", ad->token.s);
+    }
+
+    return 0;
+}
+
+
+static int v4_auth_header_callback(void *ctx, char ***hdrs) {
+    s3_auth_data *ad = (s3_auth_data *) ctx;
+    char content_hash[HASH_LENGTH_SHA256];
+    kstring_t content = KS_INITIALIZE;
+    kstring_t authorisation = KS_INITIALIZE;
+    kstring_t token_hdr = KS_INITIALIZE;
+    char *date_html = NULL;
+    time_t now;
+    int idx;
+
+    if (!hdrs) { // Closing connection
+        free_auth_data(ad);
+        return 0;
+    }
+
+    now = time(NULL);
+
+    if (update_time(ad, now)) {
+        return -1;
+    }
+
+    if (ad->creds_expiry_time > 0
+        && ad->creds_expiry_time - now < CREDENTIAL_LIFETIME) {
+        refresh_auth_data(ad);
+    }
+
+    if (!ad->id.l || !ad->secret.l) {
+        return copy_auth_headers(ad, hdrs);
+    }
+
+    hash_string("", 0, content_hash, sizeof(content_hash)); // empty hash
+
+    ad->canonical_query_string.l = 0;
+
+    if (ad->user_query_string.l > 0) {
+        kputs(ad->user_query_string.s, &ad->canonical_query_string);
+
+        if (order_query_string(&ad->canonical_query_string)) {
+            return -1;
+        }
+    } else {
+        kputs("", &ad->canonical_query_string);
+    }
+
+    if (make_authorisation(ad, "GET", content_hash, &authorisation)) {
+        return -1;
+    }
+
+    ksprintf(&content, "x-amz-content-sha256: %s", content_hash);
+    date_html = strdup(ad->date_html.s);
+
+    if (ad->token.l > 0) {
+        kputs("X-Amz-Security-Token: ", &token_hdr);
+        kputs(ad->token.s, &token_hdr);
+    }
+
+    if (content.l == 0 || date_html == NULL) {
+        ksfree(&authorisation);
+        ksfree(&content);
+        ksfree(&token_hdr);
+        free(date_html);
+        return -1;
+    }
+
+    *hdrs = &ad->headers[0];
+    idx = 0;
+    ad->headers[idx++] = ks_release(&authorisation);
+    ad->headers[idx++] = date_html;
+    ad->headers[idx++] = ks_release(&content);
+    if (token_hdr.s)
+        ad->headers[idx++] = ks_release(&token_hdr);
+    ad->headers[idx++] = NULL;
+
+    return 0;
+}
+
+static int handle_400_response(hFILE *fp, s3_auth_data *ad) {
+    // v4 signatures in virtual hosted mode return 400 Bad Request if the
+    // wrong region is used to make the signature.  The response is an xml
+    // document which includes the name of the correct region.  This can
+    // be extracted and used to generate a corrected signature.
+    // As the xml is fairly simple, go with something "good enough" instead
+    // of trying to parse it properly.
+
+    char buffer[1024], *region, *reg_end;
+    ssize_t bytes;
+
+    bytes = hread(fp, buffer, sizeof(buffer) - 1);
+    if (bytes < 0) {
+        return -1;
+    }
+    buffer[bytes] = '\0';
+    region = strstr(buffer, "<Region>");
+    if (region == NULL) {
+        return -1;
+    }
+    region += 8;
+    while (isspace((unsigned char) *region)) ++region;
+    reg_end = strchr(region, '<');
+    if (reg_end == NULL || strncmp(reg_end + 1, "/Region>", 8) != 0) {
+        return -1;
+    }
+    while (reg_end > region && isspace((unsigned char) reg_end[-1])) --reg_end;
+    ad->region.l = 0;
+    kputsn(region, reg_end - region, &ad->region);
+    if (ad->region.l == 0) {
+        return -1;
+    }
+
+    return 0;
+}
+
+static int set_region(void *adv, kstring_t *region) {
+    s3_auth_data *ad = (s3_auth_data *) adv;
+
+    ad->region.l = 0;
+    return kputsn(region->s, region->l, &ad->region) < 0;
+}
+
+static int http_status_errno(int status)
+{
+    if (status >= 500)
+        switch (status) {
+        case 501: return ENOSYS;
+        case 503: return EBUSY;
+        case 504: return ETIMEDOUT;
+        default:  return EIO;
+        }
+    else if (status >= 400)
+        switch (status) {
+        case 401: return EPERM;
+        case 403: return EACCES;
+        case 404: return ENOENT;
+        case 405: return EROFS;
+        case 407: return EPERM;
+        case 408: return ETIMEDOUT;
+        case 410: return ENOENT;
+        default:  return EINVAL;
+        }
+    else return 0;
+}
+
+static hFILE *s3_open_v4(const char *s3url, const char *mode, va_list *argsp) {
+    kstring_t url = { 0, 0, NULL };
+
+    s3_auth_data *ad = setup_auth_data(s3url, mode, 4, &url);
+    hFILE *fp = NULL;
+
+    if (ad == NULL) {
+        return NULL;
+    }
+
+    if (ad->mode == 'r') {
+        long http_response = 0;
+
+        fp = hopen(url.s, mode, "va_list", argsp,
+                   "httphdr_callback", v4_auth_header_callback,
+                   "httphdr_callback_data", ad,
+                   "redirect_callback", redirect_endpoint_callback,
+                   "redirect_callback_data", ad,
+                   "http_response_ptr", &http_response,
+                   "fail_on_error", 0,
+                   NULL);
+
+        if (fp == NULL) goto error;
+
+        if (http_response == 307) {
+            // Follow additional redirect.
+            ad->refcount = 1;
+            hclose_abruptly(fp);
+
+            url.l  = 0;
+            ksprintf(&url, "https://%s%s", ad->host.s, ad->bucket);
+
+            fp = hopen(url.s, mode, "va_list", argsp,
+                   "httphdr_callback", v4_auth_header_callback,
+                   "httphdr_callback_data", ad,
+                   "redirect_callback", redirect_endpoint_callback,
+                   "redirect_callback_data", ad,
+                   "http_response_ptr", &http_response,
+                   "fail_on_error", 0,
+                   NULL);
+        }
+
+        if (http_response == 400) {
+            ad->refcount = 1;
+            if (handle_400_response(fp, ad) != 0) {
+                goto error;
+            }
+            hclose_abruptly(fp);
+            fp = hopen(url.s, mode, "va_list", argsp,
+                       "httphdr_callback", v4_auth_header_callback,
+                       "httphdr_callback_data", ad,
+                       "redirect_callback", redirect_endpoint_callback,
+                       "redirect_callback_data", ad,
+                       NULL);
+        } else if (http_response > 400) {
+            ad->refcount = 1;
+            errno = http_status_errno(http_response);
+            goto error;
+        }
+
+        if (fp == NULL) goto error;
+    } else {
+        kstring_t final_url = KS_INITIALIZE;
+
+         // add the scheme marker
+        ksprintf(&final_url, "s3w+%s", url.s);
+
+        if(final_url.l == 0) goto error;
+
+        fp = hopen(final_url.s, mode, "va_list", argsp,
+                   "s3_auth_callback",  write_authorisation_callback,
+                   "s3_auth_callback_data", ad,
+                   "redirect_callback", redirect_endpoint_callback,
+                   "set_region_callback", set_region,
+                   NULL);
+        free(final_url.s);
+
+        if (fp == NULL) goto error;
+    }
+
+    free(url.s);
+
+    return fp;
+
+  error:
+
+    if (fp) hclose_abruptly(fp);
+    free(url.s);
+    free_auth_data(ad);
+
+    return NULL;
+}
+
+
+static hFILE *s3_open(const char *url, const char *mode)
+{
+    hFILE *fp;
+
+    kstring_t mode_colon = { 0, 0, NULL };
+    kputs(mode, &mode_colon);
+    kputc(':', &mode_colon);
+
+    if (getenv("HTS_S3_V2") == NULL) { // Force the v2 signature code
+        fp = s3_open_v4(url, mode_colon.s, NULL);
+    } else {
+        fp = s3_rewrite(url, mode_colon.s, NULL);
+    }
+
+    free(mode_colon.s);
+
+    return fp;
+}
+
+static hFILE *s3_vopen(const char *url, const char *mode_colon, va_list args0)
+{
+    hFILE *fp;
+    // Need to use va_copy() as we can only take the address of an actual
+    // va_list object, not that of a parameter whose type may have decayed.
+    va_list args;
+    va_copy(args, args0);
+
+    if (getenv("HTS_S3_V2") == NULL) { // Force the v2 signature code
+        fp = s3_open_v4(url, mode_colon, &args);
+    } else {
+        fp = s3_rewrite(url, mode_colon, &args);
+    }
+
+    va_end(args);
+    return fp;
+}
+
+int PLUGIN_GLOBAL(hfile_plugin_init,_s3)(struct hFILE_plugin *self)
+{
+    static const struct hFILE_scheme_handler handler =
+        { s3_open, hfile_always_remote, "Amazon S3", 2000 + 50, s3_vopen
+        };
+
+#ifdef ENABLE_PLUGINS
+    // Embed version string for examination via strings(1) or what(1)
+    static const char id[] = "@(#)hfile_s3 plugin (htslib)\t" HTS_VERSION_TEXT;
+    if (hts_verbose >= 9)
+        fprintf(stderr, "[M::hfile_s3.init] version %s\n", strchr(id, '\t')+1);
+#endif
+
+    self->name = "Amazon S3";
+    hfile_add_scheme_handler("s3", &handler);
+    hfile_add_scheme_handler("s3+http", &handler);
+    hfile_add_scheme_handler("s3+https", &handler);
+    return 0;
+}
diff --git a/ext/htslib/hfile_s3_write.c b/ext/htslib/hfile_s3_write.c
new file mode 100644
index 0000000..a501645
--- /dev/null
+++ b/ext/htslib/hfile_s3_write.c
@@ -0,0 +1,896 @@
+/*
+    hfile_s3_write.c - Code to handle multipart uploading to S3.
+
+    Copyright (C) 2019 Genome Research Ltd.
+
+    Author: Andrew Whitwham <aw7@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE
+
+
+S3 Multipart Upload
+-------------------
+
+There are several steps in the Mulitipart upload.
+
+
+1) Initiate Upload
+------------------
+
+Initiate the upload and get an upload ID.  This ID is used in all other steps.
+
+
+2) Upload Part
+--------------
+
+Upload a part of the data.  5Mb minimum part size (except for the last part).
+Each part is numbered and a successful upload returns an Etag header value that
+needs to used for the completion step.
+
+Step repeated till all data is uploaded.
+
+
+3) Completion
+-------------
+
+Complete the upload by sending all the part numbers along with their associated
+Etag values.
+
+
+Optional - Abort
+----------------
+
+If something goes wrong this instructs the server to delete all the partial
+uploads and abandon the upload process.
+
+
+Andrew Whitwham, January 2019
+*/
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+
+#include <stdarg.h>
+#include <stdlib.h>
+#include <string.h>
+#ifdef __MSYS__
+#include <strings.h>
+#endif
+#include <errno.h>
+#include <pthread.h>
+
+#include "hfile_internal.h"
+#ifdef ENABLE_PLUGINS
+#include "version.h"
+#endif
+#include "htslib/hts.h"
+#include "htslib/kstring.h"
+#include "htslib/khash.h"
+
+#include <curl/curl.h>
+
+#define MINIMUM_S3_WRITE_SIZE 5242880
+#define S3_MOVED_PERMANENTLY 301
+#define S3_BAD_REQUEST 400
+
+// Lets the part memory size grow to about 1Gb giving a 2.5Tb max file size.
+// Max. parts allowed by AWS is 10000, so use ceil(10000.0/9.0)
+#define EXPAND_ON 1112
+
+static struct {
+    kstring_t useragent;
+    CURLSH *share;
+    pthread_mutex_t share_lock;
+} curl = { { 0, 0, NULL }, NULL, PTHREAD_MUTEX_INITIALIZER };
+
+static void share_lock(CURL *handle, curl_lock_data data,
+                       curl_lock_access access, void *userptr) {
+    pthread_mutex_lock(&curl.share_lock);
+}
+
+static void share_unlock(CURL *handle, curl_lock_data data, void *userptr) {
+    pthread_mutex_unlock(&curl.share_lock);
+}
+
+typedef int (*s3_auth_callback) (void *auth_data, char *, kstring_t*, char*, kstring_t*, kstring_t*, kstring_t*, kstring_t*, int);
+
+typedef int (*set_region_callback) (void *auth_data, kstring_t *region);
+
+typedef struct {
+    s3_auth_callback callback;
+    redirect_callback redirect_callback;
+    set_region_callback set_region_callback;
+    void *callback_data;
+} s3_authorisation;
+
+typedef struct {
+    hFILE base;
+    CURL *curl;
+    CURLcode ret;
+    s3_authorisation *au;
+    kstring_t buffer;
+    kstring_t url;
+    kstring_t upload_id;
+    kstring_t completion_message;
+    int part_no;
+    int aborted;
+    size_t index;
+    long verbose;
+    int part_size;
+    int expand;
+} hFILE_s3_write;
+
+
+static void ksinit(kstring_t *s) {
+    s->l = 0;
+    s->m = 0;
+    s->s = NULL;
+}
+
+
+static void ksfree(kstring_t *s) {
+    free(s->s);
+    ksinit(s);
+}
+
+
+static size_t response_callback(void *contents, size_t size, size_t nmemb, void *userp) {
+    size_t realsize = size * nmemb;
+    kstring_t *resp = (kstring_t *)userp;
+
+    if (kputsn((const char *)contents, realsize, resp) == EOF) {
+        return 0;
+    }
+
+    return realsize;
+}
+
+
+static int get_entry(char *in, char *start_tag, char *end_tag, kstring_t *out) {
+    char *start;
+    char *end;
+
+    if (!in) {
+        return EOF;
+    }
+
+    start = strstr(in, start_tag);
+    if (!start) return EOF;
+
+    start += strlen(start_tag);
+    end = strstr(start, end_tag);
+
+    if (!end) return EOF;
+
+    return kputsn(start, end - start, out);
+}
+
+
+static void cleanup_local(hFILE_s3_write *fp) {
+    ksfree(&fp->buffer);
+    ksfree(&fp->url);
+    ksfree(&fp->upload_id);
+    ksfree(&fp->completion_message);
+    curl_easy_cleanup(fp->curl);
+    free(fp->au);
+
+}
+
+
+static void cleanup(hFILE_s3_write *fp) {
+    // free up authorisation data
+    fp->au->callback(fp->au->callback_data,  NULL, NULL, NULL, NULL, NULL, NULL, NULL, 0);
+    cleanup_local(fp);
+}
+
+
+static struct curl_slist *set_html_headers(hFILE_s3_write *fp, kstring_t *auth, kstring_t *date, kstring_t *content, kstring_t *token) {
+    struct curl_slist *headers = NULL;
+
+    headers = curl_slist_append(headers, "Content-Type:"); // get rid of this
+    headers = curl_slist_append(headers, "Expect:");       // and this
+    headers = curl_slist_append(headers, auth->s);
+    headers = curl_slist_append(headers, date->s);
+    headers = curl_slist_append(headers, content->s);
+
+    if (token->l) {
+        headers = curl_slist_append(headers, token->s);
+    }
+
+    curl_easy_setopt(fp->curl, CURLOPT_HTTPHEADER, headers);
+
+    return headers;
+}
+
+
+/*
+    The partially uploaded file will hang around unless the delete command is sent.
+*/
+static int abort_upload(hFILE_s3_write *fp) {
+    kstring_t content_hash = {0, 0, NULL};
+    kstring_t authorisation = {0, 0, NULL};
+    kstring_t url = {0, 0, NULL};
+    kstring_t content = {0, 0, NULL};
+    kstring_t canonical_query_string = {0, 0, NULL};
+    kstring_t date = {0, 0, NULL};
+    kstring_t token = {0, 0, NULL};
+    int ret = -1;
+    struct curl_slist *headers = NULL;
+    char http_request[] = "DELETE";
+
+    if (ksprintf(&canonical_query_string, "uploadId=%s", fp->upload_id.s) < 0) {
+        goto out;
+    }
+
+    if (fp->au->callback(fp->au->callback_data,  http_request, NULL,
+                         canonical_query_string.s, &content_hash,
+                         &authorisation, &date, &token, 0) != 0) {
+        goto out;
+    }
+
+    if (ksprintf(&url, "%s?%s", fp->url.s, canonical_query_string.s) < 0) {
+        goto out;
+    }
+
+    if (ksprintf(&content, "x-amz-content-sha256: %s", content_hash.s) < 0) {
+        goto out;
+    }
+
+    curl_easy_reset(fp->curl);
+    curl_easy_setopt(fp->curl, CURLOPT_CUSTOMREQUEST, http_request);
+    curl_easy_setopt(fp->curl, CURLOPT_USERAGENT, curl.useragent.s);
+    curl_easy_setopt(fp->curl, CURLOPT_URL, url.s);
+
+    curl_easy_setopt(fp->curl, CURLOPT_VERBOSE, fp->verbose);
+
+    headers = set_html_headers(fp, &authorisation, &date, &content, &token);
+    fp->ret = curl_easy_perform(fp->curl);
+
+    if (fp->ret == CURLE_OK) {
+        ret = 0;
+    }
+
+ out:
+    ksfree(&authorisation);
+    ksfree(&content);
+    ksfree(&content_hash);
+    ksfree(&url);
+    ksfree(&date);
+    ksfree(&canonical_query_string);
+    ksfree(&token);
+    curl_slist_free_all(headers);
+
+    fp->aborted = 1;
+    cleanup(fp);
+
+    return ret;
+}
+
+
+static int complete_upload(hFILE_s3_write *fp, kstring_t *resp) {
+    kstring_t content_hash = {0, 0, NULL};
+    kstring_t authorisation = {0, 0, NULL};
+    kstring_t url = {0, 0, NULL};
+    kstring_t content = {0, 0, NULL};
+    kstring_t canonical_query_string = {0, 0, NULL};
+    kstring_t date = {0, 0, NULL};
+    kstring_t token = {0, 0, NULL};
+    int ret = -1;
+    struct curl_slist *headers = NULL;
+    char http_request[] = "POST";
+
+    if (ksprintf(&canonical_query_string, "uploadId=%s", fp->upload_id.s) < 0) {
+        return -1;
+    }
+
+    // finish off the completion reply
+    if (kputs("</CompleteMultipartUpload>\n", &fp->completion_message) < 0) {
+        goto out;
+    }
+
+    if (fp->au->callback(fp->au->callback_data,  http_request,
+                         &fp->completion_message, canonical_query_string.s,
+                         &content_hash, &authorisation, &date, &token, 0) != 0) {
+        goto out;
+    }
+
+    if (ksprintf(&url, "%s?%s", fp->url.s, canonical_query_string.s) < 0) {
+        goto out;
+    }
+
+    if (ksprintf(&content, "x-amz-content-sha256: %s", content_hash.s) < 0) {
+        goto out;
+    }
+
+    curl_easy_reset(fp->curl);
+    curl_easy_setopt(fp->curl, CURLOPT_POST, 1L);
+    curl_easy_setopt(fp->curl, CURLOPT_POSTFIELDS, fp->completion_message.s);
+    curl_easy_setopt(fp->curl, CURLOPT_POSTFIELDSIZE, (long) fp->completion_message.l);
+    curl_easy_setopt(fp->curl, CURLOPT_WRITEFUNCTION, response_callback);
+    curl_easy_setopt(fp->curl, CURLOPT_WRITEDATA, (void *)resp);
+    curl_easy_setopt(fp->curl, CURLOPT_URL, url.s);
+    curl_easy_setopt(fp->curl, CURLOPT_USERAGENT, curl.useragent.s);
+
+    curl_easy_setopt(fp->curl, CURLOPT_VERBOSE, fp->verbose);
+
+    headers = set_html_headers(fp, &authorisation, &date, &content, &token);
+    fp->ret = curl_easy_perform(fp->curl);
+
+    if (fp->ret == CURLE_OK) {
+        ret = 0;
+    }
+
+ out:
+    ksfree(&authorisation);
+    ksfree(&content);
+    ksfree(&content_hash);
+    ksfree(&url);
+    ksfree(&date);
+    ksfree(&token);
+    ksfree(&canonical_query_string);
+    curl_slist_free_all(headers);
+
+    return ret;
+}
+
+
+static size_t upload_callback(void *ptr, size_t size, size_t nmemb, void *stream) {
+    size_t realsize = size * nmemb;
+    hFILE_s3_write *fp = (hFILE_s3_write *)stream;
+    size_t read_length;
+
+    if (realsize > (fp->buffer.l - fp->index)) {
+        read_length = fp->buffer.l - fp->index;
+    } else {
+        read_length = realsize;
+    }
+
+    memcpy(ptr, fp->buffer.s + fp->index, read_length);
+    fp->index += read_length;
+
+    return read_length;
+}
+
+
+static int upload_part(hFILE_s3_write *fp, kstring_t *resp) {
+    kstring_t content_hash = {0, 0, NULL};
+    kstring_t authorisation = {0, 0, NULL};
+    kstring_t url = {0, 0, NULL};
+    kstring_t content = {0, 0, NULL};
+    kstring_t canonical_query_string = {0, 0, NULL};
+    kstring_t date = {0, 0, NULL};
+    kstring_t token = {0, 0, NULL};
+    int ret = -1;
+    struct curl_slist *headers = NULL;
+    char http_request[] = "PUT";
+
+    if (ksprintf(&canonical_query_string, "partNumber=%d&uploadId=%s", fp->part_no, fp->upload_id.s) < 0) {
+        return -1;
+    }
+
+    if (fp->au->callback(fp->au->callback_data, http_request, &fp->buffer,
+                         canonical_query_string.s, &content_hash,
+                         &authorisation, &date, &token, 0) != 0) {
+        goto out;
+    }
+
+    if (ksprintf(&url, "%s?%s", fp->url.s, canonical_query_string.s) < 0) {
+        goto out;
+    }
+
+    fp->index = 0;
+    if (ksprintf(&content, "x-amz-content-sha256: %s", content_hash.s) < 0) {
+        goto out;
+    }
+
+    curl_easy_reset(fp->curl);
+
+    curl_easy_setopt(fp->curl, CURLOPT_UPLOAD, 1L);
+    curl_easy_setopt(fp->curl, CURLOPT_READFUNCTION, upload_callback);
+    curl_easy_setopt(fp->curl, CURLOPT_READDATA, fp);
+    curl_easy_setopt(fp->curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)fp->buffer.l);
+    curl_easy_setopt(fp->curl, CURLOPT_HEADERFUNCTION, response_callback);
+    curl_easy_setopt(fp->curl, CURLOPT_HEADERDATA, (void *)resp);
+    curl_easy_setopt(fp->curl, CURLOPT_URL, url.s);
+    curl_easy_setopt(fp->curl, CURLOPT_USERAGENT, curl.useragent.s);
+
+    curl_easy_setopt(fp->curl, CURLOPT_VERBOSE, fp->verbose);
+
+    headers = set_html_headers(fp, &authorisation, &date, &content, &token);
+    fp->ret = curl_easy_perform(fp->curl);
+
+    if (fp->ret == CURLE_OK) {
+        ret = 0;
+    }
+
+ out:
+    ksfree(&authorisation);
+    ksfree(&content);
+    ksfree(&content_hash);
+    ksfree(&url);
+    ksfree(&date);
+    ksfree(&token);
+    ksfree(&canonical_query_string);
+    curl_slist_free_all(headers);
+
+    return ret;
+}
+
+
+static ssize_t s3_write(hFILE *fpv, const void *bufferv, size_t nbytes) {
+    hFILE_s3_write *fp = (hFILE_s3_write *)fpv;
+    const char *buffer  = (const char *)bufferv;
+
+    if (kputsn(buffer, nbytes, &fp->buffer) == EOF) {
+        return -1;
+    }
+
+    if (fp->buffer.l > fp->part_size) {
+        // time to write out our data
+        kstring_t response = {0, 0, NULL};
+        int ret;
+
+        ret = upload_part(fp, &response);
+
+        if (!ret) {
+            long response_code;
+            kstring_t etag = {0, 0, NULL};
+
+            curl_easy_getinfo(fp->curl, CURLINFO_RESPONSE_CODE, &response_code);
+
+            if (response_code > 200) {
+                ret = -1;
+            } else {
+                if (get_entry(response.s, "ETag: \"", "\"", &etag) == EOF) {
+                    ret = -1;
+                } else {
+                    ksprintf(&fp->completion_message, "\t<Part>\n\t\t<PartNumber>%d</PartNumber>\n\t\t<ETag>%s</ETag>\n\t</Part>\n",
+                        fp->part_no, etag.s);
+
+                    ksfree(&etag);
+                }
+            }
+        }
+
+        ksfree(&response);
+
+        if (ret) {
+            abort_upload(fp);
+            return -1;
+        }
+
+        fp->part_no++;
+        fp->buffer.l = 0;
+
+        if (fp->expand && (fp->part_no % EXPAND_ON == 0)) {
+            fp->part_size *= 2;
+        }
+    }
+
+    return nbytes;
+}
+
+
+static int s3_close(hFILE *fpv) {
+    hFILE_s3_write *fp = (hFILE_s3_write *)fpv;
+    kstring_t response = {0, 0, NULL};
+    int ret = 0;
+
+    if (!fp->aborted) {
+
+        if (fp->buffer.l) {
+            // write the last part
+
+            ret = upload_part(fp, &response);
+
+            if (!ret) {
+                long response_code;
+                kstring_t etag = {0, 0, NULL};
+
+                curl_easy_getinfo(fp->curl, CURLINFO_RESPONSE_CODE, &response_code);
+
+                if (response_code > 200) {
+                    ret = -1;
+                } else {
+                    if (get_entry(response.s, "ETag: \"", "\"", &etag) == EOF) {
+                        ret = -1;
+                    } else {
+                        ksprintf(&fp->completion_message, "\t<Part>\n\t\t<PartNumber>%d</PartNumber>\n\t\t<ETag>%s</ETag>\n\t</Part>\n",
+                            fp->part_no, etag.s);
+
+                        ksfree(&etag);
+                    }
+                }
+            }
+
+            ksfree(&response);
+
+            if (ret) {
+                abort_upload(fp);
+                return -1;
+            }
+
+            fp->part_no++;
+        }
+
+        if (fp->part_no > 1) {
+            ret = complete_upload(fp, &response);
+
+            if (!ret) {
+                if (strstr(response.s, "CompleteMultipartUploadResult") == NULL) {
+                    ret = -1;
+                }
+            }
+        } else {
+            ret = -1;
+        }
+
+        if (ret) {
+            abort_upload(fp);
+        } else {
+            cleanup(fp);
+        }
+    }
+
+    ksfree(&response);
+
+    return ret;
+}
+
+
+static int redirect_endpoint(hFILE_s3_write *fp, kstring_t *head) {
+    int ret = -1;
+
+    if (fp->au->redirect_callback) {
+        ret = fp->au->redirect_callback(fp->au->callback_data, 301, head, &fp->url);
+    }
+
+    return ret;
+}
+
+static int handle_bad_request(hFILE_s3_write *fp, kstring_t *resp) {
+    kstring_t region = {0, 0, NULL};
+    int ret = -1;
+
+    if (fp->au->set_region_callback) {
+        if (get_entry(resp->s, "<Region>", "</Region>", &region) == EOF) {
+            return -1;
+        }
+
+        ret = fp->au->set_region_callback(fp->au->callback_data, &region);
+
+        ksfree(&region);
+    }
+
+    return ret;
+}
+
+static int initialise_upload(hFILE_s3_write *fp, kstring_t *head, kstring_t *resp, int user_query) {
+    kstring_t content_hash = {0, 0, NULL};
+    kstring_t authorisation = {0, 0, NULL};
+    kstring_t url = {0, 0, NULL};
+    kstring_t content = {0, 0, NULL};
+    kstring_t date = {0, 0, NULL};
+    kstring_t token = {0, 0, NULL};
+    int ret = -1;
+    struct curl_slist *headers = NULL;
+    char http_request[] = "POST";
+    char delimiter = '?';
+
+    if (user_query) {
+        delimiter = '&';
+    }
+
+    if (fp->au->callback(fp->au->callback_data,  http_request, NULL, "uploads=",
+                         &content_hash, &authorisation, &date, &token, user_query) != 0) {
+        goto out;
+    }
+
+    if (ksprintf(&url, "%s%cuploads", fp->url.s, delimiter) < 0) {
+        goto out;
+    }
+
+    if (ksprintf(&content, "x-amz-content-sha256: %s", content_hash.s) < 0) {
+        goto out;
+    }
+
+    curl_easy_setopt(fp->curl, CURLOPT_URL, url.s);
+    curl_easy_setopt(fp->curl, CURLOPT_POST, 1L);
+    curl_easy_setopt(fp->curl, CURLOPT_POSTFIELDS, "");  // send no data
+    curl_easy_setopt(fp->curl, CURLOPT_WRITEFUNCTION, response_callback);
+    curl_easy_setopt(fp->curl, CURLOPT_WRITEDATA, (void *)resp);
+    curl_easy_setopt(fp->curl, CURLOPT_HEADERFUNCTION, response_callback);
+    curl_easy_setopt(fp->curl, CURLOPT_HEADERDATA, (void *)head);
+    curl_easy_setopt(fp->curl, CURLOPT_USERAGENT, curl.useragent.s);
+
+    curl_easy_setopt(fp->curl, CURLOPT_VERBOSE, fp->verbose);
+
+    headers = set_html_headers(fp, &authorisation, &date, &content, &token);
+    fp->ret = curl_easy_perform(fp->curl);
+
+    if (fp->ret == CURLE_OK) {
+        ret = 0;
+    }
+
+ out:
+    ksfree(&authorisation);
+    ksfree(&content);
+    ksfree(&content_hash);
+    ksfree(&url);
+    ksfree(&date);
+    ksfree(&token);
+    curl_slist_free_all(headers);
+
+    return ret;
+}
+
+
+static int get_upload_id(hFILE_s3_write *fp, kstring_t *resp) {
+    int ret = 0;
+
+    ksinit(&fp->upload_id);
+
+    if (get_entry(resp->s, "<UploadId>", "</UploadId>", &fp->upload_id) == EOF) {
+        ret = -1;
+    }
+
+    return ret;
+}
+
+
+static const struct hFILE_backend s3_write_backend = {
+    NULL, s3_write, NULL, NULL, s3_close
+};
+
+
+static hFILE *s3_write_open(const char *url, s3_authorisation *auth) {
+    hFILE_s3_write *fp;
+    kstring_t response = {0, 0, NULL};
+    kstring_t header   = {0, 0, NULL};
+    int ret, has_user_query = 0;
+    char *query_start;
+    const char *env;
+
+
+    if (!auth || !auth->callback || !auth->callback_data) {
+        return NULL;
+    }
+
+    fp = (hFILE_s3_write *)hfile_init(sizeof(hFILE_s3_write), "w", 0);
+
+    if (fp == NULL) {
+        return NULL;
+    }
+
+    if ((fp->curl = curl_easy_init()) == NULL) {
+        errno = ENOMEM;
+        goto error;
+    }
+
+    if ((fp->au = calloc(1, sizeof(s3_authorisation))) == NULL) {
+        goto error;
+    }
+
+    memcpy(fp->au, auth, sizeof(s3_authorisation));
+
+    ksinit(&fp->buffer);
+    ksinit(&fp->url);
+    ksinit(&fp->completion_message);
+    fp->aborted = 0;
+
+    fp->part_size = MINIMUM_S3_WRITE_SIZE;
+    fp->expand = 1;
+
+    if ((env = getenv("HTS_S3_PART_SIZE")) != NULL) {
+        int part_size = atoi(env) * 1024 * 1024;
+
+        if (part_size > fp->part_size)
+            fp->part_size = part_size;
+
+        fp->expand = 0;
+    }
+
+    if (hts_verbose >= 8) {
+        fp->verbose = 1L;
+    } else {
+        fp->verbose = 0L;
+    }
+
+    kputs(url + 4, &fp->url);
+
+    if ((query_start = strchr(fp->url.s, '?'))) {
+        has_user_query = 1;;
+    }
+
+    ret = initialise_upload(fp, &header, &response, has_user_query);
+
+    if (ret == 0) {
+        long response_code;
+
+        curl_easy_getinfo(fp->curl, CURLINFO_RESPONSE_CODE, &response_code);
+
+        if (response_code == S3_MOVED_PERMANENTLY) {
+            if (redirect_endpoint(fp, &header) == 0) {
+                ksfree(&response);
+                ksfree(&header);
+
+                ret = initialise_upload(fp, &header, &response, has_user_query);
+            }
+        } else if (response_code == S3_BAD_REQUEST) {
+            if (handle_bad_request(fp, &response) == 0) {
+                ksfree(&response);
+                ksfree(&header);
+
+                ret = initialise_upload(fp, &header, &response, has_user_query);
+            }
+        }
+
+        ksfree(&header); // no longer needed
+    }
+
+    if (ret) goto error;
+
+    if (get_upload_id(fp, &response)) goto error;
+
+    // start the completion message (a formatted list of parts)
+    ksinit(&fp->completion_message);
+
+    if (kputs("<CompleteMultipartUpload>\n", &fp->completion_message) == EOF) {
+        goto error;
+    }
+
+    fp->part_no = 1;
+
+    // user query string no longer a useful part of the URL
+    if (query_start)
+         *query_start = '\0';
+
+    fp->base.backend = &s3_write_backend;
+    ksfree(&response);
+
+    return &fp->base;
+
+error:
+    ksfree(&response);
+    cleanup_local(fp);
+    hfile_destroy((hFILE *)fp);
+    return NULL;
+}
+
+
+static hFILE *hopen_s3_write(const char *url, const char *mode) {
+    if (hts_verbose >= 1) {
+        fprintf(stderr, "[E::%s] s3w:// URLs should not be used directly; use s3:// instead.\n", __func__);
+    }
+    return NULL;
+}
+
+
+static int parse_va_list(s3_authorisation *auth, va_list args) {
+    const char *argtype;
+
+    while  ((argtype = va_arg(args, const char *)) != NULL) {
+        if (strcmp(argtype, "s3_auth_callback") == 0) {
+            auth->callback = va_arg(args, s3_auth_callback);
+        } else if (strcmp(argtype, "s3_auth_callback_data") == 0) {
+            auth->callback_data = va_arg(args, void *);
+        } else if (strcmp(argtype, "redirect_callback") == 0) {
+            auth->redirect_callback = va_arg(args, redirect_callback);
+        } else if (strcmp(argtype, "set_region_callback") == 0) {
+            auth->set_region_callback = va_arg(args, set_region_callback);
+        } else if (strcmp(argtype, "va_list") == 0) {
+            va_list *args2 = va_arg(args, va_list *);
+
+            if (args2) {
+                if (parse_va_list(auth, *args2) < 0) return -1;
+            }
+        } else {
+            errno = EINVAL;
+            return -1;
+        }
+    }
+
+    return 0;
+}
+
+
+static hFILE *vhopen_s3_write(const char *url, const char *mode, va_list args) {
+    hFILE *fp = NULL;
+    s3_authorisation auth = {NULL, NULL, NULL};
+
+    if (parse_va_list(&auth, args) == 0) {
+        fp =  s3_write_open(url, &auth);
+    }
+
+    return fp;
+}
+
+
+static void s3_write_exit(void) {
+    if (curl_share_cleanup(curl.share) == CURLSHE_OK)
+        curl.share = NULL;
+
+    free(curl.useragent.s);
+    curl.useragent.l = curl.useragent.m = 0; curl.useragent.s = NULL;
+    curl_global_cleanup();
+}
+
+
+int PLUGIN_GLOBAL(hfile_plugin_init,_s3_write)(struct hFILE_plugin *self) {
+
+    static const struct hFILE_scheme_handler handler =
+        { hopen_s3_write, hfile_always_remote, "S3 Multipart Upload",
+          2000 + 50, vhopen_s3_write
+        };
+
+#ifdef ENABLE_PLUGINS
+    // Embed version string for examination via strings(1) or what(1)
+    static const char id[] =
+        "@(#)hfile_s3_write plugin (htslib)\t" HTS_VERSION_TEXT;
+    const char *version = strchr(id, '\t') + 1;
+
+    if (hts_verbose >= 9)
+        fprintf(stderr, "[M::hfile_s3_write.init] version %s\n",
+                version);
+#else
+    const char *version = hts_version();
+#endif
+
+    const curl_version_info_data *info;
+    CURLcode err;
+    CURLSHcode errsh;
+
+    err = curl_global_init(CURL_GLOBAL_ALL);
+
+    if (err != CURLE_OK) {
+        // look at putting in an errno here
+        return -1;
+    }
+
+    curl.share = curl_share_init();
+
+    if (curl.share == NULL) {
+        curl_global_cleanup();
+        errno = EIO;
+        return -1;
+    }
+
+    errsh  = curl_share_setopt(curl.share, CURLSHOPT_LOCKFUNC, share_lock);
+    errsh |= curl_share_setopt(curl.share, CURLSHOPT_UNLOCKFUNC, share_unlock);
+    errsh |= curl_share_setopt(curl.share, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS);
+
+    if (errsh != 0) {
+        curl_share_cleanup(curl.share);
+        curl_global_cleanup();
+        errno = EIO;
+        return -1;
+    }
+
+    info = curl_version_info(CURLVERSION_NOW);
+    ksprintf(&curl.useragent, "htslib/%s libcurl/%s", version, info->version);
+
+    self->name = "S3 Multipart Upload";
+    self->destroy = s3_write_exit;
+
+    hfile_add_scheme_handler("s3w",       &handler);
+    hfile_add_scheme_handler("s3w+http",  &handler);
+    hfile_add_scheme_handler("s3w+https", &handler);
+
+    return 0;
+}
diff --git a/ext/htslib/hts.c b/ext/htslib/hts.c
new file mode 100644
index 0000000..a8a8bea
--- /dev/null
+++ b/ext/htslib/hts.c
@@ -0,0 +1,5152 @@
+/*  hts.c -- format-neutral I/O, indexing, and iterator API functions.
+
+    Copyright (C) 2008, 2009, 2012-2024 Genome Research Ltd.
+    Copyright (C) 2012, 2013 Broad Institute.
+
+    Author: Heng Li <lh3@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+
+#include <zlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <strings.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <inttypes.h>
+#include <limits.h>
+#include <stdint.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <time.h>
+#include <sys/stat.h>
+#include <assert.h>
+
+#ifdef HAVE_LIBLZMA
+#ifdef HAVE_LZMA_H
+#include <lzma.h>
+#else
+#include "os/lzma_stub.h"
+#endif
+#endif
+
+#include "htslib/hts.h"
+#include "htslib/bgzf.h"
+#include "cram/cram.h"
+#include "htslib/hfile.h"
+#include "htslib/hts_endian.h"
+#include "version.h"
+#include "config_vars.h"
+#include "hts_internal.h"
+#include "hfile_internal.h"
+#include "sam_internal.h"
+#include "htslib/hts_expr.h"
+#include "htslib/hts_os.h" // drand48
+
+#include "htslib/khash.h"
+#include "htslib/kseq.h"
+#include "htslib/ksort.h"
+#include "htslib/tbx.h"
+#if defined(HAVE_EXTERNAL_LIBHTSCODECS)
+#include <htscodecs/htscodecs.h>
+#else
+#include "htscodecs/htscodecs/htscodecs.h"
+#endif
+
+#ifndef EFTYPE
+#define EFTYPE ENOEXEC
+#endif
+
+KHASH_INIT2(s2i,, kh_cstr_t, int64_t, 1, kh_str_hash_func, kh_str_hash_equal)
+
+HTSLIB_EXPORT
+int hts_verbose = HTS_LOG_WARNING;
+
+const char *hts_version(void)
+{
+    return HTS_VERSION_TEXT;
+}
+
+unsigned int hts_features(void) {
+    unsigned int feat = HTS_FEATURE_HTSCODECS; // Always present
+
+#ifdef PACKAGE_URL
+    feat |= HTS_FEATURE_CONFIGURE;
+#endif
+
+#ifdef ENABLE_PLUGINS
+    feat |= HTS_FEATURE_PLUGINS;
+#endif
+
+#ifdef HAVE_LIBCURL
+    feat |= HTS_FEATURE_LIBCURL;
+#endif
+
+#ifdef ENABLE_S3
+    feat |= HTS_FEATURE_S3;
+#endif
+
+#ifdef ENABLE_GCS
+    feat |= HTS_FEATURE_GCS;
+#endif
+
+#ifdef HAVE_LIBDEFLATE
+    feat |= HTS_FEATURE_LIBDEFLATE;
+#endif
+
+#ifdef HAVE_LIBLZMA
+    feat |= HTS_FEATURE_LZMA;
+#endif
+
+#ifdef HAVE_LIBBZ2
+    feat |= HTS_FEATURE_BZIP2;
+#endif
+
+    return feat;
+}
+
+const char *hts_test_feature(unsigned int id) {
+    unsigned int feat = hts_features();
+
+    switch (id) {
+    case HTS_FEATURE_CONFIGURE:
+        return feat & HTS_FEATURE_CONFIGURE ? "yes" : NULL;
+    case HTS_FEATURE_PLUGINS:
+        return feat & HTS_FEATURE_PLUGINS ? "yes" : NULL;
+    case HTS_FEATURE_LIBCURL:
+        return feat & HTS_FEATURE_LIBCURL ? "yes" : NULL;
+    case HTS_FEATURE_S3:
+        return feat & HTS_FEATURE_S3 ? "yes" : NULL;
+    case HTS_FEATURE_GCS:
+        return feat & HTS_FEATURE_GCS ? "yes" : NULL;
+    case HTS_FEATURE_LIBDEFLATE:
+        return feat & HTS_FEATURE_LIBDEFLATE ? "yes" : NULL;
+    case HTS_FEATURE_BZIP2:
+        return feat & HTS_FEATURE_BZIP2 ? "yes" : NULL;
+    case HTS_FEATURE_LZMA:
+        return feat & HTS_FEATURE_LZMA ? "yes" : NULL;
+
+    case HTS_FEATURE_HTSCODECS:
+        return htscodecs_version();
+
+    case HTS_FEATURE_CC:
+        return HTS_CC;
+    case HTS_FEATURE_CFLAGS:
+        return HTS_CFLAGS;
+    case HTS_FEATURE_LDFLAGS:
+        return HTS_LDFLAGS;
+    case HTS_FEATURE_CPPFLAGS:
+        return HTS_CPPFLAGS;
+
+    default:
+        fprintf(stderr, "Unknown feature code: %u\n", id);
+    }
+
+    return NULL;
+}
+
+// Note this implementation also means we can just "strings" the library
+// to find the configuration parameters.
+const char *hts_feature_string(void) {
+    static char config[1200];
+    const char *flags=
+
+#ifdef PACKAGE_URL
+    "build=configure "
+#else
+    "build=Makefile "
+#endif
+
+#ifdef HAVE_LIBCURL
+    "libcurl=yes "
+#else
+    "libcurl=no "
+#endif
+
+#ifdef ENABLE_S3
+    "S3=yes "
+#else
+    "S3=no "
+#endif
+
+#ifdef ENABLE_GCS
+    "GCS=yes "
+#else
+    "GCS=no "
+#endif
+
+#ifdef HAVE_LIBDEFLATE
+    "libdeflate=yes "
+#else
+    "libdeflate=no "
+#endif
+
+#ifdef HAVE_LIBLZMA
+    "lzma=yes "
+#else
+    "lzma=no "
+#endif
+
+#ifdef HAVE_LIBBZ2
+    "bzip2=yes "
+#else
+    "bzip2=no "
+#endif
+
+// "plugins=" must stay at the end as it is followed by "plugin-path="
+#ifdef ENABLE_PLUGINS
+    "plugins=yes";
+#else
+    "plugins=no";
+#endif
+
+#ifdef ENABLE_PLUGINS
+    snprintf(config, sizeof(config),
+             "%s plugin-path=%.1000s htscodecs=%.40s",
+             flags, hts_plugin_path(), htscodecs_version());
+#else
+    snprintf(config, sizeof(config),
+             "%s htscodecs=%.40s",
+             flags, htscodecs_version());
+#endif
+    return config;
+}
+
+
+HTSLIB_EXPORT
+const unsigned char seq_nt16_table[256] = {
+    15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,
+    15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,
+    15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,
+     1, 2, 4, 8, 15,15,15,15, 15,15,15,15, 15, 0 /*=*/,15,15,
+    15, 1,14, 2, 13,15,15, 4, 11,15,15,12, 15, 3,15,15,
+    15,15, 5, 6,  8,15, 7, 9, 15,10,15,15, 15,15,15,15,
+    15, 1,14, 2, 13,15,15, 4, 11,15,15,12, 15, 3,15,15,
+    15,15, 5, 6,  8,15, 7, 9, 15,10,15,15, 15,15,15,15,
+
+    15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,
+    15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,
+    15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,
+    15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,
+    15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,
+    15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,
+    15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,
+    15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15
+};
+
+HTSLIB_EXPORT
+const char seq_nt16_str[] = "=ACMGRSVTWYHKDBN";
+
+HTSLIB_EXPORT
+const int seq_nt16_int[] = { 4, 0, 1, 4, 2, 4, 4, 4, 3, 4, 4, 4, 4, 4, 4, 4 };
+
+/**********************
+ *** Basic file I/O ***
+ **********************/
+
+static enum htsFormatCategory format_category(enum htsExactFormat fmt)
+{
+    switch (fmt) {
+    case bam:
+    case sam:
+    case cram:
+    case fastq_format:
+    case fasta_format:
+        return sequence_data;
+
+    case vcf:
+    case bcf:
+        return variant_data;
+
+    case bai:
+    case crai:
+    case csi:
+    case fai_format:
+    case fqi_format:
+    case gzi:
+    case tbi:
+        return index_file;
+
+    case bed:
+    case d4_format:
+        return region_list;
+
+    case htsget:
+    case hts_crypt4gh_format:
+        return unknown_category;
+
+    case unknown_format:
+    case binary_format:
+    case text_format:
+    case empty_format:
+    case format_maximum:
+        break;
+    }
+
+    return unknown_category;
+}
+
+// Decompress several hundred bytes by peeking at the file, which must be
+// positioned at the start of a GZIP block.
+static ssize_t
+decompress_peek_gz(hFILE *fp, unsigned char *dest, size_t destsize)
+{
+    unsigned char buffer[2048];
+    z_stream zs;
+    ssize_t npeek = hpeek(fp, buffer, sizeof buffer);
+
+    if (npeek < 0) return -1;
+
+    zs.zalloc = NULL;
+    zs.zfree = NULL;
+    zs.next_in = buffer;
+    zs.avail_in = npeek;
+    zs.next_out = dest;
+    zs.avail_out = destsize;
+    if (inflateInit2(&zs, 31) != Z_OK) return -1;
+
+    int ret;
+    const unsigned char *last_in = buffer;
+    while (zs.avail_out > 0) {
+        ret = inflate(&zs, Z_SYNC_FLUSH);
+        if (ret == Z_STREAM_END) {
+            if (last_in == zs.next_in)
+                break; // Paranoia to avoid potential looping. Shouldn't happen
+            else
+                last_in = zs.next_in;
+            inflateReset(&zs);
+        } else if (ret != Z_OK) {
+            // eg Z_BUF_ERROR due to avail_in/out becoming zero
+            break;
+        }
+    }
+
+    // NB: zs.total_out is changed by inflateReset, so use pointer diff instead
+    destsize = zs.next_out - dest;
+    inflateEnd(&zs);
+
+    return destsize;
+}
+
+#ifdef HAVE_LIBLZMA
+// Similarly decompress a portion by peeking at the file, which must be
+// positioned at the start of the file.
+static ssize_t
+decompress_peek_xz(hFILE *fp, unsigned char *dest, size_t destsize)
+{
+    unsigned char buffer[2048];
+    ssize_t npeek = hpeek(fp, buffer, sizeof buffer);
+    if (npeek < 0) return -1;
+
+    lzma_stream ls = LZMA_STREAM_INIT;
+    if (lzma_stream_decoder(&ls, lzma_easy_decoder_memusage(9), 0) != LZMA_OK)
+        return -1;
+
+    ls.next_in = buffer;
+    ls.avail_in = npeek;
+    ls.next_out = dest;
+    ls.avail_out = destsize;
+
+    int r = lzma_code(&ls, LZMA_RUN);
+    if (! (r == LZMA_OK || r == LZMA_STREAM_END)) {
+        lzma_end(&ls);
+        return -1;
+    }
+
+    destsize = ls.total_out;
+    lzma_end(&ls);
+
+    return destsize;
+}
+#endif
+
+// Parse "x.y" text, taking care because the string is not NUL-terminated
+// and filling in major/minor only when the digits are followed by a delimiter,
+// so we don't misread "1.10" as "1.1" due to reaching the end of the buffer.
+static void
+parse_version(htsFormat *fmt, const unsigned char *u, const unsigned char *ulim)
+{
+    const char *s    = (const char *) u;
+    const char *slim = (const char *) ulim;
+    short v;
+
+    fmt->version.major = fmt->version.minor = -1;
+
+    for (v = 0; s < slim && isdigit_c(*s); s++)
+        v = 10 * v + *s - '0';
+
+    if (s < slim) {
+        fmt->version.major = v;
+        if (*s == '.') {
+            s++;
+            for (v = 0; s < slim && isdigit_c(*s); s++)
+                v = 10 * v + *s - '0';
+            if (s < slim)
+                fmt->version.minor = v;
+        }
+        else
+            fmt->version.minor = 0;
+    }
+}
+
+static int
+cmp_nonblank(const char *key, const unsigned char *u, const unsigned char *ulim)
+{
+    const unsigned char *ukey = (const unsigned char *) key;
+
+    while (*ukey)
+        if (u >= ulim) return +1;
+        else if (isspace_c(*u)) u++;
+        else if (*u != *ukey) return (*ukey < *u)? -1 : +1;
+        else u++, ukey++;
+
+    return 0;
+}
+
+static int is_text_only(const unsigned char *u, const unsigned char *ulim)
+{
+    for (; u < ulim; u++)
+        if (! (*u >= ' ' || *u == '\t' || *u == '\r' || *u == '\n'))
+            return 0;
+
+    return 1;
+}
+
+static inline int
+alternate_zeros(const unsigned char *u, const unsigned char *ulim)
+{
+    for (; u < ulim; u += 2)
+        if (*u != '\0') return 0;
+    return 1;
+}
+
+static int is_utf16_text(const unsigned char *u, const unsigned char *ulim)
+{
+    if (ulim - u >= 6 &&
+        ((u[0] == 0xfe && u[1] == 0xff && alternate_zeros(u+2, ulim)) ||
+         (u[0] == 0xff && u[1] == 0xfe && alternate_zeros(u+3, ulim))))
+        return 2;
+    else if (ulim - u >= 8 &&
+             (alternate_zeros(u, ulim) || alternate_zeros(u+1, ulim)))
+        return 1;
+    else
+        return 0;
+}
+
+static int is_fastaq(const unsigned char *u, const unsigned char *ulim)
+{
+    const unsigned char *eol = memchr(u, '\n', ulim - u);
+
+    // Check that the first line is entirely textual
+    if (! is_text_only(u, eol? eol : ulim)) return 0;
+
+    // If the first line is very long, consider the file to indeed be FASTA/Q
+    if (eol == NULL) return 1;
+
+    u = eol+1; // Now points to the first character of the second line
+
+    // Scan over all base-encoding letters (including 'N' but not SEQ's '=')
+    while (u < ulim && (seq_nt16_table[*u] != 15 || toupper(*u) == 'N')) {
+        if (*u == '=') return 0;
+        u++;
+    }
+
+    return (u == ulim || *u == '\r' || *u == '\n')? 1 : 0;
+}
+
+// Parse tab-delimited text, filling in a string of column types and returning
+// the number of columns spotted (within [u,ulim), and up to column_len) or -1
+// if non-printable characters were seen.  Column types:
+//     i: integer, s: strand sign, C: CIGAR, O: SAM optional field, Z: anything
+static int
+parse_tabbed_text(char *columns, int column_len,
+                  const unsigned char *u, const unsigned char *ulim,
+                  int *complete)
+{
+    const char *str  = (const char *) u;
+    const char *slim = (const char *) ulim;
+    const char *s;
+    int ncolumns = 0;
+
+    enum { digit = 1, leading_sign = 2, cigar_operator = 4, other = 8 };
+    unsigned seen = 0;
+    *complete = 0;
+
+    for (s = str; s < slim; s++)
+        if (*s >= ' ') {
+            if (isdigit_c(*s))
+                seen |= digit;
+            else if ((*s == '+' || *s == '-') && s == str)
+                seen |= leading_sign;
+            else if (strchr(BAM_CIGAR_STR, *s) && s > str && isdigit_c(s[-1]))
+                seen |= cigar_operator;
+            else
+                seen |= other;
+        }
+        else if (*s == '\t' || *s == '\r' || *s == '\n') {
+            size_t len = s - str;
+            char type;
+
+            if (seen == digit || seen == (leading_sign|digit)) type = 'i';
+            else if (seen == (digit|cigar_operator)) type = 'C';
+            else if (len == 1)
+                switch (str[0]) {
+                case '*': type = 'C'; break;
+                case '+': case '-': case '.': type = 's'; break;
+                default: type = 'Z'; break;
+                }
+            else if (len >= 5 && str[2] == ':' && str[4] == ':') type = 'O';
+            else type = 'Z';
+
+            columns[ncolumns++] = type;
+            if (*s != '\t' || ncolumns >= column_len - 1) {
+                *complete = 1; // finished the line or more columns than needed
+                break;
+            }
+
+            str = s + 1;
+            seen = 0;
+        }
+        else return -1;
+
+    columns[ncolumns] = '\0';
+    return ncolumns;
+}
+
+// Match COLUMNS as a prefix against PATTERN (so COLUMNS may run out first).
+// Returns len(COLUMNS) (modulo '+'), or 0 if there is a mismatched entry.
+static int colmatch(const char *columns, const char *pattern)
+{
+    int i;
+    for (i = 0; columns[i] != '\0'; i++) {
+        if (pattern[i] == '+') return i;
+        if (! (columns[i] == pattern[i] || pattern[i] == 'Z')) return 0;
+    }
+
+    return i;
+}
+
+int hts_detect_format(hFILE *hfile, htsFormat *fmt)
+{
+    return hts_detect_format2(hfile, NULL, fmt);
+}
+
+int hts_detect_format2(hFILE *hfile, const char *fname, htsFormat *fmt)
+{
+    char extension[HTS_MAX_EXT_LEN], columns[24];
+    unsigned char s[1024];
+    int complete = 0;
+    ssize_t len = hpeek(hfile, s, 18);
+    if (len < 0) return -1;
+
+    fmt->category = unknown_category;
+    fmt->format = unknown_format;
+    fmt->version.major = fmt->version.minor = -1;
+    fmt->compression = no_compression;
+    fmt->compression_level = -1;
+    fmt->specific = NULL;
+
+    if (len >= 2 && s[0] == 0x1f && s[1] == 0x8b) {
+        // The stream is either gzip-compressed or BGZF-compressed.
+        // Determine which, and decompress the first few records or lines.
+        fmt->compression = gzip;
+        if (len >= 18 && (s[3] & 4)) {
+            if (memcmp(&s[12], "BC\2\0", 4) == 0)
+                fmt->compression = bgzf;
+            else if (memcmp(&s[12], "RAZF", 4) == 0)
+                fmt->compression = razf_compression;
+        }
+        if (len >= 9 && s[2] == 8)
+            fmt->compression_level = (s[8] == 2)? 9 : (s[8] == 4)? 1 : -1;
+
+        len = decompress_peek_gz(hfile, s, sizeof s);
+    }
+    else if (len >= 10 && memcmp(s, "BZh", 3) == 0 &&
+             (memcmp(&s[4], "\x31\x41\x59\x26\x53\x59", 6) == 0 ||
+              memcmp(&s[4], "\x17\x72\x45\x38\x50\x90", 6) == 0)) {
+        fmt->compression = bzip2_compression;
+        fmt->compression_level = s[3] - '0';
+        // Decompressing via libbz2 produces no output until it has a whole
+        // block (of size 100Kb x level), which is too large for peeking.
+        // So unfortunately we can recognise bzip2 but not the contents,
+        // except that \x1772... magic indicates the stream is empty.
+        if (s[4] == '\x31') return 0;
+        else len = 0;
+    }
+    else if (len >= 6 && memcmp(s, "\xfd""7zXZ\0", 6) == 0) {
+        fmt->compression = xz_compression;
+#ifdef HAVE_LIBLZMA
+        len = decompress_peek_xz(hfile, s, sizeof s);
+#else
+        // Without liblzma, we can't recognise the decompressed contents.
+        return 0;
+#endif
+    }
+    else if (len >= 4 && memcmp(s, "\x28\xb5\x2f\xfd", 4) == 0) {
+        fmt->compression = zstd_compression;
+        return 0;
+    }
+    else {
+        len = hpeek(hfile, s, sizeof s);
+    }
+    if (len < 0) return -1;
+
+    if (len == 0) {
+        fmt->format = empty_format;
+        return 0;
+    }
+
+    // We avoid using filename extensions wherever possible (as filenames are
+    // not always available), but in a few cases they must be considered:
+    //  - FASTA/Q indexes are simply tab-separated text; files that match these
+    //    patterns but not the fai/fqi extension are usually generic BED files
+    //  - GZI indexes have no magic numbers so can only be detected by filename
+    if (fname && strcmp(fname, "-") != 0) {
+        char *s;
+        if (find_file_extension(fname, extension) < 0) extension[0] = '\0';
+        for (s = extension; *s; s++) *s = tolower_c(*s);
+    }
+    else extension[0] = '\0';
+
+    if (len >= 6 && memcmp(s,"CRAM",4) == 0 && s[4]>=1 && s[4]<=7 && s[5]<=7) {
+        fmt->category = sequence_data;
+        fmt->format = cram;
+        fmt->version.major = s[4], fmt->version.minor = s[5];
+        fmt->compression = custom;
+        return 0;
+    }
+    else if (len >= 4 && s[3] <= '\4') {
+        if (memcmp(s, "BAM\1", 4) == 0) {
+            fmt->category = sequence_data;
+            fmt->format = bam;
+            // TODO Decompress enough to pick version from @HD-VN header
+            fmt->version.major = 1, fmt->version.minor = -1;
+            return 0;
+        }
+        else if (memcmp(s, "BAI\1", 4) == 0) {
+            fmt->category = index_file;
+            fmt->format = bai;
+            fmt->version.major = -1, fmt->version.minor = -1;
+            return 0;
+        }
+        else if (memcmp(s, "BCF\4", 4) == 0) {
+            fmt->category = variant_data;
+            fmt->format = bcf;
+            fmt->version.major = 1, fmt->version.minor = -1;
+            return 0;
+        }
+        else if (memcmp(s, "BCF\2", 4) == 0) {
+            fmt->category = variant_data;
+            fmt->format = bcf;
+            fmt->version.major = s[3];
+            fmt->version.minor = (len >= 5 && s[4] <= 2)? s[4] : 0;
+            return 0;
+        }
+        else if (memcmp(s, "CSI\1", 4) == 0) {
+            fmt->category = index_file;
+            fmt->format = csi;
+            fmt->version.major = 1, fmt->version.minor = -1;
+            return 0;
+        }
+        else if (memcmp(s, "TBI\1", 4) == 0) {
+            fmt->category = index_file;
+            fmt->format = tbi;
+            return 0;
+        }
+        // GZI indexes have no magic numbers, so must be recognised solely by
+        // filename extension.
+        else if (strcmp(extension, "gzi") == 0) {
+            fmt->category = index_file;
+            fmt->format = gzi;
+            return 0;
+        }
+    }
+    else if (len >= 16 && memcmp(s, "##fileformat=VCF", 16) == 0) {
+        fmt->category = variant_data;
+        fmt->format = vcf;
+        if (len >= 21 && s[16] == 'v')
+            parse_version(fmt, &s[17], &s[len]);
+        return 0;
+    }
+    else if (len >= 4 && s[0] == '@' &&
+             (memcmp(s, "@HD\t", 4) == 0 || memcmp(s, "@SQ\t", 4) == 0 ||
+              memcmp(s, "@RG\t", 4) == 0 || memcmp(s, "@PG\t", 4) == 0 ||
+              memcmp(s, "@CO\t", 4) == 0)) {
+        fmt->category = sequence_data;
+        fmt->format = sam;
+        // @HD-VN is not guaranteed to be the first tag, but then @HD is
+        // not guaranteed to be present at all...
+        if (len >= 9 && memcmp(s, "@HD\tVN:", 7) == 0)
+            parse_version(fmt, &s[7], &s[len]);
+        else
+            fmt->version.major = 1, fmt->version.minor = -1;
+        return 0;
+    }
+    else if (len >= 8 && memcmp(s, "d4\xdd\xdd", 4) == 0) {
+        fmt->category = region_list;
+        fmt->format = d4_format;
+        // How to decode the D4 Format Version bytes is not yet specified
+        // so we don't try to set fmt->version.{major,minor}.
+        return 0;
+    }
+    else if (cmp_nonblank("{\"htsget\":", s, &s[len]) == 0) {
+        fmt->category = unknown_category;
+        fmt->format = htsget;
+        return 0;
+    }
+    else if (len > 8 && memcmp(s, "crypt4gh", 8) == 0) {
+        fmt->category = unknown_category;
+        fmt->format = hts_crypt4gh_format;
+        return 0;
+    }
+    else if (len >= 1 && s[0] == '>' && is_fastaq(s, &s[len])) {
+        fmt->category = sequence_data;
+        fmt->format = fasta_format;
+        return 0;
+    }
+    else if (len >= 1 && s[0] == '@' && is_fastaq(s, &s[len])) {
+        fmt->category = sequence_data;
+        fmt->format = fastq_format;
+        return 0;
+    }
+    else if (parse_tabbed_text(columns, sizeof columns, s,
+                               &s[len], &complete) > 0) {
+        // A complete SAM line is at least 11 columns.  On unmapped long reads may
+        // be missing two.  (On mapped long reads we must have an @ header so long
+        // CIGAR is irrelevant.)
+        if (colmatch(columns, "ZiZiiCZiiZZOOOOOOOOOOOOOOOOOOOO+")
+            >= 9 + 2*complete) {
+            fmt->category = sequence_data;
+            fmt->format = sam;
+            fmt->version.major = 1, fmt->version.minor = -1;
+            return 0;
+        }
+        else if (fmt->compression == gzip && colmatch(columns, "iiiiii") == 6) {
+            fmt->category = index_file;
+            fmt->format = crai;
+            return 0;
+        }
+        else if (strstr(extension, "fqi") && colmatch(columns, "Ziiiii") == 6) {
+            fmt->category = index_file;
+            fmt->format = fqi_format;
+            return 0;
+        }
+        else if (strstr(extension, "fai") && colmatch(columns, "Ziiii") == 5) {
+            fmt->category = index_file;
+            fmt->format = fai_format;
+            return 0;
+        }
+        else if (colmatch(columns, "Zii+") >= 3) {
+            fmt->category = region_list;
+            fmt->format = bed;
+            return 0;
+        }
+    }
+
+    // Arbitrary text files can be read using hts_getline().
+    if (is_text_only(s, &s[len])) fmt->format = text_format;
+
+    // Nothing recognised: leave unset fmt-> fields as unknown.
+    return 0;
+}
+
+char *hts_format_description(const htsFormat *format)
+{
+    kstring_t str = { 0, 0, NULL };
+
+    switch (format->format) {
+    case sam:   kputs("SAM", &str); break;
+    case bam:   kputs("BAM", &str); break;
+    case cram:  kputs("CRAM", &str); break;
+    case fasta_format:  kputs("FASTA", &str); break;
+    case fastq_format:  kputs("FASTQ", &str); break;
+    case vcf:   kputs("VCF", &str); break;
+    case bcf:
+        if (format->version.major == 1) kputs("Legacy BCF", &str);
+        else kputs("BCF", &str);
+        break;
+    case bai:   kputs("BAI", &str); break;
+    case crai:  kputs("CRAI", &str); break;
+    case csi:   kputs("CSI", &str); break;
+    case fai_format:    kputs("FASTA-IDX", &str); break;
+    case fqi_format:    kputs("FASTQ-IDX", &str); break;
+    case gzi:   kputs("GZI", &str); break;
+    case tbi:   kputs("Tabix", &str); break;
+    case bed:   kputs("BED", &str); break;
+    case d4_format:     kputs("D4", &str); break;
+    case htsget: kputs("htsget", &str); break;
+    case hts_crypt4gh_format: kputs("crypt4gh", &str); break;
+    case empty_format:  kputs("empty", &str); break;
+    default:    kputs("unknown", &str); break;
+    }
+
+    if (format->version.major >= 0) {
+        kputs(" version ", &str);
+        kputw(format->version.major, &str);
+        if (format->version.minor >= 0) {
+            kputc('.', &str);
+            kputw(format->version.minor, &str);
+        }
+    }
+
+    switch (format->compression) {
+    case bzip2_compression:  kputs(" bzip2-compressed", &str); break;
+    case razf_compression:   kputs(" legacy-RAZF-compressed", &str); break;
+    case xz_compression:     kputs(" XZ-compressed", &str); break;
+    case zstd_compression:   kputs(" Zstandard-compressed", &str); break;
+    case custom: kputs(" compressed", &str); break;
+    case gzip:   kputs(" gzip-compressed", &str); break;
+
+    case bgzf:
+        switch (format->format) {
+        case bam:
+        case bcf:
+        case csi:
+        case tbi:
+            // These are by definition BGZF, so just use the generic term
+            kputs(" compressed", &str);
+            break;
+        default:
+            kputs(" BGZF-compressed", &str);
+            break;
+        }
+        break;
+
+    case no_compression:
+        switch (format->format) {
+        case bam:
+        case bcf:
+        case cram:
+        case csi:
+        case tbi:
+            // These are normally compressed, so emphasise that this one isn't
+            kputs(" uncompressed", &str);
+            break;
+        default:
+            break;
+        }
+        break;
+
+    default: break;
+    }
+
+    switch (format->category) {
+    case sequence_data: kputs(" sequence", &str); break;
+    case variant_data:  kputs(" variant calling", &str); break;
+    case index_file:    kputs(" index", &str); break;
+    case region_list:   kputs(" genomic region", &str); break;
+    default: break;
+    }
+
+    if (format->compression == no_compression)
+        switch (format->format) {
+        case text_format:
+        case sam:
+        case crai:
+        case vcf:
+        case bed:
+        case fai_format:
+        case fqi_format:
+        case fasta_format:
+        case fastq_format:
+        case htsget:
+            kputs(" text", &str);
+            break;
+
+        case empty_format:
+            break;
+
+        default:
+            kputs(" data", &str);
+            break;
+        }
+    else
+        kputs(" data", &str);
+
+    return ks_release(&str);
+}
+
+htsFile *hts_open_format(const char *fn, const char *mode, const htsFormat *fmt)
+{
+    char smode[101], *cp, *cp2, *mode_c, *uncomp = NULL;
+    htsFile *fp = NULL;
+    hFILE *hfile = NULL;
+    char fmt_code = '\0';
+    // see enum htsExactFormat in htslib/hts.h
+    const char format_to_mode[] = "\0g\0\0b\0c\0\0b\0g\0\0\0\0\0Ff\0\0";
+
+    strncpy(smode, mode, 99);
+    smode[99]=0;
+    if ((cp = strchr(smode, ',')))
+        *cp = '\0';
+
+    // Migrate format code (b or c) to the end of the smode buffer.
+    for (cp2 = cp = smode; *cp; cp++) {
+        if (*cp == 'b')
+            fmt_code = 'b';
+        else if (*cp == 'c')
+            fmt_code = 'c';
+        else {
+            *cp2++ = *cp;
+            // Cache the uncompress flag 'u' pos if present
+            if (!uncomp && (*cp == 'u')) {
+                uncomp = cp2 - 1;
+            }
+        }
+    }
+    mode_c = cp2;
+    *cp2++ = fmt_code;
+    *cp2++ = 0;
+
+    // Set or reset the format code if opts->format is used
+    if (fmt && fmt->format > unknown_format
+        && fmt->format < sizeof(format_to_mode)) {
+        *mode_c = format_to_mode[fmt->format];
+    }
+
+    // Uncompressed bam/bcf is not supported, change 'u' to '0' on write
+    if (uncomp && *mode_c == 'b' && (strchr(smode, 'w') || strchr(smode, 'a'))) {
+        *uncomp = '0';
+    }
+
+    // If we really asked for a compressed text format then mode_c above will
+    // point to nul.  We set to 'z' to enable bgzf.
+    if (strchr(mode, 'w') && fmt && fmt->compression == bgzf) {
+        if (fmt->format == sam || fmt->format == vcf || fmt->format == text_format)
+            *mode_c = 'z';
+    }
+
+    char *rmme = NULL, *fnidx = strstr(fn, HTS_IDX_DELIM);
+    if ( fnidx ) {
+        rmme = strdup(fn);
+        if ( !rmme ) goto error;
+        rmme[fnidx-fn] = 0;
+        fn = rmme;
+    }
+
+    hfile = hopen(fn, smode);
+    if (hfile == NULL) goto error;
+
+    fp = hts_hopen(hfile, fn, smode);
+    if (fp == NULL) goto error;
+
+    // Compensate for the loss of exactness in htsExactFormat.
+    // hts_hopen returns generics such as binary or text, but we
+    // have been given something explicit here so use that instead.
+    if (fp->is_write && fmt &&
+        (fmt->format == bam || fmt->format == sam ||
+         fmt->format == vcf || fmt->format == bcf ||
+         fmt->format == bed || fmt->format == fasta_format ||
+         fmt->format == fastq_format))
+        fp->format.format = fmt->format;
+
+    if (fmt && fmt->specific) {
+        if (hts_opt_apply(fp, fmt->specific) != 0) {
+            if (((hts_opt*)fmt->specific)->opt == CRAM_OPT_REFERENCE &&
+                (errno == ENOENT || errno == EIO || errno == EBADF ||
+                  errno == EACCES || errno == EISDIR)) {
+                /* error during reference file operation
+                 for these specific errors, set the error as EINVAL */
+                errno = EINVAL;
+            }
+            goto error;
+        }
+    }
+    if ( rmme ) free(rmme);
+    return fp;
+
+error:
+    hts_log_error("Failed to open file \"%s\"%s%s", fn,
+                  errno ? " : " : "", errno ? strerror(errno) : "");
+    if ( rmme ) free(rmme);
+
+    if (hfile)
+        hclose_abruptly(hfile);
+
+    return NULL;
+}
+
+htsFile *hts_open(const char *fn, const char *mode) {
+    return hts_open_format(fn, mode, NULL);
+}
+
+/*
+ * Splits str into a prefix, delimiter ('\0' or delim), and suffix, writing
+ * the prefix in lowercase into buf and returning a pointer to the suffix.
+ * On return, buf is always NUL-terminated; thus assumes that the "keyword"
+ * prefix should be one of several known values of maximum length buflen-2.
+ * (If delim is not found, returns a pointer to the '\0'.)
+ */
+static const char *
+scan_keyword(const char *str, char delim, char *buf, size_t buflen)
+{
+    size_t i = 0;
+    while (*str && *str != delim) {
+        if (i < buflen-1) buf[i++] = tolower_c(*str);
+        str++;
+    }
+
+    buf[i] = '\0';
+    return *str? str+1 : str;
+}
+
+/*
+ * Parses arg and appends it to the option list.
+ *
+ * Returns 0 on success;
+ *        -1 on failure.
+ */
+int hts_opt_add(hts_opt **opts, const char *c_arg) {
+    hts_opt *o, *t;
+    char *val;
+
+    /*
+     * IMPORTANT!!!
+     * If you add another string option here, don't forget to also add
+     * it to the case statement in hts_opt_apply.
+     */
+
+    if (!c_arg)
+        return -1;
+
+    if (!(o =  malloc(sizeof(*o))))
+        return -1;
+
+    if (!(o->arg = strdup(c_arg))) {
+        free(o);
+        return -1;
+    }
+
+    if (!(val = strchr(o->arg, '=')))
+        val = "1"; // assume boolean
+    else
+        *val++ = '\0';
+
+    if (strcmp(o->arg, "decode_md") == 0 ||
+        strcmp(o->arg, "DECODE_MD") == 0)
+        o->opt = CRAM_OPT_DECODE_MD, o->val.i = atoi(val);
+
+    else if (strcmp(o->arg, "verbosity") == 0 ||
+             strcmp(o->arg, "VERBOSITY") == 0)
+        o->opt = CRAM_OPT_VERBOSITY, o->val.i = atoi(val);
+
+    else if (strcmp(o->arg, "seqs_per_slice") == 0 ||
+             strcmp(o->arg, "SEQS_PER_SLICE") == 0)
+        o->opt = CRAM_OPT_SEQS_PER_SLICE, o->val.i = atoi(val);
+
+    else if (strcmp(o->arg, "bases_per_slice") == 0 ||
+             strcmp(o->arg, "BASES_PER_SLICE") == 0)
+        o->opt = CRAM_OPT_BASES_PER_SLICE, o->val.i = atoi(val);
+
+    else if (strcmp(o->arg, "slices_per_container") == 0 ||
+             strcmp(o->arg, "SLICES_PER_CONTAINER") == 0)
+        o->opt = CRAM_OPT_SLICES_PER_CONTAINER, o->val.i = atoi(val);
+
+    else if (strcmp(o->arg, "embed_ref") == 0 ||
+             strcmp(o->arg, "EMBED_REF") == 0)
+        o->opt = CRAM_OPT_EMBED_REF, o->val.i = atoi(val);
+
+    else if (strcmp(o->arg, "no_ref") == 0 ||
+             strcmp(o->arg, "NO_REF") == 0)
+        o->opt = CRAM_OPT_NO_REF, o->val.i = atoi(val);
+
+    else if (strcmp(o->arg, "pos_delta") == 0 ||
+             strcmp(o->arg, "POS_DELTA") == 0)
+        o->opt = CRAM_OPT_POS_DELTA, o->val.i = atoi(val);
+
+    else if (strcmp(o->arg, "ignore_md5") == 0 ||
+             strcmp(o->arg, "IGNORE_MD5") == 0)
+        o->opt = CRAM_OPT_IGNORE_MD5, o->val.i = atoi(val);
+
+    else if (strcmp(o->arg, "use_bzip2") == 0 ||
+             strcmp(o->arg, "USE_BZIP2") == 0)
+        o->opt = CRAM_OPT_USE_BZIP2, o->val.i = atoi(val);
+
+    else if (strcmp(o->arg, "use_rans") == 0 ||
+             strcmp(o->arg, "USE_RANS") == 0)
+        o->opt = CRAM_OPT_USE_RANS, o->val.i = atoi(val);
+
+    else if (strcmp(o->arg, "use_lzma") == 0 ||
+             strcmp(o->arg, "USE_LZMA") == 0)
+        o->opt = CRAM_OPT_USE_LZMA, o->val.i = atoi(val);
+
+    else if (strcmp(o->arg, "use_tok") == 0 ||
+             strcmp(o->arg, "USE_TOK") == 0)
+        o->opt = CRAM_OPT_USE_TOK, o->val.i = atoi(val);
+
+    else if (strcmp(o->arg, "use_fqz") == 0 ||
+             strcmp(o->arg, "USE_FQZ") == 0)
+        o->opt = CRAM_OPT_USE_FQZ, o->val.i = atoi(val);
+
+    else if (strcmp(o->arg, "use_arith") == 0 ||
+             strcmp(o->arg, "USE_ARITH") == 0)
+        o->opt = CRAM_OPT_USE_ARITH, o->val.i = atoi(val);
+
+    else if (strcmp(o->arg, "fast") == 0 ||
+             strcmp(o->arg, "FAST") == 0)
+        o->opt = HTS_OPT_PROFILE, o->val.i = HTS_PROFILE_FAST;
+
+    else if (strcmp(o->arg, "normal") == 0 ||
+             strcmp(o->arg, "NORMAL") == 0)
+        o->opt = HTS_OPT_PROFILE, o->val.i = HTS_PROFILE_NORMAL;
+
+    else if (strcmp(o->arg, "small") == 0 ||
+             strcmp(o->arg, "SMALL") == 0)
+        o->opt = HTS_OPT_PROFILE, o->val.i = HTS_PROFILE_SMALL;
+
+    else if (strcmp(o->arg, "archive") == 0 ||
+             strcmp(o->arg, "ARCHIVE") == 0)
+        o->opt = HTS_OPT_PROFILE, o->val.i = HTS_PROFILE_ARCHIVE;
+
+    else if (strcmp(o->arg, "reference") == 0 ||
+             strcmp(o->arg, "REFERENCE") == 0)
+        o->opt = CRAM_OPT_REFERENCE, o->val.s = val;
+
+    else if (strcmp(o->arg, "version") == 0 ||
+             strcmp(o->arg, "VERSION") == 0)
+        o->opt = CRAM_OPT_VERSION, o->val.s =val;
+
+    else if (strcmp(o->arg, "multi_seq_per_slice") == 0 ||
+             strcmp(o->arg, "MULTI_SEQ_PER_SLICE") == 0)
+        o->opt = CRAM_OPT_MULTI_SEQ_PER_SLICE, o->val.i = atoi(val);
+
+    else if (strcmp(o->arg, "nthreads") == 0 ||
+             strcmp(o->arg, "NTHREADS") == 0)
+        o->opt = HTS_OPT_NTHREADS, o->val.i = atoi(val);
+
+    else if (strcmp(o->arg, "cache_size") == 0 ||
+             strcmp(o->arg, "CACHE_SIZE") == 0) {
+        char *endp;
+        o->opt = HTS_OPT_CACHE_SIZE;
+        o->val.i = strtol(val, &endp, 0);
+        // NB: Doesn't support floats, eg 1.5g
+        // TODO: extend hts_parse_decimal? See also samtools sort.
+        switch (*endp) {
+        case 'g': case 'G': o->val.i *= 1024; // fall through
+        case 'm': case 'M': o->val.i *= 1024; // fall through
+        case 'k': case 'K': o->val.i *= 1024; break;
+        case '\0': break;
+        default:
+            hts_log_error("Unrecognised cache size suffix '%c'", *endp);
+            free(o->arg);
+            free(o);
+            return -1;
+        }
+    }
+
+    else if (strcmp(o->arg, "required_fields") == 0 ||
+             strcmp(o->arg, "REQUIRED_FIELDS") == 0)
+        o->opt = CRAM_OPT_REQUIRED_FIELDS, o->val.i = strtol(val, NULL, 0);
+
+    else if (strcmp(o->arg, "lossy_names") == 0 ||
+             strcmp(o->arg, "LOSSY_NAMES") == 0)
+        o->opt = CRAM_OPT_LOSSY_NAMES, o->val.i = strtol(val, NULL, 0);
+
+    else if (strcmp(o->arg, "name_prefix") == 0 ||
+             strcmp(o->arg, "NAME_PREFIX") == 0)
+        o->opt = CRAM_OPT_PREFIX, o->val.s = val;
+
+    else if (strcmp(o->arg, "store_md") == 0 ||
+             strcmp(o->arg, "store_md") == 0)
+        o->opt = CRAM_OPT_STORE_MD, o->val.i = atoi(val);
+
+    else if (strcmp(o->arg, "store_nm") == 0 ||
+             strcmp(o->arg, "store_nm") == 0)
+        o->opt = CRAM_OPT_STORE_NM, o->val.i = atoi(val);
+
+    else if (strcmp(o->arg, "block_size") == 0 ||
+             strcmp(o->arg, "BLOCK_SIZE") == 0)
+        o->opt = HTS_OPT_BLOCK_SIZE, o->val.i = strtol(val, NULL, 0);
+
+    else if (strcmp(o->arg, "level") == 0 ||
+             strcmp(o->arg, "LEVEL") == 0)
+        o->opt = HTS_OPT_COMPRESSION_LEVEL, o->val.i = strtol(val, NULL, 0);
+
+    else if (strcmp(o->arg, "filter") == 0 ||
+             strcmp(o->arg, "FILTER") == 0)
+        o->opt = HTS_OPT_FILTER, o->val.s = val;
+
+    else if (strcmp(o->arg, "fastq_aux") == 0 ||
+        strcmp(o->arg, "FASTQ_AUX") == 0)
+        o->opt = FASTQ_OPT_AUX, o->val.s = val;
+
+    else if (strcmp(o->arg, "fastq_barcode") == 0 ||
+        strcmp(o->arg, "FASTQ_BARCODE") == 0)
+        o->opt = FASTQ_OPT_BARCODE, o->val.s = val;
+
+    else if (strcmp(o->arg, "fastq_rnum") == 0 ||
+        strcmp(o->arg, "FASTQ_RNUM") == 0)
+        o->opt = FASTQ_OPT_RNUM, o->val.i = 1;
+
+    else if (strcmp(o->arg, "fastq_casava") == 0 ||
+        strcmp(o->arg, "FASTQ_CASAVA") == 0)
+        o->opt = FASTQ_OPT_CASAVA, o->val.i = 1;
+
+    else if (strcmp(o->arg, "fastq_name2") == 0 ||
+        strcmp(o->arg, "FASTQ_NAME2") == 0)
+        o->opt = FASTQ_OPT_NAME2, o->val.i = 1;
+
+    else {
+        hts_log_error("Unknown option '%s'", o->arg);
+        free(o->arg);
+        free(o);
+        return -1;
+    }
+
+    o->next = NULL;
+
+    // Append; assumes small list.
+    if (*opts) {
+        t = *opts;
+        while (t->next)
+            t = t->next;
+        t->next = o;
+    } else {
+        *opts = o;
+    }
+
+    return 0;
+}
+
+/*
+ * Applies an hts_opt option list to a given htsFile.
+ *
+ * Returns 0 on success
+ *        -1 on failure
+ */
+int hts_opt_apply(htsFile *fp, hts_opt *opts) {
+    hts_opt *last = NULL;
+
+    for (; opts;  opts = (last=opts)->next) {
+        switch (opts->opt) {
+            case CRAM_OPT_REFERENCE:
+                if (!(fp->fn_aux = strdup(opts->val.s)))
+                    return -1;
+                // fall through
+            case CRAM_OPT_VERSION:
+            case CRAM_OPT_PREFIX:
+            case HTS_OPT_FILTER:
+            case FASTQ_OPT_AUX:
+            case FASTQ_OPT_BARCODE:
+                if (hts_set_opt(fp,  opts->opt,  opts->val.s) != 0)
+                    return -1;
+                break;
+            default:
+                if (hts_set_opt(fp,  opts->opt,  opts->val.i) != 0)
+                    return -1;
+                break;
+        }
+    }
+
+    return 0;
+}
+
+/*
+ * Frees an hts_opt list.
+ */
+void hts_opt_free(hts_opt *opts) {
+    hts_opt *last = NULL;
+    while (opts) {
+        opts = (last=opts)->next;
+        free(last->arg);
+        free(last);
+    }
+}
+
+
+/*
+ * Tokenise options as (key(=value)?,)*(key(=value)?)?
+ * NB: No provision for ',' appearing in the value!
+ * Add backslashing rules?
+ *
+ * This could be used as part of a general command line option parser or
+ * as a string concatenated onto the file open mode.
+ *
+ * Returns 0 on success
+ *        -1 on failure.
+ */
+int hts_parse_opt_list(htsFormat *fmt, const char *str) {
+    while (str && *str) {
+        const char *str_start;
+        int len;
+        char arg[8001];
+
+        while (*str && *str == ',')
+            str++;
+
+        for (str_start = str; *str && *str != ','; str++);
+        len = str - str_start;
+
+        // Produce a nul terminated copy of the option
+        strncpy(arg, str_start, len < 8000 ? len : 8000);
+        arg[len < 8000 ? len : 8000] = '\0';
+
+        if (hts_opt_add((hts_opt **)&fmt->specific, arg) != 0)
+            return -1;
+
+        if (*str)
+            str++;
+    }
+
+    return 0;
+}
+
+/*
+ * Accepts a string file format (sam, bam, cram, vcf, bam) optionally
+ * followed by a comma separated list of key=value options and splits
+ * these up into the fields of htsFormat struct.
+ *
+ * format is assumed to be already initialised, either to blank
+ * "unknown" values or via previous hts_opt_add calls.
+ *
+ * Returns 0 on success
+ *        -1 on failure.
+ */
+int hts_parse_format(htsFormat *format, const char *str) {
+    char fmt[9];
+    const char *cp = scan_keyword(str, ',', fmt, sizeof fmt);
+
+    format->version.minor = 0; // unknown
+    format->version.major = 0; // unknown
+
+    if (strcmp(fmt, "sam") == 0) {
+        format->category          = sequence_data;
+        format->format            = sam;
+        format->compression       = no_compression;
+        format->compression_level = 0;
+    } else if (strcmp(fmt, "sam.gz") == 0) {
+        format->category          = sequence_data;
+        format->format            = sam;
+        format->compression       = bgzf;
+        format->compression_level = -1;
+    } else if (strcmp(fmt, "bam") == 0) {
+        format->category          = sequence_data;
+        format->format            = bam;
+        format->compression       = bgzf;
+        format->compression_level = -1;
+    } else if (strcmp(fmt, "cram") == 0) {
+        format->category          = sequence_data;
+        format->format            = cram;
+        format->compression       = custom;
+        format->compression_level = -1;
+    } else if (strcmp(fmt, "vcf") == 0) {
+        format->category          = variant_data;
+        format->format            = vcf;
+        format->compression       = no_compression;
+        format->compression_level = 0;
+    } else if (strcmp(fmt, "bcf") == 0) {
+        format->category          = variant_data;
+        format->format            = bcf;
+        format->compression       = bgzf;
+        format->compression_level = -1;
+    } else if (strcmp(fmt, "fastq") == 0 || strcmp(fmt, "fq") == 0) {
+        format->category          = sequence_data;
+        format->format            = fastq_format;
+        format->compression       = no_compression;
+        format->compression_level = 0;
+    } else if (strcmp(fmt, "fastq.gz") == 0 || strcmp(fmt, "fq.gz") == 0) {
+        format->category          = sequence_data;
+        format->format            = fastq_format;
+        format->compression       = bgzf;
+        format->compression_level = 0;
+    } else if (strcmp(fmt, "fasta") == 0 || strcmp(fmt, "fa") == 0) {
+        format->category          = sequence_data;
+        format->format            = fasta_format;
+        format->compression       = no_compression;
+        format->compression_level = 0;
+    } else if (strcmp(fmt, "fasta.gz") == 0 || strcmp(fmt, "fa.gz") == 0) {
+        format->category          = sequence_data;
+        format->format            = fasta_format;
+        format->compression       = bgzf;
+        format->compression_level = 0;
+    } else {
+        return -1;
+    }
+
+    return hts_parse_opt_list(format, cp);
+}
+
+
+/*
+ * Tokenise options as (key(=value)?,)*(key(=value)?)?
+ * NB: No provision for ',' appearing in the value!
+ * Add backslashing rules?
+ *
+ * This could be used as part of a general command line option parser or
+ * as a string concatenated onto the file open mode.
+ *
+ * Returns 0 on success
+ *        -1 on failure.
+ */
+static int hts_process_opts(htsFile *fp, const char *opts) {
+    htsFormat fmt;
+
+    fmt.specific = NULL;
+    if (hts_parse_opt_list(&fmt, opts) != 0)
+        return -1;
+
+    if (hts_opt_apply(fp, fmt.specific) != 0) {
+        hts_opt_free(fmt.specific);
+        return -1;
+    }
+
+    hts_opt_free(fmt.specific);
+
+    return 0;
+}
+
+static int hts_crypt4gh_redirect(const char *fn, const char *mode,
+                                 hFILE **hfile_ptr, htsFile *fp) {
+    hFILE *hfile1 = *hfile_ptr;
+    hFILE *hfile2 = NULL;
+    char fn_buf[512], *fn2 = fn_buf;
+    char mode2[102]; // Size set by sizeof(simple_mode) in hts_hopen()
+    const char *prefix = "crypt4gh:";
+    size_t fn2_len = strlen(prefix) + strlen(fn) + 1;
+    int ret = -1;
+
+    if (fn2_len > sizeof(fn_buf)) {
+        if (fn2_len >= INT_MAX) // Silence gcc format-truncation warning
+            return -1;
+        fn2 = malloc(fn2_len);
+        if (!fn2) return -1;
+    }
+
+    // Reopen fn using the crypt4gh plug-in (if available)
+    snprintf(fn2, fn2_len, "%s%s", prefix, fn);
+    snprintf(mode2, sizeof(mode2), "%s%s", mode, strchr(mode, ':') ? "" : ":");
+    hfile2 = hopen(fn2, mode2, "parent", hfile1, NULL);
+    if (hfile2) {
+        // Replace original hfile with the new one.  The original is now
+        // enclosed within hfile2
+        *hfile_ptr = hfile2;
+        ret = 0;
+    }
+
+    if (fn2 != fn_buf)
+        free(fn2);
+    return ret;
+}
+
+htsFile *hts_hopen(hFILE *hfile, const char *fn, const char *mode)
+{
+    hFILE *hfile_orig = hfile;
+    hFILE *hfile_cleanup = hfile;
+    htsFile *fp = (htsFile*)calloc(1, sizeof(htsFile));
+    char simple_mode[101], *cp, *opts;
+    simple_mode[100] = '\0';
+
+    if (fp == NULL) goto error;
+
+    fp->fn = strdup(fn);
+    fp->is_be = ed_is_big();
+
+    // Split mode into simple_mode,opts strings
+    if ((cp = strchr(mode, ','))) {
+        strncpy(simple_mode, mode, cp-mode <= 100 ? cp-mode : 100);
+        simple_mode[cp-mode] = '\0';
+        opts = cp+1;
+    } else {
+        strncpy(simple_mode, mode, 100);
+        opts = NULL;
+    }
+
+    if (strchr(simple_mode, 'r')) {
+        const int max_loops = 5; // Should be plenty
+        int loops = 0;
+        if (hts_detect_format2(hfile, fn, &fp->format) < 0) goto error;
+
+        // Deal with formats that re-direct an underlying file via a plug-in.
+        // Loops as we may have crypt4gh served via htsget, or
+        // crypt4gh-in-crypt4gh.
+
+        while (fp->format.format == htsget ||
+               fp->format.format == hts_crypt4gh_format) {
+            // Ensure we don't get stuck in an endless redirect loop
+            if (++loops > max_loops) {
+                errno = ELOOP;
+                goto error;
+            }
+
+            if (fp->format.format == htsget) {
+                hFILE *hfile2 = hopen_htsget_redirect(hfile, simple_mode);
+                if (hfile2 == NULL) goto error;
+
+                if (hfile != hfile_cleanup) {
+                    // Close the result of an earlier redirection
+                    hclose_abruptly(hfile);
+                }
+
+                hfile = hfile2;
+            }
+            else if (fp->format.format == hts_crypt4gh_format) {
+                int should_preserve = (hfile == hfile_orig);
+                int update_cleanup = (hfile == hfile_cleanup);
+                if (hts_crypt4gh_redirect(fn, simple_mode, &hfile, fp) < 0)
+                    goto error;
+                if (should_preserve) {
+                    // The original hFILE is now contained in a crypt4gh
+                    // wrapper.  Should we need to close the wrapper due
+                    // to a later error, we need to prevent the wrapped
+                    // handle from being closed as the caller will see
+                    // this function return NULL and try to clean up itself.
+                    hfile_orig->preserve = 1;
+                }
+                if (update_cleanup) {
+                    // Update handle to close at the end if redirected by htsget
+                    hfile_cleanup = hfile;
+                }
+            }
+
+            // Re-detect format against the result of the redirection
+            if (hts_detect_format2(hfile, fn, &fp->format) < 0) goto error;
+        }
+    }
+    else if (strchr(simple_mode, 'w') || strchr(simple_mode, 'a')) {
+        htsFormat *fmt = &fp->format;
+        fp->is_write = 1;
+
+        if (strchr(simple_mode, 'b')) fmt->format = binary_format;
+        else if (strchr(simple_mode, 'c')) fmt->format = cram;
+        else if (strchr(simple_mode, 'f')) fmt->format = fastq_format;
+        else if (strchr(simple_mode, 'F')) fmt->format = fasta_format;
+        else fmt->format = text_format;
+
+        if (strchr(simple_mode, 'z')) fmt->compression = bgzf;
+        else if (strchr(simple_mode, 'g')) fmt->compression = gzip;
+        else if (strchr(simple_mode, 'u')) fmt->compression = no_compression;
+        else {
+            // No compression mode specified, set to the default for the format
+            switch (fmt->format) {
+            case binary_format: fmt->compression = bgzf; break;
+            case cram: fmt->compression = custom; break;
+            case fastq_format: fmt->compression = no_compression; break;
+            case fasta_format: fmt->compression = no_compression; break;
+            case text_format: fmt->compression = no_compression; break;
+            default: abort();
+            }
+        }
+
+        // Fill in category (if determinable; e.g. 'b' could be BAM or BCF)
+        fmt->category = format_category(fmt->format);
+
+        fmt->version.major = fmt->version.minor = -1;
+        fmt->compression_level = -1;
+        fmt->specific = NULL;
+    }
+    else { errno = EINVAL; goto error; }
+
+    switch (fp->format.format) {
+    case binary_format:
+    case bam:
+    case bcf:
+        fp->fp.bgzf = bgzf_hopen(hfile, simple_mode);
+        if (fp->fp.bgzf == NULL) goto error;
+        fp->is_bin = fp->is_bgzf = 1;
+        break;
+
+    case cram:
+        fp->fp.cram = cram_dopen(hfile, fn, simple_mode);
+        if (fp->fp.cram == NULL) goto error;
+        if (!fp->is_write)
+            cram_set_option(fp->fp.cram, CRAM_OPT_DECODE_MD, -1); // auto
+        fp->is_cram = 1;
+        break;
+
+    case empty_format:
+    case text_format:
+    case bed:
+    case fasta_format:
+    case fastq_format:
+    case sam:
+    case vcf:
+        if (fp->format.compression != no_compression) {
+            fp->fp.bgzf = bgzf_hopen(hfile, simple_mode);
+            if (fp->fp.bgzf == NULL) goto error;
+            fp->is_bgzf = 1;
+        }
+        else
+            fp->fp.hfile = hfile;
+        break;
+
+    default:
+        errno = EFTYPE;
+        goto error;
+    }
+
+    if (opts)
+        hts_process_opts(fp, opts);
+
+    // Allow original file to close if it was preserved earlier by crypt4gh
+    hfile_orig->preserve = 0;
+
+    // If redirecting via htsget, close the original hFILE now (pedantically
+    // we would instead close it in hts_close(), but this a simplifying
+    // optimisation)
+    if (hfile != hfile_cleanup) hclose_abruptly(hfile_cleanup);
+
+    return fp;
+
+error:
+    hts_log_error("Failed to open file %s", fn);
+
+    // If redirecting, close the failed redirection hFILE that we have opened
+    if (hfile != hfile_orig) hclose_abruptly(hfile);
+    hfile_orig->preserve = 0; // Allow caller to close the original hfile
+
+    if (fp) {
+        free(fp->fn);
+        free(fp->fn_aux);
+        free(fp);
+    }
+    return NULL;
+}
+
+static int hts_idx_close_otf_fp(hts_idx_t *idx);
+
+int hts_close(htsFile *fp)
+{
+    int ret = 0, save;
+    if (!fp) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    switch (fp->format.format) {
+    case binary_format:
+    case bam:
+    case bcf:
+        ret = bgzf_close(fp->fp.bgzf);
+        break;
+
+    case cram:
+        if (!fp->is_write) {
+            switch (cram_eof(fp->fp.cram)) {
+            case 2:
+                hts_log_warning("EOF marker is absent. The input is probably truncated");
+                break;
+            case 0:  /* not at EOF, but may not have wanted all seqs */
+            default: /* case 1, expected EOF */
+                break;
+            }
+        }
+        ret = cram_close(fp->fp.cram);
+        break;
+
+    case empty_format:
+    case text_format:
+    case bed:
+    case fasta_format:
+    case fastq_format:
+    case sam:
+    case vcf:
+        if (fp->format.format == sam)
+            ret = sam_state_destroy(fp);
+        else if (fp->format.format == fastq_format ||
+                 fp->format.format == fasta_format)
+            fastq_state_destroy(fp);
+
+        if (fp->format.compression != no_compression)
+            ret |= bgzf_close(fp->fp.bgzf);
+        else
+            ret |= hclose(fp->fp.hfile);
+        break;
+
+    default:
+        ret = -1;
+        break;
+    }
+
+    if (fp->idx) {
+        // Close deferred index file handle, if present.
+        // Unfortunately this means errors on the index will get mixed with
+        // those on the main file, but as we only have the EOF block left to
+        // write it hopefully won't happen that often.
+        ret |= hts_idx_close_otf_fp(fp->idx);
+    }
+
+    save = errno;
+    sam_hdr_destroy(fp->bam_header);
+    hts_idx_destroy(fp->idx);
+    hts_filter_free(fp->filter);
+    free(fp->fn);
+    free(fp->fn_aux);
+    free(fp->line.s);
+    free(fp);
+    errno = save;
+    return ret;
+}
+
+int hts_flush(htsFile *fp)
+{
+    if (fp == NULL) return 0;
+
+    switch (fp->format.format) {
+    case binary_format:
+    case bam:
+    case bcf:
+        return bgzf_flush(fp->fp.bgzf);
+
+    case cram:
+        return cram_flush(fp->fp.cram);
+
+    case empty_format:
+    case text_format:
+    case bed:
+    case fasta_format:
+    case fastq_format:
+    case sam:
+    case vcf:
+        if (fp->format.compression != no_compression)
+            return bgzf_flush(fp->fp.bgzf);
+        else
+            return hflush(fp->fp.hfile);
+
+    default:
+        break;
+    }
+
+    return 0;
+}
+
+const htsFormat *hts_get_format(htsFile *fp)
+{
+    return fp? &fp->format : NULL;
+}
+
+const char *hts_format_file_extension(const htsFormat *format) {
+    if (!format)
+        return "?";
+
+    switch (format->format) {
+    case sam:  return "sam";
+    case bam:  return "bam";
+    case bai:  return "bai";
+    case cram: return "cram";
+    case crai: return "crai";
+    case vcf:  return "vcf";
+    case bcf:  return "bcf";
+    case csi:  return "csi";
+    case fai_format:   return "fai";
+    case fqi_format:   return "fqi";
+    case gzi:  return "gzi";
+    case tbi:  return "tbi";
+    case bed:  return "bed";
+    case d4_format:    return "d4";
+    case fasta_format: return "fa";
+    case fastq_format: return "fq";
+    default:   return "?";
+    }
+}
+
+static hFILE *hts_hfile(htsFile *fp) {
+    switch (fp->format.format) {
+    case binary_format:// fall through
+    case bcf:          // fall through
+    case bam:          return bgzf_hfile(fp->fp.bgzf);
+    case cram:         return cram_hfile(fp->fp.cram);
+    case text_format:  // fall through
+    case vcf:          // fall through
+    case fastq_format: // fall through
+    case fasta_format: // fall through
+    case sam:          return fp->format.compression != no_compression
+                              ? bgzf_hfile(fp->fp.bgzf)
+                              : fp->fp.hfile;
+    default:           return NULL;
+    }
+}
+
+int hts_set_opt(htsFile *fp, enum hts_fmt_option opt, ...) {
+    int r;
+    va_list args;
+
+    switch (opt) {
+    case HTS_OPT_NTHREADS: {
+        va_start(args, opt);
+        int nthreads = va_arg(args, int);
+        va_end(args);
+        return hts_set_threads(fp, nthreads);
+    }
+
+    case HTS_OPT_BLOCK_SIZE: {
+        hFILE *hf = hts_hfile(fp);
+
+        if (hf) {
+            va_start(args, opt);
+            if (hfile_set_blksize(hf, va_arg(args, int)) != 0)
+                hts_log_warning("Failed to change block size");
+            va_end(args);
+        }
+        else {
+            // To do - implement for vcf/bcf.
+            hts_log_warning("Cannot change block size for this format");
+        }
+
+        return 0;
+    }
+
+    case HTS_OPT_THREAD_POOL: {
+        va_start(args, opt);
+        htsThreadPool *p = va_arg(args, htsThreadPool *);
+        va_end(args);
+        return hts_set_thread_pool(fp, p);
+    }
+
+    case HTS_OPT_CACHE_SIZE: {
+        va_start(args, opt);
+        int cache_size = va_arg(args, int);
+        va_end(args);
+        hts_set_cache_size(fp, cache_size);
+        return 0;
+    }
+
+    case FASTQ_OPT_CASAVA:
+    case FASTQ_OPT_RNUM:
+    case FASTQ_OPT_NAME2:
+        if (fp->format.format == fastq_format ||
+            fp->format.format == fasta_format)
+            return fastq_state_set(fp, opt);
+        return 0;
+
+    case FASTQ_OPT_AUX:
+        if (fp->format.format == fastq_format ||
+            fp->format.format == fasta_format) {
+            va_start(args, opt);
+            char *list = va_arg(args, char *);
+            va_end(args);
+            return fastq_state_set(fp, opt, list);
+        }
+        return 0;
+
+    case FASTQ_OPT_BARCODE:
+        if (fp->format.format == fastq_format ||
+            fp->format.format == fasta_format) {
+            va_start(args, opt);
+            char *bc = va_arg(args, char *);
+            va_end(args);
+            return fastq_state_set(fp, opt, bc);
+        }
+        return 0;
+
+    // Options below here flow through to cram_set_voption
+    case HTS_OPT_COMPRESSION_LEVEL: {
+        va_start(args, opt);
+        int level = va_arg(args, int);
+        va_end(args);
+        if (fp->is_bgzf)
+            fp->fp.bgzf->compress_level = level;
+        else if (fp->format.format == cram)
+            return cram_set_option(fp->fp.cram, opt, level);
+        return 0;
+    }
+
+    case HTS_OPT_FILTER: {
+        va_start(args, opt);
+        char *expr = va_arg(args, char *);
+        va_end(args);
+        return hts_set_filter_expression(fp, expr);
+    }
+
+    case HTS_OPT_PROFILE: {
+        va_start(args, opt);
+        enum hts_profile_option prof = va_arg(args, int);
+        va_end(args);
+        if (fp->is_bgzf) {
+            switch (prof) {
+#ifdef HAVE_LIBDEFLATE
+            case HTS_PROFILE_FAST:    fp->fp.bgzf->compress_level =  2; break;
+            case HTS_PROFILE_NORMAL:  fp->fp.bgzf->compress_level = -1; break;
+            case HTS_PROFILE_SMALL:   fp->fp.bgzf->compress_level = 10; break;
+            case HTS_PROFILE_ARCHIVE: fp->fp.bgzf->compress_level = 12; break;
+#else
+            case HTS_PROFILE_FAST:    fp->fp.bgzf->compress_level =  1; break;
+            case HTS_PROFILE_NORMAL:  fp->fp.bgzf->compress_level = -1; break;
+            case HTS_PROFILE_SMALL:   fp->fp.bgzf->compress_level =  8; break;
+            case HTS_PROFILE_ARCHIVE: fp->fp.bgzf->compress_level =  9; break;
+#endif
+            }
+        } // else CRAM manages this in its own way
+        break;
+    }
+
+    default:
+        break;
+    }
+
+    if (fp->format.format != cram)
+        return 0;
+
+    va_start(args, opt);
+    r = cram_set_voption(fp->fp.cram, opt, args);
+    va_end(args);
+
+    return r;
+}
+
+BGZF *hts_get_bgzfp(htsFile *fp);
+
+int hts_set_threads(htsFile *fp, int n)
+{
+    if (fp->format.format == sam) {
+        return sam_set_threads(fp, n);
+    } else if (fp->format.compression == bgzf) {
+        return bgzf_mt(hts_get_bgzfp(fp), n, 256/*unused*/);
+    } else if (fp->format.format == cram) {
+        return hts_set_opt(fp, CRAM_OPT_NTHREADS, n);
+    }
+    else return 0;
+}
+
+int hts_set_thread_pool(htsFile *fp, htsThreadPool *p) {
+    if (fp->format.format == sam || fp->format.format == text_format) {
+        return sam_set_thread_pool(fp, p);
+    } else if (fp->format.compression == bgzf) {
+        return bgzf_thread_pool(hts_get_bgzfp(fp), p->pool, p->qsize);
+    } else if (fp->format.format == cram) {
+        return hts_set_opt(fp, CRAM_OPT_THREAD_POOL, p);
+    }
+    else return 0;
+}
+
+void hts_set_cache_size(htsFile *fp, int n)
+{
+    if (fp->format.compression == bgzf)
+        bgzf_set_cache_size(hts_get_bgzfp(fp), n);
+}
+
+int hts_set_fai_filename(htsFile *fp, const char *fn_aux)
+{
+    free(fp->fn_aux);
+    if (fn_aux) {
+        fp->fn_aux = strdup(fn_aux);
+        if (fp->fn_aux == NULL) return -1;
+    }
+    else fp->fn_aux = NULL;
+
+    if (fp->format.format == cram)
+        if (cram_set_option(fp->fp.cram, CRAM_OPT_REFERENCE, fp->fn_aux))
+            return -1;
+
+    return 0;
+}
+
+int hts_set_filter_expression(htsFile *fp, const char *expr)
+{
+    if (fp->filter)
+        hts_filter_free(fp->filter);
+
+    if (!expr)
+        return 0;
+
+    return (fp->filter = hts_filter_init(expr))
+        ? 0 : -1;
+}
+
+hFILE *hts_open_tmpfile(const char *fname, const char *mode, kstring_t *tmpname)
+{
+    int pid = (int) getpid();
+    unsigned ptr = (uintptr_t) tmpname;
+    int n = 0;
+    hFILE *fp = NULL;
+
+    do {
+        // Attempt to further uniquify the temporary filename
+        unsigned t = ((unsigned) time(NULL)) ^ ((unsigned) clock()) ^ ptr;
+        n++;
+
+        ks_clear(tmpname);
+        if (ksprintf(tmpname, "%s.tmp_%d_%d_%u", fname, pid, n, t) < 0) break;
+
+        fp = hopen(tmpname->s, mode);
+    } while (fp == NULL && errno == EEXIST && n < 100);
+
+    return fp;
+}
+
+int hts_is_utf16_text(const kstring_t *str)
+{
+    const unsigned char *u = (const unsigned char *) (str->s);
+    return (str->l > 0 && str->s)? is_utf16_text(u, u + str->l) : 0;
+}
+
+// For VCF/BCF backward sweeper. Not exposing these functions because their
+// future is uncertain. Things will probably have to change with hFILE...
+BGZF *hts_get_bgzfp(htsFile *fp)
+{
+    if (fp->is_bgzf)
+        return fp->fp.bgzf;
+    else
+        return NULL;
+}
+int hts_useek(htsFile *fp, off_t uoffset, int where)
+{
+    if (fp->is_bgzf)
+        return bgzf_useek(fp->fp.bgzf, uoffset, where);
+    else
+        return (hseek(fp->fp.hfile, uoffset, SEEK_SET) >= 0)? 0 : -1;
+}
+off_t hts_utell(htsFile *fp)
+{
+    if (fp->is_bgzf)
+        return bgzf_utell(fp->fp.bgzf);
+    else
+        return htell(fp->fp.hfile);
+}
+
+int hts_getline(htsFile *fp, int delimiter, kstring_t *str)
+{
+    int ret;
+    if (! (delimiter == KS_SEP_LINE || delimiter == '\n')) {
+        hts_log_error("Unexpected delimiter %d", delimiter);
+        abort();
+    }
+
+    switch (fp->format.compression) {
+    case no_compression:
+        str->l = 0;
+        ret = kgetline2(str, (kgets_func2 *) hgetln, fp->fp.hfile);
+        if (ret >= 0) ret = (str->l <= INT_MAX)? (int) str->l : INT_MAX;
+        else if (herrno(fp->fp.hfile)) ret = -2, errno = herrno(fp->fp.hfile);
+        else ret = -1;
+        break;
+
+    case gzip:
+    case bgzf:
+        ret = bgzf_getline(fp->fp.bgzf, '\n', str);
+        break;
+
+    default:
+        abort();
+    }
+
+    ++fp->lineno;
+    return ret;
+}
+
+char **hts_readlist(const char *string, int is_file, int *_n)
+{
+    unsigned int m = 0, n = 0;
+    char **s = 0, **s_new;
+    if ( is_file )
+    {
+        BGZF *fp = bgzf_open(string, "r");
+        if ( !fp ) return NULL;
+
+        kstring_t str;
+        int ret;
+        str.s = 0; str.l = str.m = 0;
+        while ((ret = bgzf_getline(fp, '\n', &str)) >= 0)
+        {
+            if (str.l == 0) continue;
+            if (n == 0 && hts_is_utf16_text(&str))
+                hts_log_warning("'%s' appears to be encoded as UTF-16", string);
+            if (hts_resize(char*, n + 1, &m, &s, 0) < 0)
+                goto err;
+            s[n] = strdup(str.s);
+            if (!s[n])
+                goto err;
+            n++;
+        }
+        if (ret < -1) // Read error
+            goto err;
+        bgzf_close(fp);
+        free(str.s);
+    }
+    else
+    {
+        const char *q = string, *p = string;
+        while ( 1 )
+        {
+            if (*p == ',' || *p == 0)
+            {
+                if (hts_resize(char*, n + 1, &m, &s, 0) < 0)
+                    goto err;
+                s[n] = (char*)calloc(p - q + 1, 1);
+                if (!s[n])
+                    goto err;
+                strncpy(s[n++], q, p - q);
+                q = p + 1;
+            }
+            if ( !*p ) break;
+            p++;
+        }
+    }
+    // Try to shrink s to the minimum size needed
+    s_new = (char**)realloc(s, n * sizeof(char*));
+    if (!s_new)
+        goto err;
+
+    s = s_new;
+    assert(n < INT_MAX); // hts_resize() should ensure this
+    *_n = n;
+    return s;
+
+ err:
+    for (m = 0; m < n; m++)
+        free(s[m]);
+    free(s);
+    return NULL;
+}
+
+char **hts_readlines(const char *fn, int *_n)
+{
+    unsigned int m = 0, n = 0;
+    char **s = 0, **s_new;
+    BGZF *fp = bgzf_open(fn, "r");
+    if ( fp ) { // read from file
+        kstring_t str;
+        int ret;
+        str.s = 0; str.l = str.m = 0;
+        while ((ret = bgzf_getline(fp, '\n', &str)) >= 0) {
+            if (str.l == 0) continue;
+            if (n == 0 && hts_is_utf16_text(&str))
+                hts_log_warning("'%s' appears to be encoded as UTF-16", fn);
+            if (hts_resize(char *, n + 1, &m, &s, 0) < 0)
+                goto err;
+            s[n] = strdup(str.s);
+            if (!s[n])
+                goto err;
+            n++;
+        }
+        if (ret < -1) // Read error
+            goto err;
+        bgzf_close(fp);
+        free(str.s);
+    } else if (*fn == ':') { // read from string
+        const char *q, *p;
+        for (q = p = fn + 1;; ++p)
+            if (*p == ',' || *p == 0) {
+                if (hts_resize(char *, n + 1, &m, &s, 0) < 0)
+                    goto err;
+                s[n] = (char*)calloc(p - q + 1, 1);
+                if (!s[n])
+                    goto err;
+                strncpy(s[n++], q, p - q);
+                q = p + 1;
+                if (*p == 0) break;
+            }
+    } else return 0;
+    // Try to shrink s to the minimum size needed
+    s_new = (char**)realloc(s, n * sizeof(char*));
+    if (!s_new)
+        goto err;
+
+    s = s_new;
+    assert(n < INT_MAX); // hts_resize() should ensure this
+    *_n = n;
+    return s;
+
+ err:
+    for (m = 0; m < n; m++)
+        free(s[m]);
+    free(s);
+    return NULL;
+}
+
+// DEPRECATED: To be removed in a future HTSlib release
+int hts_file_type(const char *fname)
+{
+    int len = strlen(fname);
+    if ( !strcasecmp(".vcf.gz",fname+len-7) ) return FT_VCF_GZ;
+    if ( !strcasecmp(".vcf",fname+len-4) ) return FT_VCF;
+    if ( !strcasecmp(".bcf",fname+len-4) ) return FT_BCF_GZ;
+    if ( !strcmp("-",fname) ) return FT_STDIN;
+
+    hFILE *f = hopen(fname, "r");
+    if (f == NULL) return 0;
+
+    htsFormat fmt;
+    if (hts_detect_format2(f, fname, &fmt) < 0) { hclose_abruptly(f); return 0; }
+    if (hclose(f) < 0) return 0;
+
+    switch (fmt.format) {
+    case vcf: return (fmt.compression == no_compression)? FT_VCF : FT_VCF_GZ;
+    case bcf: return (fmt.compression == no_compression)? FT_BCF : FT_BCF_GZ;
+    default:  return 0;
+    }
+}
+
+int hts_check_EOF(htsFile *fp)
+{
+    if (fp->format.compression == bgzf)
+        return bgzf_check_EOF(hts_get_bgzfp(fp));
+    else if (fp->format.format == cram)
+        return cram_check_EOF(fp->fp.cram);
+    else
+        return 3;
+}
+
+
+/****************
+ *** Indexing ***
+ ****************/
+
+#define HTS_MIN_MARKER_DIST 0x10000
+
+// Finds the special meta bin
+//  ((1<<(3 * n_lvls + 3)) - 1) / 7 + 1
+#define META_BIN(idx) ((idx)->n_bins + 1)
+
+#define pair64_lt(a,b) ((a).u < (b).u)
+#define pair64max_lt(a,b) ((a).u < (b).u || \
+                           ((a).u == (b).u && (a).max < (b).max))
+
+KSORT_INIT_STATIC(_off, hts_pair64_t, pair64_lt)
+KSORT_INIT_STATIC(_off_max, hts_pair64_max_t, pair64max_lt)
+
+typedef struct {
+    int32_t m, n;
+    uint64_t loff;
+    hts_pair64_t *list;
+} bins_t;
+
+KHASH_MAP_INIT_INT(bin, bins_t)
+typedef khash_t(bin) bidx_t;
+
+typedef struct {
+    hts_pos_t n, m;
+    uint64_t *offset;
+} lidx_t;
+
+struct hts_idx_t {
+    int fmt, min_shift, n_lvls, n_bins;
+    uint32_t l_meta;
+    int32_t n, m;
+    uint64_t n_no_coor;
+    bidx_t **bidx;
+    lidx_t *lidx;
+    uint8_t *meta; // MUST have a terminating NUL on the end
+    int tbi_n, last_tbi_tid;
+    struct {
+        uint32_t last_bin, save_bin;
+        hts_pos_t last_coor;
+        int last_tid, save_tid, finished;
+        uint64_t last_off, save_off;
+        uint64_t off_beg, off_end;
+        uint64_t n_mapped, n_unmapped;
+    } z; // keep internal states
+    BGZF *otf_fp;  // Index on-the-fly output file
+};
+
+static char * idx_format_name(int fmt) {
+    switch (fmt) {
+        case HTS_FMT_CSI: return "csi";
+        case HTS_FMT_BAI: return "bai";
+        case HTS_FMT_TBI: return "tbi";
+        case HTS_FMT_CRAI: return "crai";
+        default: return "unknown";
+    }
+}
+
+#ifdef DEBUG_INDEX
+static void idx_dump(const hts_idx_t *idx) {
+    int i;
+    int64_t j;
+
+    if (!idx) fprintf(stderr, "Null index\n");
+
+    fprintf(stderr, "format='%s', min_shift=%d, n_lvls=%d, n_bins=%d, l_meta=%u ",
+            idx_format_name(idx->fmt), idx->min_shift, idx->n_lvls, idx->n_bins, idx->l_meta);
+    fprintf(stderr, "n=%d, m=%d, n_no_coor=%"PRIu64"\n", idx->n, idx->m, idx->n_no_coor);
+    for (i = 0; i < idx->n; i++) {
+        bidx_t *bidx = idx->bidx[i];
+        lidx_t *lidx = &idx->lidx[i];
+        if (bidx) {
+            fprintf(stderr, "======== BIN Index - tid=%d, n_buckets=%d, size=%d\n", i, bidx->n_buckets, bidx->size);
+            int b;
+            for (b = 0; b < META_BIN(idx); b++) {
+                khint_t k;
+                if ((k = kh_get(bin, bidx, b)) != kh_end(bidx)) {
+                    bins_t *entries = &kh_value(bidx, k);
+                    int l = hts_bin_level(b);
+                    int64_t bin_width = 1LL << ((idx->n_lvls - l) * 3 + idx->min_shift);
+                    fprintf(stderr, "\tbin=%d, level=%d, parent=%d, n_chunks=%d, loff=%"PRIu64", interval=[%"PRId64" - %"PRId64"]\n",
+                        b, l, hts_bin_parent(b), entries->n, entries->loff, (b-hts_bin_first(l))*bin_width+1, (b+1-hts_bin_first(l))*bin_width);
+                    for (j = 0; j < entries->n; j++)
+                        fprintf(stderr, "\t\tchunk=%"PRId64", u=%"PRIu64", v=%"PRIu64"\n", j, entries->list[j].u, entries->list[j].v);
+                }
+            }
+        }
+        if (lidx) {
+            fprintf(stderr, "======== LINEAR Index - tid=%d, n_values=%"PRId64"\n", i, lidx->n);
+            for (j = 0; j < lidx->n; j++) {
+                fprintf(stderr, "\t\tentry=%"PRId64", offset=%"PRIu64", interval=[%"PRId64" - %"PRId64"]\n",
+                    j, lidx->offset[j], j*(1<<idx->min_shift)+1, (j+1)*(1<<idx->min_shift));
+            }
+        }
+    }
+}
+#endif
+
+static inline int insert_to_b(bidx_t *b, int bin, uint64_t beg, uint64_t end)
+{
+    khint_t k;
+    bins_t *l;
+    int absent;
+    k = kh_put(bin, b, bin, &absent);
+    if (absent < 0) return -1; // Out of memory
+    l = &kh_value(b, k);
+    if (absent) {
+        l->m = 1; l->n = 0;
+        l->list = (hts_pair64_t*)calloc(l->m, sizeof(hts_pair64_t));
+        if (!l->list) {
+            kh_del(bin, b, k);
+            return -1;
+        }
+    } else if (l->n == l->m) {
+        uint32_t new_m = l->m ? l->m << 1 : 1;
+        hts_pair64_t *new_list = realloc(l->list, new_m * sizeof(hts_pair64_t));
+        if (!new_list) return -1;
+        l->list = new_list;
+        l->m = new_m;
+    }
+    l->list[l->n].u = beg;
+    l->list[l->n++].v = end;
+    return 0;
+}
+
+static inline int insert_to_l(lidx_t *l, int64_t _beg, int64_t _end, uint64_t offset, int min_shift)
+{
+    int i;
+    hts_pos_t beg, end;
+    beg = _beg >> min_shift;
+    end = (_end - 1) >> min_shift;
+    if (l->m < end + 1) {
+        size_t new_m = l->m * 2 > end + 1 ? l->m * 2 : end + 1;
+        uint64_t *new_offset;
+
+        new_offset = (uint64_t*)realloc(l->offset, new_m * sizeof(uint64_t));
+        if (!new_offset) return -1;
+
+        // fill unused memory with (uint64_t)-1
+        memset(new_offset + l->m, 0xff, sizeof(uint64_t) * (new_m - l->m));
+        l->m = new_m;
+        l->offset = new_offset;
+    }
+    for (i = beg; i <= end; ++i) {
+        if (l->offset[i] == (uint64_t)-1) l->offset[i] = offset;
+    }
+    if (l->n < end + 1) l->n = end + 1;
+    return 0;
+}
+
+hts_idx_t *hts_idx_init(int n, int fmt, uint64_t offset0, int min_shift, int n_lvls)
+{
+    hts_idx_t *idx;
+    idx = (hts_idx_t*)calloc(1, sizeof(hts_idx_t));
+    if (idx == NULL) return NULL;
+    idx->fmt = fmt;
+    idx->min_shift = min_shift;
+    idx->n_lvls = n_lvls;
+    idx->n_bins = ((1<<(3 * n_lvls + 3)) - 1) / 7;
+    idx->z.save_tid = idx->z.last_tid = -1;
+    idx->z.save_bin = idx->z.last_bin = 0xffffffffu;
+    idx->z.save_off = idx->z.last_off = idx->z.off_beg = idx->z.off_end = offset0;
+    idx->z.last_coor = 0xffffffffu;
+    if (n) {
+        idx->n = idx->m = n;
+        idx->bidx = (bidx_t**)calloc(n, sizeof(bidx_t*));
+        if (idx->bidx == NULL) { free(idx); return NULL; }
+        idx->lidx = (lidx_t*) calloc(n, sizeof(lidx_t));
+        if (idx->lidx == NULL) { free(idx->bidx); free(idx); return NULL; }
+    }
+    idx->tbi_n = -1;
+    idx->last_tbi_tid = -1;
+    idx->otf_fp = NULL;
+    return idx;
+}
+
+static void update_loff(hts_idx_t *idx, int i, int free_lidx)
+{
+    bidx_t *bidx = idx->bidx[i];
+    lidx_t *lidx = &idx->lidx[i];
+    khint_t k;
+    int l;
+    // the last entry is always valid
+    for (l=lidx->n-2; l >= 0; l--) {
+        if (lidx->offset[l] == (uint64_t)-1)
+            lidx->offset[l] = lidx->offset[l+1];
+    }
+    if (bidx == 0) return;
+    for (k = kh_begin(bidx); k != kh_end(bidx); ++k) // set loff
+        if (kh_exist(bidx, k))
+        {
+            if ( kh_key(bidx, k) < idx->n_bins )
+            {
+                int bot_bin = hts_bin_bot(kh_key(bidx, k), idx->n_lvls);
+                // disable linear index if bot_bin out of bounds
+                kh_val(bidx, k).loff = bot_bin < lidx->n ? lidx->offset[bot_bin] : 0;
+            }
+            else
+                kh_val(bidx, k).loff = 0;
+        }
+    if (free_lidx) {
+        free(lidx->offset);
+        lidx->m = lidx->n = 0;
+        lidx->offset = 0;
+    }
+}
+
+static int compress_binning(hts_idx_t *idx, int i)
+{
+    bidx_t *bidx = idx->bidx[i];
+    khint_t k;
+    int l, m;
+    if (bidx == 0) return 0;
+    // merge a bin to its parent if the bin is too small
+    for (l = idx->n_lvls; l > 0; --l) {
+        unsigned start = hts_bin_first(l);
+        for (k = kh_begin(bidx); k != kh_end(bidx); ++k) {
+            bins_t *p, *q;
+            if (!kh_exist(bidx, k) || kh_key(bidx, k) >= idx->n_bins || kh_key(bidx, k) < start) continue;
+            p = &kh_value(bidx, k);
+            if (l < idx->n_lvls && p->n > 1) ks_introsort(_off, p->n, p->list);
+            if ((p->list[p->n - 1].v>>16) - (p->list[0].u>>16) < HTS_MIN_MARKER_DIST) {
+                khint_t kp;
+                kp = kh_get(bin, bidx, hts_bin_parent(kh_key(bidx, k)));
+                if (kp == kh_end(bidx)) continue;
+                q = &kh_val(bidx, kp);
+                if (q->n + p->n > q->m) {
+                    uint32_t new_m = q->n + p->n;
+                    hts_pair64_t *new_list;
+                    kroundup32(new_m);
+                    if (new_m > INT32_MAX) return -1; // Limited by index format
+                    new_list = realloc(q->list, new_m * sizeof(*new_list));
+                    if (!new_list) return -1;
+                    q->m = new_m;
+                    q->list = new_list;
+                }
+                memcpy(q->list + q->n, p->list, p->n * sizeof(hts_pair64_t));
+                q->n += p->n;
+                free(p->list);
+                kh_del(bin, bidx, k);
+            }
+        }
+    }
+    k = kh_get(bin, bidx, 0);
+    if (k != kh_end(bidx)) ks_introsort(_off, kh_val(bidx, k).n, kh_val(bidx, k).list);
+    // merge adjacent chunks that start from the same BGZF block
+    for (k = kh_begin(bidx); k != kh_end(bidx); ++k) {
+        bins_t *p;
+        if (!kh_exist(bidx, k) || kh_key(bidx, k) >= idx->n_bins) continue;
+        p = &kh_value(bidx, k);
+        for (l = 1, m = 0; l < p->n; ++l) {
+            if (p->list[m].v>>16 >= p->list[l].u>>16) {
+                if (p->list[m].v < p->list[l].v) p->list[m].v = p->list[l].v;
+            } else p->list[++m] = p->list[l];
+        }
+        p->n = m + 1;
+    }
+    return 0;
+}
+
+int hts_idx_finish(hts_idx_t *idx, uint64_t final_offset)
+{
+    int i, ret = 0;
+    if (idx == NULL || idx->z.finished) return 0; // do not run this function on an empty index or multiple times
+    if (idx->z.save_tid >= 0) {
+        ret |= insert_to_b(idx->bidx[idx->z.save_tid], idx->z.save_bin, idx->z.save_off, final_offset);
+        ret |= insert_to_b(idx->bidx[idx->z.save_tid], META_BIN(idx), idx->z.off_beg, final_offset);
+        ret |= insert_to_b(idx->bidx[idx->z.save_tid], META_BIN(idx), idx->z.n_mapped, idx->z.n_unmapped);
+    }
+    for (i = 0; i < idx->n; ++i) {
+        update_loff(idx, i, (idx->fmt == HTS_FMT_CSI));
+        ret |= compress_binning(idx, i);
+    }
+    idx->z.finished = 1;
+
+    return ret;
+}
+
+static inline hts_pos_t hts_idx_maxpos(const hts_idx_t *idx)
+{
+    return hts_bin_maxpos(idx->min_shift, idx->n_lvls);
+}
+
+int hts_idx_check_range(hts_idx_t *idx, int tid, hts_pos_t beg, hts_pos_t end)
+{
+    hts_pos_t maxpos = hts_idx_maxpos(idx);
+    if (tid < 0 || (beg <= maxpos && end <= maxpos))
+        return 0;
+
+    if (idx->fmt == HTS_FMT_CSI) {
+        hts_log_error("Region %"PRIhts_pos"..%"PRIhts_pos" "
+                      "cannot be stored in a csi index with these parameters. "
+                      "Please use a larger min_shift or depth",
+                      beg, end);
+    } else {
+        hts_log_error("Region %"PRIhts_pos"..%"PRIhts_pos
+                      " cannot be stored in a %s index. Try using a csi index",
+                      beg, end, idx_format_name(idx->fmt));
+    }
+    errno = ERANGE;
+    return -1;
+}
+
+int hts_idx_push(hts_idx_t *idx, int tid, hts_pos_t beg, hts_pos_t end, uint64_t offset, int is_mapped)
+{
+    int bin;
+    if (tid<0) beg = -1, end = 0;
+    if (hts_idx_check_range(idx, tid, beg, end) < 0)
+        return -1;
+    if (tid >= idx->m) { // enlarge the index
+        uint32_t new_m = idx->m * 2 > tid + 1 ? idx->m * 2 : tid + 1;
+        bidx_t **new_bidx;
+        lidx_t *new_lidx;
+
+        new_bidx = (bidx_t**)realloc(idx->bidx, new_m * sizeof(bidx_t*));
+        if (!new_bidx) return -1;
+        idx->bidx = new_bidx;
+
+        new_lidx = (lidx_t*) realloc(idx->lidx, new_m * sizeof(lidx_t));
+        if (!new_lidx) return -1;
+        idx->lidx = new_lidx;
+
+        memset(&idx->bidx[idx->m], 0, (new_m - idx->m) * sizeof(bidx_t*));
+        memset(&idx->lidx[idx->m], 0, (new_m - idx->m) * sizeof(lidx_t));
+        idx->m = new_m;
+    }
+    if (idx->n < tid + 1) idx->n = tid + 1;
+    if (idx->z.finished) return 0;
+    if (idx->z.last_tid != tid || (idx->z.last_tid >= 0 && tid < 0)) { // change of chromosome
+        if ( tid>=0 && idx->n_no_coor )
+        {
+            hts_log_error("NO_COOR reads not in a single block at the end %d %d", tid, idx->z.last_tid);
+            return -1;
+        }
+        if (tid>=0 && idx->bidx[tid] != 0)
+        {
+            hts_log_error("Chromosome blocks not continuous");
+            return -1;
+        }
+        idx->z.last_tid = tid;
+        idx->z.last_bin = 0xffffffffu;
+    } else if (tid >= 0 && idx->z.last_coor > beg) { // test if positions are out of order
+        hts_log_error("Unsorted positions on sequence #%d: %"PRIhts_pos" followed by %"PRIhts_pos, tid+1, idx->z.last_coor+1, beg+1);
+        return -1;
+    }
+    if (end < beg) {
+        // Malformed ranges are errors. (Empty ranges (beg==end) are unusual but acceptable.)
+        hts_log_error("Invalid record on sequence #%d: end %"PRId64" < begin %"PRId64, tid+1, end, beg+1);
+        return -1;
+    }
+    if ( tid>=0 )
+    {
+        if (idx->bidx[tid] == 0) idx->bidx[tid] = kh_init(bin);
+        // shoehorn [-1,0) (VCF POS=0) into the leftmost bottom-level bin
+        if (beg < 0)  beg = 0;
+        if (end <= 0) end = 1;
+        // idx->z.last_off points to the start of the current record
+        if (insert_to_l(&idx->lidx[tid], beg, end,
+                        idx->z.last_off, idx->min_shift) < 0) return -1;
+    }
+    else idx->n_no_coor++;
+    bin = hts_reg2bin(beg, end, idx->min_shift, idx->n_lvls);
+    if ((int)idx->z.last_bin != bin) { // then possibly write the binning index
+        if (idx->z.save_bin != 0xffffffffu) { // save_bin==0xffffffffu only happens to the first record
+            if (insert_to_b(idx->bidx[idx->z.save_tid], idx->z.save_bin,
+                            idx->z.save_off, idx->z.last_off) < 0) return -1;
+        }
+        if (idx->z.last_bin == 0xffffffffu && idx->z.save_bin != 0xffffffffu) { // change of chr; keep meta information
+            idx->z.off_end = idx->z.last_off;
+            if (insert_to_b(idx->bidx[idx->z.save_tid], META_BIN(idx),
+                            idx->z.off_beg, idx->z.off_end) < 0) return -1;
+            if (insert_to_b(idx->bidx[idx->z.save_tid], META_BIN(idx),
+                            idx->z.n_mapped, idx->z.n_unmapped) < 0) return -1;
+            idx->z.n_mapped = idx->z.n_unmapped = 0;
+            idx->z.off_beg = idx->z.off_end;
+        }
+        idx->z.save_off = idx->z.last_off;
+        idx->z.save_bin = idx->z.last_bin = bin;
+        idx->z.save_tid = tid;
+    }
+    if (is_mapped) ++idx->z.n_mapped;
+    else ++idx->z.n_unmapped;
+    idx->z.last_off = offset;
+    idx->z.last_coor = beg;
+    return 0;
+}
+
+// Needed for TBI only.  Ensure 'tid' with 'name' is in the index meta data.
+// idx->meta needs to have been initialised first with an appropriate Tabix
+// configuration via hts_idx_set_meta.
+//
+// NB number of references (first 4 bytes of tabix header) aren't in
+// idx->meta, but held in idx->n instead.
+int hts_idx_tbi_name(hts_idx_t *idx, int tid, const char *name) {
+    // Horrid - we have to map incoming tid to a tbi alternative tid.
+    // This is because TBI counts tids by "covered" refs while everything
+    // else counts by Nth SQ/contig record in header.
+    if (tid == idx->last_tbi_tid || tid < 0 || !name)
+        return idx->tbi_n;
+
+    uint32_t len = strlen(name)+1;
+    uint8_t *tmp = (uint8_t *)realloc(idx->meta, idx->l_meta + len);
+    if (!tmp)
+        return -1;
+
+    // Append name
+    idx->meta = tmp;
+    strcpy((char *)idx->meta + idx->l_meta, name);
+    idx->l_meta += len;
+
+    // Update seq length
+    u32_to_le(le_to_u32(idx->meta+24)+len, idx->meta+24);
+
+    idx->last_tbi_tid = tid;
+    return ++idx->tbi_n;
+}
+
+// When doing samtools index we have a read_bam / hts_idx_push(bgzf_tell())
+// loop.  idx->z.last_off is the previous bzgf_tell location, so we know
+// the location the current bam record started at as well as where it ends.
+//
+// When building an index on the fly via a write_bam / hts_idx_push loop,
+// this isn't quite identical as we may amend the virtual coord returned
+// by bgzf_tell to the start of a new block if the next bam struct doesn't
+// fit.  It's essentially the same thing, but for bit-identical indices
+// we need to amend the idx->z.last_off when we know we're starting a new
+// block.
+void hts_idx_amend_last(hts_idx_t *idx, uint64_t offset)
+{
+    idx->z.last_off = offset;
+}
+
+void hts_idx_destroy(hts_idx_t *idx)
+{
+    khint_t k;
+    int i;
+    if (idx == 0) return;
+
+    // For HTS_FMT_CRAI, idx actually points to a different type -- see sam.c
+    if (idx->fmt == HTS_FMT_CRAI) {
+        hts_cram_idx_t *cidx = (hts_cram_idx_t *) idx;
+        cram_index_free(cidx->cram);
+        free(cidx);
+        return;
+    }
+
+    for (i = 0; i < idx->m; ++i) {
+        bidx_t *bidx = idx->bidx[i];
+        free(idx->lidx[i].offset);
+        if (bidx == 0) continue;
+        for (k = kh_begin(bidx); k != kh_end(bidx); ++k)
+            if (kh_exist(bidx, k))
+                free(kh_value(bidx, k).list);
+        kh_destroy(bin, bidx);
+    }
+    free(idx->bidx); free(idx->lidx); free(idx->meta);
+    free(idx);
+}
+
+int hts_idx_fmt(hts_idx_t *idx) {
+    return idx->fmt;
+}
+
+// The optimizer eliminates these ed_is_big() calls; still it would be good to
+// TODO Determine endianness at configure- or compile-time
+
+static inline ssize_t HTS_RESULT_USED idx_write_int32(BGZF *fp, int32_t x)
+{
+    if (ed_is_big()) x = ed_swap_4(x);
+    return bgzf_write(fp, &x, sizeof x);
+}
+
+static inline ssize_t HTS_RESULT_USED idx_write_uint32(BGZF *fp, uint32_t x)
+{
+    if (ed_is_big()) x = ed_swap_4(x);
+    return bgzf_write(fp, &x, sizeof x);
+}
+
+static inline ssize_t HTS_RESULT_USED idx_write_uint64(BGZF *fp, uint64_t x)
+{
+    if (ed_is_big()) x = ed_swap_8(x);
+    return bgzf_write(fp, &x, sizeof x);
+}
+
+static inline void swap_bins(bins_t *p)
+{
+    int i;
+    for (i = 0; i < p->n; ++i) {
+        ed_swap_8p(&p->list[i].u);
+        ed_swap_8p(&p->list[i].v);
+    }
+}
+
+static int need_idx_ugly_delay_hack(const hts_idx_t *idx)
+{
+    // Ugly hack for on-the-fly BAI indexes.  As these are uncompressed,
+    // we need to delay writing a few bytes of data until file close
+    // so that we have something to force a modification time update.
+    //
+    // (For compressed indexes like CSI, the BGZF EOF block serves the same
+    // purpose).
+    return idx->otf_fp && !idx->otf_fp->is_compressed;
+}
+
+static int idx_save_core(const hts_idx_t *idx, BGZF *fp, int fmt)
+{
+    int32_t i, j;
+
+    #define check(ret) if ((ret) < 0) return -1
+
+    // VCF TBI/CSI only writes IDs for non-empty bins (ie covered references)
+    //
+    // NOTE: CSI meta is undefined in spec, so this code has an assumption
+    // that we're only using it for Tabix data.
+    int nids = idx->n;
+    if (idx->meta && idx->l_meta >= 4 && le_to_u32(idx->meta) == TBX_VCF) {
+        for (i = nids = 0; i < idx->n; ++i) {
+            if (idx->bidx[i])
+                nids++;
+        }
+    }
+    check(idx_write_int32(fp, nids));
+    if (fmt == HTS_FMT_TBI && idx->l_meta)
+        check(bgzf_write(fp, idx->meta, idx->l_meta));
+
+    for (i = 0; i < idx->n; ++i) {
+        khint_t k;
+        bidx_t *bidx = idx->bidx[i];
+        lidx_t *lidx = &idx->lidx[i];
+
+        // write binning index
+        if (nids == idx->n || bidx)
+            check(idx_write_int32(fp, bidx? kh_size(bidx) : 0));
+        if (bidx)
+            for (k = kh_begin(bidx); k != kh_end(bidx); ++k)
+                if (kh_exist(bidx, k)) {
+                    bins_t *p = &kh_value(bidx, k);
+                    check(idx_write_uint32(fp, kh_key(bidx, k)));
+                    if (fmt == HTS_FMT_CSI) check(idx_write_uint64(fp, p->loff));
+                    //int j;for(j=0;j<p->n;++j)fprintf(stderr,"%d,%llx,%d,%llx:%llx\n",kh_key(bidx,k),kh_val(bidx, k).loff,j,p->list[j].u,p->list[j].v);
+                    check(idx_write_int32(fp, p->n));
+                    for (j = 0; j < p->n; ++j) {
+                        //fprintf(stderr, "\t%ld\t%ld\n", p->list[j].u, p->list[j].v);
+                        check(idx_write_uint64(fp, p->list[j].u));
+                        check(idx_write_uint64(fp, p->list[j].v));
+                    }
+                }
+
+        // write linear index
+        if (fmt != HTS_FMT_CSI) {
+            check(idx_write_int32(fp, lidx->n));
+            for (j = 0; j < lidx->n; ++j)
+                check(idx_write_uint64(fp, lidx->offset[j]));
+        }
+    }
+
+    if (!need_idx_ugly_delay_hack(idx)) {
+        // Write this for compressed (CSI) indexes, but for BAI we
+        // need to save a bit for later.  See hts_idx_close_otf_fp()
+        check(idx_write_uint64(fp, idx->n_no_coor));
+    }
+
+#ifdef DEBUG_INDEX
+    idx_dump(idx);
+#endif
+
+    return 0;
+    #undef check
+}
+
+int hts_idx_save(const hts_idx_t *idx, const char *fn, int fmt)
+{
+    int ret, save;
+    if (idx == NULL || fn == NULL) { errno = EINVAL; return -1; }
+    char *fnidx = (char*)calloc(1, strlen(fn) + 5);
+    if (fnidx == NULL) return -1;
+
+    strcpy(fnidx, fn);
+    switch (fmt) {
+    case HTS_FMT_BAI: strcat(fnidx, ".bai"); break;
+    case HTS_FMT_CSI: strcat(fnidx, ".csi"); break;
+    case HTS_FMT_TBI: strcat(fnidx, ".tbi"); break;
+    default: abort();
+    }
+
+    ret = hts_idx_save_as(idx, fn, fnidx, fmt);
+    save = errno;
+    free(fnidx);
+    errno = save;
+    return ret;
+}
+
+static int hts_idx_write_out(const hts_idx_t *idx, BGZF *fp, int fmt)
+{
+    #define check(ret) if ((ret) < 0) return -1
+
+    if (fmt == HTS_FMT_CSI) {
+        check(bgzf_write(fp, "CSI\1", 4));
+        check(idx_write_int32(fp, idx->min_shift));
+        check(idx_write_int32(fp, idx->n_lvls));
+        check(idx_write_uint32(fp, idx->l_meta));
+        if (idx->l_meta) check(bgzf_write(fp, idx->meta, idx->l_meta));
+    } else if (fmt == HTS_FMT_TBI) {
+        check(bgzf_write(fp, "TBI\1", 4));
+    } else if (fmt == HTS_FMT_BAI) {
+        check(bgzf_write(fp, "BAI\1", 4));
+    } else abort();
+
+    check(idx_save_core(idx, fp, fmt));
+
+    #undef check
+    return 0;
+}
+
+int hts_idx_save_as(const hts_idx_t *idx, const char *fn, const char *fnidx, int fmt)
+{
+    BGZF *fp;
+
+    if (fnidx == NULL)
+        return hts_idx_save(idx, fn, fmt);
+
+    fp = bgzf_open(fnidx, (fmt == HTS_FMT_BAI)? "wu" : "w");
+    if (fp == NULL) return -1;
+
+    if (hts_idx_write_out(idx, fp, fmt) < 0) {
+        int save_errno = errno;
+        bgzf_close(fp);
+        errno = save_errno;
+        return -1;
+    }
+
+    return bgzf_close(fp);
+}
+
+// idx_save for on-the-fly indexes.  Mostly duplicated from above, except
+// idx is not const because we want to store the file handle in it, and
+// the index file handle is not closed.  This allows the index file to be
+// closed after the EOF block on the indexed file has been written out,
+// so the modification times on the two files will be in the correct order.
+int hts_idx_save_but_not_close(hts_idx_t *idx, const char *fnidx, int fmt)
+{
+    idx->otf_fp = bgzf_open(fnidx, (fmt == HTS_FMT_BAI)? "wu" : "w");
+    if (idx->otf_fp == NULL) return -1;
+
+    if (hts_idx_write_out(idx, idx->otf_fp, fmt) < 0) {
+        int save_errno = errno;
+        bgzf_close(idx->otf_fp);
+        idx->otf_fp = NULL;
+        errno = save_errno;
+        return -1;
+    }
+
+    return bgzf_flush(idx->otf_fp);
+}
+
+static int hts_idx_close_otf_fp(hts_idx_t *idx)
+{
+    if (idx && idx->otf_fp) {
+        int ret = 0;
+        if (need_idx_ugly_delay_hack(idx)) {
+            // BAI index - write out the bytes we deferred earlier
+            ret = idx_write_uint64(idx->otf_fp, idx->n_no_coor) < 0;
+        }
+        ret |= bgzf_close(idx->otf_fp) < 0;
+        idx->otf_fp = NULL;
+        return ret == 0 ? 0 : -1;
+    }
+    return 0;
+}
+
+static int idx_read_core(hts_idx_t *idx, BGZF *fp, int fmt)
+{
+    int32_t i, n, is_be;
+    is_be = ed_is_big();
+    if (idx == NULL) return -4;
+    for (i = 0; i < idx->n; ++i) {
+        bidx_t *h;
+        lidx_t *l = &idx->lidx[i];
+        uint32_t key;
+        int j, absent;
+        bins_t *p;
+        h = idx->bidx[i] = kh_init(bin);
+        if (bgzf_read(fp, &n, 4) != 4) return -1;
+        if (is_be) ed_swap_4p(&n);
+        if (n < 0) return -3;
+        for (j = 0; j < n; ++j) {
+            khint_t k;
+            if (bgzf_read(fp, &key, 4) != 4) return -1;
+            if (is_be) ed_swap_4p(&key);
+            k = kh_put(bin, h, key, &absent);
+            if (absent <  0) return -2; // No memory
+            if (absent == 0) return -3; // Duplicate bin number
+            p = &kh_val(h, k);
+            if (fmt == HTS_FMT_CSI) {
+                if (bgzf_read(fp, &p->loff, 8) != 8) return -1;
+                if (is_be) ed_swap_8p(&p->loff);
+            } else p->loff = 0;
+            if (bgzf_read(fp, &p->n, 4) != 4) return -1;
+            if (is_be) ed_swap_4p(&p->n);
+            if (p->n < 0) return -3;
+            if ((size_t) p->n > SIZE_MAX / sizeof(hts_pair64_t)) return -2;
+            p->m = p->n;
+            p->list = (hts_pair64_t*)malloc(p->m * sizeof(hts_pair64_t));
+            if (p->list == NULL) return -2;
+            if (bgzf_read(fp, p->list, ((size_t) p->n)<<4) != ((size_t) p->n)<<4) return -1;
+            if (is_be) swap_bins(p);
+        }
+        if (fmt != HTS_FMT_CSI) { // load linear index
+            int j, k;
+            uint32_t x;
+            if (bgzf_read(fp, &x, 4) != 4) return -1;
+            if (is_be) ed_swap_4p(&x);
+            l->n = x;
+            if (l->n < 0) return -3;
+            if ((size_t) l->n > SIZE_MAX / sizeof(uint64_t)) return -2;
+            l->m = l->n;
+            l->offset = (uint64_t*)malloc(l->n * sizeof(uint64_t));
+            if (l->offset == NULL) return -2;
+            if (bgzf_read(fp, l->offset, l->n << 3) != l->n << 3) return -1;
+            if (is_be) for (j = 0; j < l->n; ++j) ed_swap_8p(&l->offset[j]);
+            for (k = j = 0; j < l->n && l->offset[j] == 0; k = ++j); // stop at the first non-zero entry
+            for (j = l->n-1; j > k; j--) // fill missing values; may happen given older samtools and tabix
+                if (l->offset[j-1] == 0) l->offset[j-1] = l->offset[j];
+            update_loff(idx, i, 0);
+        }
+    }
+    if (bgzf_read(fp, &idx->n_no_coor, 8) != 8) idx->n_no_coor = 0;
+    if (is_be) ed_swap_8p(&idx->n_no_coor);
+#ifdef DEBUG_INDEX
+    idx_dump(idx);
+#endif
+
+    return 0;
+}
+
+static hts_idx_t *idx_read(const char *fn)
+{
+    uint8_t magic[4];
+    int i, is_be;
+    hts_idx_t *idx = NULL;
+    uint8_t *meta = NULL;
+    BGZF *fp = bgzf_open(fn, "r");
+    if (fp == NULL) return NULL;
+    is_be = ed_is_big();
+    if (bgzf_read(fp, magic, 4) != 4) goto fail;
+
+    if (memcmp(magic, "CSI\1", 4) == 0) {
+        uint32_t x[3], n;
+        if (bgzf_read(fp, x, 12) != 12) goto fail;
+        if (is_be) for (i = 0; i < 3; ++i) ed_swap_4p(&x[i]);
+        if (x[2]) {
+            if (SIZE_MAX - x[2] < 1) goto fail; // Prevent possible overflow
+            if ((meta = (uint8_t*)malloc((size_t) x[2] + 1)) == NULL) goto fail;
+            if (bgzf_read(fp, meta, x[2]) != x[2]) goto fail;
+            // Prevent possible strlen past the end in tbx_index_load2
+            meta[x[2]] = '\0';
+        }
+        if (bgzf_read(fp, &n, 4) != 4) goto fail;
+        if (is_be) ed_swap_4p(&n);
+        if (n > INT32_MAX) goto fail;
+        if ((idx = hts_idx_init(n, HTS_FMT_CSI, 0, x[0], x[1])) == NULL) goto fail;
+        idx->l_meta = x[2];
+        idx->meta = meta;
+        meta = NULL;
+        if (idx_read_core(idx, fp, HTS_FMT_CSI) < 0) goto fail;
+    }
+    else if (memcmp(magic, "TBI\1", 4) == 0) {
+        uint8_t x[8 * 4];
+        uint32_t n;
+        // Read file header
+        if (bgzf_read(fp, x, sizeof(x)) != sizeof(x)) goto fail;
+        n = le_to_u32(&x[0]); // location of n_ref
+        if (n > INT32_MAX) goto fail;
+        if ((idx = hts_idx_init(n, HTS_FMT_TBI, 0, 14, 5)) == NULL) goto fail;
+        n = le_to_u32(&x[7*4]); // location of l_nm
+        if (n > UINT32_MAX - 29) goto fail; // Prevent possible overflow
+        idx->l_meta = 28 + n;
+        if ((idx->meta = (uint8_t*)malloc(idx->l_meta + 1)) == NULL) goto fail;
+        // copy format, col_seq, col_beg, col_end, meta, skip, l_nm
+        // N.B. left in little-endian byte order.
+        memcpy(idx->meta, &x[1*4], 28);
+        // Read in sequence names.
+        if (bgzf_read(fp, idx->meta + 28, n) != n) goto fail;
+        // Prevent possible strlen past the end in tbx_index_load2
+        idx->meta[idx->l_meta] = '\0';
+        if (idx_read_core(idx, fp, HTS_FMT_TBI) < 0) goto fail;
+    }
+    else if (memcmp(magic, "BAI\1", 4) == 0) {
+        uint32_t n;
+        if (bgzf_read(fp, &n, 4) != 4) goto fail;
+        if (is_be) ed_swap_4p(&n);
+        if (n > INT32_MAX) goto fail;
+        if ((idx = hts_idx_init(n, HTS_FMT_BAI, 0, 14, 5)) == NULL) goto fail;
+        if (idx_read_core(idx, fp, HTS_FMT_BAI) < 0) goto fail;
+    }
+    else { errno = EINVAL; goto fail; }
+
+    bgzf_close(fp);
+    return idx;
+
+fail:
+    bgzf_close(fp);
+    hts_idx_destroy(idx);
+    free(meta);
+    return NULL;
+}
+
+int hts_idx_set_meta(hts_idx_t *idx, uint32_t l_meta, uint8_t *meta,
+                      int is_copy)
+{
+    uint8_t *new_meta = meta;
+    if (is_copy) {
+        size_t l = l_meta;
+        if (l > SIZE_MAX - 1) {
+            errno = ENOMEM;
+            return -1;
+        }
+        new_meta = malloc(l + 1);
+        if (!new_meta) return -1;
+        memcpy(new_meta, meta, l);
+        // Prevent possible strlen past the end in tbx_index_load2
+        new_meta[l] = '\0';
+    }
+    if (idx->meta) free(idx->meta);
+    idx->l_meta = l_meta;
+    idx->meta = new_meta;
+    return 0;
+}
+
+uint8_t *hts_idx_get_meta(hts_idx_t *idx, uint32_t *l_meta)
+{
+    *l_meta = idx->l_meta;
+    return idx->meta;
+}
+
+const char **hts_idx_seqnames(const hts_idx_t *idx, int *n, hts_id2name_f getid, void *hdr)
+{
+    if ( !idx || !idx->n )
+    {
+        *n = 0;
+        return NULL;
+    }
+
+    int tid = 0, i;
+    const char **names = (const char**) calloc(idx->n,sizeof(const char*));
+    for (i=0; i<idx->n; i++)
+    {
+        bidx_t *bidx = idx->bidx[i];
+        if ( !bidx ) continue;
+        names[tid++] = getid(hdr,i);
+    }
+    *n = tid;
+    return names;
+}
+
+int hts_idx_nseq(const hts_idx_t *idx) {
+    if (!idx) return -1;
+    return idx->n;
+}
+
+int hts_idx_get_stat(const hts_idx_t* idx, int tid, uint64_t* mapped, uint64_t* unmapped)
+{
+    if (!idx) return -1;
+    if ( idx->fmt == HTS_FMT_CRAI ) {
+        *mapped = 0; *unmapped = 0;
+        return -1;
+    }
+
+    bidx_t *h = idx->bidx[tid];
+    if (!h) return -1;
+    khint_t k = kh_get(bin, h, META_BIN(idx));
+    if (k != kh_end(h)) {
+        *mapped = kh_val(h, k).list[1].u;
+        *unmapped = kh_val(h, k).list[1].v;
+        return 0;
+    } else {
+        *mapped = 0; *unmapped = 0;
+        return -1;
+    }
+}
+
+uint64_t hts_idx_get_n_no_coor(const hts_idx_t* idx)
+{
+    if (idx->fmt == HTS_FMT_CRAI) return 0;
+    return idx->n_no_coor;
+}
+
+/****************
+ *** Iterator ***
+ ****************/
+
+// Note: even with 32-bit hts_pos_t, end needs to be 64-bit here due to 1LL<<s.
+static inline int reg2bins_narrow(int64_t beg, int64_t end, hts_itr_t *itr, int min_shift, int n_lvls, bidx_t *bidx)
+{
+    int l, t, s = min_shift + (n_lvls<<1) + n_lvls;
+    for (--end, l = 0, t = 0; l <= n_lvls; s -= 3, t += 1<<((l<<1)+l), ++l) {
+        hts_pos_t b, e;
+        int i;
+        b = t + (beg>>s); e = t + (end>>s);
+        for (i = b; i <= e; ++i) {
+            if (kh_get(bin, bidx, i) != kh_end(bidx)) {
+                assert(itr->bins.n < itr->bins.m);
+                itr->bins.a[itr->bins.n++] = i;
+            }
+        }
+    }
+    return itr->bins.n;
+}
+
+static inline int reg2bins_wide(int64_t beg, int64_t end, hts_itr_t *itr, int min_shift, int n_lvls, bidx_t *bidx)
+{
+    khint_t i;
+    hts_pos_t max_shift = 3 * n_lvls + min_shift;
+    --end;
+    if (beg < 0) beg = 0;
+    for (i = kh_begin(bidx); i != kh_end(bidx); i++) {
+        if (!kh_exist(bidx, i)) continue;
+        hts_pos_t bin = (hts_pos_t) kh_key(bidx, i);
+        int level = hts_bin_level(bin);
+        if (level > n_lvls) continue; // Dodgy index?
+        hts_pos_t first = hts_bin_first(level);
+        hts_pos_t beg_at_level = first + (beg >> (max_shift - 3 * level));
+        hts_pos_t end_at_level = first + (end >> (max_shift - 3 * level));
+        if (beg_at_level <= bin && bin <= end_at_level) {
+            assert(itr->bins.n < itr->bins.m);
+            itr->bins.a[itr->bins.n++] = bin;
+        }
+    }
+    return itr->bins.n;
+}
+
+static inline int reg2bins(int64_t beg, int64_t end, hts_itr_t *itr, int min_shift, int n_lvls, bidx_t *bidx)
+{
+    int l, t, s = min_shift + (n_lvls<<1) + n_lvls;
+    size_t reg_bin_count = 0, hash_bin_count = kh_n_buckets(bidx), max_bins;
+    hts_pos_t end1;
+    if (end >= 1LL<<s) end = 1LL<<s;
+    if (beg >= end) return 0;
+    end1 = end - 1;
+
+    // Count bins to see if it's faster to iterate through the hash table
+    // or the set of bins covering the region
+    for (l = 0, t = 0; l <= n_lvls; s -= 3, t += 1<<((l<<1)+l), ++l) {
+        reg_bin_count += (end1 >> s) - (beg >> s) + 1;
+    }
+    max_bins = reg_bin_count < kh_size(bidx) ? reg_bin_count : kh_size(bidx);
+    if (itr->bins.m - itr->bins.n < max_bins) {
+        // Worst-case memory usage.  May be wasteful on very sparse
+        // data, but the bin list usually won't be too big anyway.
+        size_t new_m = max_bins + itr->bins.n;
+        if (new_m > INT_MAX || new_m > SIZE_MAX / sizeof(int)) {
+            errno = ENOMEM;
+            return -1;
+        }
+        int *new_a = realloc(itr->bins.a, new_m * sizeof(*new_a));
+        if (!new_a) return -1;
+        itr->bins.a = new_a;
+        itr->bins.m = new_m;
+    }
+    if (reg_bin_count < hash_bin_count) {
+        return reg2bins_narrow(beg, end, itr, min_shift, n_lvls, bidx);
+    } else {
+        return reg2bins_wide(beg, end, itr, min_shift, n_lvls, bidx);
+    }
+}
+
+static inline int add_to_interval(hts_itr_t *iter, bins_t *bin,
+                                  int tid, uint32_t interval,
+                                  uint64_t min_off, uint64_t max_off)
+{
+    hts_pair64_max_t *off;
+    int j;
+
+    if (!bin->n)
+        return 0;
+    off = realloc(iter->off, (iter->n_off + bin->n) * sizeof(*off));
+    if (!off)
+        return -2;
+
+    iter->off = off;
+    for (j = 0; j < bin->n; ++j) {
+        if (bin->list[j].v > min_off && bin->list[j].u < max_off) {
+            iter->off[iter->n_off].u = min_off > bin->list[j].u
+                ? min_off : bin->list[j].u;
+            iter->off[iter->n_off].v = max_off < bin->list[j].v
+                ? max_off : bin->list[j].v;
+            // hts_pair64_max_t::max is now used to link
+            // file offsets to region list entries.
+            // The iterator can use this to decide if it
+            // can skip some file regions.
+            iter->off[iter->n_off].max = ((uint64_t) tid << 32) | interval;
+            iter->n_off++;
+        }
+    }
+    return 0;
+}
+
+static inline int reg2intervals_narrow(hts_itr_t *iter, const bidx_t *bidx,
+                                       int tid, int64_t beg, int64_t end,
+                                       uint32_t interval,
+                                       uint64_t min_off, uint64_t max_off,
+                                       int min_shift, int n_lvls)
+{
+    int l, t, s = min_shift + n_lvls * 3;
+    hts_pos_t b, e, i;
+
+    for (--end, l = 0, t = 0; l <= n_lvls; s -= 3, t += 1<<((l<<1)+l), ++l) {
+        b = t + (beg>>s); e = t + (end>>s);
+        for (i = b; i <= e; ++i) {
+            khint_t k = kh_get(bin, bidx, i);
+            if (k != kh_end(bidx)) {
+                bins_t *bin = &kh_value(bidx, k);
+                int res = add_to_interval(iter, bin, tid, interval, min_off, max_off);
+                if (res < 0)
+                    return res;
+            }
+        }
+    }
+    return 0;
+}
+
+static inline int reg2intervals_wide(hts_itr_t *iter, const bidx_t *bidx,
+                                       int tid, int64_t beg, int64_t end,
+                                       uint32_t interval,
+                                       uint64_t min_off, uint64_t max_off,
+                                       int min_shift, int n_lvls)
+{
+    khint_t i;
+    hts_pos_t max_shift = 3 * n_lvls + min_shift;
+    --end;
+    if (beg < 0) beg = 0;
+    for (i = kh_begin(bidx); i != kh_end(bidx); i++) {
+        if (!kh_exist(bidx, i)) continue;
+        hts_pos_t bin = (hts_pos_t) kh_key(bidx, i);
+        int level = hts_bin_level(bin);
+        if (level > n_lvls) continue; // Dodgy index?
+        hts_pos_t first = hts_bin_first(level);
+        hts_pos_t beg_at_level = first + (beg >> (max_shift - 3 * level));
+        hts_pos_t end_at_level = first + (end >> (max_shift - 3 * level));
+        if (beg_at_level <= bin && bin <= end_at_level) {
+            bins_t *bin = &kh_value(bidx, i);
+            int res = add_to_interval(iter, bin, tid, interval, min_off, max_off);
+            if (res < 0)
+                return res;
+        }
+    }
+    return 0;
+}
+
+static inline int reg2intervals(hts_itr_t *iter, const hts_idx_t *idx, int tid, int64_t beg, int64_t end, uint32_t interval, uint64_t min_off, uint64_t max_off, int min_shift, int n_lvls)
+{
+    int l, t, s;
+    int i, j;
+    hts_pos_t end1;
+    bidx_t *bidx;
+    int start_n_off;
+    size_t reg_bin_count = 0, hash_bin_count;
+    int res;
+
+    if (!iter || !idx || (bidx = idx->bidx[tid]) == NULL || beg > end)
+        return -1;
+
+    hash_bin_count = kh_n_buckets(bidx);
+
+    s = min_shift + (n_lvls<<1) + n_lvls;
+    if (end >= 1LL<<s)
+        end = 1LL<<s;
+
+    end1 = end - 1;
+    // Count bins to see if it's faster to iterate through the hash table
+    // or the set of bins covering the region
+    for (l = 0, t = 0; l <= n_lvls; s -= 3, t += 1<<((l<<1)+l), ++l) {
+        reg_bin_count += (end1 >> s) - (beg >> s) + 1;
+    }
+
+    start_n_off = iter->n_off;
+
+    // Populate iter->off with the intervals for this region
+    if (reg_bin_count < hash_bin_count) {
+        res = reg2intervals_narrow(iter, bidx, tid, beg, end, interval,
+                                   min_off, max_off, min_shift, n_lvls);
+    } else {
+        res = reg2intervals_wide(iter, bidx, tid, beg, end, interval,
+                                 min_off, max_off, min_shift, n_lvls);
+    }
+    if (res < 0)
+        return res;
+
+    if (iter->n_off - start_n_off > 1) {
+        ks_introsort(_off_max, iter->n_off - start_n_off, iter->off + start_n_off);
+        for (i = start_n_off, j = start_n_off + 1; j < iter->n_off; j++) {
+            if (iter->off[i].v >= iter->off[j].u) {
+                if (iter->off[i].v < iter->off[j].v)
+                    iter->off[i].v = iter->off[j].v;
+            } else {
+                i++;
+                if (i < j)
+                    iter->off[i] = iter->off[j];
+            }
+        }
+        iter->n_off = i + 1;
+    }
+
+    return iter->n_off;
+}
+
+static int compare_regions(const void *r1, const void *r2) {
+    hts_reglist_t *reg1 = (hts_reglist_t *)r1;
+    hts_reglist_t *reg2 = (hts_reglist_t *)r2;
+
+    if (reg1->tid < 0 && reg2->tid >= 0)
+        return 1;
+    else if (reg1->tid >= 0 && reg2->tid < 0)
+        return -1;
+    else
+        return reg1->tid - reg2->tid;
+}
+
+uint64_t hts_itr_off(const hts_idx_t* idx, int tid) {
+
+    int i;
+    bidx_t* bidx;
+    uint64_t off0 = (uint64_t) -1;
+    khint_t k;
+    switch (tid) {
+    case HTS_IDX_START:
+        // Find the smallest offset, note that sequence ids may not be ordered sequentially
+        for (i = 0; i < idx->n; i++) {
+            bidx = idx->bidx[i];
+            k = kh_get(bin, bidx, META_BIN(idx));
+            if (k == kh_end(bidx))
+                continue;
+
+            if (off0 > kh_val(bidx, k).list[0].u)
+                off0 = kh_val(bidx, k).list[0].u;
+        }
+        if (off0 == (uint64_t) -1 && idx->n_no_coor)
+            off0 = 0;
+        // only no-coor reads in this bam
+        break;
+    case HTS_IDX_NOCOOR:
+        /* No-coor reads sort after all of the mapped reads.  The position
+           is not stored in the index itself, so need to find the end
+           offset for the last mapped read.  A loop is needed here in
+           case references at the end of the file have no mapped reads,
+           or sequence ids are not ordered sequentially.
+           See issue samtools#568 and commits b2aab8, 60c22d and cc207d. */
+        for (i = 0; i < idx->n; i++) {
+            bidx = idx->bidx[i];
+            k = kh_get(bin, bidx, META_BIN(idx));
+            if (k != kh_end(bidx)) {
+                if (off0 == (uint64_t) -1 || off0 < kh_val(bidx, k).list[0].v) {
+                    off0 = kh_val(bidx, k).list[0].v;
+                }
+            }
+        }
+        if (off0 == (uint64_t) -1 && idx->n_no_coor)
+            off0 = 0;
+        // only no-coor reads in this bam
+        break;
+    case HTS_IDX_REST:
+        off0 = 0;
+        break;
+    case HTS_IDX_NONE:
+        off0 = 0;
+        break;
+    }
+
+    return off0;
+}
+
+hts_itr_t *hts_itr_query(const hts_idx_t *idx, int tid, hts_pos_t beg, hts_pos_t end, hts_readrec_func *readrec)
+{
+    int i, n_off, l, bin;
+    hts_pair64_max_t *off;
+    khint_t k;
+    bidx_t *bidx;
+    uint64_t min_off, max_off;
+    hts_pos_t idx_maxpos;
+    hts_itr_t *iter;
+    uint32_t unmapped = 0, rel_off;
+
+    // It's possible to call this function with NULL idx iff
+    // tid is one of the special values HTS_IDX_REST or HTS_IDX_NONE
+    if (!idx && !(tid == HTS_IDX_REST || tid == HTS_IDX_NONE)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    iter = (hts_itr_t*)calloc(1, sizeof(hts_itr_t));
+    if (iter) {
+        if (tid < 0) {
+            uint64_t off = hts_itr_off(idx, tid);
+            if (off != (uint64_t) -1) {
+                iter->read_rest = 1;
+                iter->curr_off = off;
+                iter->readrec = readrec;
+                if (tid == HTS_IDX_NONE)
+                    iter->finished = 1;
+            } else {
+                free(iter);
+                iter = NULL;
+            }
+        } else if (tid >= idx->n || (bidx = idx->bidx[tid]) == NULL) {
+            iter->finished = 1;
+        } else {
+            if (beg < 0) beg = 0;
+            if (end < beg) {
+              free(iter);
+              return NULL;
+            }
+
+            k = kh_get(bin, bidx, META_BIN(idx));
+            if (k != kh_end(bidx))
+                unmapped = kh_val(bidx, k).list[1].v;
+            else
+                unmapped = 1;
+
+            iter->tid = tid, iter->beg = beg, iter->end = end; iter->i = -1;
+            iter->readrec = readrec;
+
+            if ( !kh_size(bidx) ) { iter->finished = 1; return iter; }
+
+            idx_maxpos = hts_idx_maxpos(idx);
+            if (beg >= idx_maxpos) { iter->finished = 1; return iter; }
+
+            rel_off = beg>>idx->min_shift;
+            // compute min_off
+            bin = hts_bin_first(idx->n_lvls) + rel_off;
+            do {
+                int first;
+                k = kh_get(bin, bidx, bin);
+                if (k != kh_end(bidx)) break;
+                first = (hts_bin_parent(bin)<<3) + 1;
+                if (bin > first) --bin;
+                else bin = hts_bin_parent(bin);
+            } while (bin);
+            if (bin == 0) k = kh_get(bin, bidx, bin);
+            min_off = k != kh_end(bidx)? kh_val(bidx, k).loff : 0;
+            // min_off can be calculated more accurately if the
+            // linear index is available
+            if (idx->lidx[tid].offset
+                && rel_off < idx->lidx[tid].n) {
+                if (min_off < idx->lidx[tid].offset[rel_off])
+                    min_off = idx->lidx[tid].offset[rel_off];
+                if (unmapped) {
+                    // unmapped reads are not covered by the linear index,
+                    // so search backwards for a smaller offset
+                    int tmp_off;
+                    for (tmp_off = rel_off-1; tmp_off >= 0; tmp_off--) {
+                        if (idx->lidx[tid].offset[tmp_off] < min_off) {
+                            min_off = idx->lidx[tid].offset[tmp_off];
+                            break;
+                        }
+                    }
+                    // if the search went too far back or no satisfactory entry
+                    // was found, revert to the bin index loff value
+                    if (k != kh_end(bidx) && (min_off < kh_val(bidx, k).loff || tmp_off < 0))
+                        min_off = kh_val(bidx, k).loff;
+                }
+            } else if (unmapped) { //CSI index
+                if (k != kh_end(bidx))
+                    min_off = kh_val(bidx, k).loff;
+            }
+
+            // compute max_off: a virtual offset from a bin to the right of end
+            // First check if end lies within the range of the index (it won't
+            // if it's HTS_POS_MAX)
+            if (end <= idx_maxpos) {
+                bin = hts_bin_first(idx->n_lvls) + ((end-1) >> idx->min_shift) + 1;
+                if (bin >= idx->n_bins) bin = 0;
+                while (1) {
+                    // search for an extant bin by moving right, but moving up to the
+                    // parent whenever we get to a first child (which also covers falling
+                    // off the RHS, which wraps around and immediately goes up to bin 0)
+                    while (bin % 8 == 1) bin = hts_bin_parent(bin);
+                    if (bin == 0) { max_off = UINT64_MAX; break; }
+                    k = kh_get(bin, bidx, bin);
+                    if (k != kh_end(bidx) && kh_val(bidx, k).n > 0) { max_off = kh_val(bidx, k).list[0].u; break; }
+                    bin++;
+                }
+            } else {
+                // Searching to end of reference
+                max_off = UINT64_MAX;
+            }
+
+            // retrieve bins
+            if (reg2bins(beg, end, iter, idx->min_shift, idx->n_lvls, bidx) < 0) {
+                hts_itr_destroy(iter);
+                return NULL;
+            }
+
+            for (i = n_off = 0; i < iter->bins.n; ++i)
+                if ((k = kh_get(bin, bidx, iter->bins.a[i])) != kh_end(bidx))
+                    n_off += kh_value(bidx, k).n;
+            if (n_off == 0) {
+                // No overlapping bins means the iterator has already finished.
+                iter->finished = 1;
+                return iter;
+            }
+            off = calloc(n_off, sizeof(*off));
+            for (i = n_off = 0; i < iter->bins.n; ++i) {
+                if ((k = kh_get(bin, bidx, iter->bins.a[i])) != kh_end(bidx)) {
+                    int j;
+                    bins_t *p = &kh_value(bidx, k);
+                    for (j = 0; j < p->n; ++j)
+                        if (p->list[j].v > min_off && p->list[j].u < max_off) {
+                            off[n_off].u = min_off > p->list[j].u
+                                ? min_off : p->list[j].u;
+                            off[n_off].v = max_off < p->list[j].v
+                                ? max_off : p->list[j].v;
+                            // hts_pair64_max_t::max is now used to link
+                            // file offsets to region list entries.
+                            // The iterator can use this to decide if it
+                            // can skip some file regions.
+                            off[n_off].max = ((uint64_t) tid << 32) | j;
+                            n_off++;
+                        }
+                }
+            }
+
+            if (n_off == 0) {
+                free(off);
+                iter->finished = 1;
+                return iter;
+            }
+            ks_introsort(_off_max, n_off, off);
+            // resolve completely contained adjacent blocks
+            for (i = 1, l = 0; i < n_off; ++i)
+                if (off[l].v < off[i].v) off[++l] = off[i];
+            n_off = l + 1;
+            // resolve overlaps between adjacent blocks; this may happen due to the merge in indexing
+            for (i = 1; i < n_off; ++i)
+                if (off[i-1].v >= off[i].u) off[i-1].v = off[i].u;
+            // merge adjacent blocks
+            for (i = 1, l = 0; i < n_off; ++i) {
+                if (off[l].v>>16 == off[i].u>>16) off[l].v = off[i].v;
+                else off[++l] = off[i];
+            }
+            n_off = l + 1;
+            iter->n_off = n_off; iter->off = off;
+        }
+    }
+
+    return iter;
+}
+
+int hts_itr_multi_bam(const hts_idx_t *idx, hts_itr_t *iter)
+{
+    int i, j, bin;
+    khint_t k;
+    bidx_t *bidx;
+    uint64_t min_off, max_off, t_off = (uint64_t)-1;
+    int tid;
+    hts_pos_t beg, end, idx_maxpos;
+    hts_reglist_t *curr_reg;
+    uint32_t unmapped = 0, rel_off;
+
+    if (!idx || !iter || !iter->multi)
+        return -1;
+
+    iter->i = -1;
+    for (i=0; i<iter->n_reg; i++) {
+
+        curr_reg = &iter->reg_list[i];
+        tid = curr_reg->tid;
+
+        if (tid < 0) {
+            t_off = hts_itr_off(idx, tid);
+            if (t_off != (uint64_t)-1) {
+                switch (tid) {
+                case HTS_IDX_NONE:
+                    iter->finished = 1;
+                    // fall through
+                case HTS_IDX_START:
+                case HTS_IDX_REST:
+                    iter->curr_off = t_off;
+                    iter->n_reg = 0;
+                    iter->reg_list = NULL;
+                    iter->read_rest = 1;
+                    return 0;
+                case HTS_IDX_NOCOOR:
+                    iter->nocoor = 1;
+                    iter->nocoor_off = t_off;
+                }
+            }
+        } else {
+            if (tid >= idx->n || (bidx = idx->bidx[tid]) == NULL || !kh_size(bidx))
+                continue;
+
+            k = kh_get(bin, bidx, META_BIN(idx));
+            if (k != kh_end(bidx))
+                unmapped = kh_val(bidx, k).list[1].v;
+            else
+                unmapped = 1;
+
+            idx_maxpos = hts_idx_maxpos(idx);
+
+            for(j=0; j<curr_reg->count; j++) {
+                hts_pair32_t *curr_intv = &curr_reg->intervals[j];
+                if (curr_intv->end < curr_intv->beg)
+                    continue;
+
+                beg = curr_intv->beg;
+                end = curr_intv->end;
+                if (beg >= idx_maxpos)
+                    continue;
+                rel_off = beg>>idx->min_shift;
+
+                /* Compute 'min_off' by searching the lowest level bin containing 'beg'.
+                       If the computed bin is not in the index, try the next bin to the
+                       left, belonging to the same parent. If it is the first sibling bin,
+                       try the parent bin. */
+                bin = hts_bin_first(idx->n_lvls) + rel_off;
+                do {
+                    int first;
+                    k = kh_get(bin, bidx, bin);
+                    if (k != kh_end(bidx)) break;
+                    first = (hts_bin_parent(bin)<<3) + 1;
+                    if (bin > first) --bin;
+                    else bin = hts_bin_parent(bin);
+                } while (bin);
+                if (bin == 0)
+                    k = kh_get(bin, bidx, bin);
+                min_off = k != kh_end(bidx)? kh_val(bidx, k).loff : 0;
+                // min_off can be calculated more accurately if the
+                // linear index is available
+                if (idx->lidx[tid].offset
+                    && rel_off < idx->lidx[tid].n) {
+                    if (min_off < idx->lidx[tid].offset[rel_off])
+                        min_off = idx->lidx[tid].offset[rel_off];
+                    if (unmapped) {
+                        int tmp_off;
+                        for (tmp_off = rel_off-1; tmp_off >= 0; tmp_off--) {
+                            if (idx->lidx[tid].offset[tmp_off] < min_off) {
+                                min_off = idx->lidx[tid].offset[tmp_off];
+                                break;
+                            }
+                        }
+
+                        if (k != kh_end(bidx) && (min_off < kh_val(bidx, k).loff || tmp_off < 0))
+                            min_off = kh_val(bidx, k).loff;
+                    }
+                } else if (unmapped) { //CSI index
+                    if (k != kh_end(bidx))
+                        min_off = kh_val(bidx, k).loff;
+                }
+
+                // compute max_off: a virtual offset from a bin to the right of end
+                // First check if end lies within the range of the index (it
+                // won't if it's HTS_POS_MAX)
+                if (end <= idx_maxpos) {
+                    bin = hts_bin_first(idx->n_lvls) + ((end-1) >> idx->min_shift) + 1;
+                    if (bin >= idx->n_bins) bin = 0;
+                    while (1) {
+                        // search for an extant bin by moving right, but moving up to the
+                        // parent whenever we get to a first child (which also covers falling
+                        // off the RHS, which wraps around and immediately goes up to bin 0)
+                        while (bin % 8 == 1) bin = hts_bin_parent(bin);
+                        if (bin == 0) { max_off = UINT64_MAX; break; }
+                        k = kh_get(bin, bidx, bin);
+                        if (k != kh_end(bidx) && kh_val(bidx, k).n > 0) {
+                            max_off = kh_val(bidx, k).list[0].u;
+                            break;
+                        }
+                        bin++;
+                    }
+                } else {
+                    // Searching to end of reference
+                    max_off = UINT64_MAX;
+                }
+
+                //convert coordinates to file offsets
+                if (reg2intervals(iter, idx, tid, beg, end, j,
+                                  min_off, max_off,
+                                  idx->min_shift, idx->n_lvls) < 0) {
+                    return -1;
+                }
+            }
+        }
+    }
+
+    if (iter->n_off > 1)
+        ks_introsort(_off_max, iter->n_off, iter->off);
+
+    if(!iter->n_off && !iter->nocoor)
+        iter->finished = 1;
+
+    return 0;
+}
+
+int hts_itr_multi_cram(const hts_idx_t *idx, hts_itr_t *iter)
+{
+    const hts_cram_idx_t *cidx = (const hts_cram_idx_t *) idx;
+    int tid, i, n_off = 0;
+    uint32_t j;
+    hts_pos_t beg, end;
+    hts_reglist_t *curr_reg;
+    hts_pair32_t *curr_intv;
+    hts_pair64_max_t *off = NULL, *tmp;
+    cram_index *e = NULL;
+
+    if (!cidx || !iter || !iter->multi)
+        return -1;
+
+    iter->is_cram = 1;
+    iter->read_rest = 0;
+    iter->off = NULL;
+    iter->n_off = 0;
+    iter->curr_off = 0;
+    iter->i = -1;
+
+    for (i=0; i<iter->n_reg; i++) {
+
+        curr_reg = &iter->reg_list[i];
+        tid = curr_reg->tid;
+
+        if (tid >= 0) {
+            tmp = realloc(off, (n_off + curr_reg->count) * sizeof(*off));
+            if (!tmp)
+                goto err;
+            off = tmp;
+
+            for (j=0; j < curr_reg->count; j++) {
+                curr_intv = &curr_reg->intervals[j];
+                if (curr_intv->end < curr_intv->beg)
+                    continue;
+
+                beg = curr_intv->beg;
+                end = curr_intv->end;
+
+/* First, fetch the container overlapping 'beg' and assign its file offset to u, then
+ * find the container overlapping 'end' and assign the relative end of the slice to v.
+ * The cram_ptell function will adjust with the container offset, which is not stored
+ * in the index.
+ */
+                e = cram_index_query(cidx->cram, tid, beg+1, NULL);
+                if (e) {
+                    off[n_off].u = e->offset;
+                    // hts_pair64_max_t::max is now used to link
+                    // file offsets to region list entries.
+                    // The iterator can use this to decide if it
+                    // can skip some file regions.
+                    off[n_off].max = ((uint64_t) tid << 32) | j;
+
+                    if (end >= HTS_POS_MAX) {
+                       e = cram_index_last(cidx->cram, tid, NULL);
+                    } else {
+                       e = cram_index_query_last(cidx->cram, tid, end+1);
+                    }
+
+                    if (e) {
+                        off[n_off++].v = e->e_next
+                            ? e->e_next->offset
+                            : e->offset + e->slice + e->len;
+                    } else {
+                        hts_log_warning("Could not set offset end for region %d:%"PRIhts_pos"-%"PRIhts_pos". Skipping", tid, beg, end);
+                    }
+                }
+            }
+        } else {
+            switch (tid) {
+                case HTS_IDX_NOCOOR:
+                    e = cram_index_query(cidx->cram, tid, 1, NULL);
+                    if (e) {
+                        iter->nocoor = 1;
+                        iter->nocoor_off = e->offset;
+                    } else {
+                        hts_log_warning("No index entry for NOCOOR region");
+                    }
+                    break;
+                case HTS_IDX_START:
+                    e = cram_index_query(cidx->cram, tid, 1, NULL);
+                    if (e) {
+                        iter->read_rest = 1;
+                        tmp = realloc(off, sizeof(*off));
+                        if (!tmp)
+                            goto err;
+                        off = tmp;
+                        off[0].u = e->offset;
+                        off[0].v = 0;
+                        n_off=1;
+                    } else {
+                        hts_log_warning("No index entries");
+                    }
+                    break;
+                case HTS_IDX_REST:
+                    break;
+                case HTS_IDX_NONE:
+                    iter->finished = 1;
+                    break;
+                default:
+                    hts_log_error("Query with tid=%d not implemented for CRAM files", tid);
+            }
+        }
+    }
+
+    if (n_off) {
+        ks_introsort(_off_max, n_off, off);
+        iter->n_off = n_off; iter->off = off;
+    }
+
+    if(!n_off && !iter->nocoor)
+        iter->finished = 1;
+
+    return 0;
+
+ err:
+    free(off);
+    return -1;
+}
+
+void hts_itr_destroy(hts_itr_t *iter)
+{
+    if (iter) {
+        if (iter->multi) {
+            hts_reglist_free(iter->reg_list, iter->n_reg);
+        } else {
+            free(iter->bins.a);
+        }
+
+        if (iter->off)
+            free(iter->off);
+        free(iter);
+    }
+}
+
+static inline unsigned long long push_digit(unsigned long long i, char c)
+{
+    // ensure subtraction occurs first, avoiding overflow for >= MAX-48 or so
+    int digit = c - '0';
+    return 10 * i + digit;
+}
+
+long long hts_parse_decimal(const char *str, char **strend, int flags)
+{
+    unsigned long long n = 0;
+    int digits = 0, decimals = 0, e = 0, lost = 0;
+    char sign = '+', esign = '+';
+    const char *s, *str_orig = str;
+
+    while (isspace_c(*str)) str++;
+    s = str;
+
+    if (*s == '+' || *s == '-') sign = *s++;
+    while (*s)
+        if (isdigit_c(*s)) digits++, n = push_digit(n, *s++);
+        else if (*s == ',' && (flags & HTS_PARSE_THOUSANDS_SEP)) s++;
+        else break;
+
+    if (*s == '.') {
+        s++;
+        while (isdigit_c(*s)) decimals++, digits++, n = push_digit(n, *s++);
+    }
+
+    switch (*s) {
+    case 'e': case 'E':
+        s++;
+        if (*s == '+' || *s == '-') esign = *s++;
+        while (isdigit_c(*s)) e = push_digit(e, *s++);
+        if (esign == '-') e = -e;
+        break;
+
+    case 'k': case 'K': e += 3; s++; break;
+    case 'm': case 'M': e += 6; s++; break;
+    case 'g': case 'G': e += 9; s++; break;
+    }
+
+    e -= decimals;
+    while (e > 0) n *= 10, e--;
+    while (e < 0) lost += n % 10, n /= 10, e++;
+
+    if (lost > 0) {
+        hts_log_warning("Discarding fractional part of %.*s", (int)(s - str), str);
+    }
+
+    if (strend) {
+        // Set to the original input str pointer if not valid number syntax
+        *strend = (digits > 0)? (char *)s : (char *)str_orig;
+    } else if (digits == 0) {
+        hts_log_warning("Invalid numeric value %.8s[truncated]", str);
+    } else if (*s) {
+        if ((flags & HTS_PARSE_THOUSANDS_SEP) || (!(flags & HTS_PARSE_THOUSANDS_SEP) && *s != ','))
+            hts_log_warning("Ignoring unknown characters after %.*s[%s]", (int)(s - str), str, s);
+    }
+
+    return (sign == '+')? n : -n;
+}
+
+static void *hts_memrchr(const void *s, int c, size_t n) {
+    size_t i;
+    unsigned char *u = (unsigned char *)s;
+    for (i = n; i > 0; i--) {
+        if (u[i-1] == c)
+            return u+i-1;
+    }
+
+    return NULL;
+}
+
+/*
+ * A variant of hts_parse_reg which is reference-id aware.  It uses
+ * the iterator name2id callbacks to validate the region tokenisation works.
+ *
+ * This is necessary due to GRCh38 HLA additions which have reference names
+ * like "HLA-DRB1*12:17".
+ *
+ * All parameters are mandatory.
+ *
+ * To work around ambiguous parsing issues, eg both "chr1" and "chr1:100-200"
+ * are reference names, we may quote using curly braces.
+ * Thus "{chr1}:100-200" and "{chr1:100-200}" disambiguate the above example.
+ *
+ * Flags are used to control how parsing works, and can be one of the below.
+ *
+ * HTS_PARSE_LIST:
+ *     If present, the region is assmed to be a comma separated list and
+ *     position parsing will not contain commas (this implicitly
+ *     clears HTS_PARSE_THOUSANDS_SEP in the call to hts_parse_decimal).
+ *     On success the return pointer will be the start of the next region, ie
+ *     the character after the comma.  (If *ret != '\0' then the caller can
+ *     assume another region is present in the list.)
+ *
+ *     If not set then positions may contain commas.  In this case the return
+ *     value should point to the end of the string, or NULL on failure.
+ *
+ * HTS_PARSE_ONE_COORD:
+ *     If present, X:100 is treated as the single base pair region X:100-100.
+ *     In this case X:-100 is shorthand for X:1-100 and X:100- is X:100-<end>.
+ *     (This is the standard bcftools region convention.)
+ *
+ *     When not set X:100 is considered to be X:100-<end> where <end> is
+ *     the end of chromosome X (set to HTS_POS_MAX here).  X:100- and X:-100
+ *     are invalid.
+ *     (This is the standard samtools region convention.)
+ *
+ * Note the supplied string expects 1 based inclusive coordinates, but the
+ * returned coordinates start from 0 and are half open, so pos0 is valid
+ * for use in e.g. "for (pos0 = beg; pos0 < end; pos0++) {...}"
+ *
+ * On success a pointer to the byte after the end of the entire region
+ *            specifier is returned (plus any trailing comma), and tid,
+ *            beg & end will be set.
+ * On failure NULL is returned.
+ */
+const char *hts_parse_region(const char *s, int *tid, hts_pos_t *beg,
+                             hts_pos_t *end, hts_name2id_f getid, void *hdr,
+                             int flags)
+{
+    if (!s || !tid || !beg || !end || !getid)
+        return NULL;
+
+    size_t s_len = strlen(s);
+    kstring_t ks = { 0, 0, NULL };
+
+    const char *colon = NULL, *comma = NULL;
+    int quoted = 0;
+
+    if (flags & HTS_PARSE_LIST)
+        flags &= ~HTS_PARSE_THOUSANDS_SEP;
+    else
+        flags |= HTS_PARSE_THOUSANDS_SEP;
+
+    const char *s_end = s + s_len;
+
+    // Braced quoting of references is permitted to resolve ambiguities.
+    if (*s == '{') {
+        const char *close = memchr(s, '}', s_len);
+        if (!close) {
+            hts_log_error("Mismatching braces in \"%s\"", s);
+            *tid = -1;
+            return NULL;
+        }
+        s++;
+        s_len--;
+        if (close[1] == ':')
+            colon = close+1;
+        quoted = 1; // number of trailing characters to trim
+
+        // Truncate to this item only, if appropriate.
+        if (flags & HTS_PARSE_LIST) {
+            comma = strchr(close, ',');
+            if (comma) {
+                s_len = comma-s;
+                s_end = comma+1;
+            }
+        }
+    } else {
+        // Truncate to this item only, if appropriate.
+        if (flags & HTS_PARSE_LIST) {
+            comma = strchr(s, ',');
+            if (comma) {
+                s_len = comma-s;
+                s_end = comma+1;
+            }
+        }
+
+        colon = hts_memrchr(s, ':', s_len);
+    }
+
+    // No colon is simplest case; just check and return.
+    if (colon == NULL) {
+        *beg = 0; *end = HTS_POS_MAX;
+        kputsn(s, s_len-quoted, &ks); // convert to nul terminated string
+        if (!ks.s) {
+            *tid = -2;
+            return NULL;
+        }
+
+        *tid = getid(hdr, ks.s);
+        free(ks.s);
+
+        return *tid >= 0 ? s_end : NULL;
+    }
+
+    // Has a colon, but check whole name first.
+    if (!quoted) {
+        *beg = 0; *end = HTS_POS_MAX;
+        kputsn(s, s_len, &ks); // convert to nul terminated string
+        if (!ks.s) {
+            *tid = -2;
+            return NULL;
+        }
+        if ((*tid = getid(hdr, ks.s)) >= 0) {
+            // Entire name matches, but also check this isn't
+            // ambiguous.  eg we have ref chr1 and ref chr1:100-200
+            // both present.
+            ks.l = 0;
+            kputsn(s, colon-s, &ks); // convert to nul terminated string
+            if (!ks.s) {
+                *tid = -2;
+                return NULL;
+            }
+            if (getid(hdr, ks.s) >= 0) {
+                free(ks.s);
+                *tid = -1;
+                hts_log_error("Range is ambiguous. "
+                              "Use {%s} or {%.*s}%s instead",
+                              s, (int)(colon-s), s, colon);
+                return NULL;
+            }
+            free(ks.s);
+
+            return s_end;
+        }
+        if (*tid < -1) // Failed to parse header
+            return NULL;
+    }
+
+    // Quoted, or unquoted and whole string isn't a name.
+    // Check the pre-colon part is valid.
+    ks.l = 0;
+    kputsn(s, colon-s-quoted, &ks); // convert to nul terminated string
+    if (!ks.s) {
+        *tid = -2;
+        return NULL;
+    }
+    *tid = getid(hdr, ks.s);
+    free(ks.s);
+    if (*tid < 0)
+        return NULL;
+
+    // Finally parse the post-colon coordinates
+    char *hyphen;
+    *beg = hts_parse_decimal(colon+1, &hyphen, flags) - 1;
+    if (*beg < 0) {
+        if (*beg != -1 && *hyphen == '-' && colon[1] != '\0') {
+            // User specified zero, but we're 1-based.
+            hts_log_error("Coordinates must be > 0");
+            return NULL;
+        }
+        if (isdigit_c(*hyphen) || *hyphen == '\0' || *hyphen == ',') {
+            // interpret chr:-100 as chr:1-100
+            *end = *beg==-1 ? HTS_POS_MAX : -(*beg+1);
+            *beg = 0;
+            return s_end;
+        } else if (*beg < -1) {
+            hts_log_error("Unexpected string \"%s\" after region", hyphen);
+            return NULL;
+        }
+    }
+
+    if (*hyphen == '\0' || ((flags & HTS_PARSE_LIST) && *hyphen == ',')) {
+        *end = flags & HTS_PARSE_ONE_COORD ? *beg+1 : HTS_POS_MAX;
+    } else if (*hyphen == '-') {
+        *end = hts_parse_decimal(hyphen+1, &hyphen, flags);
+        if (*hyphen != '\0' && *hyphen != ',') {
+            hts_log_error("Unexpected string \"%s\" after region", hyphen);
+            return NULL;
+        }
+    } else {
+        hts_log_error("Unexpected string \"%s\" after region", hyphen);
+        return NULL;
+    }
+
+    if (*end == 0)
+        *end = HTS_POS_MAX; // interpret chr:100- as chr:100-<end>
+
+    if (*beg >= *end) return NULL;
+
+    return s_end;
+}
+
+// Next release we should mark this as deprecated?
+// Use hts_parse_region above instead.
+const char *hts_parse_reg64(const char *s, hts_pos_t *beg, hts_pos_t *end)
+{
+    char *hyphen;
+    const char *colon = strrchr(s, ':');
+    if (colon == NULL) {
+        *beg = 0; *end = HTS_POS_MAX;
+        return s + strlen(s);
+    }
+
+    *beg = hts_parse_decimal(colon+1, &hyphen, HTS_PARSE_THOUSANDS_SEP) - 1;
+    if (*beg < 0) *beg = 0;
+
+    if (*hyphen == '\0') *end = HTS_POS_MAX;
+    else if (*hyphen == '-') *end = hts_parse_decimal(hyphen+1, NULL, HTS_PARSE_THOUSANDS_SEP);
+    else return NULL;
+
+    if (*beg >= *end) return NULL;
+    return colon;
+}
+
+const char *hts_parse_reg(const char *s, int *beg, int *end)
+{
+    hts_pos_t beg64 = 0, end64 = 0;
+    const char *colon = hts_parse_reg64(s, &beg64, &end64);
+    if (beg64 > INT_MAX) {
+        hts_log_error("Position %"PRId64" too large", beg64);
+        return NULL;
+    }
+    if (end64 > INT_MAX) {
+        if (end64 == HTS_POS_MAX) {
+            end64 = INT_MAX;
+        } else {
+            hts_log_error("Position %"PRId64" too large", end64);
+            return NULL;
+        }
+    }
+    *beg = beg64;
+    *end = end64;
+    return colon;
+}
+
+hts_itr_t *hts_itr_querys(const hts_idx_t *idx, const char *reg, hts_name2id_f getid, void *hdr, hts_itr_query_func *itr_query, hts_readrec_func *readrec)
+{
+    int tid;
+    hts_pos_t beg, end;
+
+    if (strcmp(reg, ".") == 0)
+        return itr_query(idx, HTS_IDX_START, 0, 0, readrec);
+    else if (strcmp(reg, "*") == 0)
+        return itr_query(idx, HTS_IDX_NOCOOR, 0, 0, readrec);
+
+    if (!hts_parse_region(reg, &tid, &beg, &end, getid, hdr, HTS_PARSE_THOUSANDS_SEP))
+        return NULL;
+
+    return itr_query(idx, tid, beg, end, readrec);
+}
+
+hts_itr_t *hts_itr_regions(const hts_idx_t *idx, hts_reglist_t *reglist, int count, hts_name2id_f getid, void *hdr, hts_itr_multi_query_func *itr_specific, hts_readrec_func *readrec, hts_seek_func *seek, hts_tell_func *tell) {
+
+    int i;
+
+    if (!reglist)
+        return NULL;
+
+    hts_itr_t *itr = (hts_itr_t*)calloc(1, sizeof(hts_itr_t));
+    if (itr) {
+        itr->n_reg = count;
+        itr->readrec = readrec;
+        itr->seek = seek;
+        itr->tell = tell;
+        itr->reg_list = reglist;
+        itr->finished = 0;
+        itr->nocoor = 0;
+        itr->multi = 1;
+
+        for (i = 0; i < itr->n_reg; i++) {
+            if (itr->reg_list[i].reg) {
+                if (!strcmp(itr->reg_list[i].reg, ".")) {
+                    itr->reg_list[i].tid = HTS_IDX_START;
+                    continue;
+                }
+
+                if (!strcmp(itr->reg_list[i].reg, "*")) {
+                    itr->reg_list[i].tid = HTS_IDX_NOCOOR;
+                    continue;
+                }
+
+                itr->reg_list[i].tid = getid(hdr, reglist[i].reg);
+                if (itr->reg_list[i].tid < 0) {
+                    if (itr->reg_list[i].tid < -1) {
+                        hts_log_error("Failed to parse header");
+                        hts_itr_destroy(itr);
+                        return NULL;
+                    } else {
+                        hts_log_warning("Region '%s' specifies an unknown reference name. Continue anyway", reglist[i].reg);
+                    }
+                }
+            }
+        }
+
+        qsort(itr->reg_list, itr->n_reg, sizeof(hts_reglist_t), compare_regions);
+        if (itr_specific(idx, itr) != 0) {
+            hts_log_error("Failed to create the multi-region iterator!");
+            hts_itr_destroy(itr);
+            itr = NULL;
+        }
+    }
+
+    return itr;
+}
+
+int hts_itr_next(BGZF *fp, hts_itr_t *iter, void *r, void *data)
+{
+    int ret, tid;
+    hts_pos_t beg, end;
+    if (iter == NULL || iter->finished) return -1;
+    if (iter->read_rest) {
+        if (iter->curr_off) { // seek to the start
+            if (bgzf_seek(fp, iter->curr_off, SEEK_SET) < 0) {
+                hts_log_error("Failed to seek to offset %"PRIu64"%s%s",
+                              iter->curr_off,
+                              errno ? ": " : "", strerror(errno));
+                return -2;
+            }
+            iter->curr_off = 0; // only seek once
+        }
+        ret = iter->readrec(fp, data, r, &tid, &beg, &end);
+        if (ret < 0) iter->finished = 1;
+        iter->curr_tid = tid;
+        iter->curr_beg = beg;
+        iter->curr_end = end;
+        return ret;
+    }
+    // A NULL iter->off should always be accompanied by iter->finished.
+    assert(iter->off != NULL);
+    for (;;) {
+        if (iter->curr_off == 0 || iter->curr_off >= iter->off[iter->i].v) { // then jump to the next chunk
+            if (iter->i == iter->n_off - 1) { ret = -1; break; } // no more chunks
+            if (iter->i < 0 || iter->off[iter->i].v != iter->off[iter->i+1].u) { // not adjacent chunks; then seek
+                if (bgzf_seek(fp, iter->off[iter->i+1].u, SEEK_SET) < 0) {
+                    hts_log_error("Failed to seek to offset %"PRIu64"%s%s",
+                                  iter->off[iter->i+1].u,
+                                  errno ? ": " : "", strerror(errno));
+                    return -2;
+                }
+                iter->curr_off = bgzf_tell(fp);
+            }
+            ++iter->i;
+        }
+        if ((ret = iter->readrec(fp, data, r, &tid, &beg, &end)) >= 0) {
+            iter->curr_off = bgzf_tell(fp);
+            if (tid != iter->tid || beg >= iter->end) { // no need to proceed
+                ret = -1; break;
+            } else if (end > iter->beg && iter->end > beg) {
+                iter->curr_tid = tid;
+                iter->curr_beg = beg;
+                iter->curr_end = end;
+                return ret;
+            }
+        } else break; // end of file or error
+    }
+    iter->finished = 1;
+    return ret;
+}
+
+int hts_itr_multi_next(htsFile *fd, hts_itr_t *iter, void *r)
+{
+    void *fp;
+    int ret, tid, i, cr, ci;
+    hts_pos_t beg, end;
+    hts_reglist_t *found_reg;
+
+    if (iter == NULL || iter->finished) return -1;
+
+    if (iter->is_cram) {
+        fp = fd->fp.cram;
+    } else {
+        fp = fd->fp.bgzf;
+    }
+
+    if (iter->read_rest) {
+        if (iter->curr_off) { // seek to the start
+            if (iter->seek(fp, iter->curr_off, SEEK_SET) < 0) {
+                hts_log_error("Seek at offset %" PRIu64 " failed.", iter->curr_off);
+                return -1;
+            }
+            iter->curr_off = 0; // only seek once
+        }
+
+        ret = iter->readrec(fp, fd, r, &tid, &beg, &end);
+        if (ret < 0)
+            iter->finished = 1;
+
+        iter->curr_tid = tid;
+        iter->curr_beg = beg;
+        iter->curr_end = end;
+
+        return ret;
+    }
+    // A NULL iter->off should always be accompanied by iter->finished.
+    assert(iter->off != NULL || iter->nocoor != 0);
+
+    int next_range = 0;
+    for (;;) {
+        // Note that due to the way bam indexing works, iter->off may contain
+        // file chunks that are not actually needed as they contain data
+        // beyond the end of the requested region.  These are filtered out
+        // by comparing the tid and index into hts_reglist_t::intervals
+        // (packed for reasons of convenience into iter->off[iter->i].max)
+        // associated with the file region with iter->curr_tid and
+        // iter->curr_intv.
+
+        if (next_range
+            || iter->curr_off == 0
+            || iter->i >= iter->n_off
+            || iter->curr_off >= iter->off[iter->i].v
+            || (iter->off[iter->i].max >> 32 == iter->curr_tid
+                && (iter->off[iter->i].max & 0xffffffff) < iter->curr_intv)) {
+
+            // Jump to the next chunk.  It may be necessary to skip more
+            // than one as the iter->off list can include overlapping entries.
+            do {
+                iter->i++;
+            } while (iter->i < iter->n_off
+                     && (iter->curr_off >= iter->off[iter->i].v
+                         || (iter->off[iter->i].max >> 32 == iter->curr_tid
+                             && (iter->off[iter->i].max & 0xffffffff) < iter->curr_intv)));
+
+            if (iter->is_cram && iter->i < iter->n_off) {
+                // Ensure iter->curr_reg is correct.
+                //
+                // We need this for CRAM as we shortcut some of the later
+                // logic by getting an end-of-range and continuing to the
+                // next offset.
+                //
+                // We cannot do this for BAM (and fortunately do not need to
+                // either) because in BAM world a query to genomic positions
+                // GX and GY leading to a seek offsets PX and PY may have
+                // GX > GY and PX < PY.  (This is due to the R-tree and falling
+                // between intervals, bumping up to a higher bin.)
+                // CRAM strictly follows PX >= PY if GX >= GY, so this logic
+                // works.
+                int want_tid = iter->off[iter->i].max >> 32;
+                if (!(iter->curr_reg < iter->n_reg &&
+                      iter->reg_list[iter->curr_reg].tid == want_tid)) {
+                    int j;
+                    for (j = 0; j < iter->n_reg; j++)
+                        if (iter->reg_list[j].tid == want_tid)
+                            break;
+                    if (j == iter->n_reg)
+                        return -1;
+                    iter->curr_reg = j;
+                    iter->curr_tid = iter->reg_list[iter->curr_reg].tid;
+                };
+                iter->curr_intv = iter->off[iter->i].max & 0xffffffff;
+            }
+
+            if (iter->i >= iter->n_off) { // no more chunks, except NOCOORs
+                if (iter->nocoor) {
+                    next_range = 0;
+                    if (iter->seek(fp, iter->nocoor_off, SEEK_SET) < 0) {
+                        hts_log_error("Seek at offset %" PRIu64 " failed.", iter->nocoor_off);
+                        return -1;
+                    }
+                    if (iter->is_cram) {
+                        cram_range r = { HTS_IDX_NOCOOR };
+                        cram_set_option(fp, CRAM_OPT_RANGE_NOSEEK, &r);
+                    }
+
+                    // The first slice covering the unmapped reads might
+                    // contain a few mapped reads, so scroll
+                    // forward until finding the first unmapped read.
+                    do {
+                        ret = iter->readrec(fp, fd, r, &tid, &beg, &end);
+                    } while (tid >= 0 && ret >=0);
+
+                    if (ret < 0)
+                        iter->finished = 1;
+                    else
+                        iter->read_rest = 1;
+
+                    iter->curr_off = 0; // don't seek any more
+                    iter->curr_tid = tid;
+                    iter->curr_beg = beg;
+                    iter->curr_end = end;
+
+                    return ret;
+                } else {
+                    ret = -1; break;
+                }
+            } else if (iter->i < iter->n_off) {
+                // New chunk may overlap the last one, so ensure we
+                // only seek forwards.
+                if (iter->curr_off < iter->off[iter->i].u || next_range) {
+                    iter->curr_off = iter->off[iter->i].u;
+
+                    // CRAM has the capability of setting an end location.
+                    // This means multi-threaded decodes can stop once they
+                    // reach that point, rather than pointlessly decoding
+                    // more slices than we'll be using.
+                    //
+                    // We have to be careful here.  Whenever we set the cram
+                    // range we need a corresponding seek in order to ensure
+                    // we can safely decode at that offset.  We use next_range
+                    // var to ensure this is always true; this is set on
+                    // end-of-range condition. It's never modified for BAM.
+                    if (iter->is_cram) {
+                        // Next offset.[uv] tuple, but it's already been
+                        // included in our cram range, so don't seek and don't
+                        // reset range so we can efficiently multi-thread.
+                        if (next_range || iter->curr_off >= iter->end) {
+                            if (iter->seek(fp, iter->curr_off, SEEK_SET) < 0) {
+                                hts_log_error("Seek at offset %" PRIu64
+                                        " failed.", iter->curr_off);
+                                return -1;
+                            }
+
+                            // Find the genomic range matching this interval.
+                            int j;
+                            hts_reglist_t *rl = &iter->reg_list[iter->curr_reg];
+                            cram_range r = {
+                                    rl->tid,
+                                    rl->intervals[iter->curr_intv].beg,
+                                    rl->intervals[iter->curr_intv].end
+                            };
+
+                            // Expand it up to cover neighbouring intervals.
+                            // Note we can only have a single chromosome in a
+                            // range, so if we detect our blocks span chromosomes
+                            // or we have a multi-ref mode slice, we just use
+                            // HTS_IDX_START refid instead.  This doesn't actually
+                            // seek (due to CRAM_OPT_RANGE_NOSEEK) and is simply
+                            // and indicator of decoding with no end limit.
+                            //
+                            // That isn't as efficient as it could be, but it's
+                            // no poorer than before and it works.
+                            int tid = r.refid;
+                            int64_t end = r.end;
+                            int64_t v = iter->off[iter->i].v;
+                            j = iter->i+1;
+                            while (j < iter->n_off) {
+                                if (iter->off[j].u > v)
+                                    break;
+
+                                uint64_t max = iter->off[j].max;
+                                if ((max>>32) != tid) {
+                                    tid = HTS_IDX_START; // => no range limit
+                                } else {
+                                    if (end < rl->intervals[max & 0xffffffff].end)
+                                        end = rl->intervals[max & 0xffffffff].end;
+                                }
+                                if (v < iter->off[j].v)
+                                    v = iter->off[j].v;
+                                j++;
+                            }
+                            r.refid = tid;
+                            r.end = end;
+
+                            // Remember maximum 'v' here so we don't do
+                            // unnecessary subsequent seeks for the next
+                            // regions.  We can't change curr_off, but
+                            // beg/end are used only by single region iterator so
+                            // we cache it there to avoid changing the struct.
+                            iter->end = v;
+
+                            cram_set_option(fp, CRAM_OPT_RANGE_NOSEEK, &r);
+                            next_range = 0;
+                        }
+                    } else { // Not CRAM
+                        if (iter->seek(fp, iter->curr_off, SEEK_SET) < 0) {
+                            hts_log_error("Seek at offset %" PRIu64 " failed.",
+                                          iter->curr_off);
+                            return -1;
+                        }
+                    }
+                }
+            }
+        }
+
+        ret = iter->readrec(fp, fd, r, &tid, &beg, &end);
+        if (ret < 0) {
+            if (iter->is_cram && cram_eof(fp)) {
+                // Skip to end of range
+                //
+                // We should never be adjusting curr_off manually unless
+                // we also can guarantee we'll be doing a seek after to
+                // a new location.  Otherwise we'll be reading wrong offset
+                // for the next container.
+                //
+                // We ensure this by adjusting our CRAM_OPT_RANGE
+                // accordingly above, but to double check we also
+                // set the skipped_block flag to enforce a seek also.
+                iter->curr_off = iter->off[iter->i].v;
+                next_range = 1;
+
+                // Next region
+                if (++iter->curr_intv >= iter->reg_list[iter->curr_reg].count){
+                    if (++iter->curr_reg >= iter->n_reg)
+                        break;
+                    iter->curr_intv = 0;
+                    iter->curr_tid = iter->reg_list[iter->curr_reg].tid;
+                }
+                continue;
+            } else {
+                break;
+            }
+        }
+
+        iter->curr_off = iter->tell(fp);
+
+        if (tid != iter->curr_tid) {
+            hts_reglist_t key;
+            key.tid = tid;
+
+            found_reg = (hts_reglist_t *)bsearch(&key, iter->reg_list,
+                                                 iter->n_reg,
+                                                 sizeof(hts_reglist_t),
+                                                 compare_regions);
+            if (!found_reg)
+                continue;
+
+            iter->curr_reg = (found_reg - iter->reg_list);
+            iter->curr_tid = tid;
+            iter->curr_intv = 0;
+        }
+
+        cr = iter->curr_reg;
+        ci = iter->curr_intv;
+
+        for (i = ci; i < iter->reg_list[cr].count; i++) {
+            if (end > iter->reg_list[cr].intervals[i].beg &&
+                iter->reg_list[cr].intervals[i].end > beg) {
+                iter->curr_beg = beg;
+                iter->curr_end = end;
+                iter->curr_intv = i;
+
+                return ret;
+            }
+
+            // Check if the read starts beyond intervals[i].end
+            // If so, the interval is finished so move on to the next.
+            if (beg > iter->reg_list[cr].intervals[i].end)
+                iter->curr_intv = i + 1;
+
+            // No need to keep searching if the read ends before intervals[i].beg
+            if (end < iter->reg_list[cr].intervals[i].beg)
+                break;
+        }
+    }
+    iter->finished = 1;
+
+    return ret;
+}
+
+/**********************
+ *** Retrieve index ***
+ **********************/
+// Local_fn and local_len will return a sub-region of 'fn'.
+// Eg http://elsewhere/dir/foo.bam.bai?a=b may return
+// foo.bam.bai via local_fn and local_len.
+//
+// Returns -1 if index couldn't be opened.
+//         -2 on other errors
+static int idx_test_and_fetch(const char *fn, const char **local_fn, int *local_len, int download)
+{
+    hFILE *remote_hfp = NULL;
+    hFILE *local_fp = NULL;
+    int save_errno;
+    htsFormat fmt;
+    kstring_t s = KS_INITIALIZE;
+    kstring_t tmps = KS_INITIALIZE;
+
+    if (hisremote(fn)) {
+        const int buf_size = 1 * 1024 * 1024;
+        int l;
+        const char *p, *e;
+        // Ignore ?# params: eg any file.fmt?param=val, except for S3 URLs
+        e = fn + ((strncmp(fn, "s3://", 5) && strncmp(fn, "s3+http://", 10) && strncmp(fn, "s3+https://", 11)) ? strcspn(fn, "?#") : strcspn(fn, "?"));
+        // Find the previous slash from there.
+        p = e;
+        while (p > fn && *p != '/') p--;
+        if (*p == '/') p++;
+
+        // Attempt to open local file first
+        kputsn(p, e-p, &s);
+        if (access(s.s, R_OK) == 0)
+        {
+            free(s.s);
+            *local_fn = p;
+            *local_len = e-p;
+            return 0;
+        }
+
+        // Attempt to open remote file. Stay quiet on failure, it is OK to fail when trying first .csi then .bai or .tbi index.
+        if ((remote_hfp = hopen(fn, "r")) == 0) {
+            hts_log_info("Failed to open index file '%s'", fn);
+            free(s.s);
+            return -1;
+        }
+        if (hts_detect_format2(remote_hfp, fn, &fmt)) {
+            hts_log_error("Failed to detect format of index file '%s'", fn);
+            goto fail;
+        }
+        if (fmt.category != index_file || (fmt.format != bai &&  fmt.format != csi && fmt.format != tbi
+                && fmt.format != crai && fmt.format != fai_format)) {
+            hts_log_error("Format of index file '%s' is not supported", fn);
+            goto fail;
+        }
+
+        if (download) {
+            if ((local_fp = hts_open_tmpfile(s.s, "wx", &tmps)) == NULL) {
+                hts_log_error("Failed to create file %s in the working directory", p);
+                goto fail;
+            }
+            hts_log_info("Downloading file %s to local directory", fn);
+            uint8_t *buf = (uint8_t*)calloc(buf_size, 1);
+            if (!buf) {
+                hts_log_error("%s", strerror(errno));
+                goto fail;
+            }
+            while ((l = hread(remote_hfp, buf, buf_size)) > 0) {
+                if (hwrite(local_fp, buf, l) != l) {
+                    hts_log_error("Failed to write data to %s : %s",
+                            fn, strerror(errno));
+                    free(buf);
+                    goto fail;
+                }
+            }
+            free(buf);
+            if (l < 0) {
+                hts_log_error("Error reading \"%s\"", fn);
+                goto fail;
+            }
+            if (hclose(local_fp) < 0) {
+                hts_log_error("Error closing %s : %s", fn, strerror(errno));
+                local_fp = NULL;
+                goto fail;
+            }
+            local_fp = NULL;
+            if (rename(tmps.s, s.s) < 0) {
+                hts_log_error("Error renaming %s : %s", tmps.s, strerror(errno));
+                goto fail;
+            }
+            ks_clear(&tmps);
+
+            *local_fn = p;
+            *local_len = e-p;
+        } else {
+            *local_fn = fn;
+            *local_len = e-fn;
+        }
+
+        if (hclose(remote_hfp) != 0) {
+            hts_log_error("Failed to close remote file %s", fn);
+        }
+
+        free(tmps.s);
+        free(s.s);
+        return 0;
+    } else {
+        hFILE *local_hfp;
+        if ((local_hfp = hopen(fn, "r")) == 0) return -1;
+        hclose_abruptly(local_hfp);
+        *local_fn = fn;
+        *local_len = strlen(fn);
+        return 0;
+    }
+
+ fail:
+    save_errno = errno;
+    if (remote_hfp) hclose_abruptly(remote_hfp);
+    if (local_fp) hclose_abruptly(local_fp);
+    if (tmps.l > 0) unlink(tmps.s);
+    free(tmps.s);
+    free(s.s);
+    errno = save_errno;
+    return -2;
+}
+
+/*
+ * Check the existence of a local index file using part of the alignment
+ * file name.
+ *
+ * For a filename fn of fn.fmt (eg fn.bam or fn.cram) the order of checks is
+ * fn.fmt.csi,  fn.csi,
+ * fn.fmt.bai,  fn.bai  - if fmt is HTS_FMT_BAI
+ * fn.fmt.tbi,  fn.tbi  - if fmt is HTS_FMT_TBI
+ * fn.fmt.crai, fn.crai - if fmt is HTS_FMT_CRAI
+ * fn.fmt.fai           - if fmt is HTS_FMT_FAI
+ *   also .gzi if fmt is ".gz"
+ *
+ * @param fn    - pointer to the file name
+ * @param fmt   - one of the HTS_FMT index formats
+ * @param fnidx - pointer to the index file name placeholder
+ * @return        1 for success, 0 for failure
+ */
+int hts_idx_check_local(const char *fn, int fmt, char **fnidx) {
+    int i, l_fn, l_ext;
+    const char *fn_tmp = NULL;
+    char *fnidx_tmp;
+    const char *csi_ext = ".csi";
+    const char *bai_ext = ".bai";
+    const char *tbi_ext = ".tbi";
+    const char *crai_ext = ".crai";
+    const char *fai_ext = ".fai";
+    const char *gzi_ext = ".gzi";
+
+    if (!fn)
+        return 0;
+
+    if (hisremote(fn)) {
+        for (i = strlen(fn) - 1; i >= 0; --i)
+            if (fn[i] == '/') {
+                fn_tmp = (char *)&fn[i+1];
+                break;
+            }
+    } else {
+        // Borrowed from hopen_fd_fileuri()
+        if (strncmp(fn, "file://localhost/", 17) == 0) fn_tmp = fn + 16;
+        else if (strncmp(fn, "file:///", 8) == 0) fn_tmp = fn + 7;
+        else fn_tmp = fn;
+#if defined(_WIN32) || defined(__MSYS__)
+        // For cases like C:/foo
+        if (fn_tmp[0] == '/' && fn_tmp[1] && fn_tmp[2] == ':' && fn_tmp[3] == '/')
+            fn_tmp++;
+#endif
+    }
+
+    if (!fn_tmp) return 0;
+    hts_log_info("Using alignment file '%s'", fn_tmp);
+    l_fn = strlen(fn_tmp); l_ext = 5;
+    fnidx_tmp = (char*)calloc(l_fn + l_ext + 1, 1);
+    if (!fnidx_tmp) return 0;
+
+    struct stat sbuf;
+
+    // Try alignment.bam.csi first
+    strcpy(fnidx_tmp, fn_tmp); strcpy(fnidx_tmp + l_fn, csi_ext);
+    if(stat(fnidx_tmp, &sbuf) == 0) {
+        *fnidx = fnidx_tmp;
+        return 1;
+    } else { // Then try alignment.csi
+        for (i = l_fn - 1; i > 0; --i)
+            if (fnidx_tmp[i] == '.') {
+                strcpy(fnidx_tmp + i, csi_ext);
+                if(stat(fnidx_tmp, &sbuf) == 0) {
+                    *fnidx = fnidx_tmp;
+                    return 1;
+                }
+                break;
+            }
+    }
+    if (fmt == HTS_FMT_BAI) {
+        // Next, try alignment.bam.bai
+        strcpy(fnidx_tmp, fn_tmp); strcpy(fnidx_tmp + l_fn, bai_ext);
+        if(stat(fnidx_tmp, &sbuf) == 0) {
+            *fnidx = fnidx_tmp;
+            return 1;
+        } else { // And finally, try alignment.bai
+            for (i = l_fn - 1; i > 0; --i)
+                if (fnidx_tmp[i] == '.') {
+                    strcpy(fnidx_tmp + i, bai_ext);
+                    if(stat(fnidx_tmp, &sbuf) == 0) {
+                        *fnidx = fnidx_tmp;
+                        return 1;
+                    }
+                    break;
+                }
+        }
+    } else if (fmt == HTS_FMT_TBI) { // Or .tbi
+        strcpy(fnidx_tmp, fn_tmp); strcpy(fnidx_tmp + l_fn, tbi_ext);
+        if(stat(fnidx_tmp, &sbuf) == 0) {
+            *fnidx = fnidx_tmp;
+            return 1;
+        } else {
+            for (i = l_fn - 1; i > 0; --i)
+                if (fnidx_tmp[i] == '.') {
+                    strcpy(fnidx_tmp + i, tbi_ext);
+                    if(stat(fnidx_tmp, &sbuf) == 0) {
+                        *fnidx = fnidx_tmp;
+                        return 1;
+                    }
+                    break;
+                }
+        }
+    } else if (fmt == HTS_FMT_CRAI) { // Or .crai
+        strcpy(fnidx_tmp, fn_tmp); strcpy(fnidx_tmp + l_fn, crai_ext);
+        if(stat(fnidx_tmp, &sbuf) == 0) {
+            *fnidx = fnidx_tmp;
+            return 1;
+        } else {
+            for (i = l_fn - 1; i > 0; --i)
+                if (fnidx_tmp[i] == '.') {
+                    strcpy(fnidx_tmp + i, crai_ext);
+                    if(stat(fnidx_tmp, &sbuf) == 0) {
+                        *fnidx = fnidx_tmp;
+                        return 1;
+                    }
+                    break;
+                }
+        }
+    } else if (fmt == HTS_FMT_FAI) { // Or .fai
+        // Check .gzi if we have a .gz file
+        strcpy(fnidx_tmp, fn_tmp);
+        int gzi_ok = 1;
+        if ((l_fn > 3 && strcmp(fn_tmp+l_fn-3, ".gz") == 0) ||
+            (l_fn > 5 && strcmp(fn_tmp+l_fn-5, ".bgzf") == 0)) {
+            strcpy(fnidx_tmp + l_fn, gzi_ext);
+            gzi_ok = stat(fnidx_tmp, &sbuf)==0;
+        }
+
+        // Now check for .fai.  Occurs second as we're returning this
+        // in *fnidx irrespective of whether we did gzi check.
+        strcpy(fnidx_tmp + l_fn, fai_ext);
+        *fnidx = fnidx_tmp;
+        if (stat(fnidx_tmp, &sbuf) == 0)
+            return gzi_ok;
+        else
+            return 0;
+    }
+
+    free(fnidx_tmp);
+    return 0;
+}
+
+static char *idx_filename(const char *fn, const char *ext, int download) {
+    int ret, local_len;
+    char *fnidx;
+    const char *local_fn = NULL;
+    kstring_t buffer = KS_INITIALIZE;
+
+    // First try : append `ext` to `fn`
+    if (!(fnidx = haddextension(&buffer, fn, 0, ext))) {
+        free(buffer.s);
+        return NULL;
+    }
+    if ((ret = idx_test_and_fetch(fnidx, &local_fn, &local_len, download)) == -1) {
+        // Second try : replace suffix of `fn` with `ext`
+        if (!(fnidx = haddextension(&buffer, fn, 1, ext))) {
+            free(buffer.s);
+            return NULL;
+        }
+        ret = idx_test_and_fetch(fnidx, &local_fn, &local_len, download);
+    }
+
+    if (ret < 0) {
+        free(buffer.s);
+        return NULL;
+    }
+
+    memmove(fnidx, local_fn, local_len);
+    fnidx[local_len] = 0;
+    return fnidx;
+}
+
+char *hts_idx_getfn(const char *fn, const char *ext)
+{
+    return idx_filename(fn, ext, HTS_IDX_SAVE_REMOTE);
+}
+
+char *hts_idx_locatefn(const char *fn, const char *ext)
+{
+    return idx_filename(fn, ext, 0);
+}
+
+static hts_idx_t *idx_find_and_load(const char *fn, int fmt, int flags)
+{
+    char *fnidx = strstr(fn, HTS_IDX_DELIM);
+    hts_idx_t *idx;
+
+    if ( fnidx ) {
+        char *fn2 = strdup(fn);
+        if (!fn2) {
+            hts_log_error("%s", strerror(errno));
+            return NULL;
+        }
+        fn2[fnidx - fn] = '\0';
+        fnidx += strlen(HTS_IDX_DELIM);
+        idx = hts_idx_load3(fn2, fnidx, fmt, flags);
+        free(fn2);
+        return idx;
+    }
+
+    if (hts_idx_check_local(fn, fmt, &fnidx) == 0 && hisremote(fn)) {
+        if (flags & HTS_IDX_SAVE_REMOTE) {
+            fnidx = idx_filename(fn, ".csi", HTS_IDX_SAVE_REMOTE);
+            if (!fnidx) {
+                switch (fmt) {
+                case HTS_FMT_BAI: fnidx = idx_filename(fn, ".bai", HTS_IDX_SAVE_REMOTE); break;
+                case HTS_FMT_TBI: fnidx = idx_filename(fn, ".tbi", HTS_IDX_SAVE_REMOTE); break;
+                default: break;
+                }
+            }
+        } else {
+            fnidx = idx_filename(fn, ".csi", 0);
+            if (!fnidx) {
+                switch (fmt) {
+                case HTS_FMT_BAI: fnidx = idx_filename(fn, ".bai", 0); break;
+                case HTS_FMT_TBI: fnidx = idx_filename(fn, ".tbi", 0); break;
+                default: break;
+                }
+            }
+        }
+    }
+    if (!fnidx) {
+        if (!(flags & HTS_IDX_SILENT_FAIL))
+            hts_log_error("Could not retrieve index file for '%s'", fn);
+        return 0;
+    }
+
+    if (flags & HTS_IDX_SAVE_REMOTE)
+        idx = hts_idx_load3(fn, fnidx, fmt, flags);
+    else
+        idx = idx_read(fnidx);
+    free(fnidx);
+    return idx;
+}
+
+hts_idx_t *hts_idx_load(const char *fn, int fmt) {
+    return idx_find_and_load(fn, fmt, 1);
+}
+
+hts_idx_t *hts_idx_load2(const char *fn, const char *fnidx)
+{
+    return hts_idx_load3(fn, fnidx, 0, 0);
+}
+
+hts_idx_t *hts_idx_load3(const char *fn, const char *fnidx, int fmt, int flags)
+{
+    const char *local_fn = NULL;
+    char *local_fnidx = NULL;
+    int local_len;
+    if (!fnidx)
+        return idx_find_and_load(fn, fmt, flags);
+
+    // Check that the index file is up to date, the main file might have changed
+    struct stat stat_idx,stat_main;
+    int remote_fn = hisremote(fn), remote_fnidx = hisremote(fnidx);
+    if ( !remote_fn && !remote_fnidx
+         && !stat(fn, &stat_main) && !stat(fnidx, &stat_idx) )
+    {
+        if ( stat_idx.st_mtime < stat_main.st_mtime )
+            hts_log_warning("The index file is older than the data file: %s", fnidx);
+    }
+
+    if (remote_fnidx && (flags & HTS_IDX_SAVE_REMOTE))
+    {
+        int ret = idx_test_and_fetch(fnidx, &local_fn, &local_len, 1);
+        if (ret == 0) {
+            local_fnidx = strdup(local_fn);
+            if (local_fnidx) {
+                local_fnidx[local_len] = '\0';
+                fnidx = local_fnidx;
+            }
+        }
+    }
+
+    hts_idx_t *idx = idx_read(fnidx);
+    if (!idx && !(flags & HTS_IDX_SILENT_FAIL))
+        hts_log_error("Could not load local index file '%s'%s%s", fnidx,
+                      errno ? " : " : "", errno ? strerror(errno) : "");
+
+
+    free(local_fnidx);
+
+    return idx;
+}
+
+
+
+/**********************
+ ***     Memory     ***
+ **********************/
+
+/* For use with hts_expand macros *only* */
+HTSLIB_EXPORT
+size_t hts_realloc_or_die(size_t n, size_t m, size_t m_sz, size_t size,
+                          int clear, void **ptr, const char *func) {
+    /* If new_m and size are both below this limit, multiplying them
+       together can't overflow */
+    const size_t safe = (size_t) 1 << (sizeof(size_t) * 4);
+    void *new_ptr;
+    size_t bytes, new_m;
+
+    new_m = n;
+    kroundup_size_t(new_m);
+
+    bytes = size * new_m;
+
+    /* Check for overflow.  Both ensure that new_m will fit in m (we make the
+       pessimistic assumption that m is signed), and that bytes has not
+       wrapped around. */
+    if (new_m > (((size_t) 1 << (m_sz * 8 - 1)) - 1)
+        || ((size > safe || new_m > safe)
+            && bytes / new_m != size)) {
+        errno = ENOMEM;
+        goto die;
+    }
+
+    new_ptr = realloc(*ptr, bytes);
+    if (new_ptr == NULL) goto die;
+
+    if (clear) {
+        if (new_m > m) {
+            memset((char *) new_ptr + m * size, 0, (new_m - m) * size);
+        }
+    }
+
+    *ptr = new_ptr;
+
+    return new_m;
+
+ die:
+    hts_log_error("%s", strerror(errno));
+    exit(1);
+}
+
+/*
+ * Companion to hts_resize() macro that does the actual allocation.
+ *
+ * Somewhat complicated as hts_resize() needs to write the new allocated
+ * size back into *size_in_out, and the value pointed to may either be
+ * int32_t, uint32_t or size_t depending on which array is being resized.
+ * This is solved by making `size_in_out` a void pointer, getting the macro
+ * to pass in the size of the item pointed to (in `size_sz`) and then using
+ * an appropriate cast (based on the value of size_sz).  The function
+ * ensures that the maximum size will be storable in a signed type of
+ * the given size so storing to an int32_t should work correctly.
+ *
+ * Assumes that sizeof(uint32_t) and sizeof(int32_t) is 4,
+ * sizeof(uint64_t) and sizeof(int64_t) is 8 and sizeof(size_t) is
+ * either 4 or 8.  It also assumes casting from unsigned to signed will
+ * work as long as the top bit isn't set.
+ */
+
+int hts_resize_array_(size_t item_size, size_t num, size_t size_sz,
+                      void *size_in_out, void **ptr_in_out, int flags,
+                      const char *func) {
+    /* If new_size and item_size are both below this limit, multiplying them
+       together can't overflow */
+    const size_t safe = (size_t) 1 << (sizeof(size_t) * 4);
+    void *new_ptr;
+    size_t bytes, new_size;
+
+    new_size = num;
+    kroundup_size_t(new_size);
+    bytes = item_size * new_size;
+
+    /* Check for overflow.  Both ensure that alloc will fit in alloc_in_out (we
+       make the pessimistic assumption that *alloc_in_out is signed), and that
+       bytes has not wrapped around. */
+
+    if ((new_size > (((size_t) 1 << (size_sz * 8 - 1)) - 1))
+        || (((item_size > safe) || (new_size > safe))
+            && bytes / new_size != item_size)) {
+        hts_log(HTS_LOG_ERROR, func, "Memory allocation too large");
+        errno = ENOMEM;
+        return -1;
+    }
+
+    new_ptr = realloc(*ptr_in_out, bytes);
+    if (new_ptr == NULL) {
+        int save_errno = errno;
+        hts_log(HTS_LOG_ERROR, func, "%s", strerror(errno));
+        errno = save_errno;
+        return -1;
+    }
+
+    if (flags & HTS_RESIZE_CLEAR) {
+        size_t old_size;
+        switch (size_sz) {
+        case 4: old_size = *((uint32_t *) size_in_out); break;
+        case 8: old_size = *((uint64_t *) size_in_out); break;
+        default: abort();
+        }
+        if (new_size > old_size) {
+            memset((char *) new_ptr + old_size * item_size, 0,
+                   (new_size - old_size) * item_size);
+        }
+    }
+
+    switch (size_sz) {
+    case 4: *((uint32_t *) size_in_out) = new_size; break;
+    case 8: *((uint64_t *) size_in_out) = new_size; break;
+    default: abort();
+    }
+
+    *ptr_in_out = new_ptr;
+    return 0;
+}
+
+void hts_lib_shutdown(void)
+{
+    hfile_shutdown(1);
+}
+
+void hts_free(void *ptr) {
+    free(ptr);
+}
+
+void hts_set_log_level(enum htsLogLevel level)
+{
+    hts_verbose = level;
+}
+
+enum htsLogLevel hts_get_log_level(void)
+{
+    return hts_verbose;
+}
+
+static char get_severity_tag(enum htsLogLevel severity)
+{
+    switch (severity) {
+    case HTS_LOG_ERROR:
+        return 'E';
+    case HTS_LOG_WARNING:
+        return 'W';
+    case HTS_LOG_INFO:
+        return 'I';
+    case HTS_LOG_DEBUG:
+        return 'D';
+    case HTS_LOG_TRACE:
+        return 'T';
+    default:
+        break;
+    }
+
+    return '*';
+}
+
+void hts_log(enum htsLogLevel severity, const char *context, const char *format, ...)
+{
+    int save_errno = errno;
+    if (severity <= hts_verbose) {
+        va_list argptr;
+
+        fprintf(stderr, "[%c::%s] ", get_severity_tag(severity), context);
+
+        va_start(argptr, format);
+        vfprintf(stderr, format, argptr);
+        va_end(argptr);
+
+        fprintf(stderr, "\n");
+    }
+    errno = save_errno;
+}
diff --git a/ext/htslib/hts_expr.c b/ext/htslib/hts_expr.c
new file mode 100644
index 0000000..dfd15b1
--- /dev/null
+++ b/ext/htslib/hts_expr.c
@@ -0,0 +1,927 @@
+/*  hts_expr.c -- filter expression parsing and processing.
+
+    Copyright (C) 2020-2022, 2024 Genome Research Ltd.
+
+    Author: James Bonfield <jkb@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notices and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+// TODO:
+// - ?: operator for conditionals?
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+
+#include <stdio.h>
+#include <string.h>
+#include <ctype.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <float.h>
+#include <regex.h>
+#include <math.h>
+
+#include "htslib/hts_expr.h"
+#include "htslib/hts_log.h"
+#include "textutils_internal.h"
+
+// Could also cache hts_expr_val_t stack here for kstring reuse?
+#define MAX_REGEX 10
+struct hts_filter_t {
+    char *str;
+    int parsed;
+    int curr_regex, max_regex;
+    regex_t preg[MAX_REGEX];
+};
+
+/*
+ * This is designed to be mostly C like with mostly same the precedence rules,
+ * with the exception of bit operators (widely considered as a mistake in C).
+ * It's not full C (eg no bit-shifting), but good enough for our purposes.
+ *
+ * Supported syntax, in order of precedence:
+ *
+ * Grouping:      (, ),   eg "(1+2)*3"
+ * Values:        integers, floats, strings or variables
+ * Unary ops:     +, -, !, ~  eg -10 +10, !10 (0), ~5 (bitwise not)
+ * Math ops:      *, /, %  [TODO: add // for floor division?]
+ * Math ops:      +, -
+ * Bit-wise:      &, ^, |  [NB as 3 precedence levels, in that order]
+ * Conditionals:  >, >=, <, <=,
+ * Equality:      ==, !=, =~, !~
+ * Boolean:       &&, ||
+ */
+
+// Skip to start of term
+static char *ws(char *str) {
+    while (*str && (*str == ' ' || *str == '\t'))
+        str++;
+    return str;
+}
+
+static int expression(hts_filter_t *filt, void *data, hts_expr_sym_func *fn,
+                      char *str, char **end, hts_expr_val_t *res);
+
+/*
+ * Simple functions operating on strings only.
+ * length, min, max, avg.
+ *
+ * All return 0 on success,
+ *           -1 on failure
+ */
+static int expr_func_length(hts_expr_val_t *res) {
+    if (!res->is_str)
+        return -1;
+
+    res->is_str = 0;
+    res->d = res->s.l;
+    return 0;
+}
+
+static int expr_func_min(hts_expr_val_t *res) {
+    if (!res->is_str)
+        return -1;
+
+    size_t l = res->s.l;
+    int v = INT_MAX;
+    const uint8_t *x = (uint8_t *)res->s.s;
+    for (l = 0; l < res->s.l; l++)
+        if (v > x[l])
+            v = x[l];
+
+    res->is_str = 0;
+    res->d = v == INT_MAX ? NAN : v;
+
+    return 0;
+}
+
+static int expr_func_max(hts_expr_val_t *res) {
+    if (!res->is_str)
+        return -1;
+
+    size_t l = res->s.l;
+    int v = INT_MIN;
+    const uint8_t *x = (uint8_t *)res->s.s;
+    for (l = 0; l < res->s.l; l++)
+        if (v < x[l])
+            v = x[l];
+
+    res->is_str = 0;
+    res->d = v == INT_MIN ? NAN : v;
+
+    return 0;
+}
+
+static int expr_func_avg(hts_expr_val_t *res) {
+    if (!res->is_str)
+        return -1;
+
+    size_t l = res->s.l;
+    double v = 0;
+    const uint8_t *x = (uint8_t *)res->s.s;
+    for (l = 0; l < res->s.l; l++)
+        v += x[l];
+    if (l)
+        v /= l;
+
+    res->is_str = 0;
+    res->d = v;
+
+    return 0;
+}
+
+/*
+ * functions:  FUNC(expr).
+ * Note for simplicity of parsing, the "(" must immediately follow FUNC,
+ * so "FUNC (x)" is invalid.
+ */
+static int func_expr(hts_filter_t *filt, void *data, hts_expr_sym_func *fn,
+                     char *str, char **end, hts_expr_val_t *res) {
+    int func_ok = -1;
+    switch (*str) {
+    case 'a':
+        if (strncmp(str, "avg(", 4) == 0) {
+            if (expression(filt, data, fn, str+4, end, res)) return -1;
+            func_ok = expr_func_avg(res);
+        }
+        break;
+
+    case 'd':
+        if (strncmp(str, "default(", 8) == 0) {
+            if (expression(filt, data, fn, str+8, end, res)) return -1;
+            if (**end != ',')
+                return -1;
+            (*end)++;
+            hts_expr_val_t val = HTS_EXPR_VAL_INIT;
+            if (expression(filt, data, fn, ws(*end), end, &val)) return -1;
+            func_ok = 1;
+            if (!hts_expr_val_existsT(res)) {
+                kstring_t swap = res->s;
+                *res = val;
+                val.s = swap;
+                hts_expr_val_free(&val);
+            }
+        }
+        break;
+
+    case 'e':
+        if (strncmp(str, "exists(", 7) == 0) {
+            if (expression(filt, data, fn, str+7, end, res)) return -1;
+            func_ok = 1;
+            res->is_true = res->d = hts_expr_val_existsT(res);
+            res->is_str = 0;
+        } else if (strncmp(str, "exp(", 4) == 0) {
+            if (expression(filt, data, fn, str+4, end, res)) return -1;
+            func_ok = 1;
+            res->d = exp(res->d);
+            res->is_str = 0;
+            if (isnan(res->d))
+                hts_expr_val_undef(res);
+        }
+
+        break;
+
+    case 'l':
+        if (strncmp(str, "length(", 7) == 0) {
+            if (expression(filt, data, fn, str+7, end, res)) return -1;
+            func_ok = expr_func_length(res);
+        } else if (strncmp(str, "log(", 4) == 0) {
+            if (expression(filt, data, fn, str+4, end, res)) return -1;
+            func_ok = 1;
+            res->d = log(res->d);
+            res->is_str = 0;
+            if (isnan(res->d))
+                hts_expr_val_undef(res);
+        }
+        break;
+
+    case 'm':
+        if (strncmp(str, "min(", 4) == 0) {
+            if (expression(filt, data, fn, str+4, end, res)) return -1;
+            func_ok = expr_func_min(res);
+        } else if (strncmp(str, "max(", 4) == 0) {
+            if (expression(filt, data, fn, str+4, end, res)) return -1;
+            func_ok = expr_func_max(res);
+        }
+        break;
+
+    case 'p':
+        if (strncmp(str, "pow(", 4) == 0) {
+            if (expression(filt, data, fn, str+4, end, res)) return -1;
+            func_ok = 1;
+
+            if (**end != ',')
+                return -1;
+            (*end)++;
+            hts_expr_val_t val = HTS_EXPR_VAL_INIT;
+            if (expression(filt, data, fn, ws(*end), end, &val)) return -1;
+            if (!hts_expr_val_exists(res) || !hts_expr_val_exists(&val)) {
+                hts_expr_val_undef(res);
+            } else if (res->is_str || val.is_str) {
+                hts_expr_val_free(&val); // arith on strings
+                return -1;
+            } else {
+                func_ok = 1;
+                res->d = pow(res->d, val.d);
+                hts_expr_val_free(&val);
+                res->is_str = 0;
+            }
+
+            if (isnan(res->d))
+                hts_expr_val_undef(res);
+        }
+        break;
+
+    case 's':
+        if (strncmp(str, "sqrt(", 5) == 0) {
+            if (expression(filt, data, fn, str+5, end, res)) return -1;
+            func_ok = 1;
+            res->d = sqrt(res->d);
+            res->is_str = 0;
+            if (isnan(res->d))
+                hts_expr_val_undef(res);
+        }
+        break;
+    }
+
+    if (func_ok < 0)
+        return -1;
+
+    str = ws(*end);
+    if (*str != ')') {
+        fprintf(stderr, "Missing ')'\n");
+        return -1;
+    }
+    *end = str+1;
+
+    return 0;
+}
+
+/*
+ * simple_expr
+ *     : identifier
+ *     | constant
+ *     | string
+ *     | func_expr
+ *     | '(' expression ')'
+*/
+static int simple_expr(hts_filter_t *filt, void *data, hts_expr_sym_func *fn,
+                       char *str, char **end, hts_expr_val_t *res) {
+    // Main recursion step
+    str = ws(str);
+    if (*str == '(') {
+        if (expression(filt, data, fn, str+1, end, res)) return -1;
+        str = ws(*end);
+        if (*str != ')') {
+            fprintf(stderr, "Missing ')'\n");
+            return -1;
+        }
+        *end = str+1;
+
+        return 0;
+    }
+
+    // Otherwise a basic element.
+    int fail = 0;
+    double d = hts_str2dbl(str, end, &fail);
+    if (str != *end) {
+        res->is_str = 0;
+        res->d = d;
+    } else {
+        // Not valid floating point syntax.
+        // TODO: add function call names in here; len(), sqrt(), pow(), etc
+        if (*str == '"') {
+            res->is_str = 1;
+            char *e = str+1;
+            int backslash = 0;
+            while (*e && *e != '"') {
+                if (*e == '\\')
+                    backslash=1, e+=1+(e[1]!='\0');
+                else
+                    e++;
+            }
+
+            kputsn(str+1, e-(str+1), ks_clear(&res->s));
+            if (backslash) {
+                size_t i, j;
+                for (i = j = 0; i < res->s.l; i++) {
+                    res->s.s[j++] = res->s.s[i];
+                    if (res->s.s[i] == '\\') {
+                        switch (res->s.s[++i]) {
+                        case '"': res->s.s[j-1] = '"'; break;
+                        case '\\':res->s.s[j-1] = '\\'; break;
+                        case 't': res->s.s[j-1] = '\t'; break;
+                        case 'n': res->s.s[j-1] = '\n'; break;
+                        case 'r': res->s.s[j-1] = '\r'; break;
+                        default:  res->s.s[j++] = res->s.s[i];
+                        }
+                    }
+                }
+                res->s.s[j] = 0;
+                res->s.l = j;
+            }
+            if (*e != '"')
+                return -1;
+            *end = e+1;
+        } else if (fn) {
+            // Try lookup as variable, if not as function
+            if (fn(data, str, end, res) == 0)
+                return 0;
+            else
+                return func_expr(filt, data, fn, str, end, res);
+        } else {
+            return -1;
+        }
+    }
+
+    return 0;
+}
+
+/*
+ * unary_expr
+ *     : simple_expr
+ *     | '+' simple_expr
+ *     | '-' simple_expr
+ *     | '!' unary_expr // higher precedence
+ *     | '~' unary_expr // higher precedence
+ */
+static int unary_expr(hts_filter_t *filt, void *data, hts_expr_sym_func *fn,
+                      char *str, char **end, hts_expr_val_t *res) {
+    int err;
+    str = ws(str);
+    if (*str == '+' || *str == '-') {
+        err = simple_expr(filt, data, fn, str+1, end, res);
+        if (!hts_expr_val_exists(res)) {
+            hts_expr_val_undef(res);
+        } else {
+            err |= res->is_str;
+            if (*str == '-')
+                res->d = -res->d;
+            res->is_true = res->d != 0;
+        }
+    } else if (*str == '!') {
+        err = unary_expr(filt, data, fn, str+1, end, res);
+        if (res->is_true) {
+            // Any explicitly true value becomes false
+            res->d = res->is_true = 0;
+        } else if (!hts_expr_val_exists(res)) {
+            // We can also still negate undef values by toggling the
+            // is_true override value.
+            res->d = res->is_true = !res->is_true;
+        } else if (res->is_str) {
+            // !null = true, !"foo" = false, NOTE: !"" = false also
+            res->d = res->is_true = (res->s.s == NULL);
+        } else {
+            res->d = !(int64_t)res->d;
+            res->is_true = res->d != 0;
+        }
+        res->is_str = 0;
+    } else if (*str == '~') {
+        err = unary_expr(filt, data, fn, str+1, end, res);
+        if (!hts_expr_val_exists(res)) {
+            hts_expr_val_undef(res);
+        } else {
+            err |= res->is_str;
+            if (!hts_expr_val_exists(res)) {
+                hts_expr_val_undef(res);
+            } else {
+                res->d = ~(int64_t)res->d;
+                res->is_true = res->d != 0;
+            }
+        }
+    } else {
+        err = simple_expr(filt, data, fn, str, end, res);
+    }
+    return err ? -1 : 0;
+}
+
+
+/*
+ * mul_expr
+ *     : unary_expr (
+ *           '*' unary_expr
+ *         | '/' unary_expr
+ *         | '%' unary_expr
+ *       )*
+ */
+static int mul_expr(hts_filter_t *filt, void *data, hts_expr_sym_func *fn,
+                    char *str, char **end, hts_expr_val_t *res) {
+    if (unary_expr(filt, data, fn, str, end, res))
+        return -1;
+
+    str = *end;
+    hts_expr_val_t val = HTS_EXPR_VAL_INIT;
+    while (*str) {
+        str = ws(str);
+        if (*str == '*' || *str == '/' || *str == '%') {
+            if (unary_expr(filt, data, fn, str+1, end, &val)) return -1;
+            if (!hts_expr_val_exists(&val) || !hts_expr_val_exists(res)) {
+                hts_expr_val_undef(res);
+            } else if (val.is_str || res->is_str) {
+                hts_expr_val_free(&val);
+                return -1; // arith on strings
+            }
+        }
+
+        if (*str == '*')
+            res->d *= val.d;
+        else if (*str == '/')
+            res->d /= val.d;
+        else if (*str == '%') {
+            if (val.d)
+                res->d = (int64_t)res->d % (int64_t)val.d;
+            else
+                hts_expr_val_undef(res);
+        } else
+            break;
+
+        res->is_true = hts_expr_val_exists(res) && (res->d != 0);
+        str = *end;
+    }
+
+    hts_expr_val_free(&val);
+
+    return 0;
+}
+
+/*
+ * add_expr
+ *     : mul_expr (
+ *           '+' mul_expr
+ *         | '-' mul_expr
+ *       )*
+ */
+static int add_expr(hts_filter_t *filt, void *data, hts_expr_sym_func *fn,
+                    char *str, char **end, hts_expr_val_t *res) {
+    if (mul_expr(filt, data, fn, str, end, res))
+        return -1;
+
+    str = *end;
+    hts_expr_val_t val = HTS_EXPR_VAL_INIT;
+    while (*str) {
+        str = ws(str);
+        int undef = 0;
+        if (*str == '+' || *str == '-') {
+            if (mul_expr(filt, data, fn, str+1, end, &val)) return -1;
+            if (!hts_expr_val_exists(&val) || !hts_expr_val_exists(res)) {
+                undef = 1;
+            } else if (val.is_str || res->is_str) {
+                hts_expr_val_free(&val);
+                return -1; // arith on strings
+            }
+        }
+
+        if (*str == '+')
+            res->d += val.d;
+        else if (*str == '-')
+            res->d -= val.d;
+        else
+            break;
+
+        if (undef)
+            hts_expr_val_undef(res);
+        else
+            res->is_true = res->d != 0;
+
+        str = *end;
+    }
+
+    hts_expr_val_free(&val);
+
+    return 0;
+}
+
+/*
+ * bitand_expr
+ *     : add_expr
+ *     | bitand_expr '&' add_expr
+ */
+static int bitand_expr(hts_filter_t *filt, void *data, hts_expr_sym_func *fn,
+                       char *str, char **end, hts_expr_val_t *res) {
+    if (add_expr(filt, data, fn, str, end, res)) return -1;
+
+    hts_expr_val_t val = HTS_EXPR_VAL_INIT;
+    int undef = 0;
+    for (;;) {
+        str = ws(*end);
+        if (*str == '&' && str[1] != '&') {
+            if (add_expr(filt, data, fn, str+1, end, &val)) return -1;
+            if (!hts_expr_val_exists(&val) || !hts_expr_val_exists(res)) {
+                undef = 1;
+            } else if (res->is_str || val.is_str) {
+                hts_expr_val_free(&val);
+                return -1;
+            } else {
+                res->is_true =
+                    (res->d = ((int64_t)res->d & (int64_t)val.d)) != 0;
+            }
+        } else {
+            break;
+        }
+    }
+    hts_expr_val_free(&val);
+    if (undef)
+        hts_expr_val_undef(res);
+
+    return 0;
+}
+
+/*
+ * bitxor_expr
+ *     : bitand_expr
+ *     | bitxor_expr '^' bitand_expr
+ */
+static int bitxor_expr(hts_filter_t *filt, void *data, hts_expr_sym_func *fn,
+                       char *str, char **end, hts_expr_val_t *res) {
+    if (bitand_expr(filt, data, fn, str, end, res)) return -1;
+
+    hts_expr_val_t val = HTS_EXPR_VAL_INIT;
+    int undef = 0;
+    for (;;) {
+        str = ws(*end);
+        if (*str == '^') {
+            if (bitand_expr(filt, data, fn, str+1, end, &val)) return -1;
+            if (!hts_expr_val_exists(&val) || !hts_expr_val_exists(res)) {
+                undef = 1;
+            } else if (res->is_str || val.is_str) {
+                hts_expr_val_free(&val);
+                return -1;
+            } else {
+                res->is_true =
+                    (res->d = ((int64_t)res->d ^ (int64_t)val.d)) != 0;
+            }
+        } else {
+            break;
+        }
+    }
+    hts_expr_val_free(&val);
+    if (undef)
+        hts_expr_val_undef(res);
+
+    return 0;
+}
+
+/*
+ * bitor_expr
+ *     : bitxor_expr
+ *     | bitor_expr '|' bitxor_expr
+ */
+static int bitor_expr(hts_filter_t *filt, void *data, hts_expr_sym_func *fn,
+                      char *str, char **end, hts_expr_val_t *res) {
+    if (bitxor_expr(filt, data, fn, str, end, res)) return -1;
+
+    hts_expr_val_t val = HTS_EXPR_VAL_INIT;
+    int undef = 0;
+    for (;;) {
+        str = ws(*end);
+        if (*str == '|' && str[1] != '|') {
+            if (bitxor_expr(filt, data, fn, str+1, end, &val)) return -1;
+            if (!hts_expr_val_exists(&val) || !hts_expr_val_exists(res)) {
+                undef = 1;
+            } else if (res->is_str || val.is_str) {
+                hts_expr_val_free(&val);
+                return -1;
+            } else {
+                res->is_true =
+                    (res->d = ((int64_t)res->d | (int64_t)val.d)) != 0;
+            }
+        } else {
+            break;
+        }
+    }
+    hts_expr_val_free(&val);
+    if (undef)
+        hts_expr_val_undef(res);
+
+    return 0;
+}
+
+/*
+ * cmp_expr
+ *     : bitor_expr
+ *     | cmp_expr '<=' bitor_expr
+ *     | cmp_expr '<'  bitor_expr
+ *     | cmp_expr '>=' bitor_expr
+ *     | cmp_expr '>'  bitor_expr
+ */
+static int cmp_expr(hts_filter_t *filt, void *data, hts_expr_sym_func *fn,
+                    char *str, char **end, hts_expr_val_t *res) {
+    if (bitor_expr(filt, data, fn, str, end, res)) return -1;
+
+    str = ws(*end);
+    hts_expr_val_t val = HTS_EXPR_VAL_INIT;
+    int err = 0, cmp_done = 0;
+
+    if (*str == '>' && str[1] == '=') {
+        cmp_done = 1;
+        err = cmp_expr(filt, data, fn, str+2, end, &val);
+        if (!hts_expr_val_exists(res) || !hts_expr_val_exists(&val)) {
+            hts_expr_val_undef(res);
+        } else {
+            res->is_true=res->d
+                = res->is_str && res->s.s && val.is_str && val.s.s
+                ? strcmp(res->s.s, val.s.s) >= 0
+                : !res->is_str && !val.is_str && res->d >= val.d;
+            res->is_str = 0;
+        }
+    } else if (*str == '>') {
+        cmp_done = 1;
+        err = cmp_expr(filt, data, fn, str+1, end, &val);
+        if (!hts_expr_val_exists(res) || !hts_expr_val_exists(&val)) {
+            hts_expr_val_undef(res);
+        } else {
+            res->is_true=res->d
+                = res->is_str && res->s.s && val.is_str && val.s.s
+                ? strcmp(res->s.s, val.s.s) > 0
+                : !res->is_str && !val.is_str && res->d > val.d;
+            res->is_str = 0;
+        }
+    } else if (*str == '<' && str[1] == '=') {
+        cmp_done = 1;
+        err = cmp_expr(filt, data, fn, str+2, end, &val);
+        if (!hts_expr_val_exists(res) || !hts_expr_val_exists(&val)) {
+            hts_expr_val_undef(res);
+        } else {
+            res->is_true=res->d
+                = res->is_str && res->s.s && val.is_str && val.s.s
+                ? strcmp(res->s.s, val.s.s) <= 0
+                : !res->is_str && !val.is_str && res->d <= val.d;
+            res->is_str = 0;
+        }
+    } else if (*str == '<') {
+        cmp_done = 1;
+        err = cmp_expr(filt, data, fn, str+1, end, &val);
+        if (!hts_expr_val_exists(res) || !hts_expr_val_exists(&val)) {
+            hts_expr_val_undef(res);
+        } else {
+            res->is_true=res->d
+                = res->is_str && res->s.s && val.is_str && val.s.s
+                ? strcmp(res->s.s, val.s.s) < 0
+                : !res->is_str && !val.is_str && res->d < val.d;
+            res->is_str = 0;
+        }
+    }
+
+    if (cmp_done && (!hts_expr_val_exists(&val) || !hts_expr_val_exists(res)))
+        hts_expr_val_undef(res);
+    hts_expr_val_free(&val);
+
+    return err ? -1 : 0;
+}
+
+/*
+ * eq_expr
+ *     : cmp_expr
+ *     | eq_expr '==' cmp_expr
+ *     | eq_expr '!=' cmp_expr
+ *     | eq_expr '=~' cmp_expr
+ *     | eq_expr '!~' cmp_expr
+ */
+static int eq_expr(hts_filter_t *filt, void *data, hts_expr_sym_func *fn,
+                   char *str, char **end, hts_expr_val_t *res) {
+    if (cmp_expr(filt, data, fn, str, end, res)) return -1;
+
+    str = ws(*end);
+
+    int err = 0, eq_done = 0;
+    hts_expr_val_t val = HTS_EXPR_VAL_INIT;
+
+    // numeric vs numeric comparison is as expected
+    // string vs string comparison is as expected
+    // numeric vs string is false
+    if (str[0] == '=' && str[1] == '=') {
+        eq_done = 1;
+        if ((err = eq_expr(filt, data, fn, str+2, end, &val))) {
+            res->is_true = res->d = 0;
+        } else {
+            if (!hts_expr_val_exists(res) || !hts_expr_val_exists(&val)) {
+                hts_expr_val_undef(res);
+            } else {
+                res->is_true = res->d = res->is_str
+                    ? (res->s.s && val.s.s ?strcmp(res->s.s, val.s.s)==0 :0)
+                    : !res->is_str && !val.is_str && res->d == val.d;
+            }
+        }
+        res->is_str = 0;
+
+    } else if (str[0] == '!' && str[1] == '=') {
+        eq_done = 1;
+        if ((err = eq_expr(filt, data, fn, str+2, end, &val))) {
+            res->is_true = res->d = 0;
+        } else {
+            if (!hts_expr_val_exists(res) || !hts_expr_val_exists(&val)) {
+                hts_expr_val_undef(res);
+            } else {
+                res->is_true = res->d = res->is_str
+                    ? (res->s.s && val.s.s ?strcmp(res->s.s, val.s.s) != 0 :1)
+                    : res->is_str != val.is_str || res->d != val.d;
+            }
+        }
+        res->is_str = 0;
+
+    } else if ((str[0] == '=' && str[1] == '~') ||
+               (str[0] == '!' && str[1] == '~')) {
+        eq_done = 1;
+        err = eq_expr(filt, data, fn, str+2, end, &val);
+        if (!val.is_str || !res->is_str) {
+            hts_expr_val_free(&val);
+            return -1;
+        }
+        if (val.s.s && res->s.s && val.is_true >= 0 && res->is_true >= 0) {
+            regex_t preg_, *preg;
+            if (filt->curr_regex >= filt->max_regex) {
+                // Compile regex if not seen before
+                if (filt->curr_regex >= MAX_REGEX) {
+                    preg = &preg_;
+                } else {
+                    preg = &filt->preg[filt->curr_regex];
+                    filt->max_regex++;
+                }
+
+                int ec = regcomp(preg, val.s.s, REG_EXTENDED | REG_NOSUB);
+                if (ec != 0) {
+                    char errbuf[1024];
+                    regerror(ec, preg, errbuf, 1024);
+                    fprintf(stderr, "Failed regex: %.1024s\n", errbuf);
+                    hts_expr_val_free(&val);
+                    return -1;
+                }
+            } else {
+                preg = &filt->preg[filt->curr_regex];
+            }
+            res->is_true = res->d = regexec(preg, res->s.s, 0, NULL, 0) == 0
+                ? *str == '='  // matcn
+                : *str == '!'; // no-match
+            if (preg == &preg_)
+                regfree(preg);
+
+            filt->curr_regex++;
+        } else {
+            // nul regexp or input is considered false
+            res->is_true = 0;
+        }
+        res->is_str = 0;
+    }
+
+    if (eq_done && ((!hts_expr_val_exists(&val)) || !hts_expr_val_exists(res)))
+        hts_expr_val_undef(res);
+    hts_expr_val_free(&val);
+
+    return err ? -1 : 0;
+}
+
+/*
+ * and_expr
+ *     : eq_expr
+ *     | and_expr 'and' eq_expr
+ *     | and_expr 'or'  eq_expr
+ */
+static int and_expr(hts_filter_t *filt, void *data, hts_expr_sym_func *fn,
+                    char *str, char **end, hts_expr_val_t *res) {
+    if (eq_expr(filt, data, fn, str, end, res)) return -1;
+
+    for (;;) {
+        hts_expr_val_t val = HTS_EXPR_VAL_INIT;
+        str = ws(*end);
+        if (str[0] == '&' && str[1] == '&') {
+            if (eq_expr(filt, data, fn, str+2, end, &val)) return -1;
+            if (!hts_expr_val_existsT(res) || !hts_expr_val_existsT(&val)) {
+                hts_expr_val_undef(res);
+                res->d = 0;
+            } else {
+                res->is_true = res->d =
+                    (res->is_true || (res->is_str && res->s.s) || res->d) &&
+                    (val.is_true  || (val.is_str && val.s.s) || val.d);
+                res->is_str = 0;
+            }
+        } else if (str[0] == '|' && str[1] == '|') {
+            if (eq_expr(filt, data, fn, str+2, end, &val)) return -1;
+            if (!hts_expr_val_existsT(res) && !hts_expr_val_existsT(&val)) {
+                // neither defined
+                hts_expr_val_undef(res);
+                res->d = 0;
+            } else if (!hts_expr_val_existsT(res) &&
+                       !(val.is_true  || (val.is_str  && val.s.s ) || val.d)) {
+                // LHS undef and RHS false
+                hts_expr_val_undef(res);
+                res->d = 0;
+            } else if (!hts_expr_val_existsT(&val) &&
+                       !(res->is_true || (res->is_str && res->s.s) || res->d)){
+                // RHS undef and LHS false
+                hts_expr_val_undef(res);
+                res->d = 0;
+            } else {
+                res->is_true = res->d =
+                    res->is_true || (res->is_str && res->s.s) || res->d ||
+                    val.is_true  || (val.is_str  && val.s.s ) || val.d;
+                res->is_str = 0;
+            }
+        } else {
+            break;
+        }
+        hts_expr_val_free(&val);
+    }
+
+    return 0;
+}
+
+static int expression(hts_filter_t *filt, void *data, hts_expr_sym_func *fn,
+                      char *str, char **end, hts_expr_val_t *res) {
+    return and_expr(filt, data, fn, str, end, res);
+}
+
+hts_filter_t *hts_filter_init(const char *str) {
+    hts_filter_t *f = calloc(1, sizeof(*f));
+    if (!f) return NULL;
+
+    // Oversize to permit faster comparisons with memcmp over strcmp
+    size_t len = strlen(str)+100;
+    if (!(f->str = malloc(len))) {
+        free(f);
+        return NULL;
+    }
+    strcpy(f->str, str);
+    return f;
+}
+
+void hts_filter_free(hts_filter_t *filt) {
+    if (!filt)
+        return;
+
+    int i;
+    for (i = 0; i < filt->max_regex; i++)
+        regfree(&filt->preg[i]);
+
+    free(filt->str);
+    free(filt);
+}
+
+static int hts_filter_eval_(hts_filter_t *filt,
+                            void *data, hts_expr_sym_func *fn,
+                            hts_expr_val_t *res) {
+    char *end = NULL;
+
+    filt->curr_regex = 0;
+    if (expression(filt, data, fn, filt->str, &end, res))
+        return -1;
+
+    if (end && *ws(end)) {
+        fprintf(stderr, "Unable to parse expression at %s\n", filt->str);
+        return -1;
+    }
+
+    // Strings evaluate to true.  An empty string is also true, but an
+    // absent (null) string is false, unless overriden by is_true.  An
+    // empty string has kstring length of zero, but a pointer as it's
+    // nul-terminated.
+    if (res->is_str) {
+        res->is_true |= res->s.s != NULL;
+        res->d = res->is_true;
+    } else if (hts_expr_val_exists(res)) {
+        res->is_true |= res->d != 0;
+    }
+
+    return 0;
+}
+
+int hts_filter_eval(hts_filter_t *filt,
+                    void *data, hts_expr_sym_func *fn,
+                    hts_expr_val_t *res) {
+    if (res->s.l != 0 || res->s.m != 0 || res->s.s != NULL) {
+        // As *res is cleared below, it's not safe to call this function
+        // with res->s.s set, as memory would be leaked.  It's also not
+        // possible to know is res was initialised correctly, so in
+        // either case we fail.
+        hts_log_error("Results structure must be cleared before calling this function");
+        return -1;
+    }
+
+    memset(res, 0, sizeof(*res));
+
+    return hts_filter_eval_(filt, data, fn, res);
+}
+
+int hts_filter_eval2(hts_filter_t *filt,
+                     void *data, hts_expr_sym_func *fn,
+                     hts_expr_val_t *res) {
+    ks_free(&res->s);
+    memset(res, 0, sizeof(*res));
+
+    return hts_filter_eval_(filt, data, fn, res);
+}
diff --git a/ext/htslib/hts_internal.h b/ext/htslib/hts_internal.h
new file mode 100644
index 0000000..52f29e6
--- /dev/null
+++ b/ext/htslib/hts_internal.h
@@ -0,0 +1,149 @@
+/*  hts_internal.h -- internal functions; not part of the public API.
+
+    Copyright (C) 2015-2016, 2018-2020 Genome Research Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#ifndef HTSLIB_HTS_INTERNAL_H
+#define HTSLIB_HTS_INTERNAL_H
+
+#include <stddef.h>
+#include <ctype.h>
+
+#include "htslib/hts.h"
+#include "textutils_internal.h"
+
+#define HTS_MAX_EXT_LEN 9
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct hFILE;
+
+struct hts_json_token {
+    char type;    ///< Token type
+    char *str;    ///< Value as a C string (filled in for all token types)
+    // TODO Add other fields to fill in for particular data types, e.g.
+    // int inum;
+    // float fnum;
+};
+
+struct cram_fd;
+
+/*
+ * Check the existence of a local index file using part of the alignment file name.
+ * The order is alignment.bam.csi, alignment.csi, alignment.bam.bai, alignment.bai
+ * @param fn    - pointer to the file name
+ * @param fnidx - pointer to the index file name placeholder
+ * @return        1 for success, 0 for failure
+ */
+int hts_idx_check_local(const char *fn, int fmt, char **fnidx);
+
+// Retrieve the name of the index file and also download it, if it is remote
+char *hts_idx_getfn(const char *fn, const char *ext);
+
+// Retrieve the name of the index file, but do not download it, if it is remote
+char *hts_idx_locatefn(const char *fn, const char *ext);
+
+// Used for on-the-fly indexing.  See the comments in hts.c.
+void hts_idx_amend_last(hts_idx_t *idx, uint64_t offset);
+
+int hts_idx_fmt(hts_idx_t *idx);
+
+// Internal interface to save on-the-fly indexes.  The index file handle
+// is kept open so hts_close() can close if after writing out the EOF
+// block for its own file.
+int hts_idx_save_but_not_close(hts_idx_t *idx, const char *fnidx, int fmt);
+
+// Construct a unique filename based on fname and open it.
+struct hFILE *hts_open_tmpfile(const char *fname, const char *mode, kstring_t *tmpname);
+
+// Check that index is capable of storing items in range beg..end
+int hts_idx_check_range(hts_idx_t *idx, int tid, hts_pos_t beg, hts_pos_t end);
+
+// The CRAM implementation stores the loaded index within the cram_fd rather
+// than separately as is done elsewhere in htslib.  So if p is a pointer to
+// an hts_idx_t with p->fmt == HTS_FMT_CRAI, then it actually points to an
+// hts_cram_idx_t and should be cast accordingly.
+typedef struct hts_cram_idx_t {
+    int fmt;
+    struct cram_fd *cram;
+} hts_cram_idx_t;
+
+// Determine whether the string's contents appear to be UTF-16-encoded text.
+// Returns 1 if they are, 2 if there is also a BOM, or 0 otherwise.
+int hts_is_utf16_text(const kstring_t *str);
+
+// Entry point to hFILE_multipart backend.
+struct hFILE *hopen_htsget_redirect(struct hFILE *hfile, const char *mode);
+
+struct hts_path_itr {
+    kstring_t path, entry;
+    void *dirv;  // DIR * privately
+    const char *pathdir, *prefix, *suffix;
+    size_t prefix_len, suffix_len, entry_dir_l;
+};
+
+void hts_path_itr_setup(struct hts_path_itr *itr, const char *path,
+    const char *builtin_path, const char *prefix, size_t prefix_len,
+    const char *suffix, size_t suffix_len);
+
+const char *hts_path_itr_next(struct hts_path_itr *itr);
+
+typedef void plugin_void_func(void);
+plugin_void_func *load_plugin(void **pluginp, const char *filename, const char *symbol);
+void *plugin_sym(void *plugin, const char *name, const char **errmsg);
+plugin_void_func *plugin_func(void *plugin, const char *name, const char **errmsg);
+void close_plugin(void *plugin);
+const char *hts_plugin_path(void);
+
+/*
+ * Buffers up arguments to hts_idx_push for later use, once we've written all bar
+ * this block.  This is necessary when multiple blocks are in flight (threading).
+ *
+ * Returns 0 on success,
+ *        -1 on failure
+ */
+int bgzf_idx_push(BGZF *fp, hts_idx_t *hidx, int tid, hts_pos_t beg, hts_pos_t end, uint64_t offset, int is_mapped);
+
+static inline int find_file_extension(const char *fn, char ext_out[static HTS_MAX_EXT_LEN])
+{
+    const char *delim = fn ? strstr(fn, HTS_IDX_DELIM) : NULL, *ext;
+    if (!fn) return -1;
+    if (!delim) delim = fn + strlen(fn);
+    for (ext = delim; ext > fn && *ext != '.' && *ext != '/'; --ext) {}
+    if (*ext == '.' &&
+        ((delim - ext == 3 && ext[1] == 'g' && ext[2] == 'z') || // permit .sam.gz as a valid file extension
+        (delim - ext == 4 && ext[1] == 'b' && ext[2] == 'g' && ext[3] == 'z'))) // permit .vcf.bgz as a valid file extension
+    {
+        for (ext--; ext > fn && *ext != '.' && *ext != '/'; --ext) {}
+    }
+    if (*ext != '.' || delim - ext > HTS_MAX_EXT_LEN || delim - ext < 3)
+        return -1;
+    memcpy(ext_out, ext + 1, delim - ext - 1);
+    ext_out[delim - ext - 1] = '\0';
+    return 0;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/ext/htslib/hts_os.c b/ext/htslib/hts_os.c
new file mode 100644
index 0000000..b391a41
--- /dev/null
+++ b/ext/htslib/hts_os.c
@@ -0,0 +1,59 @@
+/// @file hts_os.c
+/// Operating System specific tweaks, for compatibility with POSIX.
+/*
+   Copyright (C) 2017, 2019-2020 Genome Research Ltd.
+
+    Author: James Bonfield <jkb@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+#include "htslib/hts_defs.h"
+
+// Windows (maybe more) lack a drand48 implementation.
+#ifndef HAVE_DRAND48
+#include "os/rand.c"
+#else
+#include <stdlib.h>
+HTSLIB_EXPORT
+void hts_srand48(long seed)
+{
+#ifdef HAVE_SRAND48_DETERMINISTIC
+    srand48_deterministic(seed);
+#else
+    srand48(seed);
+#endif
+}
+
+HTSLIB_EXPORT
+double hts_erand48(unsigned short xseed[3]) { return erand48(xseed); }
+
+HTSLIB_EXPORT
+double hts_drand48(void) { return drand48(); }
+
+HTSLIB_EXPORT
+long hts_lrand48(void) { return lrand48(); }
+#endif
+
+// // On Windows when using the MSYS or Cygwin terminals, isatty fails
+// #ifdef _WIN32
+// #define USE_FILEEXTD
+// #include "os/iscygpty.c"
+// #endif
diff --git a/ext/htslib/hts_probe_cc.sh b/ext/htslib/hts_probe_cc.sh
new file mode 100755
index 0000000..c9fc0a8
--- /dev/null
+++ b/ext/htslib/hts_probe_cc.sh
@@ -0,0 +1,143 @@
+#!/bin/sh
+
+# Check compiler options for non-configure builds and create Makefile fragment
+#
+#    Copyright (C) 2022-2024 Genome Research Ltd.
+#
+#    Author: Rob Davies <rmd@sanger.ac.uk>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+# Arguments are:
+# 1. C compiler command
+# 2. Initial CFLAGS
+# 3. LDFLAGS
+
+CC=$1
+CFLAGS=$2
+LDFLAGS=$3
+
+# Try running the compiler.  Uses the same contest.* names as
+# configure for temporary files.
+run_compiler ()
+{
+    $CC $CFLAGS $1 $LDFLAGS -o conftest conftest.c 2> conftest.err
+    retval=$?
+    rm -f conftest.err conftest
+    return $retval
+}
+
+# Run a test.  $1 is the flag to try, $2 is the Makefile variable to set
+# with the flag probe result, $3 is a Makefile variable which will be
+# set to 1 if the code was built successfully.  The code to test should
+# be passed in via fd 0.
+# First try compiling conftest.c without the flag.  If that fails, try
+# again with it to see if the flag is needed.
+run_test ()
+{
+    if [ $have_cpuid -ne 1 ] ; then
+        # Only test for and build SSE / AVX code if cpuid works as
+        # otherwise it won't be executed, even if present
+        echo "$3 ="
+        return
+    fi
+    rm -f conftest conftest.err conftest.c
+    cat - > conftest.c
+    if run_compiler ; then
+        echo "$2 ="
+        echo "$3 = 1"
+    elif run_compiler "$1" ; then
+        echo "$2 = $1"
+        echo "$3 = 1"
+    else
+        echo "$3 ="
+    fi
+}
+
+echo "# Compiler probe results, generated by $0"
+
+# Check for cpuid
+rm -f conftest conftest.err conftest.c
+cat > conftest.c <<'EOF'
+#include <cpuid.h>
+#include <stddef.h>
+int main(int argc, char **argv) {
+    unsigned int a, b, c, d;
+    int level = __get_cpuid_max(0, NULL);
+    if (level > 0)
+        __cpuid_count(1, 0, a, b, c, d);
+    return 0;
+}
+EOF
+if run_compiler ; then
+    echo "HTS_HAVE_CPUID = 1"
+    have_cpuid=1
+else
+    echo "HTS_HAVE_CPUID ="
+    have_cpuid=0
+fi
+
+# Check for sse4.1 etc. support
+run_test "-msse4.1 -mpopcnt -mssse3" HTS_CFLAGS_SSE4 HTS_BUILD_SSE4 <<'EOF'
+#ifdef __x86_64__
+#include "x86intrin.h"
+int main(int argc, char **argv) {
+    __m128i a = _mm_set_epi32(1, 2, 3, 4), b = _mm_set_epi32(4, 3, 2, 1);
+    __m128i c = _mm_shuffle_epi8(_mm_max_epu32(a, b), b);
+    return _mm_popcnt_u32(*((char *) &c));
+}
+#else
+int main(int argc, char **argv) { return 0; }
+#endif
+EOF
+
+# Check for avx2
+
+run_test "-mavx2 -mpopcnt" HTS_CFLAGS_AVX2 HTS_BUILD_AVX2 <<'EOF'
+#ifdef __x86_64__
+#include "x86intrin.h"
+int main(int argc, char **argv) {
+    __m256i a = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8);
+    __m256i b = _mm256_add_epi32(a, a);
+    long long c = _mm256_extract_epi64(b, 0);
+    return _mm_popcnt_u32((int) c);
+}
+#else
+int main(int argc, char **argv) { return 0; }
+#endif
+EOF
+
+# Check for avx512
+
+run_test "-mavx512f -mpopcnt" HTS_CFLAGS_AVX512 HTS_BUILD_AVX512 <<'EOF'
+#ifdef __x86_64__
+#include "x86intrin.h"
+int main(int argc, char **argv) {
+    __m512i a = _mm512_set1_epi32(1);
+    __m512i b = _mm512_add_epi32(a, a);
+    __m256i c = _mm512_castsi512_si256(b);
+    __m256i d = _mm512_extracti64x4_epi64(a, 1);
+    return _mm_popcnt_u32(*((char *) &c)) + (*(char *) &d);
+}
+#else
+int main(int argc, char **argv) { return 0; }
+#endif
+EOF
+
+rm -f conftest.c
diff --git a/ext/htslib/hts_time_funcs.h b/ext/htslib/hts_time_funcs.h
new file mode 100644
index 0000000..2a05084
--- /dev/null
+++ b/ext/htslib/hts_time_funcs.h
@@ -0,0 +1,170 @@
+/*  hts_time_funcs.h -- Implementations of non-standard time functions
+
+    Copyright (C) 2022 Genome Research Ltd.
+
+    Author: Rob Davies <rmd@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+/*
+  This mainly exists because timegm() is not a standard function, and so
+  Cannot be used in portable code.  Unfortunately the standard one (mktime)
+  always takes the local timezone into accout so doing a UTC conversion
+  with it involves changing the TZ environment variable, which is rather
+  messy and not likely to go well with threaded code.
+
+  The code here is a much simplified version of the BSD timegm() implementation.
+  It currently rejects dates before 1970, avoiding problems with -ve time_t.
+  It also works strictly in UTC, so doesn't have to worry about tm_isdst
+  which makes the calculation much easier.
+
+  Some of this is derived from BSD sources, for example
+  https://github.com/NetBSD/src/blob/trunk/lib/libc/time/localtime.c
+  which state:
+
+  ** This file is in the public domain, so clarified as of
+  ** 1996-06-05 by Arthur David Olson.
+
+  Non-derived code is copyright as above.
+*/
+
+#include <stdint.h>
+#include <limits.h>
+#include <errno.h>
+#include <time.h>
+
+static inline int hts_time_normalise(int *tens, int *units, int base) {
+    if (*units < 0 || *units >= base) {
+        int delta = *units >= 0 ? *units / base : (-1 - (-1 - *units) / base);
+        int64_t tmp = (int64_t) (*tens) + delta;
+        if (tmp < INT_MIN || tmp > INT_MAX) return 1;
+        *tens = tmp;
+        *units -= delta * base;
+    }
+    return 0;
+}
+
+static inline int hts_year_is_leap(int64_t year) {
+    return ((year % 4 == 0) && (year % 100 != 0)) || (year % 400 == 0);
+}
+
+// Number of leap years to start of year
+// Only works for year >= 1.
+static inline int64_t hts_leaps_to_year_start(int64_t year) {
+    --year;
+    return year / 4 - year / 100 + year / 400;
+}
+
+static inline int hts_time_normalise_tm(struct tm *t)
+{
+    const int days_per_mon[2][12] = {
+        { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 },
+        { 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }
+    };
+    const int year_days[2] = { 365, 366 };
+    int overflow = 0;
+    int64_t year;
+
+    if (t->tm_sec > 62) {
+        overflow |= hts_time_normalise(&t->tm_min, &t->tm_sec, 60);
+    }
+    overflow |= hts_time_normalise(&t->tm_hour, &t->tm_min,  60);
+    overflow |= hts_time_normalise(&t->tm_mday, &t->tm_hour, 24);
+    overflow |= hts_time_normalise(&t->tm_year, &t->tm_mon,  12);
+    if (overflow)
+        return 1;
+
+    year = (int64_t) t->tm_year + 1900LL;
+    while (t->tm_mday <= 0) {
+        --year;
+        t->tm_mday += year_days[hts_year_is_leap(year + (1 < t->tm_mon))];
+    }
+    while (t->tm_mday > 366) {
+        t->tm_mday -= year_days[hts_year_is_leap(year + (1 < t->tm_mon))];
+        ++year;
+    }
+    for (;;) {
+        int mdays = days_per_mon[hts_year_is_leap(year)][t->tm_mon];
+        if (t->tm_mday <= mdays)
+            break;
+        t->tm_mday -= mdays;
+        t->tm_mon++;
+        if (t->tm_mon >= 12) {
+            year++;
+            t->tm_mon = 0;
+        }
+    }
+    year -= 1900;
+    if (year != t->tm_year) {
+        if (year < INT_MIN || year > INT_MAX)
+            return 1;
+        t->tm_year = year;
+    }
+    return 0;
+}
+
+/**
+ *  Convert broken-down time to an equivalent time_t value
+ *  @param target  Target broken-down time structure
+ *  @return Equivalent time_t value on success; -1 on failure
+ *
+ *  This function first normalises the time in @p target so that the
+ *  structure members are in the valid range.  It then calculates the
+ *  number of seconds (ignoring leap seconds) between midnight Jan 1st 1970
+ *  and the target date.
+ *
+ *  If @p target is outside the range that can be represented in a time_t,
+ *  or tm_year is less than 70 (which would return a negative value) then
+ *  it returns -1 and sets errno to EOVERFLOW.
+ */
+
+static inline time_t hts_time_gm(struct tm *target)
+{
+    int month_start[2][12] = {
+        { 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334 },
+        { 0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335 }
+    };
+    int years_from_epoch, leaps, days;
+    int64_t secs;
+
+    if (hts_time_normalise_tm(target) != 0)
+        goto overflow;
+
+    if (target->tm_year < 70)
+        goto overflow;
+
+    years_from_epoch = target->tm_year - 70;
+    leaps = (hts_leaps_to_year_start(target->tm_year + 1900)
+        - hts_leaps_to_year_start(1970));
+    days = ((365 * (years_from_epoch - leaps) + 366 * leaps)
+        + month_start[hts_year_is_leap(target->tm_year + 1900)][target->tm_mon]
+        + target->tm_mday - 1);
+    secs = ((int64_t) days * 86400LL
+        + target->tm_hour * 3600
+        + target->tm_min * 60
+        + target->tm_sec);
+    if (sizeof(time_t) < 8 && secs > INT_MAX)
+        goto overflow;
+
+    return (time_t) secs;
+
+ overflow:
+    errno = EOVERFLOW;
+    return (time_t) -1;
+}
diff --git a/ext/htslib/htscodecs/BENCHMARKS.md b/ext/htslib/htscodecs/BENCHMARKS.md
new file mode 100644
index 0000000..18d5765
--- /dev/null
+++ b/ext/htslib/htscodecs/BENCHMARKS.md
@@ -0,0 +1,146 @@
+-c<XXYY> option species decode method XX encode method YY where
+
+    00 is scalar
+    01 is SSE4
+    02 is AVX2
+    04 is AVX512
+
+Input data is 10MB worth of NovaSeq quality values; approx 100k
+records.  Performance is data specific, so these figures are purely a
+snapshot and not indicative of all data types.  The test machine
+reports as:
+
+    Intel(R) Xeon(R) Gold 6142 CPU @ 2.60GHz
+
+The -o<order> field is a bit field where.
+
+    0/1   Order 0 or 1
+    4     32-way variant (permits SIMD)
+    64    RLE
+    128   Bit packing (4 novaseq quals to a byte)
+
+Hence -o133-c0202 is pack 4 quals to a bit and order-1 encode with AVX2
+32-way encode/decode.
+
+
+      r4x8-o0           10000000 uncomp,  665848 comp  395.3 enc MB/s  718.3 dec MB/s
+     r4x16-o0           10000000 uncomp,  665415 comp  400.3 enc MB/s  716.1 dec MB/s
+     arith-o0           10000000 uncomp,  660701 comp  105.0 enc MB/s   86.3 dec MB/s
+    
+      r4x8-o1           10000000 uncomp,  615304 comp  274.8 enc MB/s  385.1 dec MB/s
+     r4x16-o1           10000000 uncomp,  616134 comp  289.6 enc MB/s  536.6 dec MB/s
+     arith-o1           10000000 uncomp,  613736 comp   75.4 enc MB/s   87.1 dec MB/s
+    
+     r4x16-o64          10000000 uncomp,  712335 comp  382.0 enc MB/s  749.2 dec MB/s
+     arith-o64          10000000 uncomp,  744000 comp  153.1 enc MB/s  112.2 dec MB/s
+    
+     r4x16-o65          10000000 uncomp,  591457 comp  360.6 enc MB/s  705.8 dec MB/s
+     arith-o65          10000000 uncomp,  585233 comp  161.3 enc MB/s  117.7 dec MB/s
+    
+     r4x16-o128         10000000 uncomp,  615915 comp  780.2 enc MB/s 2092.5 dec MB/s
+     arith-o128         10000000 uncomp,  609977 comp  257.4 enc MB/s  219.0 dec MB/s
+    
+     r4x16-o129         10000000 uncomp,  553081 comp  645.1 enc MB/s 1394.1 dec MB/s
+     arith-o129         10000000 uncomp,  550377 comp  165.1 enc MB/s  180.7 dec MB/s
+    
+     r4x16-o192         10000000 uncomp,  621771 comp  513.1 enc MB/s 1003.0 dec MB/s
+     arith-o192         10000000 uncomp,  621415 comp  217.9 enc MB/s  180.8 dec MB/s
+    
+     r4x16-o193         10000000 uncomp,  550325 comp  474.1 enc MB/s  920.6 dec MB/s
+     arith-o193         10000000 uncomp,  543687 comp  195.7 enc MB/s  158.0 dec MB/s
+    
+    r32x16-o4-c0000     10000000 uncomp,  665501 comp  399.0 enc MB/s  613.9 dec MB/s
+    r32x16-o4-c0101     10000000 uncomp,  665501 comp  402.1 enc MB/s  968.0 dec MB/s
+    r32x16-o4-c0202     10000000 uncomp,  665501 comp  690.8 enc MB/s 1796.0 dec MB/s
+    r32x16-o4-c0404     10000000 uncomp,  665501 comp  866.9 enc MB/s 2098.6 dec MB/s
+    
+    r32x16-o5-c0000     10000000 uncomp,  616223 comp  274.6 enc MB/s  426.5 dec MB/s
+    r32x16-o5-c0101     10000000 uncomp,  616223 comp  274.1 enc MB/s  626.8 dec MB/s
+    r32x16-o5-c0202     10000000 uncomp,  616223 comp  391.8 enc MB/s 1472.8 dec MB/s
+    r32x16-o5-c0404     10000000 uncomp,  616223 comp  563.5 enc MB/s 1673.9 dec MB/s
+    
+    r32x16-o68-c0000    10000000 uncomp,  712513 comp  363.8 enc MB/s  717.4 dec MB/s
+    r32x16-o68-c0101    10000000 uncomp,  712513 comp  384.7 enc MB/s  836.5 dec MB/s
+    r32x16-o68-c0202    10000000 uncomp,  712513 comp  438.8 enc MB/s  913.6 dec MB/s
+    r32x16-o68-c0404    10000000 uncomp,  712513 comp  450.8 enc MB/s  918.0 dec MB/s
+    
+    r32x16-o69-c0000    10000000 uncomp,  591639 comp  369.7 enc MB/s  684.2 dec MB/s
+    r32x16-o69-c0101    10000000 uncomp,  591639 comp  370.2 enc MB/s  780.1 dec MB/s
+    r32x16-o69-c0202    10000000 uncomp,  591639 comp  408.5 enc MB/s  894.9 dec MB/s
+    r32x16-o69-c0404    10000000 uncomp,  591639 comp  431.6 enc MB/s  906.5 dec MB/s
+    
+    r32x16-o132-c0000   10000000 uncomp,  615999 comp  659.2 enc MB/s 1861.9 dec MB/s
+    r32x16-o132-c0101   10000000 uncomp,  615999 comp  660.0 enc MB/s 2580.6 dec MB/s
+    r32x16-o132-c0202   10000000 uncomp,  615999 comp  971.6 enc MB/s 3679.2 dec MB/s
+    r32x16-o132-c0404   10000000 uncomp,  615999 comp 1050.6 enc MB/s 3947.9 dec MB/s
+    
+    r32x16-o133-c0000   10000000 uncomp,  553181 comp  573.2 enc MB/s  848.8 dec MB/s
+    r32x16-o133-c0101   10000000 uncomp,  553181 comp  566.3 enc MB/s 1517.0 dec MB/s
+    r32x16-o133-c0202   10000000 uncomp,  553181 comp  759.1 enc MB/s 1923.8 dec MB/s
+    r32x16-o133-c0404   10000000 uncomp,  553181 comp  914.4 enc MB/s 1981.4 dec MB/s
+    
+    r32x16-o194-c0000   10000000 uncomp,  621771 comp  558.0 enc MB/s 1085.0 dec MB/s
+    r32x16-o194-c0101   10000000 uncomp,  621771 comp  559.2 enc MB/s 1088.6 dec MB/s
+    r32x16-o194-c0202   10000000 uncomp,  621771 comp  552.9 enc MB/s 1091.2 dec MB/s
+    r32x16-o194-c0404   10000000 uncomp,  621771 comp  550.1 enc MB/s 1070.3 dec MB/s
+    
+    r32x16-o197-c0000   10000000 uncomp,  550497 comp  484.2 enc MB/s  791.8 dec MB/s
+    r32x16-o197-c0101   10000000 uncomp,  550497 comp  487.2 enc MB/s 1004.4 dec MB/s
+    r32x16-o197-c0202   10000000 uncomp,  550497 comp  488.0 enc MB/s 1033.9 dec MB/s
+    r32x16-o197-c0404   10000000 uncomp,  550497 comp  502.0 enc MB/s 1027.6 dec MB/s
+
+For completeness, a couple other tools are also shown below.  Note
+fqzcomp here is slightly smaller as it has been trimmed to end on a
+whole line.
+
+    fqzcomp -s1          9999975 uncomp,  494485 comp   27.4 enc MB/s   27.1 dec MB/s
+
+    bsc -m3e1tT         10000000 uncomp,  553958 comp   43.7 enc MB/s   31.6 dec MB/s
+    bsc -m0e2tT         10000000 uncomp,  531536 comp   19.0 enc MB/s   25.5 dec MB/s
+
+-----------------------------------------------------------------------------
+
+10MB worth of Illumina HiSeq data with 40 distinct quality values.
+Note this sequencing run had a few erratic cycles, leading to
+unusually good performance from fqzcomp.  The bit-packing modes of
+rANS are not relevant (nor shown) here due to the cardinality of the
+data.
+
+      r4x8-o0             10000000 uncomp,    5092977 comp  303.9 enc MB/s  553.3 dec MB/s
+     r4x16-o0             10000000 uncomp,    5092608 comp  357.4 enc MB/s  579.8 dec MB/s
+     arith-o0             10000000 uncomp,    5079029 comp   51.9 enc MB/s   33.1 dec MB/s
+
+      r4x8-o1             10000000 uncomp,    4911113 comp  278.1 enc MB/s  356.4 dec MB/s
+     r4x16-o1             10000000 uncomp,    4918609 comp  290.5 enc MB/s  542.4 dec MB/s
+     arith-o1             10000000 uncomp,    4911347 comp   42.1 enc MB/s   32.3 dec MB/s
+
+     r4x16-o64            10000000 uncomp,    5092608 comp  215.5 enc MB/s  782.7 dec MB/s
+     arith-o64            10000000 uncomp,    5194241 comp   36.8 enc MB/s   26.6 dec MB/s
+
+     r4x16-o65            10000000 uncomp,    4918609 comp  167.0 enc MB/s  484.0 dec MB/s
+     arith-o65            10000000 uncomp,    4909925 comp   33.4 enc MB/s   23.8 dec MB/s
+
+    r32x16-o4-c0000       10000000 uncomp,    5092684 comp  367.2 enc MB/s  642.1 dec MB/s
+    r32x16-o4-c0101       10000000 uncomp,    5092684 comp  340.7 enc MB/s 1005.1 dec MB/s
+    r32x16-o4-c0202       10000000 uncomp,    5092684 comp  666.8 enc MB/s 1777.5 dec MB/s
+    r32x16-o4-c0404       10000000 uncomp,    5092684 comp  827.0 enc MB/s 2158.9 dec MB/s
+
+    r32x16-o5-c0000       10000000 uncomp,    4918685 comp  273.9 enc MB/s  391.5 dec MB/s
+    r32x16-o5-c0101       10000000 uncomp,    4918685 comp  268.5 enc MB/s  524.0 dec MB/s
+    r32x16-o5-c0202       10000000 uncomp,    4918685 comp  396.0 enc MB/s 1218.2 dec MB/s
+    r32x16-o5-c0404       10000000 uncomp,    4918685 comp  553.4 enc MB/s 1418.4 dec MB/s
+
+    r32x16-o68-c0000      10000000 uncomp,    5092684 comp  216.3 enc MB/s  646.6 dec MB/s
+    r32x16-o68-c0101      10000000 uncomp,    5092684 comp  235.2 enc MB/s 1016.3 dec MB/s
+    r32x16-o68-c0202      10000000 uncomp,    5092684 comp  336.4 enc MB/s 1804.4 dec MB/s
+    r32x16-o68-c0404      10000000 uncomp,    5092684 comp  376.5 enc MB/s 2162.2 dec MB/s
+
+    r32x16-o69-c0000      10000000 uncomp,    4918685 comp  194.3 enc MB/s  390.1 dec MB/s
+    r32x16-o69-c0101      10000000 uncomp,    4918685 comp  195.3 enc MB/s  593.4 dec MB/s
+    r32x16-o69-c0202      10000000 uncomp,    4918685 comp  251.6 enc MB/s 1212.7 dec MB/s
+    r32x16-o69-c0404      10000000 uncomp,    4918685 comp  306.3 enc MB/s 1415.6 dec MB/s
+
+    fqzcomp -s1           10000000 uncomp,    3196746 comp   16.6 enc MB/s  16.0 dec MB/s
+
+    bsc -m3e1tT           10000000 uncomp,    4762846 comp   12.9 enc MB/s  17.5 dec MB/s
+    bsc -m0e2tT           10000000 uncomp,    4477056 comp    6.1 enc MB/s   8.8 dec MB/s
diff --git a/ext/htslib/htscodecs/LICENSE.md b/ext/htslib/htscodecs/LICENSE.md
new file mode 100644
index 0000000..14d3778
--- /dev/null
+++ b/ext/htslib/htscodecs/LICENSE.md
@@ -0,0 +1,45 @@
+All files except those explicitly listed below are copyright Genome
+Research Limited and are made available under the BSD license.
+
+> Redistribution and use in source and binary forms, with or without
+> modification, are permitted provided that the following conditions
+> are met:
+> 
+>     (1) Redistributions of source code must retain the above copyright
+>     notice, this list of conditions and the following disclaimer. 
+> 
+>     (2) Redistributions in binary form must reproduce the above copyright
+>     notice, this list of conditions and the following disclaimer in
+>     the documentation and/or other materials provided with the distribution.  
+>     
+>     (3)The name of the author may not be used to endorse or promote
+>     products derived from this software without specific prior written
+>     permission.
+> 
+> THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+> IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+> WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+> DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+> INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+> (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+> SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+> HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+> STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+> IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+> POSSIBILITY OF SUCH DAMAGE. 
+
+c_range_coder.h is Public Domain, derived from work by Eugene
+Shelwien.
+
+rANS_byte.h and rANS_word.h are derived from Fabien Giesen's work and
+is Public Domain.  https://github.com/rygorous/ryg_rans This work was
+in turn based on the ANS family of entropy encoders as described by
+Jarek Duda's paper: http://arxiv.org/abs/1311.2540
+
+> To the extent possible under law, Fabian Giesen has waived all
+> copyright and related or neighboring rights to ryg_rans, as
+> per the terms of the CC0 license:
+> 
+>   https://creativecommons.org/publicdomain/zero/1.0
+> 
+> This work is published from the United States.
diff --git a/ext/htslib/htscodecs/MAINTAINERS.md b/ext/htslib/htscodecs/MAINTAINERS.md
new file mode 100644
index 0000000..7ec161b
--- /dev/null
+++ b/ext/htslib/htscodecs/MAINTAINERS.md
@@ -0,0 +1,55 @@
+Notes to maintainers for building releases.
+This is best done as a release PR so we can check it first.
+
+1. Places to update the version number include:
+
+   - htscodecs/htscodecs.h (used for program introspection)
+
+   - configure.ac AC_INIT macro
+
+   - configure.ac VERS_CURRENT, VERS_REVISION and VERS_AGE variables.
+     See the long comment above for instructions of how these change.
+
+   - NEWS files.
+
+
+2. Ensure NEWS and README files are up to date.  NEWS is a git log
+   summary.  README likely doesn't change unless something major needs
+   mentioning.
+
+   - At time of merging, set the date at the top of NEWS.
+
+
+3. Test it all.
+   - Push to github PR so the CI can validate for us.
+
+   - make distcheck
+     This also makes the tarball htscodecs-${vers}.tar.gz.
+
+
+4. Merge into master
+
+
+5. Add an annotated tag with minimal message, eg:
+
+   - git tag -a v1.1 -m v1.1
+
+
+6. Push master and --tags upstream to github
+
+
+7. Make a new release on github.
+
+   - Title: "htscodecs ${vers}"
+
+   - Message: this is just a copy of NEWS.
+     It's already in Markdown format, but double check the preview panel.
+
+   - Upload the tarball produced from distcheck to the assets.
+
+
+8. Finally, consider updating any packages that use this as a
+   submodule to ensure they have the latest tagged release.
+
+   This will invariably help OS distributions keep their package
+   dependencies neatly in sync.
diff --git a/ext/htslib/htscodecs/NEWS.md b/ext/htslib/htscodecs/NEWS.md
new file mode 100644
index 0000000..ff0177c
--- /dev/null
+++ b/ext/htslib/htscodecs/NEWS.md
@@ -0,0 +1,438 @@
+Release 1.6.1: 22nd August 2024
+-------------------------------
+
+This release is primarily portability and minor bug fixes.
+
+Changes
+
+- Improve warning levels by the compiler in CI. (#125)
+
+- Switch to GitHub actions for some CI builds. (#121, #123)
+
+- Add configure check for cpuid systems. (#115, #116.  Reported by
+  Ryan Carsten Schmidt)
+
+Bug fixes
+
+- Use unsigned chars for ctype macros in the name tokeniser.
+  On many systems this was already mitigated against, but on some OSes
+  a char > 128 could trigger a buffer underrun. (#124)
+
+- Fix interaction between _XOPEN_SOURCE and FreeBSD.
+  (#119, John Marshall)
+
+- Improve AVX512 compiler support, notably MacOS El Capitan's XCode.
+  (#118, Rob Davies)
+
+- Fix -std=c99 -pendantic pedantry (#117)
+
+
+Release 1.6.0: 7th December 2023
+--------------------------------
+
+This release is primarily bug fixes, mostly spotted through improved fuzz
+testing.
+
+One big change however is the SIMD rANS codecs are now performant on Intel
+CPUs with the DownFall mitigation microcode applied.
+
+
+Changes
+
+- Replaced the rANS codec SIMD gathers with simulated gathers via scalar
+  memory fetches.  This helps AMD Zen4, but importantly it also fixes a
+  disastrous performance regression caused by Intel's DownFall microcode fix.
+
+  There is an impact on pre-DownFall speeds, but we should focus on patched
+  CPUs as a priority.
+
+- A small speed up to the rans_F_to_s3 function used by order-0 rans decode.
+
+- Small speed up to SIMD rans32x16 order-1 encoder by reducing cache misses.
+  Also sped up the rans4x8 order-1 encoder, particularly on AMD Zen4.
+
+- Now supports building with "zig cc"
+  (Issue #109, reported by David Jackson)
+
+
+Bug fixes
+
+- Improve robustness of name tokeniser when given non 7-bit ASCII and on
+  machines where "char" defaults to unsigned.
+  (Issue #105, reported by Shubham Chandak)
+
+- Also fixed a 1 byte buffer read-overrun in name tokeniser.
+
+- Fix name tokeniser encoder failure with some duplicated streams.
+
+- Fixed rans_set_cpu to work multiple times, as well as reinstating the
+  ability to change decode and encode side independently (accidentally lost in
+  commit 958032c).  No effect on usage, but it improves the test coverage.
+
+- Added a round-trip fuzz tester to test the ability to encode.  The old fuzz
+  testing was decode streams only.
+
+- Fixed bounds checking in rans_uncompress_O0_32x16_avx2, fixing buffer read
+  overruns.
+
+- Removed undefined behaviour in transpose_and_copy(), fixing zig cc builds.
+
+
+Release 1.5.2: 6th October 2023
+-------------------------------
+
+*** SECURITY FIXES ***
+
+This release contains multiple bug fixes, including a couple
+buffer overruns that could corrupt memory when used in specific
+scenarios.  These have not been observed with real data, but could
+represent an attack vector for a malicious user.  (We know of no
+exploit.)
+
+
+Changes
+
+- The range coder has been extended to do bounds checking if the
+  new RC_SetOutputEnd() is called.  This has a small performance hit
+  for the encoder, depending on compiler, but tests showed within 10%
+  at worst.
+
+Bug fixes
+
+- Fix write-buffer overruns in fqzcomp and name tokeniser.
+
+  SECURITY ISSUE: FQZComp could overflow the computed maximum growth
+  size, causing writes beyond the ends of the allocated memory.  This
+  is triggered by many very small 1bp reads.  Fixed the maximum
+  bounds for compressed data.
+
+  SECURITY ISSUE: The name tokeniser using the maximum number of
+  tokens (128) would erroneously write a 129th token.  This is a
+  restricted overflow of a few bytes.
+
+  (PR#97, reported by Shubham Chandak)
+
+- Fix an maximum 8-byte read overflow in the AVX2 rans decoder.
+  SECURITY ISSUE: This was only present when using gcc.
+  (PR#100, reported by Rob Davies)
+
+- The rANS Order-1 SSE4 decoder could decode incorrectly.
+  When a single symbol only occurs and we're using 12-bit freqs, the
+  frequency of 4096 was interpreted as freq 0.  This only happens in
+  the non-SIMD tidy-up stage at the end of the decode, so at worst the
+  final 31 bytes may be incorrect. (PR#102)
+
+- Fixed a 1-byte heap read-buffer overflow. Existed since 6a87ead2
+  (Oct 2021).  Low severity security due to size and high likelihood
+  it's just malloc meta-data. (PR#95; OSS-Fuzz 62270)
+
+- rans_compress_4x16 now works on zero length input.
+  Previously this was giving divide-by-zero errors.
+  (PR#101, reported by Shubham Chandak)
+
+- Remove asserts which caused warnings about unused variables when
+  building with -DNDEBUG.
+
+- Fix ARM builds when HWCAP_ASIMD is missing (on Conda) (PR#91)
+
+- Improve FreeBSD CI testing
+
+- Fix undefined behaviour from signed bit-shifting (PR#90).
+
+
+Release 1.5.1: 19th July 2023
+-----------------------------
+
+This release is mainly small updates and bug fixes focusing on
+specific platforms, with no new features added.
+
+Changes
+
+- Be more selective in use of AVX512 on AMD Zen4 processors.  This can
+  be faster (e.g. with 64-way unrolling), but in the current rANS codec
+  implementations AVX2 is faster for certain operations (PR#85).
+
+- Add config.h to test programs to help them pick up definitions such
+  as XOPEN_SOURCE (PR#84)
+
+- Add FreeBSD to CI testing (PR#83)
+
+Bug fixes
+
+- Trivial bug fix to the rans4x16pr test harness when given
+  incompressible data (PR#86).
+
+- Make ARM NEON checks specific to AArch64 and exclude AArch32 systems.
+  (PR#82 to fix issue#81, reported by Robert Clausecker)
+
+
+Release 1.5.0: 14th April 2023
+------------------------------
+
+Changes
+
+- Significant speed ups to the fqzcomp codec via code restructuring
+  and use of memory prefetch instructions.  Encode is 30-40% faster
+  and decode 5-8% faster. (PR#75 James Bonfield)
+
+- Improve multiarch builds on MacOS, fixing issues with getting the
+  various SIMD implementations integrated. (Issue#76 John Marshall,
+  PR#77/#78 Rob Davies)
+
+- Remove unused ax_with_libdeflate.m4 file from build system.
+
+
+Release 1.4.0: Februrary 2023
+-----------------------------
+
+This is almost entirely minor bug fixing with a few small updates.
+
+Changes
+
+- Optimise compression / speed of the name tokeniser.
+  - In arithmetic coding mode, it can now utilise bzip2 at higher levels.
+  - For both rans / arith entropy encoders, the choice of method / order
+    is now optimised per token type, giving faster compression.
+  - Culled a pointless zlib check in the configure script.
+  - Made lack of bzip2 a hard failure in configure, unless an explicit
+    --disable-bz2 option is given.
+  (#72, #73)
+
+- Switch CI to use ARM for MacOS builds
+  (#69, thanks to Rob Davies)
+
+
+Bug fixes
+
+- Remove some newer compiler warnings (#61)
+
+- Improvements for Intel -m32 builds, including better AVX2 validation
+  (m32 misses _mm256_extract_epi64) and improved data alignment.
+  (#62. See also samtools/htslib#1500)
+
+- Detect Neon capability at runtime via operating system APIs.
+  (#63, thanks to John Marshall)
+
+- Improve FreeBSD diagnostics when neglecting to use -lpthread / -lthr. 
+  Plus additional extra error checking too.
+  (#68, #64, thanks to John Marshall)
+
+- Update hts_pack to operate in line with CRAMcodecs spec, where the
+  number of symbols > 16.
+  (#65/#66, reported by Michael Macias)
+
+- Fixed too-stringent buffer overflow checking in O1 rans decoder.
+  (#71, reported by Divon Lan)
+
+
+Release 1.3.0: 9th August 2022
+------------------------------
+
+The primary change in this release is a new SIMD enabled rANS codec.
+
+Changes
+
+- There is a 32-way unrolled rANS implementation.  This is accessed
+  using the existing rans 4x16 API with the RANS_ORDER_X32 bit set.
+  Implementations exist for SSE4.1, AVX2, AVX512 and ARM Neon, as
+  well as traditional non-SIMD scalar code in C and JavaScript. See
+  the commit logs for benchmarks.
+
+- Improved memory allocation via a new htscodecs_tls_alloc function.
+  This uses Thread Local Storage (TLS) to avoid multiple malloc/free
+  calls, reducing system CPU time.
+
+- Some external functions have been renamed, with the old ones still
+  existing in a deprecated fashion.  Every symbol should now start
+  hts_, rans_, arith_, fqz_ or tok3_*.
+
+- Improved test framework with an "entropy" tool that iterates over
+  all entropy encoders.
+
+- Updated the Appveyor CI image to user a newer gcc.  Also added ARM
+  to the list of processors to test on.
+
+- Tab vs space code changes.  Use "git diff -w" to see through these.
+
+- Reworked fuzzing infrastructure.
+
+- Small speed improvements to various rANS encoders and decoders.
+  These were tested on a broad range of compilers, versions and
+  systems.  The new code may be slightly slower with some combinations,
+  but is faster overall and removes a few outliers with considerably
+  degraded performance.
+
+- Substantial memory reduction to the name tokeniser (tok3).
+
+Bug fixes
+
+- Fixed undefined behaviour in our use of _builtin_clz().
+
+- Fixed a few redundant #includes.
+
+- Work around strict aliasing bugs, uncovered with gcc -O2.
+
+- Fixed an issue with encoding data blocks close to 2GB in size.
+  (Additionally blocks above 2GB now error, rather than crashing or
+  returning incorrect results.)
+
+- Fix encode error with large blocks using RANS_ORDER_STRIPE.
+
+
+Release 1.2.2: 1st April 2022
+-----------------------------
+
+This release contains some fixes found during fuzzing with Clang's
+memory-sanitizer.  None of these are involving writing memory so there
+is no possibility for code execution vulnerabilities.  However some do
+could access uninitialised elements in locally allocated memory, which
+could leak private data if the library was used in conjunction with
+other tools which don't zero sensitive data before freeing.
+
+Bug fixes:
+
+- The name tokeniser now validates the stored length in the data
+  stream matches the actual decoded length.  Discovered by Taotao Gu.
+
+- Fixed an endless loop in arith_dynamic and rans4x16pr involving
+  X_STRIPE with 0 stripes.
+
+- Avoid a harmless (and wrong?) undefined behaviour sanitizer error
+  when calling memcpy(ptr, NULL, 0) in the name tokeniser.
+
+- Fixed possible uninitialised memory access in
+  rans_uncompress_O1_4x16.  If the frequency table didn't add up to
+  the correct amount, parts of the "fb" table were left unpopulated.
+  It was then possible to use these array elements in some of the rANS
+  calculations.
+
+- Similarly rans_uncompress_O0 could access an uninitialised element
+  4095 of the decoder tables if the frequencies summed to 4095 instead
+  of the expected 4096.
+
+- Improved error detection from fqzcomp's read_array function.
+
+- Reject fqzcomp parameters with inconsistent "sel" parameters, which
+  could lead to uninitialised access to the model.sel range coder.
+
+
+Release 1.2.1: 15th February 2022
+---------------------------------
+
+The only change in this release is a minor adjustment to the histogram
+code so it works on systems with small stacks.  This was detected on
+Windows Mingw builds.
+
+
+Release 1.2: 10th February 2022
+-------------------------------
+
+This release contains the following minor changes.
+Please see the "git log" for the full details.
+
+Improvements / changes:
+
+- Speed up of rANS4x16 order-0.  We now use a branchless encoder
+  renormalisation step.  For complex data it's between 13 and 50%
+  speed up depending on compiler.
+
+- Improve rANS4x16 compute_shift estimates.  The entropy calculation
+  is now more accurate.  This leads to more frequent use of the 10-bit
+  frequency mode, at an expense of up to 1% size growth.
+
+- Speed improvements to the striped rANS mode, both encoding and
+  decoding.  Encoder gains ~8% and decoder ~5%, but varies
+  considerably by compiler and data.
+
+- Added new var_put_u64_safe and var_put_u32_safe interfaces.
+  These are automatically used by var_put_u64 and var_put_u32 when
+  near the end of the buffer, but may also be called directly.
+
+- Small speed ups to the hist8 and hist1_4 functions.
+
+- Minor speed up to RLE decoding.
+
+Bug fixes:
+
+- Work around an icc-2021 compiler bug, but also speed up the varint
+  encoding too (#29).
+
+- Fix an off-by-one error in the initial size check in arith_dynamic.
+  This meant the very smallest of blocks could fail to decode.
+  Reported by Divon Lan.
+
+- Fixed hist1_4 to also count the last byte when computing T0[].
+
+- Fixed overly harsh bounds checking in the fqzcomp read_array
+  function, which meant it failed to decode some configurations.
+
+
+Release 1.1.1: 6th July 2021
+----------------------------
+
+This release contains the following minor changes.
+Please see the "git log" for the full details.
+
+Improvements / changes:
+
+- Modernised autoconf usage to avoid warnings with newer versions.
+  (John Marshall)
+
+- Avoid using awk with large records, due to some systems
+  (e.g. Solaris / OpenIndiana) with line length limits .
+  (John Marshall)
+
+- Applied Debian patch to make the library link against -lm.
+
+Bug fixes:
+
+- Fixed an issue with the name tokeniser when a slice (name_context)
+  has exactly 1 more name than the previous call. (James Bonfield)
+
+- Removed access to an uninitialised variable in the name tokeniser
+  decode when given malformed data.  This occurs when we use delta
+  encoding for the very first name. (James Bonfield, OSS-Fuzz)
+
+- Minor fixes to distcheck and distclean targets
+
+
+Release 1.0: 23rd Feb 2021
+--------------------------
+
+This marks the first non-beta release of htscodecs, following a
+perioid of integration with Htslib and automated fuzzing by Google's
+OSS-Fuzz program.
+
+[Note this testing only applies to the C implementation.  The
+JavaScript code should still be considered as examples of the codecs,
+more for purposes of understanding and clarity than as a fully
+optimised and tested release.]
+
+Since the last release (0.5) the key changes are:
+
+- Improved support for big endian platforms
+
+- Speed improvements to CRAM 3.0 4x8 rANS order-1 encoding.
+  It's between 10 and 50% faster at encoding, based on input data.
+
+- Improved autoconf bzip2 checks and tidy up "make test" output.
+
+- Added some more files into "make install", so that "make distcheck"
+  now passes.
+
+- Replaced Travis with Cirrus-CI testing.
+
+- Removed various C undefined behaviour, such as left shifting of
+  negative values and integer overflows.  As far as we know these were
+  currently harmless on the supported platforms, but may break future
+  compiler optimisations.
+
+- Fixed numerous OSS-Fuzz identified flaws.  Some of these were
+  potential security issues such as small buffer overruns.
+
+- Tidied up some code to prevent warnings.
+
+- The name tokeniser now has a limit on the size of data it can encode
+  (10 million records).  This may still be too high given the memory
+  it will require, so it may be reduced again.
+
diff --git a/ext/htslib/htscodecs/htscodecs/arith_dynamic.c b/ext/htslib/htscodecs/htscodecs/arith_dynamic.c
new file mode 100644
index 0000000..37aca77
--- /dev/null
+++ b/ext/htslib/htscodecs/htscodecs/arith_dynamic.c
@@ -0,0 +1,1234 @@
+/*
+ * Copyright (c) 2019-2022 Genome Research Ltd.
+ * Author(s): James Bonfield
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above
+ *       copyright notice, this list of conditions and the following
+ *       disclaimer in the documentation and/or other materials provided
+ *       with the distribution.
+ *
+ *    3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+ *       Institute nor the names of its contributors may be used to endorse
+ *       or promote products derived from this software without specific
+ *       prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH
+ * LTD OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+// As per standard rANS_static but using optional RLE or bit-packing
+// techniques prior to entropy encoding.  This is a significant
+// reduction in some data sets.
+
+// top bits in order byte
+#define X_PACK   0x80    // Pack 2,4,8 or infinite symbols into a byte.
+#define X_RLE    0x40    // Run length encoding with runs & lits encoded separately
+#define X_CAT    0x20    // Nop; for tiny segments where rANS overhead is too big
+#define X_NOSZ   0x10    // Don't store the original size; used by STRIPE mode
+#define X_STRIPE 0x08    // For 4-byte integer data; rotate & encode 4 streams.
+#define X_EXT    0x04    // External compression codec via magic num (gz, xz, bz2)
+#define X_ORDER  0x03    // Mask to obtain order
+
+#include "config.h"
+
+#ifdef HAVE_LIBBZ2
+#include <bzlib.h>
+#endif
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <assert.h>
+#include <string.h>
+#include <sys/time.h>
+#include <limits.h>
+
+#include "arith_dynamic.h"
+#include "varint.h"
+#include "pack.h"
+#include "utils.h"
+
+#define MIN(a,b) ((a)<(b)?(a):(b))
+
+/*-----------------------------------------------------------------------------
+ * Memory to memory compression functions.
+ *
+ * These are original versions without any manual loop unrolling. They
+ * are easier to understand, but can be up to 2x slower.
+ */
+#define MAGIC 8
+
+unsigned int arith_compress_bound(unsigned int size, int order) {
+    int N = (order>>8) & 0xff;
+    if (!N) N=4;
+    return (order == 0
+        ? 1.05*size + 257*3 + 4
+        : 1.05*size + 257*257*3 + 4 + 257*3+4) + 5 +
+        ((order & X_PACK) ? 1 : 0) +
+        ((order & X_RLE) ? 1 + 257*3+4: 0) +
+        ((order & X_STRIPE) ? 7 + 5*N: 0);
+}
+
+#ifndef MODEL_256 // see fqzcomp_qual_fuzz.c
+#define NSYM 256
+#include "c_simple_model.h"
+#endif
+
+// Compresses in_size bytes from 'in' to *out_size bytes in 'out'.
+//
+// NB: The output buffer does not hold the original size, so it is up to
+// the caller to store this.
+static
+unsigned char *arith_compress_O0(unsigned char *in, unsigned int in_size,
+                                 unsigned char *out, unsigned int *out_size) {
+    int i, bound = arith_compress_bound(in_size,0)-5; // -5 for order/size
+    unsigned char *out_free = NULL;
+
+    if (!out) {
+        *out_size = bound;
+        out_free = out = malloc(*out_size);
+    }
+    if (!out || bound > *out_size)
+        return NULL;
+
+    unsigned int m = 0;
+    for (i = 0; i < in_size; i++)
+        if (m < in[i])
+            m = in[i];
+    m++;
+    *out = m;
+
+    SIMPLE_MODEL(256,_) byte_model;
+    SIMPLE_MODEL(256,_init)(&byte_model, m);
+
+    RangeCoder rc;
+    RC_SetOutput(&rc, (char *)out+1);
+    RC_SetOutputEnd(&rc, (char *)out + *out_size);
+    RC_StartEncode(&rc);
+
+    for (i = 0; i < in_size; i++)
+        SIMPLE_MODEL(256, _encodeSymbol)(&byte_model, &rc, in[i]);
+
+    if (RC_FinishEncode(&rc) < 0) {
+        free(out_free);
+        return NULL;
+    }
+
+    // Finalise block size and return it
+    *out_size = RC_OutSize(&rc)+1;
+
+    return out;
+}
+
+static
+unsigned char *arith_uncompress_O0(unsigned char *in, unsigned int in_size,
+                                   unsigned char *out, unsigned int out_sz) {
+    RangeCoder rc;
+    int i;
+    unsigned int m = in[0] ? in[0] : 256;
+
+    SIMPLE_MODEL(256,_) byte_model;
+    SIMPLE_MODEL(256,_init)(&byte_model, m);
+
+    unsigned char *out_free = NULL;
+    if (!out)
+        out_free = out = malloc(out_sz);
+    if (!out)
+        return NULL;
+
+    RC_SetInput(&rc, (char *)in+1, (char *)in+in_size);
+    RC_StartDecode(&rc);
+
+    for (i = 0; i < out_sz; i++)
+        out[i] = SIMPLE_MODEL(256, _decodeSymbol)(&byte_model, &rc);
+
+    if (RC_FinishDecode(&rc) < 0) {
+        free(out_free);
+        return NULL;
+    }
+    
+    return out;
+}
+
+
+//-----------------------------------------------------------------------------
+static
+unsigned char *arith_compress_O1(unsigned char *in, unsigned int in_size,
+                                 unsigned char *out, unsigned int *out_size) {
+    int i, bound = arith_compress_bound(in_size,0)-5; // -5 for order/size
+    unsigned char *out_free = NULL;
+
+    if (!out) {
+        *out_size = bound;
+        out_free = out = malloc(*out_size);
+    }
+    if (!out || bound > *out_size)
+        return NULL;
+
+    SIMPLE_MODEL(256,_) *byte_model =
+        htscodecs_tls_alloc(256 * sizeof(*byte_model));
+    if (!byte_model) {
+        free(out_free);
+        return NULL;
+    }
+    unsigned int m = 0;
+    if (1 || in_size > 1000) {
+        for (i = 0; i < in_size; i++)
+            if (m < in[i])
+                m = in[i];
+        //fprintf(stderr, "%d max %d\n", in_size, m);
+        m++;
+    }
+    *out = m;
+    for (i = 0; i < 256; i++)
+        SIMPLE_MODEL(256,_init)(&byte_model[i], m);
+
+    RangeCoder rc;
+    RC_SetOutput(&rc, (char *)out+1);
+    RC_SetOutputEnd(&rc, (char *)out + *out_size);
+    RC_StartEncode(&rc);
+
+    uint8_t last = 0;
+    for (i = 0; i < in_size; i++) {
+        SIMPLE_MODEL(256, _encodeSymbol)(&byte_model[last], &rc, in[i]);
+        last = in[i];
+    }
+
+    if (RC_FinishEncode(&rc) < 0) {
+        free(out_free);
+        htscodecs_tls_free(byte_model);
+        return NULL;
+    }
+
+    // Finalise block size and return it
+    *out_size = RC_OutSize(&rc)+1;
+
+    htscodecs_tls_free(byte_model);
+    return out;
+}
+
+static
+unsigned char *arith_uncompress_O1(unsigned char *in, unsigned int in_size,
+                                   unsigned char *out, unsigned int out_sz) {
+    RangeCoder rc;
+    unsigned char *out_free = NULL;
+
+    if (!out)
+        out_free = out = malloc(out_sz);
+    if (!out)
+        return NULL;
+
+
+    SIMPLE_MODEL(256,_) *byte_model =
+        htscodecs_tls_alloc(256 * sizeof(*byte_model));
+    if (!byte_model) {
+        free(out_free);
+        return NULL;
+    }
+
+    unsigned int m = in[0] ? in[0] : 256, i;
+    for (i = 0; i < 256; i++)
+        SIMPLE_MODEL(256,_init)(&byte_model[i], m);
+
+    RC_SetInput(&rc, (char *)in+1, (char *)in+in_size);
+    RC_StartDecode(&rc);
+
+    unsigned char last = 0;
+    for (i = 0; i < out_sz; i++) {
+        out[i] = SIMPLE_MODEL(256, _decodeSymbol)(&byte_model[last], &rc);
+        last = out[i];
+    }
+
+    if (RC_FinishDecode(&rc) < 0) {
+        htscodecs_tls_free(byte_model);
+        free(out_free);
+        return NULL;
+    }
+    
+    htscodecs_tls_free(byte_model);
+    return out;
+}
+
+//-----------------------------------------------------------------------------
+
+// Disable O2 for now
+#if 0
+
+#if 0
+unsigned char *arith_compress_O2(unsigned char *in, unsigned int in_size,
+                                 unsigned char *out, unsigned int *out_size) {
+    fprintf(stderr, "WARNING: using undocumented O2 arith\n");
+
+    int i, j;
+    int bound = arith_compress_bound(in_size,0)-5; // -5 for order/size
+    unsigned char *out_free = NULL;
+
+    if (!out) {
+        *out_size = bound;
+        out_free = out = malloc(*out_size);
+    }
+    if (!out || bound > *out_size)
+        return NULL;
+
+    unsigned int m = 0;
+    if (1 || in_size > 1000) {
+        for (i = 0; i < in_size; i++)
+            if (m < in[i])
+                m = in[i];
+        //fprintf(stderr, "%d max %d\n", in_size, m);
+        m++;
+    }
+    *out = m;
+
+    SIMPLE_MODEL(256,_) *byte_model;
+    byte_model = malloc(256*256*sizeof(*byte_model));
+    for (i = 0; i < 256; i++)
+        for (j = 0; j < 256; j++)
+            SIMPLE_MODEL(256,_init)(&byte_model[i*256+j], m);
+
+    RangeCoder rc;
+    RC_SetOutput(&rc, (char *)out+1);
+    RC_SetOutputEnd(&rc, (char *)out + *out_size);
+    RC_StartEncode(&rc);
+
+    unsigned char last1 = 0, last2 = 0;
+    for (i = 0; i < in_size; i++) {
+        SIMPLE_MODEL(256, _encodeSymbol)(&byte_model[last1*256 + last2], &rc, in[i]);
+        last2 = last1;
+        last1 = in[i];
+    }
+
+    free(byte_model);
+    if (RC_FinishEncode(&rc) < 0) {
+        free(out_free);
+        return NULL;
+    }
+
+    // Finalise block size and return it
+    *out_size = RC_OutSize(&rc)+1;
+
+    return out;
+}
+#else
+unsigned char *arith_compress_O2(unsigned char *in, unsigned int in_size,
+                                 unsigned char *out, unsigned int *out_size) {
+    fprintf(stderr, "WARNING: using undocumented O2 arith\n");
+
+    int i, j;
+    int bound = arith_compress_bound(in_size,0)-5; // -5 for order/size
+
+    unsigned char *out_free = NULL;
+    if (!out) {
+        *out_size = bound;
+        out_free = out = malloc(*out_size);
+    }
+    if (!out || bound > *out_size)
+        return NULL;
+
+    unsigned int m = 0;
+    if (1 || in_size > 1000) {
+        for (i = 0; i < in_size; i++)
+            if (m < in[i])
+                m = in[i];
+        //fprintf(stderr, "%d max %d\n", in_size, m);
+        m++;
+    }
+    *out = m;
+
+    SIMPLE_MODEL(256,_) *byte_model;
+    byte_model = malloc(256*256*sizeof(*byte_model));
+    for (i = 0; i < 256; i++)
+        for (j = 0; j < 256; j++)
+            SIMPLE_MODEL(256,_init)(&byte_model[i*256+j], m);
+    SIMPLE_MODEL(256,_) byte_model1[256];
+    for (i = 0; i < 256; i++)
+        SIMPLE_MODEL(256,_init)(&byte_model1[i], m);
+
+    RangeCoder rc;
+    RC_SetOutput(&rc, (char *)out+1);
+    RC_SetOutputEnd(&rc, (char *)out + *out_size);
+    RC_StartEncode(&rc);
+
+    unsigned char last1 = 0, last2 = 0;
+    for (i = 0; i < in_size; i++) {
+        // Use Order-1 is order-2 isn't sufficiently advanced yet (75+ symbols)
+        if (byte_model[last1*256+last2].TotFreq <= m+75*16) {
+            SIMPLE_MODEL(256, _encodeSymbol)(&byte_model1[last1], &rc, in[i]);
+            SIMPLE_MODEL(256, _updateSymbol)(&byte_model[last1*256 + last2], &rc, in[i]);
+        } else {
+            SIMPLE_MODEL(256, _encodeSymbol)(&byte_model[last1*256 + last2], &rc, in[i]);
+            //SIMPLE_MODEL(256, _updateSymbol)(&byte_model1[last1], &rc, in[i]);
+        }
+        last2 = last1;
+        last1 = in[i];
+    }
+
+    free(byte_model);
+    if (RC_FinishEncode(&rc) < 0) {
+        free(out_free);
+        return NULL;
+    }
+
+    // Finalise block size and return it
+    *out_size = RC_OutSize(&rc)+1;
+
+    return out;
+}
+#endif
+
+unsigned char *arith_uncompress_O2(unsigned char *in, unsigned int in_size,
+                                   unsigned char *out, unsigned int out_sz) {
+    RangeCoder rc;
+
+    SIMPLE_MODEL(256,_) *byte_model;
+    byte_model = malloc(256*256*sizeof(*byte_model));
+    unsigned int m = in[0] ? in[0] : 256, i, j;
+    for (i = 0; i < 256; i++)
+        for (j = 0; j < 256; j++)
+            SIMPLE_MODEL(256,_init)(&byte_model[i*256+j], m);
+    
+    unsigned char *out_free = NULL;
+    if (!out)
+        out_free = out = malloc(out_sz);
+    if (!out)
+        return NULL;
+
+    RC_SetInput(&rc, (char *)in+1, (char *)in+in_size);
+    RC_StartDecode(&rc);
+
+    unsigned char last1 = 0, last2 = 0;
+    for (i = 0; i < out_sz; i++) {
+        out[i] = SIMPLE_MODEL(256, _decodeSymbol)(&byte_model[last1*256 + last2], &rc);
+        last2 = last1;
+        last1 = out[i];
+    }
+
+    free(byte_model);
+    if (RC_FinishDecode(&rc) < 0) {
+        free(out_free);
+        return NULL;
+    }
+    
+    return out;
+}
+
+#endif // Disable O2
+/*-----------------------------------------------------------------------------
+ */
+
+#undef NSYM
+#define NSYM 258
+#include "c_simple_model.h"
+#define MAX_RUN 4
+
+static
+unsigned char *arith_compress_O0_RLE(unsigned char *in, unsigned int in_size,
+                                     unsigned char *out, unsigned int *out_size) {
+    int i, bound = arith_compress_bound(in_size,0)-5; // -5 for order/size
+    unsigned char *out_free = NULL;
+
+    if (!out) {
+        *out_size = bound;
+        out_free = out = malloc(*out_size);
+    }
+    if (!out || bound > *out_size)
+        return NULL;
+
+    unsigned int m = 0;
+    for (i = 0; i < in_size; i++)
+        if (m < in[i])
+            m = in[i];
+    m++;
+    *out = m;
+
+    SIMPLE_MODEL(256,_) byte_model;
+    SIMPLE_MODEL(256,_init)(&byte_model, m);
+
+    SIMPLE_MODEL(NSYM,_) *run_model =
+        htscodecs_tls_alloc(NSYM * sizeof(*run_model));
+    if (!run_model) {
+        free(out_free);
+        return NULL;
+    }
+
+    for (i = 0; i < NSYM; i++)
+        SIMPLE_MODEL(NSYM,_init)(&run_model[i], MAX_RUN);
+
+    RangeCoder rc;
+    RC_SetOutput(&rc, (char *)out+1);
+    RC_SetOutputEnd(&rc, (char *)out + *out_size);
+    RC_StartEncode(&rc);
+
+    unsigned char last = 0;
+    for (i = 0; i < in_size;) {
+        //SIMPLE_MODEL(256, _encodeSymbol)(&byte_model, &rc, in[i]);
+        SIMPLE_MODEL(256, _encodeSymbol)(&byte_model, &rc, in[i]);
+        //fprintf(stderr, "lit %c (ctx %c)\n", in[i], last);
+        int run = 0;
+        last = in[i++];
+        while (i < in_size && in[i] == last/* && run < MAX_RUN-1*/)
+            run++, i++;
+        int rctx = last;
+        do {
+            int c = run < MAX_RUN ? run : MAX_RUN-1;
+            SIMPLE_MODEL(NSYM, _encodeSymbol)(&run_model[rctx], &rc, c);
+            run -= c;
+
+            if (rctx == last)
+                rctx = 256;
+            else
+                rctx += (rctx < NSYM-1);
+            if (c == MAX_RUN-1 && run == 0)
+                SIMPLE_MODEL(NSYM, _encodeSymbol)(&run_model[rctx], &rc, 0);
+        } while (run);
+    }
+
+    if (RC_FinishEncode(&rc) < 0) {
+        htscodecs_tls_free(run_model);
+        free(out_free);
+        return NULL;
+    }
+
+    // Finalise block size and return it
+    *out_size = RC_OutSize(&rc)+1;
+
+    //fprintf(stderr, "RLE %d to %d\n", in_size, *out_size);
+
+    htscodecs_tls_free(run_model);
+    return out;
+}
+
+static
+unsigned char *arith_uncompress_O0_RLE(unsigned char *in, unsigned int in_size,
+                                       unsigned char *out, unsigned int out_sz) {
+    RangeCoder rc;
+    int i;
+    unsigned int m = in[0] ? in[0] : 256;
+    unsigned char *out_free = NULL;
+
+    if (!out)
+        out_free = out = malloc(out_sz);
+    if (!out)
+        return NULL;
+
+    SIMPLE_MODEL(256,_) byte_model;
+    SIMPLE_MODEL(256,_init)(&byte_model, m);
+
+    SIMPLE_MODEL(NSYM,_) *run_model =
+        htscodecs_tls_alloc(NSYM * sizeof(*run_model));
+    if (!run_model) {
+        free(out_free);
+        return NULL;
+    }
+
+    for (i = 0; i < NSYM; i++)
+        SIMPLE_MODEL(NSYM,_init)(&run_model[i], MAX_RUN);
+
+    RC_SetInput(&rc, (char *)in+1, (char *)in+in_size);
+    RC_StartDecode(&rc);
+
+    for (i = 0; i < out_sz; i++) {
+        unsigned char last;
+        last = out[i] = SIMPLE_MODEL(256, _decodeSymbol)(&byte_model, &rc);
+        //fprintf(stderr, "lit %c\n", last);
+        int run = 0, r = 0, rctx = out[i];
+        do {
+            r = SIMPLE_MODEL(NSYM, _decodeSymbol)(&run_model[rctx], &rc);
+            if (rctx == last)
+                rctx = 256;
+            else
+                rctx += (rctx < NSYM-1);
+            //fprintf(stderr, "run %d (ctx %d, %d)\n", r, last, l);
+            run += r;
+        } while (r == MAX_RUN-1 && run < out_sz);
+        while (run-- && i+1 < out_sz)
+            out[++i] = last;
+    }
+
+    if (RC_FinishDecode(&rc) < 0) {
+        htscodecs_tls_free(run_model);
+        free(out_free);
+        return NULL;
+    }
+
+    htscodecs_tls_free(run_model);
+    return out;
+}
+
+static
+unsigned char *arith_compress_O1_RLE(unsigned char *in, unsigned int in_size,
+                                     unsigned char *out, unsigned int *out_size) {
+    int i, bound = arith_compress_bound(in_size,0)-5; // -5 for order/size
+    unsigned char *out_free = NULL;
+
+    if (!out) {
+        *out_size = bound;
+        out_free = out = malloc(*out_size);
+    }
+    if (!out || bound > *out_size)
+        return NULL;
+
+    unsigned int m = 0;
+    for (i = 0; i < in_size; i++)
+        if (m < in[i])
+            m = in[i];
+    m++;
+    *out = m;
+
+    SIMPLE_MODEL(256,_) *byte_model =
+        htscodecs_tls_alloc(256 * sizeof(*byte_model));
+    if (!byte_model) {
+        free(out_free);
+        return NULL;
+    }
+    for (i = 0; i < 256; i++)
+        SIMPLE_MODEL(256,_init)(&byte_model[i], m);
+
+    SIMPLE_MODEL(NSYM,_) *run_model =
+        htscodecs_tls_alloc(NSYM * sizeof(*run_model));
+    if (!run_model) {
+        htscodecs_tls_free(byte_model);
+        free(out_free);
+        return NULL;
+    }
+    for (i = 0; i < NSYM; i++)
+        SIMPLE_MODEL(NSYM,_init)(&run_model[i], MAX_RUN);
+
+    RangeCoder rc;
+    RC_SetOutput(&rc, (char *)out+1);
+    RC_SetOutputEnd(&rc, (char *)out + *out_size);
+    RC_StartEncode(&rc);
+
+    unsigned char last = 0;
+    for (i = 0; i < in_size;) {
+        //SIMPLE_MODEL(256, _encodeSymbol)(&byte_model, &rc, in[i]);
+        SIMPLE_MODEL(256, _encodeSymbol)(&byte_model[last], &rc, in[i]);
+        //fprintf(stderr, "lit %c (ctx %c)\n", in[i], last);
+        int run = 0;
+        last = in[i++];
+        while (i < in_size && in[i] == last/* && run < MAX_RUN-1*/)
+            run++, i++;
+        int rctx = last;
+        do {
+            int c = run < MAX_RUN ? run : MAX_RUN-1;
+            SIMPLE_MODEL(NSYM, _encodeSymbol)(&run_model[rctx], &rc, c);
+            run -= c;
+
+            if (rctx == last)
+                rctx = 256;
+            else
+                rctx += (rctx < NSYM-1);
+            if (c == MAX_RUN-1 && run == 0)
+                SIMPLE_MODEL(NSYM, _encodeSymbol)(&run_model[rctx], &rc, 0);
+        } while (run);
+    }
+
+    if (RC_FinishEncode(&rc) < 0) {
+        htscodecs_tls_free(byte_model);
+        htscodecs_tls_free(run_model);
+        free(out_free);
+        return NULL;
+    }
+
+    // Finalise block size and return it
+    *out_size = RC_OutSize(&rc)+1;
+
+    //fprintf(stderr, "RLE %d to %d\n", in_size, *out_size);
+
+    htscodecs_tls_free(byte_model);
+    htscodecs_tls_free(run_model);
+    return out;
+}
+
+static
+unsigned char *arith_uncompress_O1_RLE(unsigned char *in, unsigned int in_size,
+                                       unsigned char *out, unsigned int out_sz) {
+    RangeCoder rc;
+    int i;
+    unsigned int m = in[0] ? in[0] : 256;
+    unsigned char *out_free = NULL;
+
+    if (!out)
+        out_free = out = malloc(out_sz);
+    if (!out)
+        return NULL;
+
+    SIMPLE_MODEL(256,_) *byte_model =
+        htscodecs_tls_alloc(256 * sizeof(*byte_model));
+    if (!byte_model) {
+        free(out_free);
+        return NULL;
+    }
+    for (i = 0; i < 256; i++)
+        SIMPLE_MODEL(256,_init)(&byte_model[i], m);
+
+    SIMPLE_MODEL(NSYM,_) *run_model =
+        htscodecs_tls_alloc(NSYM * sizeof(*run_model));
+    if (!run_model) {
+        htscodecs_tls_free(byte_model);
+        free(out_free);
+        return NULL;
+    }
+    for (i = 0; i < NSYM; i++)
+        SIMPLE_MODEL(NSYM,_init)(&run_model[i], MAX_RUN);
+
+    RC_SetInput(&rc, (char *)in+1, (char *)in+in_size);
+    RC_StartDecode(&rc);
+
+    unsigned char last = 0;
+    for (i = 0; i < out_sz; i++) {
+        out[i] = SIMPLE_MODEL(256, _decodeSymbol)(&byte_model[last], &rc);
+        //fprintf(stderr, "lit %c (ctx %c)\n", out[i], last);
+        last = out[i];
+        int run = 0, r = 0, rctx = last;
+
+        do {
+            r = SIMPLE_MODEL(NSYM, _decodeSymbol)(&run_model[rctx], &rc);
+            if (rctx == last)
+                rctx = 256;
+            else
+                rctx += (rctx < NSYM-1);
+            run += r;
+        } while (r == MAX_RUN-1 && run < out_sz);
+        while (run-- && i+1 < out_sz)
+            out[++i] = last;
+    }
+
+    if (RC_FinishDecode(&rc) < 0) {
+        htscodecs_tls_free(byte_model);
+        htscodecs_tls_free(run_model);
+        free(out_free);
+        return NULL;
+    }
+
+    htscodecs_tls_free(byte_model);
+    htscodecs_tls_free(run_model);
+    return out;
+}
+
+/*-----------------------------------------------------------------------------
+ * Simple interface to the order-0 vs order-1 encoders and decoders.
+ *
+ * Smallest is method, <in_size> <input>, so worst case 2 bytes longer.
+ */
+unsigned char *arith_compress_to(unsigned char *in,  unsigned int in_size,
+                                 unsigned char *out, unsigned int *out_size,
+                                 int order) {
+    unsigned int c_meta_len;
+    uint8_t *rle = NULL, *packed = NULL;
+
+    if (in_size > INT_MAX) {
+        *out_size = 0;
+        return NULL;
+    }
+
+    if (!out) {
+        *out_size = arith_compress_bound(in_size, order);
+        if (!(out = malloc(*out_size)))
+            return NULL;
+    }
+    unsigned char *out_end = out + *out_size;
+
+    if (in_size <= 20)
+        order &= ~X_STRIPE;
+
+    if (order & X_CAT) {
+        out[0] = X_CAT;
+        c_meta_len = 1 + var_put_u32(&out[1], out_end, in_size);
+        memcpy(out+c_meta_len, in, in_size);
+        *out_size = in_size+c_meta_len;
+    }
+
+    if (order & X_STRIPE) {
+        int N = (order>>8);
+        if (N == 0) N = 4; // default for compatibility with old tests
+
+        if (N > 255)
+            return NULL;
+
+        unsigned char *transposed = malloc(in_size);
+        unsigned int part_len[256];
+        unsigned int idx[256];
+        if (!transposed)
+            return NULL;
+        int i, j, x;
+
+        for (i = 0; i < N; i++) {
+            part_len[i] = in_size / N + ((in_size % N) > i);
+            idx[i] = i ? idx[i-1] + part_len[i-1] : 0; // cumulative index
+        }
+
+        for (i = x = 0; i < in_size-N; i += N, x++) {
+            for (j = 0; j < N; j++)
+                transposed[idx[j]+x] = in[i+j];
+        }
+        for (; i < in_size; i += N, x++) {
+            for (j = 0; i+j < in_size; j++)
+                transposed[idx[j]+x] = in[i+j];
+        }
+
+        unsigned int olen2;
+        unsigned char *out2, *out2_start;
+        c_meta_len = 1;
+        *out = order & ~X_NOSZ;
+        c_meta_len += var_put_u32(out+c_meta_len, out_end, in_size);
+        out[c_meta_len++] = N;
+
+        out2_start = out2 = out+7+5*N; // shares a buffer with c_meta
+        for (i = 0; i < N; i++) {
+            // Brute force try all methods.
+            // FIXME: optimise this bit.  Maybe learn over time?
+            int j, best_j = 0, best_sz = INT_MAX;
+
+            // Works OK with read names. The first byte is the most important,
+            // as it has most variability (little-endian).  After that it's
+            // often quite predictable.
+            //
+            // Do we gain in any other context in CRAM? Aux tags maybe?
+            int m[][4] = {{3, 1,64,0},
+                          {2, 1,0},
+                          {2, 1,128},
+                          {2, 1,128}};
+
+//          int m[][6] = {{4, 1,64,2,0},  //test of adding in an order-2 codec
+//                        {3, 1,2,0},
+//                        {3, 1,2,128},
+//                        {3, 1,2,128}};
+
+// Other possibilities for methods to try.
+//          int m[][10] = {{8, 1,128,129,64,65,192,193,4,0},
+//                         {8, 1,128,129,64,65,192,193,4,0},
+//                         {8, 1,128,129,64,65,192,193,4,0},
+//                         {8, 1,128,129,64,65,192,193,4,0}};
+
+//          int m[][9] = {{5, 1,128,64,65,0},
+//                        {5, 1,128,64,65,0},
+//                        {5, 1,128,64,65,0},
+//                        {5, 1,128,64,65,0}};
+
+//          int m[][6] = {{4, 0,1,128,64},
+//                        {5, 0,1,128,65,193},
+//                        {3, 0,1,128},
+//                        {3, 0,1,128}};
+
+//          int m[][6] = {{4, 1,128,64,0},
+//                        {4, 1,128,65,0},
+//                        {2, 128,0},
+//                        {2, 128,0}};
+
+//          int m[][6] = {{2, 64,0},
+//                        {1, 0},
+//                        {1, 128},
+//                        {1, 128}};
+
+//          int m[][6] = {{1, 0},
+//                        {2, 128,0},
+//                        {1, 128},
+//                        {1, 128}};
+
+            for (j = 1; j <= m[MIN(i,3)][0]; j++) {
+                olen2 = *out_size - (out2 - out);
+                //fprintf(stderr, "order=%d m=%d\n", order&3, m[MIN(i,4)][j]);
+                if ((order&3) == 0 && (m[MIN(i,3)][j]&1))
+                    continue;
+
+                arith_compress_to(transposed+idx[i], part_len[i],
+                                  out2, &olen2, m[MIN(i,3)][j] | X_NOSZ);
+                if (best_sz > olen2) {
+                    best_sz = olen2;
+                    best_j = j;
+                }
+            }
+//          if (best_j == 0) // none desireable
+//              return NULL;
+            if (best_j != j-1) {
+                olen2 = *out_size - (out2 - out);
+                arith_compress_to(transposed+idx[i], part_len[i],
+                                  out2, &olen2, m[MIN(i,3)][best_j] | X_NOSZ);
+            }
+            out2 += olen2;
+            c_meta_len += var_put_u32(out+c_meta_len, out_end, olen2);
+        }
+        memmove(out+c_meta_len, out2_start, out2-out2_start);
+        free(transposed);
+        *out_size = c_meta_len + out2-out2_start;
+        return out;
+    }
+
+    int do_pack = order & X_PACK;
+    int do_rle  = order & X_RLE;
+    int no_size = order & X_NOSZ;
+    int do_ext  = order & X_EXT;
+
+    out[0] = order;
+    c_meta_len = 1;
+
+    if (!no_size)
+        c_meta_len += var_put_u32(&out[1], out_end, in_size);
+
+    order &= 0x3;
+
+    // Format is compressed meta-data, compressed data.
+    // Meta-data can be empty, pack, rle lengths, or pack + rle lengths.
+    // Data is either the original data, bit-packed packed, rle literals or
+    // packed + rle literals.
+
+    if (do_pack && in_size) {
+        // PACK 2, 4 or 8 symbols into one byte.
+        int pmeta_len;
+        uint64_t packed_len;
+        packed = hts_pack(in, in_size, out+c_meta_len, &pmeta_len, &packed_len);
+        if (!packed) {
+            out[0] &= ~X_PACK;
+            do_pack = 0;
+            free(packed);
+            packed = NULL;
+        } else {
+            in = packed;
+            in_size = packed_len;
+            c_meta_len += pmeta_len;
+
+            // Could derive this rather than storing verbatim.
+            // Orig size * 8/nbits (+1 if not multiple of 8/n)
+            int sz = var_put_u32(out+c_meta_len, out_end, in_size);
+            c_meta_len += sz;
+            *out_size -= sz;
+        }
+    } else if (do_pack) {
+        out[0] &= ~X_PACK;
+    }
+
+    if (do_rle && !in_size) {
+        out[0] &= ~X_RLE;
+    }
+
+    *out_size -= c_meta_len;
+    if (order && in_size < 8) {
+        out[0] &= ~3;
+        order  &= ~3;
+    }
+
+    if (do_ext) {
+        // Use an external compression library instead.
+        // For now, bzip2
+#ifdef HAVE_LIBBZ2
+        if (BZ_OK != BZ2_bzBuffToBuffCompress((char *)out+c_meta_len, out_size,
+                                              (char *)in, in_size, 9, 0, 30))
+            *out_size = in_size; // Didn't fit with bz2; force X_CAT below instead
+#else
+        fprintf(stderr, "Htscodecs has been compiled without libbz2 support\n");
+        free(out);
+        return NULL;
+#endif
+
+//      // lzma doesn't help generally, at least not for the name tokeniser
+//      size_t lzma_size = 0;
+//      lzma_easy_buffer_encode(9, LZMA_CHECK_CRC32, NULL,
+//                              in, in_size, out+c_meta_len, &lzma_size,
+//                              *out_size);
+//      *out_size = lzma_size;
+
+    } else {
+        if (do_rle) {
+            if (order == 0)
+                arith_compress_O0_RLE(in, in_size, out+c_meta_len, out_size);
+            else
+                arith_compress_O1_RLE(in, in_size, out+c_meta_len, out_size);
+        } else {
+            //if (order == 2)
+            //  arith_compress_O2(in, in_size, out+c_meta_len, out_size);
+            //else
+            if (order == 1)
+                arith_compress_O1(in, in_size, out+c_meta_len, out_size);
+            else
+                arith_compress_O0(in, in_size, out+c_meta_len, out_size);
+        }
+    }
+
+    if (*out_size >= in_size) {
+        out[0] &= ~(3|X_EXT); // no entropy encoding, but keep e.g. PACK
+        out[0] |= X_CAT | no_size;
+        memcpy(out+c_meta_len, in, in_size);
+        *out_size = in_size;
+    }
+
+    free(rle);
+    free(packed);
+
+    *out_size += c_meta_len;
+
+    return out;
+}
+
+unsigned char *arith_compress(unsigned char *in, unsigned int in_size,
+                              unsigned int *out_size, int order) {
+    return arith_compress_to(in, in_size, NULL, out_size, order);
+}
+
+unsigned char *arith_uncompress_to(unsigned char *in,  unsigned int in_size,
+                                   unsigned char *out, unsigned int *out_size) {
+    unsigned char *in_end = in + in_size;
+    unsigned char *out_free = NULL;
+    unsigned char *tmp_free = NULL;
+
+    if (in_size == 0)
+        return NULL;
+
+    if (*in & X_STRIPE) {
+        unsigned int ulen, olen, c_meta_len = 1;
+        int i;
+        uint64_t clen_tot = 0;
+
+        // Decode lengths
+        c_meta_len += var_get_u32(in+c_meta_len, in_end, &ulen);
+        if (c_meta_len >= in_size)
+            return NULL;
+        unsigned int N = in[c_meta_len++];
+        if (N < 1)  // Must be at least one stripe
+            return NULL;
+        unsigned int clenN[256], ulenN[256], idxN[256];
+        if (!out) {
+            if (ulen >= INT_MAX)
+                return NULL;
+            if (!(out_free = out = malloc(ulen))) {
+                return NULL;
+            }
+            *out_size = ulen;
+        }
+        if (ulen != *out_size) {
+            free(out_free);
+            return NULL;
+        }
+
+        for (i = 0; i < N; i++) {
+            ulenN[i] = ulen / N + ((ulen % N) > i);
+            idxN[i] = i ? idxN[i-1] + ulenN[i-1] : 0;
+            c_meta_len += var_get_u32(in+c_meta_len, in_end, &clenN[i]);
+            clen_tot += clenN[i];
+            if (c_meta_len > in_size || clenN[i] > in_size || clenN[i] < 1) {
+                free(out_free);
+                return NULL;
+            }
+        }
+
+        // We can call this with a larger buffer, but once we've determined
+        // how much we really use we limit it so the recursion becomes easier
+        // to limit.
+        if (c_meta_len + clen_tot > in_size) {
+            free(out_free);
+            return NULL;
+        }
+        in_size = c_meta_len + clen_tot;
+
+        //fprintf(stderr, "    stripe meta %d\n", c_meta_len); //c-size
+
+        // Uncompress the N streams
+        unsigned char *outN = malloc(ulen);
+        if (!outN) {
+            free(out_free);
+            return NULL;
+        }
+        for (i = 0; i < N; i++) {
+            olen = ulenN[i];
+            if (in_size < c_meta_len) {
+                free(out_free);
+                free(outN);
+                return NULL;
+            }
+            if (!arith_uncompress_to(in+c_meta_len, in_size-c_meta_len, outN + idxN[i], &olen)
+                || olen != ulenN[i]) {
+                free(out_free);
+                free(outN);
+                return NULL;
+            }
+            c_meta_len += clenN[i];
+        }
+
+        unstripe(out, outN, ulen, N, idxN);
+
+        free(outN);
+        *out_size = ulen;
+        return out;
+    }
+
+    int order = *in++;  in_size--;
+    int do_pack = order & X_PACK;
+    int do_rle  = order & X_RLE;
+    int do_cat  = order & X_CAT;
+    int no_size = order & X_NOSZ;
+    int do_ext  = order & X_EXT;
+    order &= 3;
+
+    int sz = 0;
+    unsigned int osz;
+    if (!no_size)
+        sz = var_get_u32(in, in_end, &osz);
+    else
+        sz = 0, osz = *out_size;
+    in += sz;
+    in_size -= sz;
+
+    if (osz >= INT_MAX)
+        return NULL;
+
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+        // Limit maximum size to get fast turnaround on fuzzing test cases
+        if (osz > 100000)
+            goto err;
+#endif
+
+    if (no_size && !out)
+        return NULL; // Need one or the other
+
+    if (!out) {
+        *out_size = osz;
+        if (!(out_free = out = malloc(*out_size)))
+            return NULL;
+    } else {
+        if (*out_size < osz)
+            return NULL;
+        *out_size = osz;
+    }
+
+    uint32_t c_meta_size = 0;
+    unsigned int tmp1_size = *out_size;
+    unsigned int tmp2_size = *out_size;
+    unsigned char *tmp1 = NULL, *tmp2 = NULL, *tmp = NULL;
+
+    // Need In, Out and Tmp buffers with temporary buffer of the same size
+    // as output.  Our entropy decode is either arithmetic (with/without RLE)
+    // or external (bz2, gzip, lzma) but with an optional unPACK transform
+    // at the end.
+    //
+    // To avoid pointless memcpy when unpacking we switch around which
+    // buffers we're writing to accordingly.
+
+    // Format is pack meta data if present, followed by compressed data.
+    if (do_pack) {
+        if (!(tmp_free = tmp = malloc(*out_size)))
+            goto err;
+        tmp1 = tmp;  // uncompress
+        tmp2 = out;  // unpack
+    } else {
+        // no pack
+        tmp  = NULL;
+        tmp1 = out;  // uncompress
+        tmp2 = out;  // NOP
+    }
+
+    
+    // Decode the bit-packing map.
+    uint8_t map[16] = {0};
+    int npacked_sym = 0;
+    uint64_t unpacked_sz = 0; // FIXME: rename to packed_per_byte
+    if (do_pack) {
+        c_meta_size = hts_unpack_meta(in, in_size, *out_size, map, &npacked_sym);
+        if (c_meta_size == 0)
+            goto err;
+
+        unpacked_sz = osz;
+        in      += c_meta_size;
+        in_size -= c_meta_size;
+
+        // New unpacked size.  We could derive this bit from *out_size
+        // and npacked_sym.
+        unsigned int osz;
+        sz = var_get_u32(in, in_end, &osz);
+        in += sz;
+        in_size -= sz;
+        if (osz > tmp1_size)
+            goto err;
+        tmp1_size = osz;
+    }
+
+    //fprintf(stderr, "    meta_size %d bytes\n", (int)(in - orig_in)); //c-size
+
+    // uncompress RLE data.  in -> tmp1
+    if (in_size) {
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+        // Limit maximum size to get fast turnaround on fuzzing test cases
+        if (tmp1_size > 100000)
+            goto err;
+#endif
+        if (do_cat) {
+            //fprintf(stderr, "    CAT %d\n", tmp1_size); //c-size
+            if (tmp1_size > in_size)
+                goto err;
+            if (tmp1_size > *out_size)
+                goto err;
+            memcpy(tmp1, in, tmp1_size);
+        } else if (do_ext) {
+#ifdef HAVE_LIBBZ2
+            if (BZ_OK != BZ2_bzBuffToBuffDecompress((char *)tmp1, &tmp1_size,
+                                                    (char *)in, in_size, 0, 0))
+                goto err;
+#else
+            fprintf(stderr, "Htscodecs has been compiled without libbz2 support\n");
+            goto err;
+#endif
+          } else {
+            // in -> tmp1
+            if (do_rle) {
+                tmp1 = order == 1
+                    ? arith_uncompress_O1_RLE(in, in_size, tmp1, tmp1_size)
+                    : arith_uncompress_O0_RLE(in, in_size, tmp1, tmp1_size);
+            } else {
+                //if (order == 2)
+                //    tmp1 = arith_uncompress_O2(in, in_size, tmp1, tmp1_size)
+                //else
+                tmp1 = order == 1
+                    ? arith_uncompress_O1(in, in_size, tmp1, tmp1_size)
+                    : arith_uncompress_O0(in, in_size, tmp1, tmp1_size);
+            }
+            if (!tmp1)
+                goto err;
+        }
+    } else {
+        tmp1_size = 0;
+    }
+
+    if (do_pack) {
+        // Unpack bits via pack-map.  tmp1 -> tmp2
+        if (npacked_sym == 1)
+            unpacked_sz = tmp1_size;
+        //uint8_t *porig = unpack(tmp2, tmp2_size, unpacked_sz, npacked_sym, map);
+        //memcpy(tmp3, porig, unpacked_sz);
+        if (!hts_unpack(tmp1, tmp1_size, tmp2, unpacked_sz, npacked_sym, map))
+            goto err;
+        tmp2_size = unpacked_sz;
+    } else {
+        tmp2_size = tmp1_size;
+    }
+
+    if (tmp)
+        free(tmp);
+
+    *out_size = tmp2_size;
+    return tmp2;
+
+ err:
+    free(tmp_free);
+    free(out_free);
+    return NULL;
+}
+
+unsigned char *arith_uncompress(unsigned char *in, unsigned int in_size,
+                                unsigned int *out_size) {
+    return arith_uncompress_to(in, in_size, NULL, out_size);
+}
diff --git a/ext/htslib/htscodecs/htscodecs/arith_dynamic.h b/ext/htslib/htscodecs/htscodecs/arith_dynamic.h
new file mode 100644
index 0000000..2ae2033
--- /dev/null
+++ b/ext/htslib/htscodecs/htscodecs/arith_dynamic.h
@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2019 Genome Research Ltd.
+ * Author(s): James Bonfield
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above
+ *       copyright notice, this list of conditions and the following
+ *       disclaimer in the documentation and/or other materials provided
+ *       with the distribution.
+ *
+ *    3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+ *       Institute nor the names of its contributors may be used to endorse
+ *       or promote products derived from this software without specific
+ *       prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH
+ * LTD OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef ARITH_DYNAMIC_H
+#define ARITH_DYNAMIC_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+unsigned char *arith_compress(unsigned char *in, unsigned int in_size,
+                              unsigned int *out_size, int order);
+
+unsigned char *arith_uncompress(unsigned char *in, unsigned int in_size,
+                                unsigned int *out_size);
+
+unsigned char *arith_compress_to(unsigned char *in,  unsigned int in_size,
+                                 unsigned char *out, unsigned int *out_size,
+                                 int order);
+
+unsigned char *arith_uncompress_to(unsigned char *in, unsigned int in_size,
+                                   unsigned char *out, unsigned int *out_sz);
+
+unsigned int arith_compress_bound(unsigned int size, int order);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* ARITH_DYNAMIC_H */
diff --git a/ext/htslib/htscodecs/htscodecs/c_range_coder.h b/ext/htslib/htscodecs/htscodecs/c_range_coder.h
new file mode 100644
index 0000000..df29968
--- /dev/null
+++ b/ext/htslib/htscodecs/htscodecs/c_range_coder.h
@@ -0,0 +1,166 @@
+// Copyright Eugene Shelwien.
+// Release into public domain.
+
+// Modifications by James Bonfield (2019)
+
+
+/*
+ * Note it is up to the calling code to ensure that no overruns on input and
+ * output buffers occur.
+ *
+ * Call the input() and output() functions to set and query the current
+ * buffer locations.
+ *
+
+ */
+
+#ifndef C_RANGER_CODER_H
+#define C_RANGER_CODER_H
+
+#define  DO(n)     int _;for (_=0; _<n; _++)
+#define  TOP       (1<<24)
+#define  Thres (unsigned)255*TOP
+
+typedef unsigned char uc;
+
+typedef struct {
+    uint32_t low, code, range;
+    uint32_t FFNum;  // Number of consecutive FFs
+    uint32_t Cache;  // Top 8-bits of low ready to emit
+    uint32_t Carry;  // Flag to indicate if we emit Cache or Cache+1
+    uc *in_buf;
+    uc *out_buf;
+    uc *in_end;
+    uc *out_end;
+    int err;
+} RangeCoder;
+
+static inline void RC_SetInput(RangeCoder *rc, char *in, char *in_end) {
+    rc->out_buf = rc->in_buf = (uc *)in;
+    rc->in_end = (uc *)in_end;
+}
+
+// NB: call RC_SetOutput first, and then RC_SetOutputEnd
+static inline void RC_SetOutput(RangeCoder *rc, char *out) { rc->in_buf = rc->out_buf = (uc *)out; rc->out_end = NULL;}
+static inline void RC_SetOutputEnd(RangeCoder *rc, char *out_end) { rc->out_end = (uc *)out_end; }
+static inline char *RC_GetInput(RangeCoder *rc) { return (char *)rc->in_buf; }
+static inline char *RC_GetOutput(RangeCoder *rc) { return (char *)rc->out_buf; }
+static inline size_t RC_OutSize(RangeCoder *rc) { return rc->out_buf - rc->in_buf; }
+static inline size_t RC_InSize(RangeCoder *rc) { return rc->in_buf - rc->out_buf; }
+
+static inline void RC_StartEncode(RangeCoder *rc)
+{ 
+    rc->range = 0xFFFFFFFF;
+    rc->low   = 0;
+    rc->FFNum = 0;
+    rc->Carry = 0;
+    rc->Cache = 0;
+    rc->code  = 0;
+    rc->err   = 0;
+}
+
+static inline void RC_StartDecode(RangeCoder *rc)
+{ 
+    rc->range = 0xFFFFFFFF;
+    rc->low   = 0;
+    rc->FFNum = 0;
+    rc->Carry = 0;
+    rc->Cache = 0;
+    rc->code  = 0;
+    rc->err   = 0;
+    if (rc->in_buf+5 > rc->in_end) {
+        rc->in_buf = rc->in_end; // prevent decode
+        return;
+    }
+    DO(5) rc->code = (rc->code<<8) | *rc->in_buf++;
+}
+
+static inline void RC_ShiftLowCheck(RangeCoder *rc) {
+    if (rc->low < Thres || rc->Carry) {
+        if (rc->out_end && rc->FFNum >= rc->out_end - rc->out_buf) {
+            rc->err = -1;
+            return;
+        }
+
+        *rc->out_buf++ = rc->Cache + rc->Carry;
+
+        // Flush any stored FFs
+        while (rc->FFNum) {
+            *rc->out_buf++ = rc->Carry-1; // (Carry-1)&255;
+            rc->FFNum--;
+        }
+
+        // Take copy of top byte ready for next flush
+        rc->Cache = rc->low >> 24;
+        rc->Carry = 0;
+    } else {
+        // Low if FFxx xxxx.  Bump FF count and shift in as before
+        rc->FFNum++;
+    }
+    rc->low = rc->low<<8;
+}
+
+static inline void RC_ShiftLow(RangeCoder *rc) {
+    if (rc->low < Thres || rc->Carry) {
+        *rc->out_buf++ = rc->Cache + rc->Carry;
+
+        // Flush any stored FFs
+        while (rc->FFNum) {
+            *rc->out_buf++ = rc->Carry-1; // (Carry-1)&255;
+            rc->FFNum--;
+        }
+
+        // Take copy of top byte ready for next flush
+        rc->Cache = rc->low >> 24;
+        rc->Carry = 0;
+    } else {
+        // Low if FFxx xxxx.  Bump FF count and shift in as before
+        rc->FFNum++;
+    }
+    rc->low = rc->low<<8;
+}
+
+static inline int RC_FinishEncode(RangeCoder *rc)
+{ 
+    DO(5) RC_ShiftLowCheck(rc);
+    return rc->err;
+}
+
+static inline int RC_FinishDecode(RangeCoder *rc) {
+    return rc->err;
+}
+
+static inline void RC_Encode (RangeCoder *rc, uint32_t cumFreq, uint32_t freq, uint32_t totFreq) 
+{
+    uint32_t tmp = rc->low;
+    rc->low  += cumFreq * (rc->range/= totFreq);
+    rc->range*= freq;
+
+    rc->Carry += rc->low<tmp; // Overflow
+
+    while (rc->range < TOP) {
+        rc->range <<= 8;
+        RC_ShiftLowCheck(rc);
+    }
+}
+
+static inline uint32_t RC_GetFreq (RangeCoder *rc, uint32_t totFreq) {
+    //return rc->code/(rc->range/=totFreq);
+    return (totFreq && rc->range >= totFreq) ? rc->code/(rc->range/=totFreq) : 0;
+}
+
+static inline void RC_Decode (RangeCoder *rc, uint32_t cumFreq, uint32_t freq, uint32_t totFreq) 
+{
+    rc->code -= cumFreq * rc->range;
+    rc->range *= freq;
+    while (rc->range < TOP) {
+        if (rc->in_buf >= rc->in_end) {
+            rc->err = -1;
+            return;
+        }
+        rc->code = (rc->code<<8) + *rc->in_buf++;
+        rc->range <<= 8;
+    }
+}
+
+#endif /* C_RANGER_CODER_H */
diff --git a/ext/htslib/htscodecs/htscodecs/c_simple_model.h b/ext/htslib/htscodecs/htscodecs/c_simple_model.h
new file mode 100644
index 0000000..0c81430
--- /dev/null
+++ b/ext/htslib/htscodecs/htscodecs/c_simple_model.h
@@ -0,0 +1,171 @@
+/*
+ * Copyright (c) 2012, 2018-2019 Genome Research Ltd.
+ * Author(s): James Bonfield
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above
+ *       copyright notice, this list of conditions and the following
+ *       disclaimer in the documentation and/or other materials provided
+ *       with the distribution.
+ *
+ *    3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+ *       Institute nor the names of its contributors may be used to endorse
+ *       or promote products derived from this software without specific
+ *       prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH
+ * LTD OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdint.h>
+#include "c_range_coder.h"
+
+/*
+ *--------------------------------------------------------------------------
+ * A simple frequency model.
+ *
+ * Define NSYM to be an integer value before including this file.
+ * It will then generate types and functions specific to that
+ * maximum number of symbols.
+ *
+ * This keeps a list of symbols and their frequencies, approximately
+ * sorted by symbol frequency. We allow for a single symbol to periodically
+ * move up the list when emitted, effectively doing a single step of
+ * bubble sort periodically. This means it's largely the same complexity
+ * irrespective of alphabet size.
+ * It's more efficient on strongly biased distributions than random data.
+ *
+ * There is no escape symbol, so the model is tailored to relatively
+ * stationary samples (although we do have occasional normalisation to
+ * avoid frequency counters getting too high).
+ *--------------------------------------------------------------------------
+ */
+
+//-----------------------------------------------------------------------------
+// Bits we want included once only - constants, types, etc
+#ifndef C_SIMPLE_MODEL_H
+#define C_SIMPLE_MODEL_H
+
+#define MAX_FREQ (1<<16)-17
+#define PASTE3(a,b,c) a##b##c
+#define SIMPLE_MODEL(a,b) PASTE3(SIMPLE_MODEL,a,b)
+#define STEP 16
+typedef struct {
+    uint16_t Freq;
+    uint16_t Symbol;
+} SymFreqs;
+#endif /* C_SIMPLE_MODEL_H */
+
+
+//-----------------------------------------------------------------------------
+// Bits we regenerate for each NSYM value.
+
+typedef struct {
+    uint32_t TotFreq;  // Total frequency
+
+    // Array of Symbols approximately sorted by Freq. 
+    SymFreqs sentinel, F[NSYM+1], terminal;
+} SIMPLE_MODEL(NSYM,_);
+
+
+static inline void SIMPLE_MODEL(NSYM,_init)(SIMPLE_MODEL(NSYM,_) *m, int max_sym) {
+    int i;
+    
+    for (i=0; i<max_sym; i++) {
+        m->F[i].Symbol = i;
+        m->F[i].Freq   = 1;
+    }
+    for (; i<NSYM; i++) {
+        m->F[i].Symbol = i;
+        m->F[i].Freq   = 0;
+    }
+
+    m->TotFreq         = max_sym;
+    m->sentinel.Symbol = 0;
+    m->sentinel.Freq   = MAX_FREQ; // Always first; simplifies sorting.
+    m->terminal.Symbol = 0;
+    m->terminal.Freq   = MAX_FREQ;
+    m->F[NSYM].Freq    = 0; // terminates normalize() loop. See below.
+}
+
+
+static inline void SIMPLE_MODEL(NSYM,_normalize)(SIMPLE_MODEL(NSYM,_) *m) {
+    SymFreqs *s;
+
+    /* Faster than F[i].Freq for 0 <= i < NSYM */
+    m->TotFreq=0;
+    for (s = m->F; s->Freq; s++) {
+        s->Freq -= s->Freq>>1;
+        m->TotFreq += s->Freq;
+    }
+}
+
+static inline void SIMPLE_MODEL(NSYM,_encodeSymbol)(SIMPLE_MODEL(NSYM,_) *m,
+                                                    RangeCoder *rc, uint16_t sym) {
+    SymFreqs *s = m->F;
+    uint32_t AccFreq  = 0;
+
+    while (s->Symbol != sym)
+        AccFreq += s++->Freq;
+
+    RC_Encode(rc, AccFreq, s->Freq, m->TotFreq);
+    s->Freq    += STEP;
+    m->TotFreq += STEP;
+
+    if (m->TotFreq > MAX_FREQ)
+        SIMPLE_MODEL(NSYM,_normalize)(m);
+
+    /* Keep approx sorted */
+    if (s[0].Freq > s[-1].Freq) {
+        SymFreqs t = s[0];
+        s[0] = s[-1];
+        s[-1] = t;
+    }
+}
+
+static inline uint16_t SIMPLE_MODEL(NSYM,_decodeSymbol)(SIMPLE_MODEL(NSYM,_) *m, RangeCoder *rc) {
+    SymFreqs* s = m->F;
+    uint32_t freq = RC_GetFreq(rc, m->TotFreq);
+    uint32_t AccFreq;
+
+    if (freq > MAX_FREQ)
+        return 0; // error
+
+    for (AccFreq = 0; (AccFreq += s->Freq) <= freq; s++)
+        ;
+    if (s - m->F > NSYM)
+        return 0; // error
+
+    AccFreq -= s->Freq;
+
+    RC_Decode(rc, AccFreq, s->Freq, m->TotFreq);
+    s->Freq    += STEP;
+    m->TotFreq += STEP;
+
+    if (m->TotFreq > MAX_FREQ)
+        SIMPLE_MODEL(NSYM,_normalize)(m);
+
+    /* Keep approx sorted */
+    if (s[0].Freq > s[-1].Freq) {
+        SymFreqs t = s[0];
+        s[0] = s[-1];
+        s[-1] = t;
+        return t.Symbol;
+    }
+
+    return s->Symbol;
+}
diff --git a/ext/htslib/htscodecs/htscodecs/fqzcomp_qual.c b/ext/htslib/htscodecs/htscodecs/fqzcomp_qual.c
new file mode 100644
index 0000000..a5b6687
--- /dev/null
+++ b/ext/htslib/htscodecs/htscodecs/fqzcomp_qual.c
@@ -0,0 +1,1630 @@
+/*
+ * Copyright (c) 2011-2013, 2018-2022 Genome Research Ltd.
+ * Author(s): James Bonfield
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above
+ *       copyright notice, this list of conditions and the following
+ *       disclaimer in the documentation and/or other materials provided
+ *       with the distribution.
+ *
+ *    3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+ *       Institute nor the names of its contributors may be used to endorse
+ *       or promote products derived from this software without specific
+ *       prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH
+ * LTD OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+// We use generic maps to turn 0-M into 0-N where N <= M
+// before adding these into the context.  These are used
+// for positions, running-diffs and quality values.
+//
+// This can be used as a simple divisor, eg pos/24 to get
+// 2 bits of positional data for each quarter along a 100bp
+// read, or it can be tailored for specific such as noting
+// the first 5 cycles are poor, then we have stability and
+// a gradual drop off in the last 20 or so.  Perhaps we then
+// map pos 0-4=0, 5-79=1, 80-89=2, 90-99=3.
+//
+// We don't need to specify how many bits of data we are
+// using (2 in the above example), as that is just implicit
+// in the values in the map.  Specify not to use a map simply
+// disables that context type (our map is essentially 0-M -> 0).
+
+// Example of command line usage:
+//
+// f=~/scratch/data/q4
+// cc -Wall -DTEST_MAIN -O3 -g fqzcomp_qual2.c -lm
+// ./a.out $f > /tmp/_ && ./a.out -d < /tmp/_ > /tmp/__ && cmp /tmp/__ $f
+
+#include "config.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <string.h>
+#include <math.h>
+#include <limits.h>
+#include <ctype.h>
+#include <math.h>
+#include <inttypes.h>
+#include <sys/types.h>
+
+#include "fqzcomp_qual.h"
+#include "varint.h"
+#include "utils.h"
+
+#define CTX_BITS 16
+#define CTX_SIZE (1<<CTX_BITS)
+
+#ifndef MIN
+#  define MIN(a,b) ((a)<(b)?(a):(b))
+#  define MAX(a,b) ((a)>(b)?(a):(b))
+#endif
+
+#define QMAX 256
+#define QBITS 12
+#define QSIZE (1<<QBITS)
+
+#define NSYM 2
+#include "c_simple_model.h"
+
+#undef NSYM
+#define NSYM QMAX
+//#include "c_escape_model.h"
+#include "c_simple_model.h"
+//#include "c_cdf_model.h"
+//#include "c_cdf16_model.h"
+
+// An array of 0,0,0, 1,1,1,1, 3, 5,5
+// is turned into a run-length of 3x0, 4x1, 0x2, 1x4, 0x4, 2x5,
+// which then becomes 3 4 0 1 0 2.
+//
+// NB: size > 255 therefore means we need to repeatedly read to find
+// the actual run length.
+// Alternatively we could bit-encode instead of byte encode, eg BETA.
+static int store_array(unsigned char *out, unsigned int *array, int size) {
+    unsigned char tmp[2048];
+
+    int i, j, k;
+    for (i = j = k = 0; i < size; j++) {
+        int run_len = i;
+        while (i < size && array[i] == j)
+            i++;
+        run_len = i-run_len;
+
+        int r;
+        do {
+            r = MIN(255, run_len);
+            tmp[k++] = r;
+            run_len -= r;
+        } while (r == 255);
+    }
+    while (i < size)
+        tmp[k++] = 0, j++;
+
+    // RLE on out.
+    //    1 2 3 3 3 3 3 4 4    5
+    // => 1 2 3 3 +3... 4 4 +0 5
+    int last = -1;
+    for (i = j = 0; j < k; i++) {
+        out[i] = tmp[j++];
+        if (out[i] == last) {
+            int n = j;
+            while (j < k && tmp[j] == last)
+                j++;
+            out[++i] = j-n;
+        } else {
+            last = out[i];
+        }
+    }
+    k = i;
+
+//    fprintf(stderr, "Store_array %d => %d {", size, k);
+//    for (i = 0; i < k; i++)
+//      fprintf(stderr, "%d,", out[i]);
+//    fprintf(stderr, "}\n");
+    return k;
+}
+
+static int read_array(unsigned char *in, size_t in_size, unsigned int *array, int size) {
+    unsigned char R[1024];
+    int i, j, z, last = -1, nb = 0;
+
+    size = MIN(1024, size);
+
+    // Remove level one of run-len encoding
+    for (i = j = z = 0; z < size && i < in_size; i++) {
+        int run = in[i];
+        R[j++] = run;
+        z += run;
+        if (run == last) {
+            if (i+1 >= in_size)
+                return -1;
+            int copy = in[++i];
+            z += run * copy;
+            while (copy-- && z <= size && j < 1024)
+                R[j++] = run;
+        }
+        if (j >= 1024)
+            return -1;
+        last = run;
+    }
+    nb = i;
+
+    // Now expand inner level of run-length encoding
+    int R_max = j;
+    for (i = j = z = 0; j < size; i++) {
+        int run_len = 0;
+        int run_part;
+        if (z >= R_max)
+            return -1;
+        do {
+            run_part = R[z++];
+            run_len += run_part;
+        } while (run_part == 255 && z < R_max);
+        if (run_part == 255)
+            return -1;
+
+        while (run_len && j < size)
+            run_len--, array[j++] = i;
+    }
+
+    return nb;
+}
+
+// FIXME: how to auto-tune these rather than trial and error?
+// r2 = READ2
+// qa = qual avg (0, 2, 4)
+static int strat_opts[][12] = {
+//   qb  qs pb ps db ds ql sl pl  dl  r2 qa
+    {10, 5, 4,-1, 2, 1, 0, 14, 10, 14, 0,-1}, // basic options (level < 7)
+    {8,  5, 7, 0, 0, 0, 0, 14, 8,  14, 1,-1}, // e.g. HiSeq 2000
+    {12, 6, 2, 0, 2, 3, 0, 9,  12, 14, 0, 0}, // e.g. MiSeq
+    {12, 6, 0, 0, 0, 0, 0, 12, 0,  0,  0, 0}, // e.g. IonTorrent; adaptive O1
+    {0,  0, 0, 0, 0, 0, 0, 0,  0,  0,  0, 0}, // custom
+};
+static int nstrats = sizeof(strat_opts) / sizeof(*strat_opts);
+
+#ifdef HAVE_BUILTIN_PREFETCH
+static inline void mm_prefetch(void *x) {
+    __builtin_prefetch(x);
+}
+#else
+static inline void mm_prefetch(void *x) {
+    // Fetch and discard is quite close to a genuine prefetch
+    *(volatile int *)x;
+}
+#endif
+
+typedef struct {
+    unsigned int qctx;  // quality sub-context
+    unsigned int p;     // pos (bytes remaining)
+    unsigned int delta; // delta running total
+    unsigned int prevq; // previous quality
+    unsigned int s;     // selector
+    unsigned int qtot, qlen;
+    unsigned int first_len;
+    unsigned int last_len;
+    ssize_t rec;
+    unsigned int ctx;
+} fqz_state;
+
+static void dump_table(unsigned int *tab, int size, char *name) {
+    int i, last = -99, run = 0;
+    fprintf(stderr, "\t%s\t{", name);
+    for (i = 0; i < size; i++) {
+        if (tab[i] == last) {
+            run++;
+        } else if (run == 1 && tab[i] == last+1) {
+            int first = last;
+            do {
+                last = tab[i];
+                i++;
+            } while (i < size && tab[i] == last+1);
+            i--;
+
+            // Want 0,1,2,3,3,3 as 0..2 3x3, not 0..3 3x2
+            if (tab[i] == tab[i+1])
+                i--;
+            if (tab[i] != first)
+                fprintf(stderr, "..%d", tab[i]);
+            run = 1;
+            last = -99;
+        } else {
+            if (run > 1)
+                fprintf(stderr, " x %d%s%d", run, i?", ":"", tab[i]);
+            else
+                fprintf(stderr, "%s%d", i?", ":"", tab[i]);
+            run = 1;
+            last = tab[i];
+        }
+    }
+    if (run > 1)
+        fprintf(stderr, " x %d", run);
+    fprintf(stderr, "}\n");
+}
+
+static void dump_map(unsigned int *map, int size, char *name) {
+    int i, c = 0;
+    fprintf(stderr, "\t%s\t{", name);
+    for (i = 0; i < size; i++)
+        if (map[i] != INT_MAX)
+            fprintf(stderr, "%s%d=%d", c++?", ":"", i, map[i]);
+    fprintf(stderr, "}\n");
+}
+
+#pragma GCC diagnostic ignored "-Wunused-function"
+static void dump_params(fqz_gparams *gp) {
+    fprintf(stderr, "Global params = {\n");
+    fprintf(stderr, "\tvers\t%d\n", gp->vers);
+    fprintf(stderr, "\tgflags\t0x%02x\n", gp->gflags);
+    fprintf(stderr, "\tnparam\t%d\n", gp->nparam);
+    fprintf(stderr, "\tmax_sel\t%d\n", gp->max_sel);
+    fprintf(stderr, "\tmax_sym\t%d\n", gp->max_sym);
+    if (gp->gflags & GFLAG_HAVE_STAB)
+        dump_table(gp->stab, 256, "stab");
+    fprintf(stderr, "}\n");
+
+    int i;
+    for (i = 0; i < gp->nparam; i++) {
+        fqz_param *pm = &gp->p[i];
+        fprintf(stderr, "\nParam[%d] = {\n", i);
+        fprintf(stderr, "\tcontext\t0x%04x\n", pm->context);
+        fprintf(stderr, "\tpflags\t0x%02x\n",  pm->pflags);
+        fprintf(stderr, "\tmax_sym\t%d\n",  pm->max_sym);
+        fprintf(stderr, "\tqbits\t%d\n",   pm->qbits);
+        fprintf(stderr, "\tqshift\t%d\n",  pm->qshift);
+        fprintf(stderr, "\tqloc\t%d\n",    pm->qloc);
+        fprintf(stderr, "\tsloc\t%d\n",    pm->sloc);
+        fprintf(stderr, "\tploc\t%d\n",    pm->ploc);
+        fprintf(stderr, "\tdloc\t%d\n",    pm->dloc);
+
+        if (pm->pflags & PFLAG_HAVE_QMAP)
+            dump_map(pm->qmap, 256, "qmap");
+
+        if (pm->pflags & PFLAG_HAVE_QTAB)
+            dump_table(pm->qtab, 256, "qtab");
+        if (pm->pflags & PFLAG_HAVE_PTAB)
+            dump_table(pm->ptab, 1024, "ptab");
+        if (pm->pflags & PFLAG_HAVE_DTAB)
+            dump_table(pm->dtab, 256, "dtab");
+        fprintf(stderr, "}\n");
+    }
+}
+
+typedef struct {
+    SIMPLE_MODEL(QMAX,_) *qual;
+    SIMPLE_MODEL(256,_)   len[4];
+    SIMPLE_MODEL(2,_)     revcomp;
+    SIMPLE_MODEL(256,_)   sel;
+    SIMPLE_MODEL(2,_)     dup;
+} fqz_model;
+
+static int fqz_create_models(fqz_model *m, fqz_gparams *gp) {
+    int i;
+
+    if (!(m->qual = htscodecs_tls_alloc(sizeof(*m->qual) * CTX_SIZE)))
+        return -1;
+
+    for (i = 0; i < CTX_SIZE; i++)
+        SIMPLE_MODEL(QMAX,_init)(&m->qual[i], gp->max_sym+1);
+
+    for (i = 0; i < 4; i++)
+        SIMPLE_MODEL(256,_init)(&m->len[i],256);
+
+    SIMPLE_MODEL(2,_init)(&m->revcomp,2);
+    SIMPLE_MODEL(2,_init)(&m->dup,2);
+    if (gp->max_sel > 0)
+        SIMPLE_MODEL(256,_init)(&m->sel, gp->max_sel+1);
+
+    return 0;
+}
+
+static void fqz_destroy_models(fqz_model *m) {
+    htscodecs_tls_free(m->qual);
+}
+
+static inline unsigned int fqz_update_ctx(fqz_param *pm, fqz_state *state, int q) {
+    unsigned int last = 0; // pm->context
+    state->qctx = (state->qctx << pm->qshift) + pm->qtab[q];
+    last += (state->qctx & pm->qmask) << pm->qloc;
+
+    // The final shifts have been factored into the tables already.
+    last += pm->ptab[MIN(1023, state->p)];      // << pm->ploc
+    last += pm->dtab[MIN(255,  state->delta)];  // << pm->dloc
+    last += state->s << pm->sloc;
+
+    // On the fly average is slow work.
+    // However it can be slightly better than using a selector bit
+    // as it's something we can compute on the fly and thus doesn't
+    // consume output bits for storing the selector itself.
+    //
+    // Q4 (novaseq.bam)
+    // qtot+=q*q -DQ1=8.84 -DQ2=8.51 -DQ3=7.70; 7203598 (-0.7%)
+    // qtot+=q   -DQ1=2.96 -DQ2=2.85 -DQ3=2.69; 7207315
+    // vs old delta;                            7255614 (default params)
+    // vs 2 bit selector (no delta)             7203006 (-x 0x8261000e80)
+    // vs 2 bit selector (no delta)             7199153 (-x 0x7270000e70) -0.8%
+    // vs 2 bit selector (no delta)             7219668 (-x 0xa243000ea0)
+    //{
+    //  double qa = state->qtot / (state->qlen+.01);
+    //  //fprintf(stderr, "%f\n", qa);
+    //  int x = 0;
+    //  if (qa>=Q1) x=3;
+    //  else if (qa>=Q2) x=2;
+    //  else if (qa>=Q3) x=1;
+    //  else x=0;
+    //  last += x << pm->dloc; // tmp reuse of delta pos
+    //  state->qtot += q*q;
+    //  state->qlen++;
+    //}
+
+    // Only update delta after 1st base.
+    state->delta += (state->prevq != q);
+    state->prevq = q;
+
+    state->p--;
+
+    return last & (CTX_SIZE-1);
+}
+
+// Build quality stats for qhist and set nsym, do_dedup and do_sel params.
+// One_param is -1 to gather stats on all data, or >= 0 to gather data
+// on one specific selector parameter.  Used only in TEST_MAIN via
+// fqz_manual_parameters at the moment.
+void fqz_qual_stats(fqz_slice *s,
+                    unsigned char *in, size_t in_size,
+                    fqz_param *pm,
+                    uint32_t qhist[256],
+                    int one_param) {
+#define NP 32
+    uint32_t qhistb[NP][256] = {{0}};  // both
+    uint32_t qhist1[NP][256] = {{0}};  // READ1 only
+    uint32_t qhist2[NP][256] = {{0}};  // READ2 only
+    uint64_t t1[NP] = {0};             // Count for READ1
+    uint64_t t2[NP] = {0};             // COUNT for READ2
+    uint32_t avg[2560] = {0};          // Avg qual *and later* avg-to-selector map.
+
+    int dir = 0;
+    int last_len = 0;
+    int do_dedup = 0;
+    size_t rec;
+    size_t i, j;
+    int num_rec = 0;
+
+    // See what info we've been given.
+    // Do we have READ1 / READ2?
+    // Do we have selector hidden in the top bits of flag?
+    int max_sel = 0;
+    int has_r2 = 0;
+    for (rec = 0; rec < s->num_records; rec++) {
+        if (one_param >= 0 && (s->flags[rec] >> 16) != one_param)
+            continue;
+        num_rec++;
+        if (max_sel < (s->flags[rec] >> 16))
+            max_sel = (s->flags[rec] >> 16);
+        if (s->flags[rec] & FQZ_FREAD2)
+            has_r2 = 1;
+    }
+
+    // Dedup detection and histogram stats gathering
+    int *avg_qual = calloc((s->num_records+1), sizeof(int));
+    if (!avg_qual)
+        return;
+
+    rec = i = j = 0;
+    while (i < in_size) {
+        if (one_param >= 0 && (s->flags[rec] >> 16) != one_param) {
+            avg_qual[rec] = 0;
+            i += s->len[rec++];
+            continue;
+        }
+        if (rec < s->num_records) {
+            j = s->len[rec];
+            dir = s->flags[rec] & FQZ_FREAD2 ? 1 : 0;
+            if (i > 0 && j == last_len
+                && !memcmp(in+i-last_len, in+i, j))
+                do_dedup++; // cache which records are dup?
+        } else {
+            j = in_size - i;
+            dir = 0;
+        }
+        last_len = j;
+
+        uint32_t (*qh)[256] = dir ? qhist2 : qhist1;
+        uint64_t *th        = dir ? t2     : t1;
+
+        uint32_t tot = 0;
+        for (; i < in_size && j > 0; i++, j--) {
+            tot += in[i];
+            qhist[in[i]]++;
+            qhistb[j & (NP-1)][in[i]]++;
+            qh[j & (NP-1)][in[i]]++;
+            th[j & (NP-1)]++;
+        }
+        tot = last_len ? (tot*10.0)/last_len+.5 : 0;
+
+        avg_qual[rec] = tot;
+        avg[MIN(2559, tot)]++;
+
+        rec++;
+    }
+    pm->do_dedup = ((rec+1)/(do_dedup+1) < 500);
+
+    last_len = 0;
+
+    // Unique symbol count
+    for (i = pm->max_sym = pm->nsym = 0; i < 256; i++) {
+        if (qhist[i])
+            pm->max_sym = i, pm->nsym++;
+    }
+
+
+    // Auto tune: does average quality helps us?
+    if (pm->do_qa != 0) {
+        // Histogram of average qual in avg[]
+        // NB: we convert avg[] from count to selector index
+
+        // Few symbols means high compression which means
+        // selector bits become more significant fraction.
+        // Reduce selector bits by skewing the distribution
+        // to not be even binning.
+        double qf0 = pm->nsym > 8 ? 0.2 : 0.05;
+        double qf1 = pm->nsym > 8 ? 0.5 : 0.22;
+        double qf2 = pm->nsym > 8 ? 0.8 : 0.60;
+
+        int total = 0;
+        i = 0;
+        while (i < 2560) {
+            total += avg[i];
+            if (total > qf0 * num_rec) {
+                //fprintf(stderr, "Q1=%d\n", (int)i);
+                break;
+            }
+            avg[i++] = 0;
+        }
+        while (i < 2560) {
+            total += avg[i];
+            if (total > qf1 * num_rec) {
+                //fprintf(stderr, "Q2=%d\n", (int)i);
+                break;
+            }
+            avg[i++] = 1;
+        }
+        while (i < 2560) {
+            total += avg[i];
+            if (total > qf2 * num_rec) {
+                //fprintf(stderr, "Q3=%d\n", (int)i);
+                break;
+            }
+            avg[i++] = 2;
+        }
+        while (i < 2560)
+            avg[i++] = 3;
+
+        // Compute simple entropy of merged signal vs split signal.
+        i = 0;
+        rec = 0;
+
+        int qbin4[4][NP][256] = {{{0}}};
+        int qbin2[2][NP][256] = {{{0}}};
+        int qbin1   [NP][256] = {{0}};
+        int qcnt4[4][NP] = {{0}};
+        int qcnt2[4][NP] = {{0}};
+        int qcnt1   [NP] = {0};
+        while (i < in_size) {
+            if (one_param >= 0 && (s->flags[rec] >> 16) != one_param) {
+                i += s->len[rec++];
+                continue;
+            }
+            if ((rec & 7) && rec < s->num_records) {
+                // subsample for speed
+                i += s->len[rec++];
+                continue;
+            }
+            if (rec < s->num_records)
+                j = s->len[rec];
+            else
+                j = in_size - i;
+            last_len = j;
+
+            uint32_t tot = avg_qual[rec];
+            int qb4 = avg[MIN(2559, tot)];
+            int qb2 = qb4/2;
+
+            for (; i < in_size && j > 0; i++, j--) {
+                int x = j & (NP-1);
+                qbin4[qb4][x][in[i]]++;  qcnt4[qb4][x]++;
+                qbin2[qb2][x][in[i]]++;  qcnt2[qb2][x]++;
+                qbin1     [x][in[i]]++;  qcnt1     [x]++;
+            }
+            rec++;
+        }
+
+        double e1 = 0, e2 = 0, e4 = 0;
+        for (j = 0; j < NP; j++) {
+            for (i = 0; i < 256; i++) {
+                if (qbin1   [j][i]) e1 += qbin1   [j][i] * fast_log(qbin1   [j][i] / (double)qcnt1   [j]);
+                if (qbin2[0][j][i]) e2 += qbin2[0][j][i] * fast_log(qbin2[0][j][i] / (double)qcnt2[0][j]);
+                if (qbin2[1][j][i]) e2 += qbin2[1][j][i] * fast_log(qbin2[1][j][i] / (double)qcnt2[1][j]);
+                if (qbin4[0][j][i]) e4 += qbin4[0][j][i] * fast_log(qbin4[0][j][i] / (double)qcnt4[0][j]);
+                if (qbin4[1][j][i]) e4 += qbin4[1][j][i] * fast_log(qbin4[1][j][i] / (double)qcnt4[1][j]);
+                if (qbin4[2][j][i]) e4 += qbin4[2][j][i] * fast_log(qbin4[2][j][i] / (double)qcnt4[2][j]);
+                if (qbin4[3][j][i]) e4 += qbin4[3][j][i] * fast_log(qbin4[3][j][i] / (double)qcnt4[3][j]);
+            }
+        }
+        e1 /= -log(2)/8;
+        e2 /= -log(2)/8;
+        e4 /= -log(2)/8;
+        //fprintf(stderr, "E1=%f E2=%f E4=%f %f\n", e1, e2+s->num_records/8, e4+s->num_records/4, (e4+s->num_records/4)/(e2+s->num_records/8));
+
+        // Note by using the selector we're robbing bits from elsewhere in
+        // the context, which may reduce compression better.
+        // We don't know how much by, so this is basically a guess!
+        // For now we just say need 5% saving here.
+        double qm = pm->do_qa > 0 ? 1 : 0.98;
+        if ((pm->do_qa == -1 || pm->do_qa >= 4) &&
+            e4 + s->num_records/4 < e2*qm + s->num_records/8 &&
+            e4 + s->num_records/4 < e1*qm) {
+            //fprintf(stderr, "do q4\n");
+            for (i = 0; i < s->num_records; i++) {
+                //fprintf(stderr, "%d -> %d -> %d, %d\n", (int)i, avg_qual[i], avg[MIN(2559, avg_qual[i])], s->flags[i]>>16);
+                s->flags[i] |= avg[MIN(2559, avg_qual[i])] <<16;
+            }
+            pm->do_sel = 1;
+            max_sel = 3;
+        } else if ((pm->do_qa == -1 || pm->do_qa >= 2) && e2 + s->num_records/8 < e1*qm) {
+            //fprintf(stderr, "do q2\n");
+            for (i = 0; i < s->num_records; i++)
+                s->flags[i] |= (avg[MIN(2559, avg_qual[i])]>>1) <<16;
+            pm->do_sel = 1;
+            max_sel = 1;
+        }
+
+        if (pm->do_qa == -1) {
+            // assume qual, pos, delta in that order.
+            if (pm->pbits > 0 && pm->dbits > 0) {
+                // 1 from pos/delta
+                pm->sloc = pm->dloc-1;
+                pm->pbits--;
+                pm->dbits--;
+                pm->dloc++;
+            } else if (pm->dbits >= 2) {
+                // 2 from delta
+                pm->sloc = pm->dloc;
+                pm->dbits -= 2;
+                pm->dloc += 2;
+            } else if (pm->qbits >= 2) {
+                pm->qbits -= 2;
+                pm->ploc -= 2;
+                pm->sloc = 16-2 - pm->do_r2;
+                if (pm->qbits == 6 && pm->qshift == 5)
+                    pm->qbits--;
+            }
+            pm->do_qa = 4;
+        }
+    }
+
+    // Auto tune: does splitting up READ1 and READ2 help us?
+    if (has_r2 || pm->do_r2) { // FIXME: && but debug for now
+        double e1 = 0, e2 = 0; // entropy sum
+
+        for (j = 0; j < NP; j++) {
+            if (!t1[j] || !t2[j]) continue;
+            for (i = 0; i < 256; i++) {
+                if (!qhistb[j][i]) continue;
+                e1 -= (qhistb[j][i])*log(qhistb[j][i] / (double)(t1[j]+t2[j]));
+                if (qhist1[j][i])
+                    e2 -= qhist1[j][i] * log(qhist1[j][i] / (double)t1[j]);
+                if (qhist2[j][i])
+                    e2 -= qhist2[j][i] * log(qhist2[j][i] / (double)t2[j]);
+            }
+        }
+        e1 /= log(2)*8; // bytes
+        e2 /= log(2)*8;
+
+        //fprintf(stderr, "read1/2 entropy merge %f split %f\n", e1, e2);
+
+        // Note by using the selector we're robbing bits from elsewhere in
+        // the context, which may reduce compression better.
+        // We don't know how much by, so this is basically a guess!
+        // For now we just say need 5% saving here.
+        double qm = pm->do_r2 > 0 ? 1 : 0.95;
+        if (e2 + (8+s->num_records/8) < e1*qm) {
+            for (rec = 0; rec < s->num_records; rec++) {
+                if (one_param >= 0 && (s->flags[rec] >> 16) != one_param)
+                    continue;
+                int sel = s->flags[rec] >> 16;
+                s->flags[rec] =  (s->flags[rec] & 0xffff)
+                    | ((s->flags[rec] & FQZ_FREAD2)
+                       ? ((sel*2)+1) << 16
+                       : ((sel*2)+0) << 16);
+                if (max_sel < (s->flags[rec]>>16))
+                    max_sel = (s->flags[rec]>>16);
+            }
+        }
+    }
+
+    // We provided explicit selector data or auto-tuned it
+    if (max_sel > 0) {
+        pm->do_sel = 1;
+        pm->max_sel = max_sel;
+    }
+
+    free(avg_qual);
+}
+
+static inline
+int fqz_store_parameters1(fqz_param *pm, unsigned char *comp) {
+    int comp_idx = 0, i, j;
+
+    // Starting context
+    comp[comp_idx++] = pm->context;
+    comp[comp_idx++] = pm->context >> 8;
+
+    comp[comp_idx++] = pm->pflags;
+    comp[comp_idx++] = pm->max_sym;
+
+    comp[comp_idx++] = (pm->qbits<<4)|pm->qshift;
+    comp[comp_idx++] = (pm->qloc<<4)|pm->sloc;
+    comp[comp_idx++] = (pm->ploc<<4)|pm->dloc;
+
+    if (pm->store_qmap) {
+        for (i = j = 0; i < 256; i++)
+            if (pm->qmap[i] != INT_MAX)
+                comp[comp_idx++] = i;
+    }
+
+    if (pm->qbits && pm->use_qtab)
+        // custom qtab
+        comp_idx += store_array(comp+comp_idx, pm->qtab, 256);
+
+    if (pm->pbits && pm->use_ptab)
+        // custom ptab
+        comp_idx += store_array(comp+comp_idx, pm->ptab, 1024);
+
+    if (pm->dbits && pm->use_dtab)
+        // custom dtab
+        comp_idx += store_array(comp+comp_idx, pm->dtab, 256);
+
+    return comp_idx;
+}
+
+static 
+int fqz_store_parameters(fqz_gparams *gp, unsigned char *comp) {
+    int comp_idx = 0;
+    comp[comp_idx++] = gp->vers; // Format number
+
+    comp[comp_idx++] = gp->gflags;
+
+    if (gp->gflags & GFLAG_MULTI_PARAM)
+        comp[comp_idx++] = gp->nparam;
+
+    if (gp->gflags & GFLAG_HAVE_STAB) {
+        comp[comp_idx++] = gp->max_sel;
+        comp_idx += store_array(comp+comp_idx, gp->stab, 256);
+    }
+
+    int i;
+    for (i = 0; i < gp->nparam; i++)
+        comp_idx += fqz_store_parameters1(&gp->p[i], comp+comp_idx);
+
+    //fprintf(stderr, "Encoded %d bytes of param\n", comp_idx);
+    return comp_idx;
+}
+
+// Choose a set of parameters based on quality statistics and
+// some predefined options (selected via "strat").
+static inline
+int fqz_pick_parameters(fqz_gparams *gp,
+                        int vers,
+                        int strat,
+                        fqz_slice *s,
+                        unsigned char *in,
+                        size_t in_size) {
+    //approx sqrt(delta), must be sequential
+    int dsqr[] = {
+        0, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3,
+        4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5,
+        5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
+        6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7
+    };
+    uint32_t qhist[256] = {0};
+
+    if (strat >= nstrats) strat = nstrats-1;
+
+    // Start with 1 set of parameters.
+    // FIXME: add support for multiple params later.
+    memset(gp, 0, sizeof(*gp));
+    gp->vers = FQZ_VERS;
+
+    if (!(gp->p = calloc(1, sizeof(fqz_param))))
+        return -1;
+    gp->nparam = 1;
+    gp->max_sel = 0;
+
+    if (vers == 3) // V3.0 doesn't store qual in original orientation
+        gp->gflags |= GFLAG_DO_REV;
+
+    fqz_param *pm = gp->p;
+
+    // Programmed strategies, which we then amend based on our
+    // statistical analysis of the quality stream.
+    pm->qbits  = strat_opts[strat][0];
+    pm->qshift = strat_opts[strat][1];
+    pm->pbits  = strat_opts[strat][2];
+    pm->pshift = strat_opts[strat][3];
+    pm->dbits  = strat_opts[strat][4];
+    pm->dshift = strat_opts[strat][5];
+    pm->qloc   = strat_opts[strat][6];
+    pm->sloc   = strat_opts[strat][7];
+    pm->ploc   = strat_opts[strat][8];
+    pm->dloc   = strat_opts[strat][9];
+
+    // Params for controlling behaviour here.
+    pm->do_r2 = strat_opts[strat][10];
+    pm->do_qa = strat_opts[strat][11];
+
+    // Validity check input lengths and buffer size
+    size_t tlen = 0, i;
+    for (i = 0; i < s->num_records; i++) {
+        if (tlen + s->len[i] > in_size)
+            // Oversized buffer
+            s->len[i] = in_size - tlen;
+        tlen += s->len[i];
+    }
+    if (s->num_records > 0 && tlen < in_size)
+        // Undersized buffer
+        s->len[s->num_records-1] += in_size - tlen;
+
+    // Quality metrics, for all recs
+    fqz_qual_stats(s, in, in_size, pm, qhist, -1);
+
+    pm->store_qmap = (pm->nsym <= 8 && pm->nsym*2 < pm->max_sym);
+
+    // Check for fixed length.
+    uint32_t first_len = s->len[0];
+    for (i = 1; i < s->num_records; i++) {
+        if (s->len[i] != first_len)
+            break;
+    }
+    pm->fixed_len = (i == s->num_records);
+    pm->use_qtab = 0; // unused by current encoder
+
+    if (strat >= nstrats-1)
+        goto manually_set; // used in TEST_MAIN for debugging
+
+    if (pm->pshift < 0)
+        pm->pshift = MAX(0, log((double)s->len[0]/(1<<pm->pbits))/log(2)+.5);
+
+    if (pm->nsym <= 4) {
+        // NovaSeq
+        pm->qshift = 2; // qmax 64, although we can store up to 256 if needed
+        if (in_size < 5000000) {
+            pm->pbits =2;
+            pm->pshift=5;
+        }
+    } else if (pm->nsym <= 8) {
+        // HiSeqX
+        pm->qbits =MIN(pm->qbits,9);
+        pm->qshift=3;
+        if (in_size < 5000000)
+            pm->qbits =6;
+    }
+
+    if (in_size < 300000) {
+        pm->qbits=pm->qshift;
+        pm->dbits=2;
+    }
+
+ manually_set:
+//    fprintf(stderr, "-x 0x%x%x%x%x%x%x%x%x%x%x%x%x\n",
+//          pm->qbits, pm->qshift,
+//          pm->pbits, pm->pshift,
+//          pm->dbits, pm->dshift,
+//          pm->qloc, pm->sloc, pm->ploc, pm->dloc,
+//          pm->do_r2, pm->do_qa);
+
+    for (i = 0; i < sizeof(dsqr)/sizeof(*dsqr); i++)
+        if (dsqr[i] > (1<<pm->dbits)-1)
+            dsqr[i] = (1<<pm->dbits)-1;
+
+    if (pm->store_qmap) {
+        int j;
+        for (i = j = 0; i < 256; i++)
+            if (qhist[i])
+                pm->qmap[i] = j++;
+            else
+                pm->qmap[i] = INT_MAX;
+        pm->max_sym = pm->nsym;
+    } else {
+        pm->nsym = 255;
+        for (i = 0; i < 256; i++)
+            pm->qmap[i] = i;
+    }
+    if (gp->max_sym < pm->max_sym)
+        gp->max_sym = pm->max_sym;
+
+    // Produce ptab from pshift.
+    if (pm->qbits) {
+        for (i = 0; i < 256; i++) {
+            pm->qtab[i] = i; // 1:1
+
+            // Alternative mappings:
+            //qtab[i] = i > 30 ? MIN(max_sym,i)-15 : i/2;  // eg for 9827 BAM
+        }
+
+    }
+    pm->qmask = (1<<pm->qbits)-1;
+
+    if (pm->pbits) {
+        for (i = 0; i < 1024; i++)
+            pm->ptab[i] = MIN((1<<pm->pbits)-1, i>>pm->pshift);
+
+        // Alternatively via analysis of quality distributions we
+        // may select a bunch of positions that are special and
+        // have a non-uniform ptab[].
+        // Manual experimentation on a NovaSeq run saved 2.8% here.
+    }
+
+    if (pm->dbits) {
+        for (i = 0; i < 256; i++)
+            pm->dtab[i] = dsqr[MIN(sizeof(dsqr)/sizeof(*dsqr)-1, i>>pm->dshift)];
+    }
+
+    pm->use_ptab = (pm->pbits > 0);
+    pm->use_dtab = (pm->dbits > 0);
+
+    pm->pflags =
+        (pm->use_qtab   ?PFLAG_HAVE_QTAB :0)|
+        (pm->use_dtab   ?PFLAG_HAVE_DTAB :0)|
+        (pm->use_ptab   ?PFLAG_HAVE_PTAB :0)|
+        (pm->do_sel     ?PFLAG_DO_SEL    :0)|
+        (pm->fixed_len  ?PFLAG_DO_LEN    :0)|
+        (pm->do_dedup   ?PFLAG_DO_DEDUP  :0)|
+        (pm->store_qmap ?PFLAG_HAVE_QMAP :0);
+
+    gp->max_sel = 0;
+    if (pm->do_sel) {
+        // 2 selectors values, but 1 parameter block.
+        // We'll use the sloc instead to encode the selector bits into
+        // the context.
+        gp->max_sel = 1; // indicator to check recs
+        gp->gflags |= GFLAG_HAVE_STAB;
+        // NB: stab is already all zero
+    }
+
+    if (gp->max_sel && s->num_records) {
+        int max = 0;
+        for (i = 0; i < s->num_records; i++) {
+            if (max < (s->flags[i] >> 16))
+                max = (s->flags[i] >> 16);
+        }
+        gp->max_sel = max;
+    }
+
+    return 0;
+}
+
+static void fqz_free_parameters(fqz_gparams *gp) {
+    if (gp && gp->p) free(gp->p);
+}
+
+static int compress_new_read(fqz_slice *s,
+                             fqz_state *state,
+                             fqz_gparams *gp,
+                             fqz_param *pm,
+                             fqz_model *model,
+                             RangeCoder *rc,
+                             unsigned char *in,
+                             size_t *in_i, // in[in_i],
+                             unsigned int *last) {
+    ssize_t rec = state->rec;
+    size_t i = *in_i;
+    if (pm->do_sel || (gp->gflags & GFLAG_MULTI_PARAM)) {
+        state->s = rec < s->num_records
+            ? s->flags[rec] >> 16 // reuse spare bits
+            : 0;
+        SIMPLE_MODEL(256,_encodeSymbol)(&model->sel, rc, state->s);
+    } else {
+        state->s = 0;
+    }
+    int x = (gp->gflags & GFLAG_HAVE_STAB) ? gp->stab[state->s] : state->s;
+    pm = &gp->p[x];
+
+    int len = s->len[rec];
+    if (!pm->fixed_len || state->first_len) {
+        SIMPLE_MODEL(256,_encodeSymbol)(&model->len[0], rc, (len>> 0) & 0xff);
+        SIMPLE_MODEL(256,_encodeSymbol)(&model->len[1], rc, (len>> 8) & 0xff);
+        SIMPLE_MODEL(256,_encodeSymbol)(&model->len[2], rc, (len>>16) & 0xff);
+        SIMPLE_MODEL(256,_encodeSymbol)(&model->len[3], rc, (len>>24) & 0xff);
+        state->first_len = 0;
+    }
+
+    if (gp->gflags & GFLAG_DO_REV) {
+        // no need to reverse complement for V4.0 as the core format
+        // already has this feature.
+        if (s->flags[rec] & FQZ_FREVERSE)
+            SIMPLE_MODEL(2,_encodeSymbol)(&model->revcomp, rc, 1);
+        else
+            SIMPLE_MODEL(2,_encodeSymbol)(&model->revcomp, rc, 0);
+    }
+
+    state->rec++;
+
+    state->qtot = 0;
+    state->qlen = 0;
+
+    state->p = len;
+    state->delta = 0;
+    state->qctx = 0;
+    state->prevq = 0;
+
+    *last = pm->context;
+
+    if (pm->do_dedup) {
+        // Possible dup of previous read?
+        if (i && len == state->last_len &&
+            !memcmp(in+i-state->last_len, in+i, len)) {
+            SIMPLE_MODEL(2,_encodeSymbol)(&model->dup, rc, 1);
+            i += len-1;
+            state->p = 0;
+            *in_i = i;
+            return 1; // is a dup
+        } else {
+            SIMPLE_MODEL(2,_encodeSymbol)(&model->dup, rc, 0);
+        }
+
+        state->last_len = len;
+    }
+
+    *in_i = i;
+
+    return 0; // not dup
+}
+
+static
+unsigned char *compress_block_fqz2f(int vers,
+                                    int strat,
+                                    fqz_slice *s,
+                                    unsigned char *in,
+                                    size_t in_size,
+                                    size_t *out_size,
+                                    fqz_gparams *gp) {
+    fqz_gparams local_gp;
+    int free_params = 0;
+
+    unsigned int last = 0;
+    size_t i, j;
+    ssize_t rec = 0;
+
+    int comp_idx = 0;
+    RangeCoder rc;
+
+    // Pick and store params
+    if (!gp) {
+        gp = &local_gp;
+        if (fqz_pick_parameters(gp, vers, strat, s, in, in_size) < 0)
+            return NULL;
+        free_params = 1;
+    }
+
+    // Worst case scenario assuming random input data and no way to compress
+    // is NBytes*growth for some small growth factor (arith_dynamic uses 1.05),
+    // plus fixed overheads for the header / params.  Growth can be high
+    // here as we're modelling things and pathological cases may trigger a
+    // bad probability model.
+    //
+    // Per read is 4-byte len if not fixed length (but less if avg smaller)
+    //             up to 1 byte for selection state (log2(max_sel) bits)
+    //             1-bit for reverse flag
+    //             1-bit for dup-last flag (but then no quals)
+    // Per qual is 1-byte (assuming QMAX==256)
+    //
+    // Header size is total guess, as depends on params, but it's almost
+    // always tiny, so a few K extra should be sufficient.
+    //
+    // => Total of (s->num_records*4.25 + in_size)*growth + hdr
+    int sel_bits = 0, sel = gp->max_sel;
+    while (sel) {
+        sel_bits++;
+        sel >>= 1;
+    }
+    double len_sz = gp->p[0].fixed_len ? 0.25 : 4.25;
+    len_sz += sel_bits / 8.0;
+    size_t comp_sz = (s->num_records*len_sz + in_size)*1.1 + 10000;
+
+    unsigned char *comp = (unsigned char *)malloc(comp_sz);
+    unsigned char *compe = comp + (size_t)comp_sz;
+    if (!comp)
+        return NULL;
+
+    //dump_params(gp);
+    comp_idx = var_put_u32(comp, compe, in_size);
+    comp_idx += fqz_store_parameters(gp, comp+comp_idx);
+
+    fqz_param *pm;
+
+    // Optimise tables to remove shifts in loop (NB: cannot do this in next vers)
+    for (j = 0; j < gp->nparam; j++) {
+        pm = &gp->p[j];
+
+        for (i = 0; i < 1024; i++)
+            pm->ptab[i] <<= pm->ploc;
+
+        for (i = 0; i < 256; i++)
+            pm->dtab[i] <<= pm->dloc;
+    }
+
+    // Create models and initialise range coder
+    fqz_model model;
+    if (fqz_create_models(&model, gp) < 0)
+        return NULL;
+
+    RC_SetOutput(&rc, (char *)comp+comp_idx);
+    RC_SetOutputEnd(&rc, (char *)comp+comp_sz);
+    RC_StartEncode(&rc);
+
+    // For CRAM3.1, reverse upfront if needed
+    pm = &gp->p[0];
+    if (gp->gflags & GFLAG_DO_REV) {
+        i = rec = j = 0;
+        while (i < in_size) {
+            int len = rec < s->num_records-1
+                ? s->len[rec] : in_size - i;
+
+            if (s->flags[rec] & FQZ_FREVERSE) {
+                // Reverse complement sequence - note: modifies buffer
+                int I,J;
+                unsigned char *cp = in+i;
+                for (I = 0, J = len-1; I < J; I++, J--) {
+                    unsigned char c;
+                    c = cp[I];
+                    cp[I] = cp[J];
+                    cp[J] = c;
+                }
+            }
+
+            i += len;
+            rec++;
+        }
+        rec = 0;
+    }
+
+    fqz_state state = {0};
+    pm = &gp->p[0];
+    state.p = 0;
+    state.first_len = 1;
+    state.last_len = 0;
+    state.rec = rec;
+
+    for (i = 0; i < in_size; i++) {
+        if (state.p == 0) {
+            if (state.rec >= s->num_records || s->len[state.rec] <= 0) {
+                free(comp);
+                comp = NULL;
+                goto err;
+            }
+
+            if (compress_new_read(s, &state, gp, pm, &model, &rc,
+                                  in, &i, /*&rec,*/ &last))
+                continue;
+        }
+
+#if 0
+        //                        fqz_qual_stats imp.
+        // q40 6.876  6.852       5.96
+        // q4  6.566              5.07
+        // _Q  1.383              1.11
+        unsigned char q = in[i];
+        unsigned char qm = pm->qmap[q];
+
+        SIMPLE_MODEL(QMAX,_encodeSymbol)(&model.qual[last], &rc, qm);
+        last = fqz_update_ctx(pm, &state, qm);
+#else
+        //     gcc    clang            gcc+fqz_qual_stats imp.
+        // q40 5.033  5.026     -27%   4.137 -38%
+        // q4  5.595            -15%   4.011 -36%
+        // _Q  1.225            -11%   0.956
+        int j = -1;
+
+        while (state.p >= 4 && i+j+4 < in_size) {
+            int l1 = last, l2, l3, l4;
+            // Model has symbols sorted by frequency, so most common are at
+            // start.  So while model is approx 1Kb, the first cache line is
+            // a big win.
+            mm_prefetch(&model.qual[l1]);
+            unsigned char qm1 = pm->qmap[in[i + ++j]];
+            last = fqz_update_ctx(pm, &state, qm1); l2 = last;
+
+            mm_prefetch(&model.qual[l2]);
+            unsigned char qm2 = pm->qmap[in[i + ++j]];
+            last = fqz_update_ctx(pm, &state, qm2); l3 = last;
+
+            mm_prefetch(&model.qual[l3]);
+            unsigned char qm3 = pm->qmap[in[i + ++j]];
+            last = fqz_update_ctx(pm, &state, qm3); l4 = last;
+
+            mm_prefetch(&model.qual[l4]);
+            unsigned char qm4 = pm->qmap[in[i + ++j]];
+            last = fqz_update_ctx(pm, &state, qm4);
+
+            SIMPLE_MODEL(QMAX,_encodeSymbol)(&model.qual[l1], &rc, qm1);
+            SIMPLE_MODEL(QMAX,_encodeSymbol)(&model.qual[l2], &rc, qm2);
+            SIMPLE_MODEL(QMAX,_encodeSymbol)(&model.qual[l3], &rc, qm3);
+            SIMPLE_MODEL(QMAX,_encodeSymbol)(&model.qual[l4], &rc, qm4);
+        }
+
+        while (state.p > 0) {
+            int l2 = last;
+            mm_prefetch(&model.qual[last]);
+            unsigned char qm = pm->qmap[in[i + ++j]];
+            last = fqz_update_ctx(pm, &state, qm);
+            SIMPLE_MODEL(QMAX,_encodeSymbol)(&model.qual[l2], &rc, qm);
+        }
+        i += j;
+#endif
+    }
+
+    if (RC_FinishEncode(&rc) < 0) {
+        free(comp);
+        comp = NULL;
+        *out_size = 0;
+        goto err;
+    }
+
+    // For CRAM3.1, undo our earlier reversal step
+    rec = state.rec;
+    if (gp->gflags & GFLAG_DO_REV) {
+        i = rec = j = 0;
+        while (i < in_size) {
+            int len = rec < s->num_records-1
+                ? s->len[rec]
+                : in_size - i;
+
+            if (s->flags[rec] & FQZ_FREVERSE) {
+                // Reverse complement sequence - note: modifies buffer
+                int I,J;
+                unsigned char *cp = in+i;
+                for (I = 0, J = len-1; I < J; I++, J--) {
+                    unsigned char c;
+                    c = cp[I];
+                    cp[I] = cp[J];
+                    cp[J] = c;
+                }
+            }
+
+            i += len;
+            rec++;
+        }
+    }
+
+    // Clear selector abuse of flags
+    for (rec = 0; rec < s->num_records; rec++)
+        s->flags[rec] &= 0xffff;
+
+    *out_size = comp_idx + RC_OutSize(&rc);
+    //fprintf(stderr, "%d -> %d\n", (int)in_size, (int)*out_size);
+
+ err:
+    fqz_destroy_models(&model);
+    if (free_params)
+        fqz_free_parameters(gp);
+
+    return comp;
+}
+
+// Read fqz paramaters.
+//
+// FIXME: pass in and check in_size.
+//
+// Returns number of bytes read on success,
+//         -1 on failure.
+static inline
+int fqz_read_parameters1(fqz_param *pm, unsigned char *in, size_t in_size) {
+    int in_idx = 0;
+    size_t i;
+
+    if (in_size < 7)
+        return -1;
+
+    // Starting context
+    pm->context = in[in_idx] + (in[in_idx+1]<<8);
+    in_idx += 2;
+
+    // Bit flags
+    pm->pflags     = in[in_idx++];
+    pm->use_qtab   = pm->pflags & PFLAG_HAVE_QTAB;
+    pm->use_dtab   = pm->pflags & PFLAG_HAVE_DTAB;
+    pm->use_ptab   = pm->pflags & PFLAG_HAVE_PTAB;
+    pm->do_sel     = pm->pflags & PFLAG_DO_SEL;
+    pm->fixed_len  = pm->pflags & PFLAG_DO_LEN;
+    pm->do_dedup   = pm->pflags & PFLAG_DO_DEDUP;
+    pm->store_qmap = pm->pflags & PFLAG_HAVE_QMAP;
+    pm->max_sym    = in[in_idx++];
+
+    // Sub-context sizes and locations
+    pm->qbits      = in[in_idx]>>4;
+    pm->qmask      = (1<<pm->qbits)-1;
+    pm->qshift     = in[in_idx++]&15;
+    pm->qloc       = in[in_idx]>>4;
+    pm->sloc       = in[in_idx++]&15;
+    pm->ploc       = in[in_idx]>>4;
+    pm->dloc       = in[in_idx++]&15;
+
+    // Maps and tables
+    if (pm->store_qmap) {
+        for (i = 0; i < 256; i++) pm->qmap[i] = INT_MAX; // so dump_map works
+        if (in_idx + pm->max_sym > in_size)
+            return -1;
+        for (i = 0; i < pm->max_sym; i++)
+            pm->qmap[i] = in[in_idx++];
+    } else {
+        for (i = 0; i < 256; i++)
+            pm->qmap[i] = i;
+    }
+
+    if (pm->qbits) {
+        if (pm->use_qtab) {
+            int used = read_array(in+in_idx, in_size-in_idx, pm->qtab, 256);
+            if (used < 0)
+                return -1;
+            in_idx += used;
+        } else {
+            for (i = 0; i < 256; i++)
+                pm->qtab[i] = i;
+        }
+    }
+
+    if (pm->use_ptab) {
+        int used = read_array(in+in_idx, in_size-in_idx, pm->ptab, 1024);
+        if (used < 0)
+            return -1;
+        in_idx += used;
+    } else {
+        for (i = 0; i < 1024; i++)
+            pm->ptab[i] = 0;
+    }
+
+    if (pm->use_dtab) {
+        int used = read_array(in+in_idx, in_size-in_idx, pm->dtab, 256);
+        if (used < 0)
+            return -1;
+        in_idx += used;
+    } else {
+        for (i = 0; i < 256; i++)
+            pm->dtab[i] = 0;
+    }
+
+    return in_idx;
+}
+
+static
+int fqz_read_parameters(fqz_gparams *gp, unsigned char *in, size_t in_size) {
+    int in_idx = 0;
+    int i;
+
+    if (in_size < 10)
+        return -1;
+
+    // Format version
+    gp->vers = in[in_idx++];
+    if (gp->vers != FQZ_VERS)
+        return -1;
+
+    // Global glags
+    gp->gflags = in[in_idx++];
+
+    // Number of param blocks and param selector details
+    gp->nparam = (gp->gflags & GFLAG_MULTI_PARAM) ? in[in_idx++] : 1;
+    if (gp->nparam <= 0)
+        return -1;
+    gp->max_sel = gp->nparam > 1 ? gp->nparam : 0;
+
+    if (gp->gflags & GFLAG_HAVE_STAB) {
+        gp->max_sel = in[in_idx++];
+        int used = read_array(in+in_idx, in_size-in_idx, gp->stab, 256);
+        if (used < 0)
+            goto err;
+        in_idx += used;
+    } else {
+        for (i = 0; i < gp->nparam; i++)
+            gp->stab[i] = i;
+        for (; i < 256; i++)
+            gp->stab[i] = gp->nparam-1;
+    }
+
+    // Load the individual parameter locks
+    if (!(gp->p = malloc(gp->nparam * sizeof(*gp->p))))
+        return -1;
+
+    gp->max_sym = 0;
+    for (i = 0; i < gp->nparam; i++) {
+        int e = fqz_read_parameters1(&gp->p[i], in + in_idx, in_size-in_idx);
+        if (e < 0)
+            goto err;
+        if (gp->p[i].do_sel && gp->max_sel == 0)
+            goto err; // Inconsistent
+        in_idx += e;
+
+        if (gp->max_sym < gp->p[i].max_sym)
+            gp->max_sym = gp->p[i].max_sym;
+    }
+
+    //fprintf(stderr, "Decoded %d bytes of param\n", in_idx);
+    return in_idx;
+
+ err:
+    fqz_free_parameters(gp);
+    gp->nparam = 0;
+    return -1;
+}
+
+// Handles the state.p==0 section of uncompress_block_fqz2f
+static int decompress_new_read(fqz_slice *s,
+                               fqz_state *state,
+                               fqz_gparams *gp,
+                               fqz_param *pm,
+                               fqz_model *model,
+                               RangeCoder *rc,
+                               unsigned char *in, ssize_t *in_i, // in[in_i],
+                               unsigned char *uncomp, size_t *out_size,
+                               int *rev, char *rev_a, int *len_a,
+                               int *lengths, int nlengths) {
+    size_t i = *in_i;
+    ssize_t rec = state->rec;
+
+    if (pm->do_sel) {
+        state->s = SIMPLE_MODEL(256,_decodeSymbol)(&model->sel, rc);
+    } else {
+        state->s = 0;
+    }
+
+    int x = (gp->gflags & GFLAG_HAVE_STAB)
+        ? gp->stab[MIN(255, state->s)]
+        : state->s;
+    if (x >= gp->nparam)
+        return -1;
+    pm = &gp->p[x];
+
+    unsigned int len = state->last_len;
+    if (!pm->fixed_len || state->first_len) {
+        len  = SIMPLE_MODEL(256,_decodeSymbol)(&model->len[0], rc);
+        len |= SIMPLE_MODEL(256,_decodeSymbol)(&model->len[1], rc)<<8;
+        len |= SIMPLE_MODEL(256,_decodeSymbol)(&model->len[2], rc)<<16;
+        len |= ((unsigned)SIMPLE_MODEL(256,_decodeSymbol)(&model->len[3], rc))<<24;
+        state->first_len = 0;
+        state->last_len = len;
+    }
+    if (len > *out_size-i || len <= 0)
+        return -1;
+
+    if (lengths && rec < nlengths)
+        lengths[rec] = len;
+
+    if (gp->gflags & GFLAG_DO_REV) {
+        *rev = SIMPLE_MODEL(2,_decodeSymbol)(&model->revcomp, rc);
+        rev_a[rec] = *rev;
+        len_a[rec] = len;
+    }
+
+    if (pm->do_dedup) {
+        if (SIMPLE_MODEL(2,_decodeSymbol)(&model->dup, rc)) {
+            // Dup of last line
+            if (len > i)
+                return -1;
+            memcpy(uncomp+i, uncomp+i-len, len);
+            i += len;
+            state->p = 0;
+            state->rec++;
+            *in_i = i;
+            return 1; // dup => continue
+        }
+    }
+
+    state->rec++;
+    state->p = len;
+    state->delta = 0;
+    state->prevq = 0;
+    state->qctx = 0;
+    state->ctx = pm->context;
+
+    *in_i = i;
+
+    return 0;
+}
+
+
+static
+unsigned char *uncompress_block_fqz2f(fqz_slice *s,
+                                      unsigned char *in,
+                                      size_t in_size,
+                                      size_t *out_size,
+                                      int *lengths,
+                                      int nlengths) {
+    fqz_gparams gp;
+    fqz_param *pm;
+    char *rev_a = NULL;
+    int *len_a = NULL;
+    memset(&gp, 0, sizeof(gp));
+
+    uint32_t len;
+    ssize_t i, rec = 0, in_idx;
+    in_idx = var_get_u32(in, in+in_size, &len);
+    *out_size = len;
+
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    if (len > 100000)
+        return NULL;
+#endif
+
+    unsigned char *uncomp = NULL;
+    RangeCoder rc;
+    unsigned int last = 0;
+
+    // Decode parameter blocks
+    if ((i = fqz_read_parameters(&gp, in+in_idx, in_size-in_idx)) < 0)
+        return NULL;
+    //dump_params(&gp);
+    in_idx += i;
+
+    // Optimisations to remove shifts from main loop
+    for (i = 0; i < gp.nparam; i++) {
+        int j;
+        pm = &gp.p[i];
+        for (j = 0; j < 1024; j++)
+            pm->ptab[j] <<= pm->ploc;
+        for (j = 0; j < 256; j++)
+            pm->dtab[j] <<= pm->dloc;
+    }
+
+    // Initialise models and entropy coder
+    fqz_model model;
+    if (fqz_create_models(&model, &gp) < 0)
+        return NULL;
+
+    RC_SetInput(&rc, (char *)in+in_idx, (char *)in+in_size);
+    RC_StartDecode(&rc);
+
+
+    // Allocate buffers
+    uncomp = (unsigned char *)malloc(*out_size);
+    if (!uncomp)
+        goto err;
+
+    int nrec = 1000;
+    rev_a = malloc(nrec);
+    len_a = malloc(nrec * sizeof(int));
+    if (!rev_a || !len_a)
+        goto err;
+
+    // Main decode loop
+    fqz_state state;
+    state.delta = 0;
+    state.prevq = 0;
+    state.qctx = 0;
+    state.p = 0;
+    state.s = 0;
+    state.first_len = 1;
+    state.last_len = 0;
+    state.rec = 0;
+    state.ctx = last;
+
+    int rev = 0;
+    int x = 0;
+    pm = &gp.p[x];
+    for (i = 0; i < len; ) {
+        if (state.rec >= nrec) {
+            nrec *= 2;
+            rev_a = realloc(rev_a, nrec);
+            len_a = realloc(len_a, nrec*sizeof(int));
+            if (!rev_a || !len_a)
+                goto err;
+        }
+
+        if (state.p == 0) {
+            int r = decompress_new_read(s, &state, &gp, pm, &model, &rc,
+                                        in, &i, uncomp, out_size,
+                                        &rev, rev_a, len_a,
+                                        lengths, nlengths);
+            if (r < 0)
+                goto err;
+            if (r > 0)
+                continue;
+            last = state.ctx;
+        }
+
+        // Decode and update context
+        do {
+            unsigned char Q = SIMPLE_MODEL(QMAX,_decodeSymbol)
+                (&model.qual[last], &rc);
+
+            last = fqz_update_ctx(pm, &state, Q);
+            uncomp[i++] = pm->qmap[Q];
+        } while (state.p != 0 && i < len);
+    }
+
+    rec = state.rec;
+    if (rec >= nrec) {
+        nrec *= 2;
+        rev_a = realloc(rev_a, nrec);
+        len_a = realloc(len_a, nrec*sizeof(int));
+        if (!rev_a || !len_a)
+            goto err;
+    }
+    rev_a[rec] = rev;
+    len_a[rec] = len;
+
+    if (gp.gflags & GFLAG_DO_REV) {
+        for (i = rec = 0; i < len && rec < nrec; i += len_a[rec++]) {
+            if (!rev_a[rec])
+                continue;
+
+            int I, J;
+            unsigned char *cp = uncomp+i;
+            for (I = 0, J = len_a[rec]-1; I < J; I++, J--) {
+                unsigned char c;
+                c = cp[I];
+                cp[I] = cp[J];
+                cp[J] = c;
+            }
+        }
+    }
+
+    if (RC_FinishDecode(&rc) < 0)
+        goto err;
+
+    fqz_destroy_models(&model);
+    free(rev_a);
+    free(len_a);
+    fqz_free_parameters(&gp);
+
+#ifdef TEST_MAIN
+    s->num_records = rec;
+#endif
+
+    return uncomp;
+
+ err:
+    fqz_destroy_models(&model);
+    free(rev_a);
+    free(len_a);
+    fqz_free_parameters(&gp);
+    free(uncomp);
+
+    return NULL;
+}
+
+char *fqz_compress(int vers, fqz_slice *s, char *in, size_t uncomp_size,
+                   size_t *comp_size, int strat, fqz_gparams *gp) {
+    if (uncomp_size > INT_MAX) {
+        *comp_size = 0;
+        return NULL;
+    }
+
+    return (char *)compress_block_fqz2f(vers, strat, s, (unsigned char *)in,
+                                        uncomp_size, comp_size, gp);
+}
+
+char *fqz_decompress(char *in, size_t comp_size, size_t *uncomp_size,
+                     int *lengths, int nlengths) {
+    return (char *)uncompress_block_fqz2f(NULL, (unsigned char *)in,
+                                          comp_size, uncomp_size, lengths, nlengths);
+}
diff --git a/ext/htslib/htscodecs/htscodecs/fqzcomp_qual.h b/ext/htslib/htscodecs/htscodecs/fqzcomp_qual.h
new file mode 100644
index 0000000..d3aa267
--- /dev/null
+++ b/ext/htslib/htscodecs/htscodecs/fqzcomp_qual.h
@@ -0,0 +1,184 @@
+/*
+ * Copyright (c) 2011-2013, 2018-2019 Genome Research Ltd.
+ * Author(s): James Bonfield
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above
+ *       copyright notice, this list of conditions and the following
+ *       disclaimer in the documentation and/or other materials provided
+ *       with the distribution.
+ *
+ *    3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+ *       Institute nor the names of its contributors may be used to endorse
+ *       or promote products derived from this software without specific
+ *       prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH
+ * LTD OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef FQZ_COMP_QUAL_H
+#define FQZ_COMP_QUAL_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+
+/* Bit flags, deliberately mirroring BAM ones */
+#define FQZ_FREVERSE 16
+#define FQZ_FREAD2 128
+
+/* Current FQZ format version */
+#define FQZ_VERS 5
+
+#define FQZ_MAX_STRAT 3
+
+/*
+ * Minimal per-record information taken from a cram slice.
+ *
+ * To compress we need to know the junction from one quality string to
+ * the next (len), whether it is first/second read and whether it is
+ * reverse complemented (flags).
+ */
+typedef struct {
+    int num_records;
+    uint32_t *len;    // of size num_records
+    uint32_t *flags;  // of size num_records
+} fqz_slice;
+
+
+// Global flags
+static const int GFLAG_MULTI_PARAM = 1;
+static const int GFLAG_HAVE_STAB   = 2;
+static const int GFLAG_DO_REV      = 4;
+
+// Param flags
+// Add PFLAG_HAVE_DMAP and a dmap[] for delta incr?
+static const int PFLAG_DO_DEDUP    = 2;
+static const int PFLAG_DO_LEN      = 4;
+static const int PFLAG_DO_SEL      = 8;
+static const int PFLAG_HAVE_QMAP   = 16;
+static const int PFLAG_HAVE_PTAB   = 32;
+static const int PFLAG_HAVE_DTAB   = 64;
+static const int PFLAG_HAVE_QTAB   = 128;
+
+/*
+ * FQZ parameters.  These may be simply passed in as NULL to fqz_compress
+ * and it'll automatically choose, but if we wish to have complete control
+ * then this (long) struct contains all the details.
+ *
+ * TODO: document all this!
+ */
+
+// A single parameter block
+typedef struct {
+    // Starting context value
+    uint16_t context;
+
+    // flags
+    unsigned int pflags;
+    unsigned int do_sel, do_dedup, store_qmap, fixed_len;
+    unsigned char use_qtab, use_dtab, use_ptab;
+
+    // context bits and locations
+    unsigned int qbits, qloc;
+    unsigned int pbits, ploc;
+    unsigned int dbits, dloc;
+    unsigned int sbits, sloc;
+
+    // models
+    int max_sym, nsym, max_sel;
+
+    // tables / maps
+    unsigned int qmap[256];
+    unsigned int qtab[256];
+    unsigned int ptab[1024];
+    unsigned int dtab[256];
+
+    // Not stored paramters, but computed as part of encoder
+    // parameterisation.
+    int qshift;
+    int pshift;
+    int dshift;
+    int sshift;
+    unsigned int qmask; // (1<<qbits)-1
+    int do_r2, do_qa;
+} fqz_param;
+
+// The global params, which is a collection of parameter blocks plus
+// a few pieces of meta-data.
+typedef struct {
+    int vers;               // Format version; Set to FQZ_VERS
+    unsigned int gflags;    // global param flags
+    int nparam;             // Number of fqz_param blocks
+    int max_sel;            // Number of selector values
+    unsigned int stab[256]; // Selector to parameter no. table
+
+    int max_sym;            // max symbol value across all sub-params
+
+    fqz_param *p;           // 1 or more parameter blocks
+} fqz_gparams;
+
+
+/** Compress a block of quality values.
+ *
+ * @param vers          The CRAM version number (<<8) plus fqz strategy (0-3)
+ * @param s             Length and flag data CRAM per-record
+ * @param in            Buffer of concatenated quality values (no separator)
+ * @param in_size       Size of in buffer
+ * @param out_size      Size of returned output
+ * @param strat         FQZ compression strategy (0 to FQZ_MAX_STRAT)
+ * @param gp            Optional fqzcomp paramters (may be NULL).
+ *
+ * @return              The compressed quality buffer on success,
+ *                      NULL on failure.
+ */
+char *fqz_compress(int vers, fqz_slice *s, char *in, size_t in_size,
+                   size_t *out_size, int strat, fqz_gparams *gp);
+
+/** Decompress a block of quality values.
+ *
+ * @param in            Buffer of compressed quality values
+ * @param in_size       Size of in buffer
+ * @param out_size      Size of returned output
+ * @param lengths       Optional array filled out with record lengths.
+ *                      May be NULL.  If not, preallocate it to correct size.
+ *
+ * @return              The uncompressed concatenated qualities on success,
+ *                      NULL on failure.
+ */
+char *fqz_decompress(char *in, size_t in_size, size_t *out_size,
+                     int *lengths, int nlengths);
+
+/** A utlity function to analyse a quality buffer to gather statistical
+ *  information.  This is written into qhist and pm.  This function is only
+ *  useful if you intend on passing your own fqz_gparams block to
+ *  fqz_compress.
+ */
+void fqz_qual_stats(fqz_slice *s,
+                    unsigned char *in, size_t in_size,
+                    fqz_param *pm,
+                    uint32_t qhist[256],
+                    int one_param);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/ext/htslib/htscodecs/htscodecs/htscodecs.c b/ext/htslib/htscodecs/htscodecs/htscodecs.c
new file mode 100644
index 0000000..35fc6b4
--- /dev/null
+++ b/ext/htslib/htscodecs/htscodecs/htscodecs.c
@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2021 Genome Research Ltd.
+ * Author(s): James Bonfield
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above
+ *       copyright notice, this list of conditions and the following
+ *       disclaimer in the documentation and/or other materials provided
+ *       with the distribution.
+ *
+ *    3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+ *       Institute nor the names of its contributors may be used to endorse
+ *       or promote products derived from this software without specific
+ *       prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH
+ * LTD OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "htscodecs.h"
+#include "version.h"
+
+/*
+ * A const string form of the HTSCODECS_VERSION define.
+ * NB: This is obtained from the auto-generated version.h, so
+ * we can include release number and git hash.
+ */
+const char *htscodecs_version(void) {
+    return HTSCODECS_VERSION_TEXT;
+}
diff --git a/ext/htslib/htscodecs/htscodecs/htscodecs.h b/ext/htslib/htscodecs/htscodecs/htscodecs.h
new file mode 100644
index 0000000..8d67e67
--- /dev/null
+++ b/ext/htslib/htscodecs/htscodecs/htscodecs.h
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2021-2024 Genome Research Ltd.
+ * Author(s): James Bonfield
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above
+ *       copyright notice, this list of conditions and the following
+ *       disclaimer in the documentation and/or other materials provided
+ *       with the distribution.
+ *
+ *    3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+ *       Institute nor the names of its contributors may be used to endorse
+ *       or promote products derived from this software without specific
+ *       prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH
+ * LTD OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef HTSCODECS_H
+#define HTSCODECS_H
+
+/*
+ * Version X.Y.Z encoded as XYYYZZ.
+ * We mainly increment X and Y.  Z *may* get bumped in between official
+ * releases in order to distinguish untagged github checkouts from
+ * official release tarballs.
+ *
+ * Note currently this needs manually editing as it isn't automatically
+ * updated by autoconf.
+ */
+#define HTSCODECS_VERSION 100601
+
+/*
+ * A const string form of the HTSCODECS_VERSION define.
+ * NB: This is obtained from the auto-generated version.h, so
+ * we can include release number and git hash.
+ */
+const char *htscodecs_version(void);
+
+#endif /* HTSCODECS_H */
diff --git a/ext/htslib/htscodecs/htscodecs/htscodecs_endian.h b/ext/htslib/htscodecs/htscodecs/htscodecs_endian.h
new file mode 100644
index 0000000..e038aec
--- /dev/null
+++ b/ext/htslib/htscodecs/htscodecs/htscodecs_endian.h
@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2020 Genome Research Ltd.
+ * Author(s): James Bonfield
+ * 
+ * Redistribution and use in source and binary forms, with or without 
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ *    1. Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimer.
+ * 
+ *    2. Redistributions in binary form must reproduce the above
+ *       copyright notice, this list of conditions and the following
+ *       disclaimer in the documentation and/or other materials provided
+ *       with the distribution.
+ * 
+ *    3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+ *    Institute nor the names of its contributors may be used to endorse
+ *    or promote products derived from this software without specific
+ *    prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH
+ * LTD OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef HTSCODECS_ENDIAN_H
+#define HTSCODECS_ENDIAN_H
+
+// Endianness checking.
+
+// Sets HTSCODECS_ENDIAN_KNOWN if system type detected and either
+// HTSCODECS_LITTLE_ENDIAN or HTSCODECS_BIG_ENDIAN.
+
+/*
+ * In general our preferred route is to write code in an endian agnostic
+ * fashion, but our data formats are natively little endian.  Therefore
+ * in time critical code it's sometimes best to exploit that.
+ *
+ * Therefore we'll optimise code along the lines of:
+ *
+ * #ifdef HTSCODECS_LITTLE_ENDIAN
+ *     // do something little endian specific
+ * #else
+ *     // do something in an endian agnostic fashion
+ * #endif
+ *
+ * This means our code works even if we fail to recognise the
+ * specific machine type.
+ */
+
+#if (defined(__i386__)      \
+ ||  defined(__i386)        \
+ ||  defined(__amd64__)     \
+ ||  defined(__amd64)       \
+ ||  defined(__x86_64__)    \
+ ||  defined(__x86_64)      \
+ ||  defined(__i686__)      \
+ ||  defined(__i686))       \
+ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) \
+ ||  defined(__LITTLE_ENDIAN__)                                            \
+ ||  defined(__ARMEL__)     \
+ ||  defined(__THUMBEL__)   \
+ ||  defined(__AARCH64EL__) \
+ ||  defined(_MIPSEL)       \
+ ||  defined(__MIPSEL)      \
+ ||  defined(__MIPSEL__) 
+    // Little endian
+#   define HTSCODECS_LITTLE_ENDIAN
+#   define HTSCODECS_ENDIAN_KNOWN
+#elif (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) \
+   ||  defined(__BIG_ENDIAN__) \
+   || defined(__ARMEB__)       \
+   || defined(__THUMBEB__)     \
+   || defined(__AAARCHEB__)    \
+   || defined(_MIPSEB)         \
+   || defined(__MIPSEB)        \
+   || defined(__MIPSEB__)
+    // Big endian
+#   define HTSCODECS_BIG_ENDIAN
+#   define HTSCODECS_ENDIAN_KNOWN
+#else
+//    Unknown - code will need to check HTSCODES_ENDIAN_KNOWN and do endian agnostic
+#endif
+
+#endif /* HTSCODECS_ENDIAN_H */
diff --git a/ext/htslib/htscodecs/htscodecs/pack.c b/ext/htslib/htscodecs/htscodecs/pack.c
new file mode 100644
index 0000000..eb8dac4
--- /dev/null
+++ b/ext/htslib/htscodecs/htscodecs/pack.c
@@ -0,0 +1,394 @@
+/*
+ * Copyright (c) 2019-2020, 2022 Genome Research Ltd.
+ * Author(s): James Bonfield
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above
+ *       copyright notice, this list of conditions and the following
+ *       disclaimer in the documentation and/or other materials provided
+ *       with the distribution.
+ *
+ *    3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+ *       Institute nor the names of its contributors may be used to endorse
+ *       or promote products derived from this software without specific
+ *       prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH
+ * LTD OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#include <stdint.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+#include "pack.h"
+
+//-----------------------------------------------------------------------------
+
+/*
+ * Packs multiple symbols into a single byte if the total alphabet of symbols
+ * used is <= 16.  Each new symbol takes up 1, 2, 4 or 8 bits, or 0 if the
+ * alphabet used is 1 (constant).
+ *
+ * If successful, out_meta/out_meta_len are set to hold the mapping table
+ * to be used during decompression.
+ *
+ * Returns the packed buffer on success with new length in out_len,
+ *         NULL of failure
+ */
+uint8_t *hts_pack(uint8_t *data, int64_t len,
+                  uint8_t *out_meta, int *out_meta_len, uint64_t *out_len) {
+    int p[256] = {0}, n;
+    uint64_t i, j;
+
+    // count syms
+    for (i = 0; i < len; i++)
+        p[data[i]]=1;
+    
+    for (i = n = 0; i < 256; i++) {
+        if (p[i]) {
+            p[i] = n++; // p[i] is now the code number
+            out_meta[n] = i;
+        }
+    }
+    out_meta[0] = n; // 256 wraps to 0
+    j = n+1;
+
+    // 1 value per byte
+    if (n > 16)
+        return NULL;
+
+    uint8_t *out = malloc(len+1);
+    if (!out)
+        return NULL;
+
+    // Work out how many values per byte to encode.
+    int val_per_byte;
+    if (n > 4)
+        val_per_byte = 2;
+    else if (n > 2)
+        val_per_byte = 4;
+    else if (n > 1)
+        val_per_byte = 8;
+    else
+        val_per_byte = 0; // infinite
+
+    *out_meta_len = j;
+    j = 0;
+
+    switch (val_per_byte) {
+    case 2:
+        for (i = 0; i < (len & ~1); i+=2)
+            out[j++] = (p[data[i]]<<0) | (p[data[i+1]]<<4);
+        switch (len-i) {
+        case 1: out[j++] = p[data[i]];
+        }
+        *out_len = j;
+        return out;
+
+    case 4: {
+        for (i = 0; i < (len & ~3); i+=4)
+            out[j++] = (p[data[i]]<<0) | (p[data[i+1]]<<2) | (p[data[i+2]]<<4) | (p[data[i+3]]<<6);
+        out[j] = 0;
+        int s = len-i, x = 0;
+        switch (s) {
+        case 3: out[j] |= p[data[i++]] << x; x+=2; // fall-through
+        case 2: out[j] |= p[data[i++]] << x; x+=2; // fall-through
+        case 1: out[j] |= p[data[i++]] << x; x+=2;
+            j++;
+        }
+        *out_len = j;
+        return out;
+    }
+
+    case 8: {
+        for (i = 0; i < (len & ~7); i+=8)
+            out[j++] = (p[data[i+0]]<<0) | (p[data[i+1]]<<1) | (p[data[i+2]]<<2) | (p[data[i+3]]<<3)
+                     | (p[data[i+4]]<<4) | (p[data[i+5]]<<5) | (p[data[i+6]]<<6) | (p[data[i+7]]<<7);
+        out[j] = 0;
+        int s = len-i, x = 0;
+        switch (s) {
+        case 7: out[j] |= p[data[i++]] << x++; // fall-through
+        case 6: out[j] |= p[data[i++]] << x++; // fall-through
+        case 5: out[j] |= p[data[i++]] << x++; // fall-through
+        case 4: out[j] |= p[data[i++]] << x++; // fall-through
+        case 3: out[j] |= p[data[i++]] << x++; // fall-through
+        case 2: out[j] |= p[data[i++]] << x++; // fall-through
+        case 1: out[j] |= p[data[i++]] << x++;
+            j++;
+        }
+        *out_len = j;
+        return out;
+    }
+
+    case 0:
+        *out_len = j;
+        return out;
+    }
+
+    return NULL;
+}
+
+
+/*
+ * Unpacks the meta-data portions of the hts_pack algorithm.
+ * This consists of the count of symbols and their values.
+ *
+ * The "map" array is filled out with the used symbols.
+ * "nsym" is set to contain the number of symbols per byte;
+ * 0, 1, 2, 4 or 8.
+ *
+ * Returns number of bytes of data[] consumed on success,
+ *         zero on failure.
+ */
+uint8_t hts_unpack_meta(uint8_t *data, uint32_t data_len,
+                        uint64_t udata_len, uint8_t *map, int *nsym) {
+    if (data_len == 0)
+        return 0;
+
+    // Number of symbols used
+    unsigned int n = data[0];
+    if (n == 0)
+        n = 256;
+
+    // Symbols per byte
+    if (n <= 1)
+        *nsym = 0;
+    else if (n <= 2)
+        *nsym = 8;
+    else if (n <= 4)
+        *nsym = 4;
+    else if (n <= 16)
+        *nsym = 2;
+    else {
+        *nsym = 1; // no packing
+        return 1;
+    }
+
+    if (data_len <= 1)
+        return 0;
+
+    int j = 1, c = 0;
+    do {
+        map[c++] = data[j++];
+    } while (c < n && j < data_len);
+
+    return c < n ? 0 : j;
+}
+
+/*
+ * Unpacks a packed data steam (given the unpacked meta-data).
+ *
+ * "map" is the pack map, mapping 0->n to the expanded symbols.
+ * The "out" buffer must be preallocated by the caller to be the correct
+ * size.  For error checking purposes, out_len is set to the size of
+ * this buffer.
+ *
+ * Returns uncompressed data (out) on success,
+ *         NULL on failure.
+ */
+uint8_t *hts_unpack(uint8_t *data, int64_t len, uint8_t *out, uint64_t out_len, int nsym, uint8_t *p) {
+    //uint8_t *out;
+    uint8_t c = 0;
+    int64_t i, j = 0, olen;
+
+    if (nsym == 1) {
+        // raw data; FIXME: shortcut the need for malloc & memcpy here
+        memcpy(out, data, len);
+        return out;
+    }
+
+    switch(nsym) {
+    case 8: {
+        union {
+            uint64_t w;
+            uint8_t c[8];
+        } map[256];
+        int x;
+        for (x = 0; x < 256; x++) {
+            map[x].c[0] = p[x>>0&1];
+            map[x].c[1] = p[x>>1&1];
+            map[x].c[2] = p[x>>2&1];
+            map[x].c[3] = p[x>>3&1];
+            map[x].c[4] = p[x>>4&1];
+            map[x].c[5] = p[x>>5&1];
+            map[x].c[6] = p[x>>6&1];
+            map[x].c[7] = p[x>>7&1];
+        }
+        if ((out_len+7)/8 > len)
+            return NULL;
+        olen = out_len & ~7;
+
+        for (i = 0; i < olen; i+=8)
+            memcpy(&out[i], &map[data[j++]].w, 8);
+
+        if (out_len != olen) {
+            c = data[j++];
+            while (i < out_len) {
+                out[i++] = p[c & 1];
+                c >>= 1;
+            }
+        }
+        break;
+    }
+
+    case 4: {
+        union {
+            uint32_t w;
+            uint8_t c[4];
+        } map[256];
+
+        int x, y, z, _, P=0;
+        for (x = 0; x < 4; x++)
+            for (y = 0; y < 4; y++)
+                for (z = 0; z < 4; z++)
+                    for (_ = 0; _ < 4; _++, P++) {
+                        map[P].c[0] = p[_];
+                        map[P].c[1] = p[z];
+                        map[P].c[2] = p[y];
+                        map[P].c[3] = p[x];
+                    }
+
+        if ((out_len+3)/4 > len)
+            return NULL;
+        olen = out_len & ~3;
+
+        for (i = 0; i < olen-12; i+=16) {
+            uint32_t w[] = {
+                map[data[j+0]].w,
+                map[data[j+1]].w,
+                map[data[j+2]].w,
+                map[data[j+3]].w
+            };
+            j += 4;
+            memcpy(&out[i], &w, 16);
+        }
+
+        for (; i < olen; i+=4)
+            memcpy(&out[i], &map[data[j++]].w, 4);
+
+        if (out_len != olen) {
+            c = data[j++];
+            while (i < out_len) {
+                out[i++] = p[c & 3];
+                c >>= 2;
+            }
+        }
+        break;
+    }
+
+    case 2: {
+        union {
+            uint16_t w;
+            uint8_t c[2];
+        } map[256];
+
+        int x, y;
+        for (x = 0; x < 16; x++) {
+            for (y = 0; y < 16; y++) {
+                map[x*16+y].c[0] = p[y];
+                map[x*16+y].c[1] = p[x];
+            }
+        }
+
+        if ((out_len+1)/2 > len)
+            return NULL;
+        olen = out_len & ~1;
+
+        for (i = j = 0; i+2 < olen; i+=4) {
+            uint16_t w[] = {
+                map[data[j+0]].w,
+                map[data[j+1]].w
+            };
+            memcpy(&out[i], &w, 4);
+
+            j += 2;
+        }
+
+        for (; i < olen; i+=2)
+            memcpy(&out[i], &map[data[j++]].w, 2);
+
+        if (out_len != olen) {
+            c = data[j++];
+            out[i+0] = p[c&15];
+        }
+        break;
+    }
+
+    case 0:
+        memset(out, p[0], out_len);
+        break;
+
+    default:
+        return NULL;
+    }
+
+    return out;
+}
+
+
+uint8_t *hts_unpack_(uint8_t *data, int64_t len, uint8_t *out, uint64_t out_len, int nsym, uint8_t *p) {
+    //uint8_t *out;
+    uint8_t c = 0;
+    int64_t i, j = 0, olen;
+
+    if (nsym == 1) {
+        // raw data; FIXME: shortcut the need for malloc & memcpy here
+        memcpy(out, data, len);
+        return out;
+    }
+
+    switch(nsym) {
+    case 2: {
+        uint16_t map[256], x, y;
+        for (x = 0; x < 16; x++)
+            for (y = 0; y < 16; y++)
+                map[x*16+y] = p[x]*256+p[y];
+
+        if ((out_len+1)/2 > len)
+            return NULL;
+        olen = out_len & ~1;
+
+        uint16_t *o16 = (uint16_t *)out;
+        for (i = 0; i+4 < olen/2; i+=4) {
+            int k;
+            for (k = 0; k < 4; k++)
+                o16[i+k] = map[data[i+k]];
+        }
+        j = i; i *= 2;
+
+        for (; i < olen; i+=2) {
+            uint16_t w1 = map[data[j++]];
+            *(uint16_t *)&out[i] = w1;
+        }
+
+        if (out_len != olen) {
+            c = data[j++];
+            out[i+0] = p[c&15];
+        }
+        break;
+    }
+
+    default:
+        return NULL;
+    }
+
+    return out;
+}
diff --git a/ext/htslib/htscodecs/htscodecs/pack.h b/ext/htslib/htscodecs/htscodecs/pack.h
new file mode 100644
index 0000000..79b05df
--- /dev/null
+++ b/ext/htslib/htscodecs/htscodecs/pack.h
@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2019 Genome Research Ltd.
+ * Author(s): James Bonfield
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above
+ *       copyright notice, this list of conditions and the following
+ *       disclaimer in the documentation and/or other materials provided
+ *       with the distribution.
+ *
+ *    3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+ *       Institute nor the names of its contributors may be used to endorse
+ *       or promote products derived from this software without specific
+ *       prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH
+ * LTD OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef HTS_PACK_H
+#define HTS_PACK_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Packs multiple symbols into a single byte if the total alphabet of symbols
+ * used is <= 16.  Each new symbol takes up 1, 2, 4 or 8 bits, or 0 if the
+ * alphabet used is 1 (constant).
+ *
+ * If successful, out_meta/out_meta_len are set to hold the mapping table
+ * to be used during decompression.
+ *
+ * Returns the packed buffer on success with new length in out_len,
+ *         NULL of failure
+ */
+uint8_t *hts_pack(uint8_t *data, int64_t len,
+                  uint8_t *out_meta, int *out_meta_len, uint64_t *out_len);
+
+/*
+ * Unpacks the meta-data portions of the hts_pack algorithm.
+ * This consists of the count of symbols and their values.
+ *
+ * The "map" array is filled out with the used symbols.
+ * "nsym" is set to contain the number of symbols per byte;
+ * 0, 1, 2, 4 or 8.
+ *
+ * Returns number of bytes of data[] consumed on success,
+ *         zero on failure.
+ */
+uint8_t hts_unpack_meta(uint8_t *data, uint32_t data_len,
+                        uint64_t udata_len, uint8_t *map, int *nsym);
+
+/*
+ * Unpacks a packed data steam (given the unpacked meta-data).
+ *
+ * "map" is the pack map, mapping 0->n to the expanded symbols.
+ * The "out" buffer must be preallocated by the caller to be the correct
+ * size.  For error checking purposes, out_len is set to the size of
+ * this buffer.
+ *
+ * Returns uncompressed data (out) on success,
+ *         NULL on failure.
+ */
+uint8_t *hts_unpack(uint8_t *data, int64_t len, uint8_t *out, uint64_t out_len, int nsym, uint8_t *map);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* HTS_PACK_H */
diff --git a/ext/htslib/htscodecs/htscodecs/permute.h b/ext/htslib/htscodecs/htscodecs/permute.h
new file mode 100644
index 0000000..71a4b7e
--- /dev/null
+++ b/ext/htslib/htscodecs/htscodecs/permute.h
@@ -0,0 +1,605 @@
+#ifdef MAIN
+#include <stdio.h>
+
+/*
+ * Shuffle based on input bits.
+ * So bit N true  => keep Nth byte.
+ *    bit N false => skip Nth byte.
+ */
+
+int main(void) {
+    int i, j;
+
+    FILE *fp = fopen(__FILE__, "r");
+    char line[8192];
+    while(fgets(line, 8192, fp)) {
+        printf("%s", line);
+    }
+    close(fp);
+    printf("\n");
+
+    // Decode table; distributes N adjacent values across lanes
+    printf("#define _ 9\n");
+    printf("static uint32_t permute[256][8] = { // reverse binary bit order\n");
+    for (i = 0; i < 256; i++) {
+        int b = 0;
+        int v[8] = {0};
+        for (j = 0; j < 8; j++) {
+            if (i & (1<<j)) {
+                v[j] = ++b;
+            }
+        }
+        printf("  { ");
+        for (j = 0; j < 8; j++) {
+            if (v[j])
+                printf("%d,", v[j]-1);
+            else
+                printf("_,");
+        }
+        printf("},\n");
+    }
+    printf("};\n\n");
+
+    // Encode table; collapses N values spread across lanes
+    printf("static uint32_t permutec[256][8] = { // reverse binary bit order\n"); 
+    for (i = 0; i < 256; i++) {
+        int b = 0;
+        int v[9] = {0};
+        for (j = 0; j < 8; j++) {
+            if (i & (1<<j)) {
+                v[b++] = j+1;
+            }
+        }
+        printf("  { ");
+        for (j = b-8; j < b; j++) {
+            if (j >= 0 && v[j])
+                printf("%d,", v[j]-1);
+            else
+                printf("_,");
+        }
+        printf("},\n");
+    }
+    printf("};\n");
+
+   return 0;
+}
+#endif
+
+/*
+ * These tables are 8k.  On older systems with small L1 cache, this may be
+ * a problem.
+ *
+ * #define PM(a,b,c,d,e,f,g,h) ((a<<0)|(b<<4)|(c<<8)|(d<<12)|(e<<16)|(f<<20)|(g<<24)|(h<<28))
+ *
+ * Instead of permute via
+ *    __m256i idx1 = _mm256_load_si256((const __m256i*)permute[imask1]);
+ *
+ * we can pack the indices and shift them back again
+ *   __m256i idx1 = _mm256_srlv_epi32(_mm256_set1_epi32(permute2[imask1]),
+ *                                    _mm256_set_epi32(28,24,20,16,12,8,4,0));
+ *
+ * However on my Haswell system this slows down r32x16b_avx2 from 1440 to
+ * 1200 MB/s decode speeds.
+ * It's much closer for order-1 decoder, but still doesn't help.
+ *
+ * The encoder side seems to make no difference either way or be very marginal.
+ */
+
+#define _ 9
+static uint32_t permute[256][8] __attribute__((aligned(32))) = { // reverse binary bit order
+  { _,_,_,_,_,_,_,_,},
+  { 0,_,_,_,_,_,_,_,},
+  { _,0,_,_,_,_,_,_,},
+  { 0,1,_,_,_,_,_,_,},
+  { _,_,0,_,_,_,_,_,},
+  { 0,_,1,_,_,_,_,_,},
+  { _,0,1,_,_,_,_,_,},
+  { 0,1,2,_,_,_,_,_,},
+  { _,_,_,0,_,_,_,_,},
+  { 0,_,_,1,_,_,_,_,},
+  { _,0,_,1,_,_,_,_,},
+  { 0,1,_,2,_,_,_,_,},
+  { _,_,0,1,_,_,_,_,},
+  { 0,_,1,2,_,_,_,_,},
+  { _,0,1,2,_,_,_,_,},
+  { 0,1,2,3,_,_,_,_,},
+  { _,_,_,_,0,_,_,_,},
+  { 0,_,_,_,1,_,_,_,},
+  { _,0,_,_,1,_,_,_,},
+  { 0,1,_,_,2,_,_,_,},
+  { _,_,0,_,1,_,_,_,},
+  { 0,_,1,_,2,_,_,_,},
+  { _,0,1,_,2,_,_,_,},
+  { 0,1,2,_,3,_,_,_,},
+  { _,_,_,0,1,_,_,_,},
+  { 0,_,_,1,2,_,_,_,},
+  { _,0,_,1,2,_,_,_,},
+  { 0,1,_,2,3,_,_,_,},
+  { _,_,0,1,2,_,_,_,},
+  { 0,_,1,2,3,_,_,_,},
+  { _,0,1,2,3,_,_,_,},
+  { 0,1,2,3,4,_,_,_,},
+  { _,_,_,_,_,0,_,_,},
+  { 0,_,_,_,_,1,_,_,},
+  { _,0,_,_,_,1,_,_,},
+  { 0,1,_,_,_,2,_,_,},
+  { _,_,0,_,_,1,_,_,},
+  { 0,_,1,_,_,2,_,_,},
+  { _,0,1,_,_,2,_,_,},
+  { 0,1,2,_,_,3,_,_,},
+  { _,_,_,0,_,1,_,_,},
+  { 0,_,_,1,_,2,_,_,},
+  { _,0,_,1,_,2,_,_,},
+  { 0,1,_,2,_,3,_,_,},
+  { _,_,0,1,_,2,_,_,},
+  { 0,_,1,2,_,3,_,_,},
+  { _,0,1,2,_,3,_,_,},
+  { 0,1,2,3,_,4,_,_,},
+  { _,_,_,_,0,1,_,_,},
+  { 0,_,_,_,1,2,_,_,},
+  { _,0,_,_,1,2,_,_,},
+  { 0,1,_,_,2,3,_,_,},
+  { _,_,0,_,1,2,_,_,},
+  { 0,_,1,_,2,3,_,_,},
+  { _,0,1,_,2,3,_,_,},
+  { 0,1,2,_,3,4,_,_,},
+  { _,_,_,0,1,2,_,_,},
+  { 0,_,_,1,2,3,_,_,},
+  { _,0,_,1,2,3,_,_,},
+  { 0,1,_,2,3,4,_,_,},
+  { _,_,0,1,2,3,_,_,},
+  { 0,_,1,2,3,4,_,_,},
+  { _,0,1,2,3,4,_,_,},
+  { 0,1,2,3,4,5,_,_,},
+  { _,_,_,_,_,_,0,_,},
+  { 0,_,_,_,_,_,1,_,},
+  { _,0,_,_,_,_,1,_,},
+  { 0,1,_,_,_,_,2,_,},
+  { _,_,0,_,_,_,1,_,},
+  { 0,_,1,_,_,_,2,_,},
+  { _,0,1,_,_,_,2,_,},
+  { 0,1,2,_,_,_,3,_,},
+  { _,_,_,0,_,_,1,_,},
+  { 0,_,_,1,_,_,2,_,},
+  { _,0,_,1,_,_,2,_,},
+  { 0,1,_,2,_,_,3,_,},
+  { _,_,0,1,_,_,2,_,},
+  { 0,_,1,2,_,_,3,_,},
+  { _,0,1,2,_,_,3,_,},
+  { 0,1,2,3,_,_,4,_,},
+  { _,_,_,_,0,_,1,_,},
+  { 0,_,_,_,1,_,2,_,},
+  { _,0,_,_,1,_,2,_,},
+  { 0,1,_,_,2,_,3,_,},
+  { _,_,0,_,1,_,2,_,},
+  { 0,_,1,_,2,_,3,_,},
+  { _,0,1,_,2,_,3,_,},
+  { 0,1,2,_,3,_,4,_,},
+  { _,_,_,0,1,_,2,_,},
+  { 0,_,_,1,2,_,3,_,},
+  { _,0,_,1,2,_,3,_,},
+  { 0,1,_,2,3,_,4,_,},
+  { _,_,0,1,2,_,3,_,},
+  { 0,_,1,2,3,_,4,_,},
+  { _,0,1,2,3,_,4,_,},
+  { 0,1,2,3,4,_,5,_,},
+  { _,_,_,_,_,0,1,_,},
+  { 0,_,_,_,_,1,2,_,},
+  { _,0,_,_,_,1,2,_,},
+  { 0,1,_,_,_,2,3,_,},
+  { _,_,0,_,_,1,2,_,},
+  { 0,_,1,_,_,2,3,_,},
+  { _,0,1,_,_,2,3,_,},
+  { 0,1,2,_,_,3,4,_,},
+  { _,_,_,0,_,1,2,_,},
+  { 0,_,_,1,_,2,3,_,},
+  { _,0,_,1,_,2,3,_,},
+  { 0,1,_,2,_,3,4,_,},
+  { _,_,0,1,_,2,3,_,},
+  { 0,_,1,2,_,3,4,_,},
+  { _,0,1,2,_,3,4,_,},
+  { 0,1,2,3,_,4,5,_,},
+  { _,_,_,_,0,1,2,_,},
+  { 0,_,_,_,1,2,3,_,},
+  { _,0,_,_,1,2,3,_,},
+  { 0,1,_,_,2,3,4,_,},
+  { _,_,0,_,1,2,3,_,},
+  { 0,_,1,_,2,3,4,_,},
+  { _,0,1,_,2,3,4,_,},
+  { 0,1,2,_,3,4,5,_,},
+  { _,_,_,0,1,2,3,_,},
+  { 0,_,_,1,2,3,4,_,},
+  { _,0,_,1,2,3,4,_,},
+  { 0,1,_,2,3,4,5,_,},
+  { _,_,0,1,2,3,4,_,},
+  { 0,_,1,2,3,4,5,_,},
+  { _,0,1,2,3,4,5,_,},
+  { 0,1,2,3,4,5,6,_,},
+  { _,_,_,_,_,_,_,0,},
+  { 0,_,_,_,_,_,_,1,},
+  { _,0,_,_,_,_,_,1,},
+  { 0,1,_,_,_,_,_,2,},
+  { _,_,0,_,_,_,_,1,},
+  { 0,_,1,_,_,_,_,2,},
+  { _,0,1,_,_,_,_,2,},
+  { 0,1,2,_,_,_,_,3,},
+  { _,_,_,0,_,_,_,1,},
+  { 0,_,_,1,_,_,_,2,},
+  { _,0,_,1,_,_,_,2,},
+  { 0,1,_,2,_,_,_,3,},
+  { _,_,0,1,_,_,_,2,},
+  { 0,_,1,2,_,_,_,3,},
+  { _,0,1,2,_,_,_,3,},
+  { 0,1,2,3,_,_,_,4,},
+  { _,_,_,_,0,_,_,1,},
+  { 0,_,_,_,1,_,_,2,},
+  { _,0,_,_,1,_,_,2,},
+  { 0,1,_,_,2,_,_,3,},
+  { _,_,0,_,1,_,_,2,},
+  { 0,_,1,_,2,_,_,3,},
+  { _,0,1,_,2,_,_,3,},
+  { 0,1,2,_,3,_,_,4,},
+  { _,_,_,0,1,_,_,2,},
+  { 0,_,_,1,2,_,_,3,},
+  { _,0,_,1,2,_,_,3,},
+  { 0,1,_,2,3,_,_,4,},
+  { _,_,0,1,2,_,_,3,},
+  { 0,_,1,2,3,_,_,4,},
+  { _,0,1,2,3,_,_,4,},
+  { 0,1,2,3,4,_,_,5,},
+  { _,_,_,_,_,0,_,1,},
+  { 0,_,_,_,_,1,_,2,},
+  { _,0,_,_,_,1,_,2,},
+  { 0,1,_,_,_,2,_,3,},
+  { _,_,0,_,_,1,_,2,},
+  { 0,_,1,_,_,2,_,3,},
+  { _,0,1,_,_,2,_,3,},
+  { 0,1,2,_,_,3,_,4,},
+  { _,_,_,0,_,1,_,2,},
+  { 0,_,_,1,_,2,_,3,},
+  { _,0,_,1,_,2,_,3,},
+  { 0,1,_,2,_,3,_,4,},
+  { _,_,0,1,_,2,_,3,},
+  { 0,_,1,2,_,3,_,4,},
+  { _,0,1,2,_,3,_,4,},
+  { 0,1,2,3,_,4,_,5,},
+  { _,_,_,_,0,1,_,2,},
+  { 0,_,_,_,1,2,_,3,},
+  { _,0,_,_,1,2,_,3,},
+  { 0,1,_,_,2,3,_,4,},
+  { _,_,0,_,1,2,_,3,},
+  { 0,_,1,_,2,3,_,4,},
+  { _,0,1,_,2,3,_,4,},
+  { 0,1,2,_,3,4,_,5,},
+  { _,_,_,0,1,2,_,3,},
+  { 0,_,_,1,2,3,_,4,},
+  { _,0,_,1,2,3,_,4,},
+  { 0,1,_,2,3,4,_,5,},
+  { _,_,0,1,2,3,_,4,},
+  { 0,_,1,2,3,4,_,5,},
+  { _,0,1,2,3,4,_,5,},
+  { 0,1,2,3,4,5,_,6,},
+  { _,_,_,_,_,_,0,1,},
+  { 0,_,_,_,_,_,1,2,},
+  { _,0,_,_,_,_,1,2,},
+  { 0,1,_,_,_,_,2,3,},
+  { _,_,0,_,_,_,1,2,},
+  { 0,_,1,_,_,_,2,3,},
+  { _,0,1,_,_,_,2,3,},
+  { 0,1,2,_,_,_,3,4,},
+  { _,_,_,0,_,_,1,2,},
+  { 0,_,_,1,_,_,2,3,},
+  { _,0,_,1,_,_,2,3,},
+  { 0,1,_,2,_,_,3,4,},
+  { _,_,0,1,_,_,2,3,},
+  { 0,_,1,2,_,_,3,4,},
+  { _,0,1,2,_,_,3,4,},
+  { 0,1,2,3,_,_,4,5,},
+  { _,_,_,_,0,_,1,2,},
+  { 0,_,_,_,1,_,2,3,},
+  { _,0,_,_,1,_,2,3,},
+  { 0,1,_,_,2,_,3,4,},
+  { _,_,0,_,1,_,2,3,},
+  { 0,_,1,_,2,_,3,4,},
+  { _,0,1,_,2,_,3,4,},
+  { 0,1,2,_,3,_,4,5,},
+  { _,_,_,0,1,_,2,3,},
+  { 0,_,_,1,2,_,3,4,},
+  { _,0,_,1,2,_,3,4,},
+  { 0,1,_,2,3,_,4,5,},
+  { _,_,0,1,2,_,3,4,},
+  { 0,_,1,2,3,_,4,5,},
+  { _,0,1,2,3,_,4,5,},
+  { 0,1,2,3,4,_,5,6,},
+  { _,_,_,_,_,0,1,2,},
+  { 0,_,_,_,_,1,2,3,},
+  { _,0,_,_,_,1,2,3,},
+  { 0,1,_,_,_,2,3,4,},
+  { _,_,0,_,_,1,2,3,},
+  { 0,_,1,_,_,2,3,4,},
+  { _,0,1,_,_,2,3,4,},
+  { 0,1,2,_,_,3,4,5,},
+  { _,_,_,0,_,1,2,3,},
+  { 0,_,_,1,_,2,3,4,},
+  { _,0,_,1,_,2,3,4,},
+  { 0,1,_,2,_,3,4,5,},
+  { _,_,0,1,_,2,3,4,},
+  { 0,_,1,2,_,3,4,5,},
+  { _,0,1,2,_,3,4,5,},
+  { 0,1,2,3,_,4,5,6,},
+  { _,_,_,_,0,1,2,3,},
+  { 0,_,_,_,1,2,3,4,},
+  { _,0,_,_,1,2,3,4,},
+  { 0,1,_,_,2,3,4,5,},
+  { _,_,0,_,1,2,3,4,},
+  { 0,_,1,_,2,3,4,5,},
+  { _,0,1,_,2,3,4,5,},
+  { 0,1,2,_,3,4,5,6,},
+  { _,_,_,0,1,2,3,4,},
+  { 0,_,_,1,2,3,4,5,},
+  { _,0,_,1,2,3,4,5,},
+  { 0,1,_,2,3,4,5,6,},
+  { _,_,0,1,2,3,4,5,},
+  { 0,_,1,2,3,4,5,6,},
+  { _,0,1,2,3,4,5,6,},
+  { 0,1,2,3,4,5,6,7,},
+};
+
+static uint32_t permutec[256][8] __attribute__((aligned(32))) = { // reverse binary bit order
+  { _,_,_,_,_,_,_,_,},
+  { _,_,_,_,_,_,_,0,},
+  { _,_,_,_,_,_,_,1,},
+  { _,_,_,_,_,_,0,1,},
+  { _,_,_,_,_,_,_,2,},
+  { _,_,_,_,_,_,0,2,},
+  { _,_,_,_,_,_,1,2,},
+  { _,_,_,_,_,0,1,2,},
+  { _,_,_,_,_,_,_,3,},
+  { _,_,_,_,_,_,0,3,},
+  { _,_,_,_,_,_,1,3,},
+  { _,_,_,_,_,0,1,3,},
+  { _,_,_,_,_,_,2,3,},
+  { _,_,_,_,_,0,2,3,},
+  { _,_,_,_,_,1,2,3,},
+  { _,_,_,_,0,1,2,3,},
+  { _,_,_,_,_,_,_,4,},
+  { _,_,_,_,_,_,0,4,},
+  { _,_,_,_,_,_,1,4,},
+  { _,_,_,_,_,0,1,4,},
+  { _,_,_,_,_,_,2,4,},
+  { _,_,_,_,_,0,2,4,},
+  { _,_,_,_,_,1,2,4,},
+  { _,_,_,_,0,1,2,4,},
+  { _,_,_,_,_,_,3,4,},
+  { _,_,_,_,_,0,3,4,},
+  { _,_,_,_,_,1,3,4,},
+  { _,_,_,_,0,1,3,4,},
+  { _,_,_,_,_,2,3,4,},
+  { _,_,_,_,0,2,3,4,},
+  { _,_,_,_,1,2,3,4,},
+  { _,_,_,0,1,2,3,4,},
+  { _,_,_,_,_,_,_,5,},
+  { _,_,_,_,_,_,0,5,},
+  { _,_,_,_,_,_,1,5,},
+  { _,_,_,_,_,0,1,5,},
+  { _,_,_,_,_,_,2,5,},
+  { _,_,_,_,_,0,2,5,},
+  { _,_,_,_,_,1,2,5,},
+  { _,_,_,_,0,1,2,5,},
+  { _,_,_,_,_,_,3,5,},
+  { _,_,_,_,_,0,3,5,},
+  { _,_,_,_,_,1,3,5,},
+  { _,_,_,_,0,1,3,5,},
+  { _,_,_,_,_,2,3,5,},
+  { _,_,_,_,0,2,3,5,},
+  { _,_,_,_,1,2,3,5,},
+  { _,_,_,0,1,2,3,5,},
+  { _,_,_,_,_,_,4,5,},
+  { _,_,_,_,_,0,4,5,},
+  { _,_,_,_,_,1,4,5,},
+  { _,_,_,_,0,1,4,5,},
+  { _,_,_,_,_,2,4,5,},
+  { _,_,_,_,0,2,4,5,},
+  { _,_,_,_,1,2,4,5,},
+  { _,_,_,0,1,2,4,5,},
+  { _,_,_,_,_,3,4,5,},
+  { _,_,_,_,0,3,4,5,},
+  { _,_,_,_,1,3,4,5,},
+  { _,_,_,0,1,3,4,5,},
+  { _,_,_,_,2,3,4,5,},
+  { _,_,_,0,2,3,4,5,},
+  { _,_,_,1,2,3,4,5,},
+  { _,_,0,1,2,3,4,5,},
+  { _,_,_,_,_,_,_,6,},
+  { _,_,_,_,_,_,0,6,},
+  { _,_,_,_,_,_,1,6,},
+  { _,_,_,_,_,0,1,6,},
+  { _,_,_,_,_,_,2,6,},
+  { _,_,_,_,_,0,2,6,},
+  { _,_,_,_,_,1,2,6,},
+  { _,_,_,_,0,1,2,6,},
+  { _,_,_,_,_,_,3,6,},
+  { _,_,_,_,_,0,3,6,},
+  { _,_,_,_,_,1,3,6,},
+  { _,_,_,_,0,1,3,6,},
+  { _,_,_,_,_,2,3,6,},
+  { _,_,_,_,0,2,3,6,},
+  { _,_,_,_,1,2,3,6,},
+  { _,_,_,0,1,2,3,6,},
+  { _,_,_,_,_,_,4,6,},
+  { _,_,_,_,_,0,4,6,},
+  { _,_,_,_,_,1,4,6,},
+  { _,_,_,_,0,1,4,6,},
+  { _,_,_,_,_,2,4,6,},
+  { _,_,_,_,0,2,4,6,},
+  { _,_,_,_,1,2,4,6,},
+  { _,_,_,0,1,2,4,6,},
+  { _,_,_,_,_,3,4,6,},
+  { _,_,_,_,0,3,4,6,},
+  { _,_,_,_,1,3,4,6,},
+  { _,_,_,0,1,3,4,6,},
+  { _,_,_,_,2,3,4,6,},
+  { _,_,_,0,2,3,4,6,},
+  { _,_,_,1,2,3,4,6,},
+  { _,_,0,1,2,3,4,6,},
+  { _,_,_,_,_,_,5,6,},
+  { _,_,_,_,_,0,5,6,},
+  { _,_,_,_,_,1,5,6,},
+  { _,_,_,_,0,1,5,6,},
+  { _,_,_,_,_,2,5,6,},
+  { _,_,_,_,0,2,5,6,},
+  { _,_,_,_,1,2,5,6,},
+  { _,_,_,0,1,2,5,6,},
+  { _,_,_,_,_,3,5,6,},
+  { _,_,_,_,0,3,5,6,},
+  { _,_,_,_,1,3,5,6,},
+  { _,_,_,0,1,3,5,6,},
+  { _,_,_,_,2,3,5,6,},
+  { _,_,_,0,2,3,5,6,},
+  { _,_,_,1,2,3,5,6,},
+  { _,_,0,1,2,3,5,6,},
+  { _,_,_,_,_,4,5,6,},
+  { _,_,_,_,0,4,5,6,},
+  { _,_,_,_,1,4,5,6,},
+  { _,_,_,0,1,4,5,6,},
+  { _,_,_,_,2,4,5,6,},
+  { _,_,_,0,2,4,5,6,},
+  { _,_,_,1,2,4,5,6,},
+  { _,_,0,1,2,4,5,6,},
+  { _,_,_,_,3,4,5,6,},
+  { _,_,_,0,3,4,5,6,},
+  { _,_,_,1,3,4,5,6,},
+  { _,_,0,1,3,4,5,6,},
+  { _,_,_,2,3,4,5,6,},
+  { _,_,0,2,3,4,5,6,},
+  { _,_,1,2,3,4,5,6,},
+  { _,0,1,2,3,4,5,6,},
+  { _,_,_,_,_,_,_,7,},
+  { _,_,_,_,_,_,0,7,},
+  { _,_,_,_,_,_,1,7,},
+  { _,_,_,_,_,0,1,7,},
+  { _,_,_,_,_,_,2,7,},
+  { _,_,_,_,_,0,2,7,},
+  { _,_,_,_,_,1,2,7,},
+  { _,_,_,_,0,1,2,7,},
+  { _,_,_,_,_,_,3,7,},
+  { _,_,_,_,_,0,3,7,},
+  { _,_,_,_,_,1,3,7,},
+  { _,_,_,_,0,1,3,7,},
+  { _,_,_,_,_,2,3,7,},
+  { _,_,_,_,0,2,3,7,},
+  { _,_,_,_,1,2,3,7,},
+  { _,_,_,0,1,2,3,7,},
+  { _,_,_,_,_,_,4,7,},
+  { _,_,_,_,_,0,4,7,},
+  { _,_,_,_,_,1,4,7,},
+  { _,_,_,_,0,1,4,7,},
+  { _,_,_,_,_,2,4,7,},
+  { _,_,_,_,0,2,4,7,},
+  { _,_,_,_,1,2,4,7,},
+  { _,_,_,0,1,2,4,7,},
+  { _,_,_,_,_,3,4,7,},
+  { _,_,_,_,0,3,4,7,},
+  { _,_,_,_,1,3,4,7,},
+  { _,_,_,0,1,3,4,7,},
+  { _,_,_,_,2,3,4,7,},
+  { _,_,_,0,2,3,4,7,},
+  { _,_,_,1,2,3,4,7,},
+  { _,_,0,1,2,3,4,7,},
+  { _,_,_,_,_,_,5,7,},
+  { _,_,_,_,_,0,5,7,},
+  { _,_,_,_,_,1,5,7,},
+  { _,_,_,_,0,1,5,7,},
+  { _,_,_,_,_,2,5,7,},
+  { _,_,_,_,0,2,5,7,},
+  { _,_,_,_,1,2,5,7,},
+  { _,_,_,0,1,2,5,7,},
+  { _,_,_,_,_,3,5,7,},
+  { _,_,_,_,0,3,5,7,},
+  { _,_,_,_,1,3,5,7,},
+  { _,_,_,0,1,3,5,7,},
+  { _,_,_,_,2,3,5,7,},
+  { _,_,_,0,2,3,5,7,},
+  { _,_,_,1,2,3,5,7,},
+  { _,_,0,1,2,3,5,7,},
+  { _,_,_,_,_,4,5,7,},
+  { _,_,_,_,0,4,5,7,},
+  { _,_,_,_,1,4,5,7,},
+  { _,_,_,0,1,4,5,7,},
+  { _,_,_,_,2,4,5,7,},
+  { _,_,_,0,2,4,5,7,},
+  { _,_,_,1,2,4,5,7,},
+  { _,_,0,1,2,4,5,7,},
+  { _,_,_,_,3,4,5,7,},
+  { _,_,_,0,3,4,5,7,},
+  { _,_,_,1,3,4,5,7,},
+  { _,_,0,1,3,4,5,7,},
+  { _,_,_,2,3,4,5,7,},
+  { _,_,0,2,3,4,5,7,},
+  { _,_,1,2,3,4,5,7,},
+  { _,0,1,2,3,4,5,7,},
+  { _,_,_,_,_,_,6,7,},
+  { _,_,_,_,_,0,6,7,},
+  { _,_,_,_,_,1,6,7,},
+  { _,_,_,_,0,1,6,7,},
+  { _,_,_,_,_,2,6,7,},
+  { _,_,_,_,0,2,6,7,},
+  { _,_,_,_,1,2,6,7,},
+  { _,_,_,0,1,2,6,7,},
+  { _,_,_,_,_,3,6,7,},
+  { _,_,_,_,0,3,6,7,},
+  { _,_,_,_,1,3,6,7,},
+  { _,_,_,0,1,3,6,7,},
+  { _,_,_,_,2,3,6,7,},
+  { _,_,_,0,2,3,6,7,},
+  { _,_,_,1,2,3,6,7,},
+  { _,_,0,1,2,3,6,7,},
+  { _,_,_,_,_,4,6,7,},
+  { _,_,_,_,0,4,6,7,},
+  { _,_,_,_,1,4,6,7,},
+  { _,_,_,0,1,4,6,7,},
+  { _,_,_,_,2,4,6,7,},
+  { _,_,_,0,2,4,6,7,},
+  { _,_,_,1,2,4,6,7,},
+  { _,_,0,1,2,4,6,7,},
+  { _,_,_,_,3,4,6,7,},
+  { _,_,_,0,3,4,6,7,},
+  { _,_,_,1,3,4,6,7,},
+  { _,_,0,1,3,4,6,7,},
+  { _,_,_,2,3,4,6,7,},
+  { _,_,0,2,3,4,6,7,},
+  { _,_,1,2,3,4,6,7,},
+  { _,0,1,2,3,4,6,7,},
+  { _,_,_,_,_,5,6,7,},
+  { _,_,_,_,0,5,6,7,},
+  { _,_,_,_,1,5,6,7,},
+  { _,_,_,0,1,5,6,7,},
+  { _,_,_,_,2,5,6,7,},
+  { _,_,_,0,2,5,6,7,},
+  { _,_,_,1,2,5,6,7,},
+  { _,_,0,1,2,5,6,7,},
+  { _,_,_,_,3,5,6,7,},
+  { _,_,_,0,3,5,6,7,},
+  { _,_,_,1,3,5,6,7,},
+  { _,_,0,1,3,5,6,7,},
+  { _,_,_,2,3,5,6,7,},
+  { _,_,0,2,3,5,6,7,},
+  { _,_,1,2,3,5,6,7,},
+  { _,0,1,2,3,5,6,7,},
+  { _,_,_,_,4,5,6,7,},
+  { _,_,_,0,4,5,6,7,},
+  { _,_,_,1,4,5,6,7,},
+  { _,_,0,1,4,5,6,7,},
+  { _,_,_,2,4,5,6,7,},
+  { _,_,0,2,4,5,6,7,},
+  { _,_,1,2,4,5,6,7,},
+  { _,0,1,2,4,5,6,7,},
+  { _,_,_,3,4,5,6,7,},
+  { _,_,0,3,4,5,6,7,},
+  { _,_,1,3,4,5,6,7,},
+  { _,0,1,3,4,5,6,7,},
+  { _,_,2,3,4,5,6,7,},
+  { _,0,2,3,4,5,6,7,},
+  { _,1,2,3,4,5,6,7,},
+  { 0,1,2,3,4,5,6,7,},
+};
diff --git a/ext/htslib/htscodecs/htscodecs/pooled_alloc.h b/ext/htslib/htscodecs/htscodecs/pooled_alloc.h
new file mode 100644
index 0000000..fa1218e
--- /dev/null
+++ b/ext/htslib/htscodecs/htscodecs/pooled_alloc.h
@@ -0,0 +1,145 @@
+/*
+ * Copyright (c) 2009-2010, 2013 Genome Research Ltd.
+ * Author(s): James Bonfield
+ * 
+ * Redistribution and use in source and binary forms, with or without 
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ *    1. Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimer.
+ * 
+ *    2. Redistributions in binary form must reproduce the above
+ *       copyright notice, this list of conditions and the following
+ *       disclaimer in the documentation and/or other materials provided
+ *       with the distribution.
+ * 
+ *    3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+ *    Institute nor the names of its contributors may be used to endorse
+ *    or promote products derived from this software without specific
+ *    prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH
+ * LTD OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+// Defined static here as we only use in one file for now and don't
+// want to pollute the library name space (io_lib has the same named
+// functions).
+
+#ifndef _POOLED_ALLOC_H_
+#define _POOLED_ALLOC_H_
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <stdint.h>
+
+/*
+ * Implements a pooled block allocator where all items are the same size,
+ * but we need many of them.
+ */
+typedef struct {
+    void   *pool;
+    size_t  used;
+} pool_t;
+
+typedef struct {
+    size_t dsize;
+    size_t npools;
+    pool_t *pools;
+    void *free;
+} pool_alloc_t;
+
+#define PSIZE 1024*1024
+
+static pool_alloc_t *pool_create(size_t dsize) {
+    pool_alloc_t *p;
+
+    if (NULL == (p = (pool_alloc_t *)malloc(sizeof(*p))))
+        return NULL;
+
+    /* Minimum size is a pointer, for free list */
+    dsize = (dsize + sizeof(void *) - 1) & ~(sizeof(void *)-1);
+    if (dsize < sizeof(void *))
+        dsize = sizeof(void *);
+    p->dsize = dsize;
+
+    p->npools = 0;
+    p->pools = NULL;
+    p->free  = NULL;
+
+    return p;
+}
+
+static pool_t *new_pool(pool_alloc_t *p) {
+    size_t n = PSIZE / p->dsize;
+    pool_t *pool;
+    
+    pool = realloc(p->pools, (p->npools + 1) * sizeof(*p->pools));
+    if (NULL == pool) return NULL;
+    p->pools = pool;
+    pool = &p->pools[p->npools];
+
+    pool->pool = malloc(n * p->dsize);
+    if (NULL == pool->pool) return NULL;
+
+    pool->used = 0;
+
+    p->npools++;
+
+    return pool;
+}
+
+static void pool_destroy(pool_alloc_t *p) {
+    size_t i;
+
+    for (i = 0; i < p->npools; i++) {
+        free(p->pools[i].pool);
+    }
+    free(p->pools);
+    free(p);
+}
+
+static void *pool_alloc(pool_alloc_t *p) {
+    pool_t *pool;
+    void *ret;
+
+    /* Look on free list */
+    if (NULL != p->free) {
+        ret = p->free;
+        p->free = *((void **)p->free);
+        return ret;
+    }
+
+    /* Look for space in the last pool */
+    if (p->npools) {
+        pool = &p->pools[p->npools - 1];
+        if (pool->used + p->dsize < PSIZE) {
+            ret = ((char *) pool->pool) + pool->used;
+            pool->used += p->dsize;
+            return ret;
+        }
+    }
+
+    /* Need a new pool */
+    pool = new_pool(p);
+    if (NULL == pool) return NULL;
+
+    pool->used = p->dsize;
+    return pool->pool;
+}
+
+// static void pool_free(pool_alloc_t *p, void *ptr) {
+//     *(void **)ptr = p->free;
+//     p->free = ptr;
+// }
+
+#endif /*_POOLED_ALLOC_H_*/
diff --git a/ext/htslib/htscodecs/htscodecs/rANS_byte.h b/ext/htslib/htscodecs/htscodecs/rANS_byte.h
new file mode 100644
index 0000000..968d157
--- /dev/null
+++ b/ext/htslib/htscodecs/htscodecs/rANS_byte.h
@@ -0,0 +1,569 @@
+/* rans_byte.h originally from https://github.com/rygorous/ryg_rans
+ *
+ * This is a public-domain implementation of several rANS variants. rANS is an
+ * entropy coder from the ANS family, as described in Jarek Duda's paper
+ * "Asymmetric numeral systems" (http://arxiv.org/abs/1311.2540).
+ */
+
+/*-------------------------------------------------------------------------- */
+/* rans_byte.h from https://github.com/rygorous/ryg_rans */
+
+// Simple byte-aligned rANS encoder/decoder - public domain - Fabian 'ryg' Giesen 2014
+//
+// Not intended to be "industrial strength"; just meant to illustrate the general
+// idea.
+
+#ifndef RANS_BYTE_HEADER
+#define RANS_BYTE_HEADER
+
+#include <stdio.h>
+#include <stdint.h>
+#include <assert.h>
+
+#include "utils.h"
+
+#ifdef assert
+#define RansAssert assert
+#else
+#define RansAssert(x)
+#endif
+
+// READ ME FIRST:
+//
+// This is designed like a typical arithmetic coder API, but there's three
+// twists you absolutely should be aware of before you start hacking:
+//
+// 1. You need to encode data in *reverse* - last symbol first. rANS works
+//    like a stack: last in, first out.
+// 2. Likewise, the encoder outputs bytes *in reverse* - that is, you give
+//    it a pointer to the *end* of your buffer (exclusive), and it will
+//    slowly move towards the beginning as more bytes are emitted.
+// 3. Unlike basically any other entropy coder implementation you might
+//    have used, you can interleave data from multiple independent rANS
+//    encoders into the same bytestream without any extra signaling;
+//    you can also just write some bytes by yourself in the middle if
+//    you want to. This is in addition to the usual arithmetic encoder
+//    property of being able to switch models on the fly. Writing raw
+//    bytes can be useful when you have some data that you know is
+//    incompressible, and is cheaper than going through the rANS encode
+//    function. Using multiple rANS coders on the same byte stream wastes
+//    a few bytes compared to using just one, but execution of two
+//    independent encoders can happen in parallel on superscalar and
+//    Out-of-Order CPUs, so this can be *much* faster in tight decoding
+//    loops.
+//
+//    This is why all the rANS functions take the write pointer as an
+//    argument instead of just storing it in some context struct.
+
+// --------------------------------------------------------------------------
+
+// L ('l' in the paper) is the lower bound of our normalization interval.
+// Between this and our byte-aligned emission, we use 31 (not 32!) bits.
+// This is done intentionally because exact reciprocals for 31-bit uints
+// fit in 32-bit uints: this permits some optimizations during encoding.
+#define RANS_BYTE_L (1u << 23)  // lower bound of our normalization interval
+
+// State for a rANS encoder. Yep, that's all there is to it.
+typedef uint32_t RansState;
+
+// Initialize a rANS encoder.
+static inline void RansEncInit(RansState* r)
+{
+    *r = RANS_BYTE_L;
+}
+
+#if 0 /* Curently unused */
+// Renormalize the encoder. Internal function.
+static inline RansState RansEncRenorm(RansState x, uint8_t** pptr, uint32_t freq, uint32_t scale_bits)
+{
+    uint32_t x_max = ((RANS_BYTE_L >> scale_bits) << 8) * freq; // this turns into a shift.
+    if (x >= x_max) {
+        uint8_t* ptr = *pptr;
+        do {
+            *--ptr = (uint8_t) (x & 0xff);
+            x >>= 8;
+        } while (x >= x_max);
+        *pptr = ptr;
+    }
+    return x;
+}
+
+// Encodes a single symbol with range start "start" and frequency "freq".
+// All frequencies are assumed to sum to "1 << scale_bits", and the
+// resulting bytes get written to ptr (which is updated).
+//
+// NOTE: With rANS, you need to encode symbols in *reverse order*, i.e. from
+// beginning to end! Likewise, the output bytestream is written *backwards*:
+// ptr starts pointing at the end of the output buffer and keeps decrementing.
+static inline void RansEncPut(RansState* r, uint8_t** pptr, uint32_t start, uint32_t freq, uint32_t scale_bits)
+{
+    // renormalize
+    RansState x = RansEncRenorm(*r, pptr, freq, scale_bits);
+
+    // x = C(s,x)
+    *r = ((x / freq) << scale_bits) + (x % freq) + start;
+}
+#endif /* Curently unused */
+
+// Flushes the rANS encoder.
+static inline void RansEncFlush(RansState* r, uint8_t** pptr)
+{
+    uint32_t x = *r;
+    uint8_t* ptr = *pptr;
+
+    ptr -= 4;
+    ptr[0] = (uint8_t) (x >> 0);
+    ptr[1] = (uint8_t) (x >> 8);
+    ptr[2] = (uint8_t) (x >> 16);
+    ptr[3] = (uint8_t) (x >> 24);
+
+    *pptr = ptr;
+}
+
+// Initializes a rANS decoder.
+// Unlike the encoder, the decoder works forwards as you'd expect.
+static inline void RansDecInit(RansState* r, uint8_t** pptr)
+{
+    uint32_t x;
+    uint8_t* ptr = *pptr;
+
+    x  = ptr[0] << 0;
+    x |= ptr[1] << 8;
+    x |= ptr[2] << 16;
+    x |= ((uint32_t)ptr[3]) << 24;
+    ptr += 4;
+
+    *pptr = ptr;
+    *r = x;
+}
+
+// Returns the current cumulative frequency (map it to a symbol yourself!)
+static inline uint32_t RansDecGet(RansState* r, uint32_t scale_bits)
+{
+    return *r & ((1u << scale_bits) - 1);
+}
+
+// Advances in the bit stream by "popping" a single symbol with range start
+// "start" and frequency "freq". All frequencies are assumed to sum to "1 << scale_bits",
+// and the resulting bytes get written to ptr (which is updated).
+static inline void RansDecAdvance(RansState* r, uint8_t** pptr, uint32_t start, uint32_t freq, uint32_t scale_bits)
+{
+    uint32_t mask = (1u << scale_bits) - 1;
+
+    // s, x = D(x)
+    uint32_t x = *r;
+    x = freq * (x >> scale_bits) + (x & mask) - start;
+
+    // renormalize
+    if (x < RANS_BYTE_L) {
+        uint8_t* ptr = *pptr;
+        do x = (x << 8) | *ptr++; while (x < RANS_BYTE_L);
+        *pptr = ptr;
+    }
+
+    *r = x;
+}
+
+// --------------------------------------------------------------------------
+
+// That's all you need for a full encoder; below here are some utility
+// functions with extra convenience or optimizations.
+
+// Encoder symbol description
+// This (admittedly odd) selection of parameters was chosen to make
+// RansEncPutSymbol as cheap as possible.
+typedef struct {
+    uint32_t x_max;     // (Exclusive) upper bound of pre-normalization interval
+    uint32_t rcp_freq;  // Fixed-point reciprocal frequency
+    uint32_t bias;      // Bias
+    uint16_t cmpl_freq; // Complement of frequency: (1 << scale_bits) - freq
+    uint16_t rcp_shift; // Reciprocal shift
+} RansEncSymbol;
+
+// Decoder symbols are straightforward.
+// 32-bit means more memory, but oddly faster on old gcc? Why?
+// 322MB/s vs 309MB/s for order-1.
+typedef struct {
+    uint16_t freq;      // Symbol frequency.
+    uint16_t start;     // Start of range.
+} RansDecSymbol;
+
+typedef struct {
+    uint32_t freq;      // Symbol frequency.
+    uint32_t start;     // Start of range.
+} RansDecSymbol32;
+
+// Initializes an encoder symbol to start "start" and frequency "freq"
+static inline void RansEncSymbolInit(RansEncSymbol* s, uint32_t start, uint32_t freq, uint32_t scale_bits)
+{
+    RansAssert(scale_bits <= 16);
+    RansAssert(start <= (1u << scale_bits));
+    RansAssert(freq <= (1u << scale_bits) - start);
+
+    // Say M := 1 << scale_bits.
+    //
+    // The original encoder does:
+    //   x_new = (x/freq)*M + start + (x%freq)
+    //
+    // The fast encoder does (schematically):
+    //   q     = mul_hi(x, rcp_freq) >> rcp_shift   (division)
+    //   r     = x - q*freq                         (remainder)
+    //   x_new = q*M + bias + r                     (new x)
+    // plugging in r into x_new yields:
+    //   x_new = bias + x + q*(M - freq)
+    //        =: bias + x + q*cmpl_freq             (*)
+    //
+    // and we can just precompute cmpl_freq. Now we just need to
+    // set up our parameters such that the original encoder and
+    // the fast encoder agree.
+
+    s->x_max = ((RANS_BYTE_L >> scale_bits) << 8) * freq;
+    s->cmpl_freq = (uint16_t) ((1 << scale_bits) - freq);
+    if (freq < 2) {
+        // freq=0 symbols are never valid to encode, so it doesn't matter what
+        // we set our values to.
+        //
+        // freq=1 is tricky, since the reciprocal of 1 is 1; unfortunately,
+        // our fixed-point reciprocal approximation can only multiply by values
+        // smaller than 1.
+        //
+        // So we use the "next best thing": rcp_freq=0xffffffff, rcp_shift=0.
+        // This gives:
+        //   q = mul_hi(x, rcp_freq) >> rcp_shift
+        //     = mul_hi(x, (1<<32) - 1)) >> 0
+        //     = floor(x - x/(2^32))
+        //     = x - 1 if 1 <= x < 2^32
+        // and we know that x>0 (x=0 is never in a valid normalization interval).
+        //
+        // So we now need to choose the other parameters such that
+        //   x_new = x*M + start
+        // plug it in:
+        //     x*M + start                   (desired result)
+        //   = bias + x + q*cmpl_freq        (*)
+        //   = bias + x + (x - 1)*(M - 1)    (plug in q=x-1, cmpl_freq)
+        //   = bias + 1 + (x - 1)*M
+        //   = x*M + (bias + 1 - M)
+        //
+        // so we have start = bias + 1 - M, or equivalently
+        //   bias = start + M - 1.
+        s->rcp_freq = ~0u;
+        s->rcp_shift = 0;
+        s->bias = start + (1 << scale_bits) - 1;
+    } else {
+        // Alverson, "Integer Division using reciprocals"
+        // shift=ceil(log2(freq))
+        uint32_t shift = 0;
+        while (freq > (1u << shift))
+            shift++;
+
+        s->rcp_freq = (uint32_t) (((1ull << (shift + 31)) + freq-1) / freq);
+        s->rcp_shift = shift - 1;
+
+        // With these values, 'q' is the correct quotient, so we
+        // have bias=start.
+        s->bias = start;
+    }
+
+    s->rcp_shift += 32; // Avoid the extra >>32 in RansEncPutSymbol
+}
+
+// Initialize a decoder symbol to start "start" and frequency "freq"
+static inline void RansDecSymbolInit(RansDecSymbol* s, uint32_t start, uint32_t freq)
+{
+    RansAssert(start <= (1 << 16));
+    RansAssert(freq <= (1 << 16) - start);
+    s->start = (uint16_t) start;
+    s->freq = (uint16_t) freq;
+}
+
+// Encodes a given symbol. This is faster than straight RansEnc since we can do
+// multiplications instead of a divide.
+//
+// See RansEncSymbolInit for a description of how this works.
+static inline void RansEncPutSymbol(RansState* r, uint8_t** pptr, RansEncSymbol const* sym)
+{
+    RansAssert(sym->x_max != 0); // can't encode symbol with freq=0
+
+    // renormalize
+    uint32_t x = *r;
+    uint32_t x_max = sym->x_max;
+
+    // This is better for 40-qual illumina (3.7% quicker overall CRAM).
+    // The old method was better for low complexity data such as NovaSeq
+    // quals (2.6% quicker overall CRAM).
+    int o = x >= x_max;
+    uint8_t* ptr = *pptr;
+    ptr[-1] = x & 0xff;
+    ptr -= o;
+    x >>= o*8;
+
+    if (unlikely(x >= x_max)) {
+        *--ptr = (uint8_t) (x & 0xff);
+        x >>= 8;
+    }
+    *pptr = ptr;
+
+    //uint32_t q = (uint32_t) (((uint64_t)x * sym->rcp_freq) >> sym->rcp_shift);
+    //*r = q * sym->cmpl_freq + x + sym->bias;
+
+    // x = C(s,x)
+    // NOTE: written this way so we get a 32-bit "multiply high" when
+    // available. If you're on a 64-bit platform with cheap multiplies
+    // (e.g. x64), just bake the +32 into rcp_shift.
+    //uint32_t q = (uint32_t) (((uint64_t)x * sym->rcp_freq) >> 32) >> sym->rcp_shift;
+
+    // The extra >>32 has already been added to RansEncSymbolInit
+    uint32_t q = (uint32_t) (((uint64_t)x * sym->rcp_freq) >> sym->rcp_shift);
+    *r = q * sym->cmpl_freq + x + sym->bias;
+}
+
+// A 4-way version of RansEncPutSymbol, renormalising 4 states
+// simulatenously with their results written to the same ptr buffer.
+// (This is perhaps a failing as it makes optmisation tricky.)
+static inline void RansEncPutSymbol4(RansState *r0,
+                                     RansState *r1,
+                                     RansState *r2,
+                                     RansState *r3,
+                                     uint8_t** pptr,
+                                     RansEncSymbol const *sym0,
+                                     RansEncSymbol const *sym1,
+                                     RansEncSymbol const *sym2,
+                                     RansEncSymbol const *sym3)
+{
+    RansAssert(sym0->x_max != 0); // can't encode symbol with freq=0
+    RansAssert(sym1->x_max != 0); // can't encode symbol with freq=0
+    RansAssert(sym2->x_max != 0); // can't encode symbol with freq=0
+    RansAssert(sym3->x_max != 0); // can't encode symbol with freq=0
+
+    // renormalize
+    uint32_t x0, x1, x2, x3;
+    uint8_t* ptr = *pptr;
+
+    int o;
+    uint32_t m[4] = {
+        sym0->x_max,
+        sym1->x_max,
+        sym2->x_max,
+        sym3->x_max
+    };
+
+    x0 = *r0;
+    o = x0 >= m[0];
+    ptr[-1] = x0;
+    ptr -= o;
+    x0 >>= o*8;
+    if (x0 >= m[0]) {
+        *--ptr = x0;
+        x0 >>= 8;
+    }
+
+    x1 = *r1;
+    o = x1 >= m[1];
+    ptr[-1] = x1;
+    ptr -= o;
+    x1 >>= o*8;
+    if (x1 >= m[1]) {
+        *--ptr = x1;
+        x1 >>= 8;
+    }
+
+    x2 = *r2;
+    o = x2 >= m[2];
+    ptr[-1] = x2;
+    ptr -= o;
+    x2 >>= o*8;
+    if (x2 >= m[2]) {
+        *--ptr = x2;
+        x2 >>= 8;
+    }
+
+    x3 = *r3;
+    o = x3 >= m[3];
+    ptr[-1] = x3;
+    ptr -= o;
+    x3 >>= o*8;
+    if (x3 >= m[3]) {
+        *--ptr = x3;
+        x3 >>= 8;
+    }
+
+    *pptr = ptr;
+
+    // x = C(s,x)
+    uint32_t qa, qb;
+    qa = (uint32_t) (((uint64_t)x0 * sym0->rcp_freq) >> sym0->rcp_shift);
+    uint32_t X0 = qa * sym0->cmpl_freq;
+    qb = (uint32_t) (((uint64_t)x1 * sym1->rcp_freq) >> sym1->rcp_shift);
+    uint32_t X1 = qb * sym1->cmpl_freq;
+
+    *r0 = X0 + x0 + sym0->bias;
+    *r1 = X1 + x1 + sym1->bias;
+
+    qa = (uint32_t) (((uint64_t)x2 * sym2->rcp_freq) >> sym2->rcp_shift);
+    uint32_t X2 = qa * sym2->cmpl_freq;
+    qb = (uint32_t) (((uint64_t)x3 * sym3->rcp_freq) >> sym3->rcp_shift);
+    uint32_t X3 = qb * sym3->cmpl_freq;
+
+    *r2 = X2 + x2 + sym2->bias;
+    *r3 = X3 + x3 + sym3->bias;
+}
+
+// Equivalent to RansDecAdvance that takes a symbol.
+static inline void RansDecAdvanceSymbol(RansState* r, uint8_t** pptr, RansDecSymbol const* sym, uint32_t scale_bits)
+{
+    RansDecAdvance(r, pptr, sym->start, sym->freq, scale_bits);
+}
+
+// Advances in the bit stream by "popping" a single symbol with range start
+// "start" and frequency "freq". All frequencies are assumed to sum to "1 << scale_bits".
+// No renormalization or output happens.
+static inline void RansDecAdvanceStep(RansState* r, uint32_t start, uint32_t freq, uint32_t scale_bits)
+{
+    uint32_t mask = (1u << scale_bits) - 1;
+
+    // s, x = D(x)
+    uint32_t x = *r;
+    *r = freq * (x >> scale_bits) + (x & mask) - start;
+}
+
+// Equivalent to RansDecAdvanceStep that takes a symbol.
+static inline void RansDecAdvanceSymbolStep(RansState* r, RansDecSymbol const* sym, uint32_t scale_bits)
+{
+    RansDecAdvanceStep(r, sym->start, sym->freq, scale_bits);
+}
+
+// Renormalize.
+#if defined(__x86_64) && !defined(__ILP32__)
+/*
+ * Assembly variants of the RansDecRenorm code.
+ * These are based on joint ideas from Rob Davies and from looking at
+ * the clang assembly output.
+ */
+static inline void RansDecRenorm(RansState* r, uint8_t** pptr) {
+    uint32_t  x   = *r;
+    uint8_t  *ptr = *pptr;
+
+    __asm__ ("movzbl (%0), %%eax\n\t"
+             "mov    %1, %%edx\n\t"
+             "shl    $0x8,%%edx\n\t"
+             "or     %%eax,%%edx\n\t"
+             "cmp    $0x800000,%1\n\t"
+             "cmovb  %%edx,%1\n\t"
+             "adc    $0x0,%0\n\t"
+             : "=r" (ptr), "=r" (x)
+             : "0" (ptr), "1" (x)
+             : "eax", "edx"
+             );
+    if (x < 0x800000) x = (x << 8) | *ptr++;
+    *pptr = ptr;
+    *r = x;
+}
+
+/*
+ * A variant that normalises two rans states.
+ * The only minor tweak here is to adjust the reorder a few opcodes
+ * to reduce dependency delays.
+ */
+static inline void RansDecRenorm2(RansState* r1, RansState* r2, uint8_t** pptr) {
+    uint32_t  x1   = *r1;
+    uint32_t  x2   = *r2;
+    uint8_t  *ptr = *pptr;
+
+    __asm__ ("movzbl (%0), %%eax\n\t"
+             "mov    %1, %%edx\n\t"
+             "shl    $0x8, %%edx\n\t"
+             "or     %%eax, %%edx\n\t"
+             "cmp    $0x800000, %1\n\t"
+             "cmovb  %%edx, %1\n\t"
+             "adc    $0x0, %0\n\t"
+             "mov    %2, %%edx\n\t"
+             "shl    $0x8, %%edx\n\t"
+             "cmp    $0x800000, %1\n\t"
+             "jae    1f\n\t"
+             "movzbl (%0), %%eax\n\t"
+             "shl    $0x8, %1\n\t"
+             "or     %%eax, %1\n\t"
+             "add    $0x1, %0\n\t"
+             "1:\n\t"
+             "movzbl (%0), %%eax\n\t"
+             "or     %%eax, %%edx\n\t"
+             "cmp    $0x800000, %2\n\t"
+             "cmovb  %%edx, %2\n\t"
+             "adc    $0x0, %0\n\t"
+             "cmp    $0x800000, %2\n\t"
+             "jae    2f\n\t"
+             "movzbl (%0), %%eax\n\t"
+             "shl    $0x8, %2\n\t"
+             "or     %%eax, %2\n\t"
+             "add    $0x1, %0\n\t"
+             "2:\n\t"
+             : "=r" (ptr), "=r" (x1), "=r" (x2)
+             : "0" (ptr), "1" (x1), "2" (x2)
+             : "eax", "edx"
+             );
+
+    *pptr = ptr;
+    *r1 = x1;
+    *r2 = x2;
+}
+
+#else /* __x86_64 */
+
+static inline void RansDecRenorm(RansState* r, uint8_t** pptr)
+{
+    // renormalize
+    uint32_t x = *r;
+
+#ifdef __clang__
+    // Generates cmov instructions on clang, but alas not gcc
+    uint8_t* ptr = *pptr;
+    uint32_t y = (x << 8) | *ptr;
+    uint32_t cond = x < RANS_BYTE_L;
+    x    = cond ? y : x;
+    ptr += cond ? 1 : 0;
+    if (x < RANS_BYTE_L) x = (x<<8) | *ptr++;
+    *pptr = ptr;
+#else
+    if (x >= RANS_BYTE_L) return;
+    uint8_t* ptr = *pptr;
+    x = (x << 8) | *ptr++;
+    if (x < RANS_BYTE_L) x = (x << 8) | *ptr++;
+    *pptr = ptr;
+#endif /* __clang__ */
+
+    *r = x;
+}
+
+static inline void RansDecRenorm2(RansState* r1, RansState* r2, uint8_t** pptr) {
+    RansDecRenorm(r1, pptr);
+    RansDecRenorm(r2, pptr);
+}
+
+#endif /* __x86_64 */
+
+static inline void RansDecRenormSafe(RansState* r, uint8_t** pptr, uint8_t *ptr_end)
+{
+    uint32_t x = *r;
+    uint8_t* ptr = *pptr;
+    if (x >= RANS_BYTE_L || ptr >= ptr_end) return;
+    x = (x << 8) | *ptr++;
+    if (x < RANS_BYTE_L && ptr < ptr_end)
+        x = (x << 8) | *ptr++;
+    *pptr = ptr;
+    *r = x;
+}
+
+static inline void RansDecSymbolInit32(RansDecSymbol32* s, uint32_t start, uint32_t freq)
+{
+    RansAssert(start <= (1 << 16));
+    RansAssert(freq <= (1 << 16) - start);
+    s->start = (uint16_t) start;
+    s->freq = (uint16_t) freq;
+}
+
+static inline void RansDecAdvanceSymbol32(RansState* r, uint8_t** pptr, RansDecSymbol32 const* sym, uint32_t scale_bits)
+{
+    RansDecAdvance(r, pptr, sym->start, sym->freq, scale_bits);
+}
+
+#endif // RANS_BYTE_HEADER
diff --git a/ext/htslib/htscodecs/htscodecs/rANS_static.c b/ext/htslib/htscodecs/htscodecs/rANS_static.c
new file mode 100644
index 0000000..1399ee7
--- /dev/null
+++ b/ext/htslib/htscodecs/htscodecs/rANS_static.c
@@ -0,0 +1,850 @@
+/*
+ * Copyright (c) 2014-2022 Genome Research Ltd.
+ * Author(s): James Bonfield
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above
+ *       copyright notice, this list of conditions and the following
+ *       disclaimer in the documentation and/or other materials provided
+ *       with the distribution.
+ *
+ *    3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+ *       Institute nor the names of its contributors may be used to endorse
+ *       or promote products derived from this software without specific
+ *       prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH
+ * LTD OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+// Use 11 for order-1?
+#define TF_SHIFT 12
+#define TOTFREQ (1<<TF_SHIFT)
+
+#include "rANS_byte.h"
+#include "utils.h"
+
+/*-------------------------------------------------------------------------- */
+/*
+ * Example wrapper to use the rans_byte.h functions included above.
+ *
+ * This demonstrates how to use, and unroll, an order-0 and order-1 frequency
+ * model.
+ */
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <assert.h>
+#include <string.h>
+#include <limits.h>
+#include <sys/time.h>
+#ifndef NO_THREADS
+#include <pthread.h>
+#endif
+
+#include "rANS_static.h"
+
+#define ABS(a) ((a)>0?(a):-(a))
+
+/*-----------------------------------------------------------------------------
+ * Memory to memory compression functions.
+ *
+ * These are original versions without any manual loop unrolling. They
+ * are easier to understand, but can be up to 2x slower.
+ */
+
+static
+unsigned char *rans_compress_O0(unsigned char *in, unsigned int in_size,
+                                unsigned int *out_size) {
+    unsigned char *out_buf = malloc(1.05*in_size + 257*257*3 + 9);
+    unsigned char *cp, *out_end;
+    RansEncSymbol syms[256];
+    RansState rans0;
+    RansState rans2;
+    RansState rans1;
+    RansState rans3;
+    uint8_t* ptr;
+    int F[256+MAGIC] = {0}, i, j, tab_size, rle, x, fsum = 0;
+    int m = 0, M = 0;
+    uint64_t tr;
+
+    if (!out_buf)
+        return NULL;
+
+    ptr = out_end = out_buf + (uint32_t)(1.05*in_size) + 257*257*3 + 9;
+
+    // Compute statistics
+    if (hist8(in, in_size, (uint32_t *)F) < 0) {
+        free(out_buf);
+        return NULL;
+    }
+    tr = in_size ? ((uint64_t)TOTFREQ<<31)/in_size + (1<<30)/in_size : 0;
+
+ normalise_harder:
+    // Normalise so T[i] == TOTFREQ
+    for (fsum = m = M = j = 0; j < 256; j++) {
+        if (!F[j])
+            continue;
+
+        if (m < F[j])
+            m = F[j], M = j;
+
+        if ((F[j] = (F[j]*tr)>>31) == 0)
+            F[j] = 1;
+        fsum += F[j];
+    }
+
+    fsum++;
+    if (fsum < TOTFREQ) {
+        F[M] += TOTFREQ-fsum;
+    } else if (fsum-TOTFREQ > F[M]/2) {
+        // Corner case to avoid excessive frequency reduction
+        tr = 2104533975; goto normalise_harder; // equiv to *0.98.
+    } else {
+        F[M] -= fsum-TOTFREQ;
+    }
+
+    //printf("F[%d]=%d\n", M, F[M]);
+    assert(F[M]>0);
+
+    // Encode statistics.
+    cp = out_buf+9;
+
+    for (x = rle = j = 0; j < 256; j++) {
+        if (F[j]) {
+            // j
+            if (rle) {
+                rle--;
+            } else {
+                *cp++ = j;
+                if (!rle && j && F[j-1])  {
+                    for(rle=j+1; rle<256 && F[rle]; rle++)
+                        ;
+                    rle -= j+1;
+                    *cp++ = rle;
+                }
+                //fprintf(stderr, "%d: %d %d\n", j, rle, N[j]);
+            }
+            
+            // F[j]
+            if (F[j]<128) {
+                *cp++ = F[j];
+            } else {
+                *cp++ = 128 | (F[j]>>8);
+                *cp++ = F[j]&0xff;
+            }
+            RansEncSymbolInit(&syms[j], x, F[j], TF_SHIFT);
+            x += F[j];
+        }
+    }
+    *cp++ = 0;
+
+    //write(2, out_buf+4, cp-(out_buf+4));
+    tab_size = cp-out_buf;
+
+    RansEncInit(&rans0);
+    RansEncInit(&rans1);
+    RansEncInit(&rans2);
+    RansEncInit(&rans3);
+
+    switch (i=(in_size&3)) {
+    case 3: RansEncPutSymbol(&rans2, &ptr, &syms[in[in_size-(i-2)]]);
+        // fall-through
+    case 2: RansEncPutSymbol(&rans1, &ptr, &syms[in[in_size-(i-1)]]);
+        // fall-through
+    case 1: RansEncPutSymbol(&rans0, &ptr, &syms[in[in_size-(i-0)]]);
+        // fall-through
+    case 0:
+        break;
+    }
+    for (i=(in_size &~3); likely(i>0); i-=4) {
+        RansEncSymbol *s3 = &syms[in[i-1]];
+        RansEncSymbol *s2 = &syms[in[i-2]];
+        RansEncSymbol *s1 = &syms[in[i-3]];
+        RansEncSymbol *s0 = &syms[in[i-4]];
+
+        RansEncPutSymbol(&rans3, &ptr, s3);
+        RansEncPutSymbol(&rans2, &ptr, s2);
+        RansEncPutSymbol(&rans1, &ptr, s1);
+        RansEncPutSymbol(&rans0, &ptr, s0);
+    }
+
+    RansEncFlush(&rans3, &ptr);
+    RansEncFlush(&rans2, &ptr);
+    RansEncFlush(&rans1, &ptr);
+    RansEncFlush(&rans0, &ptr);
+
+    // Finalise block size and return it
+    *out_size = (out_end - ptr) + tab_size;
+
+    cp = out_buf;
+
+    *cp++ = 0; // order
+    *cp++ = ((*out_size-9)>> 0) & 0xff;
+    *cp++ = ((*out_size-9)>> 8) & 0xff;
+    *cp++ = ((*out_size-9)>>16) & 0xff;
+    *cp++ = ((*out_size-9)>>24) & 0xff;
+
+    *cp++ = (in_size>> 0) & 0xff;
+    *cp++ = (in_size>> 8) & 0xff;
+    *cp++ = (in_size>>16) & 0xff;
+    *cp++ = (in_size>>24) & 0xff;
+
+    memmove(out_buf + tab_size, ptr, out_end-ptr);
+
+    return out_buf;
+}
+
+typedef struct {
+    unsigned char R[TOTFREQ];
+} ari_decoder;
+
+static
+unsigned char *rans_uncompress_O0(unsigned char *in, unsigned int in_size,
+                                  unsigned int *out_size) {
+    /* Load in the static tables */
+    unsigned char *cp = in + 9;
+    unsigned char *cp_end = in + in_size;
+    const uint32_t mask = (1u << TF_SHIFT)-1;
+    int i, j, rle;
+    unsigned int x, y;
+    unsigned int out_sz, in_sz;
+    char *out_buf;
+    RansState R[4];
+    RansState m[4];
+    uint16_t sfreq[TOTFREQ+32];
+    uint16_t ssym [TOTFREQ+32]; // faster, but only needs uint8_t
+    uint32_t sbase[TOTFREQ+16]; // faster, but only needs uint16_t
+
+    if (in_size < 26) // Need at least this many bytes just to start
+        return NULL;
+
+    if (*in++ != 0) // Order-0 check
+        return NULL;
+    
+    in_sz  = ((in[0])<<0) | ((in[1])<<8) | ((in[2])<<16) | (((uint32_t)in[3])<<24);
+    out_sz = ((in[4])<<0) | ((in[5])<<8) | ((in[6])<<16) | (((uint32_t)in[7])<<24);
+    if (in_sz != in_size-9)
+        return NULL;
+
+    if (out_sz >= INT_MAX)
+        return NULL; // protect against some overflow cases
+
+    // For speeding up the fuzzer only.
+    // Small input can lead to large uncompressed data.
+    // We reject this as it just slows things up instead of testing more code
+    // paths (once we've verified a few times for large data).
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    if (out_sz > 100000)
+        return NULL;
+#endif
+
+    out_buf = malloc(out_sz);
+    if (!out_buf)
+        return NULL;
+
+    //fprintf(stderr, "out_sz=%d\n", out_sz);
+
+    // Precompute reverse lookup of frequency.
+    rle = x = y = 0;
+    j = *cp++;
+    do {
+        int F, C;
+        if (cp > cp_end - 16) goto cleanup; // Not enough input bytes left
+        if ((F = *cp++) >= 128) {
+            F &= ~128;
+            F = ((F & 127) << 8) | *cp++;
+        }
+        C = x;
+
+        if (x + F > TOTFREQ)
+            goto cleanup;
+
+        for (y = 0; y < F; y++) {
+            ssym [y + C] = j;
+            sfreq[y + C] = F;
+            sbase[y + C] = y;
+        }
+        x += F;
+
+        if (!rle && j+1 == *cp) {
+            j = *cp++;
+            rle = *cp++;
+        } else if (rle) {
+            rle--;
+            j++;
+            if (j > 255)
+                goto cleanup;
+        } else {
+            j = *cp++;
+        }
+    } while(j);
+
+    if (x < TOTFREQ-1 || x > TOTFREQ)
+        goto cleanup;
+    if (x != TOTFREQ) {
+        // Protection against accessing uninitialised memory in the case
+        // where SUM(freqs) == 4095 and not 4096.
+        ssym [x] = ssym [x-1];
+        sfreq[x] = sfreq[x-1];
+        sbase[x] = sbase[x-1]+1;
+    }
+
+    // 16 bytes of cp here. Also why cp - 16 in above loop.
+    if (cp > cp_end - 16) goto cleanup; // Not enough input bytes left
+
+    RansDecInit(&R[0], &cp); if (R[0] < RANS_BYTE_L) goto cleanup;
+    RansDecInit(&R[1], &cp); if (R[1] < RANS_BYTE_L) goto cleanup;
+    RansDecInit(&R[2], &cp); if (R[2] < RANS_BYTE_L) goto cleanup;
+    RansDecInit(&R[3], &cp); if (R[3] < RANS_BYTE_L) goto cleanup;
+
+    int out_end = (out_sz&~3);
+    cp_end -= 8; // within 8 for simplicity of loop below
+    // 2 x likely() here harms gcc 7.5 by about 8% rate drop, but only in O2
+    for (i=0; likely(i < out_end); i+=4) {
+        //                              /curr code
+        // gcc7  O2 513/497   562/556++ 556/547 ok
+        // gcc7  O3 566/552   569/553   581/563+
+        // gcc10 O2 544/538   563/547   541/537-?
+        // gcc10 O3 531/519   546/530   575/546+
+        // gcc11 O2 512/490   588/540   540/535 mid
+        // gcc11 O3 482/471   553/541   549/535
+        // gcc12 O2 533/526   544/534   539/535
+        // gcc12 O3 548/533   502/497-- 553/527 ok
+        // clang10  555/542   564/549   560/541
+        // clang13  560/553   572/559   556/559
+        m[0] = R[0] & mask;
+        R[0] = sfreq[m[0]] * (R[0] >> TF_SHIFT) + sbase[m[0]];
+
+        m[1] = R[1] & mask;
+        R[1] = sfreq[m[1]] * (R[1] >> TF_SHIFT) + sbase[m[1]];
+
+        m[2] = R[2] & mask;
+        R[2] = sfreq[m[2]] * (R[2] >> TF_SHIFT) + sbase[m[2]];
+
+        m[3] = R[3] & mask;
+        R[3] = sfreq[m[3]] * (R[3] >> TF_SHIFT) + sbase[m[3]];
+
+        // likely() here harms gcc12 -O3
+        if (cp<cp_end) {
+            RansDecRenorm2(&R[0], &R[1], &cp);
+            RansDecRenorm2(&R[2], &R[3], &cp);
+        } else {
+            RansDecRenormSafe(&R[0], &cp, cp_end+8);
+            RansDecRenormSafe(&R[1], &cp, cp_end+8);
+            RansDecRenormSafe(&R[2], &cp, cp_end+8);
+            RansDecRenormSafe(&R[3], &cp, cp_end+8);
+        }
+
+        out_buf[i+0] = ssym[m[0]];
+        out_buf[i+1] = ssym[m[1]];
+        out_buf[i+2] = ssym[m[2]];
+        out_buf[i+3] = ssym[m[3]];
+    }
+
+
+    switch(out_sz&3) {
+    case 3:
+        out_buf[out_end + 2] = ssym[R[2] & mask];
+        // fall-through
+    case 2:
+        out_buf[out_end + 1] = ssym[R[1] & mask];
+        // fall-through
+    case 1:
+        out_buf[out_end] = ssym[R[0] & mask];
+        // fall-through
+    default:
+        break;
+    }
+    
+    *out_size = out_sz;
+    return (unsigned char *)out_buf;
+
+ cleanup:
+    free(out_buf);
+    return NULL;
+}
+
+static
+unsigned char *rans_compress_O1(unsigned char *in, unsigned int in_size,
+                                unsigned int *out_size) {
+    unsigned char *out_buf = NULL, *out_end, *cp;
+    unsigned int tab_size, rle_i, rle_j;
+
+
+    if (in_size < 4)
+        return rans_compress_O0(in, in_size, out_size);
+
+    int (*F)[256];
+    RansEncSymbol (*syms)[256];
+
+    uint8_t *mem = htscodecs_tls_alloc(256 * (sizeof(*syms) + sizeof(*F)));
+    if (!mem)
+        return NULL;
+    syms = (RansEncSymbol (*)[256])mem;
+    F = (int (*)[256])(mem + 256*sizeof(*syms));
+    memset(F, 0, 256*sizeof(*F));
+
+    if (!syms) goto cleanup;
+    int T[256+MAGIC] = {0};
+    int i, j;
+
+    out_buf = malloc(1.05*in_size + 257*257*3 + 9);
+    if (!out_buf) goto cleanup;
+
+    out_end = out_buf + (uint32_t)(1.05*in_size) + 257*257*3 + 9;
+    cp = out_buf+9;
+
+    if (hist1_4(in, in_size, (uint32_t (*)[256])F, (uint32_t *)T) < 0) {
+        free(out_buf);
+        out_buf = NULL;
+        goto cleanup;
+    }
+
+    F[0][in[1*(in_size>>2)]]++;
+    F[0][in[2*(in_size>>2)]]++;
+    F[0][in[3*(in_size>>2)]]++;
+    T[0]+=3;
+
+    
+    // Normalise so T[i] == TOTFREQ
+    for (rle_i = i = 0; i < 256; i++) {
+        int t2, m, M;
+        unsigned int x;
+
+        if (T[i] == 0)
+            continue;
+
+        //uint64_t p = (TOTFREQ * TOTFREQ) / t;
+        double p = ((double)TOTFREQ)/T[i];
+    normalise_harder:
+        for (t2 = m = M = j = 0; j < 256; j++) {
+            if (!F[i][j])
+                continue;
+
+            if (m < F[i][j])
+                m = F[i][j], M = j;
+
+            //if ((F[i][j] = (F[i][j] * p) / TOTFREQ) == 0)
+            if ((F[i][j] *= p) == 0)
+                F[i][j] = 1;
+            t2 += F[i][j];
+        }
+
+        t2++;
+        if (t2 < TOTFREQ) {
+            F[i][M] += TOTFREQ-t2;
+        } else if (t2-TOTFREQ >= F[i][M]/2) {
+            // Corner case to avoid excessive frequency reduction
+            p = .98; goto normalise_harder;
+        } else {
+            F[i][M] -= t2-TOTFREQ;
+        }
+
+        // Store frequency table
+        // i
+        if (rle_i) {
+            rle_i--;
+        } else {
+            *cp++ = i;
+            // FIXME: could use order-0 statistics to observe which alphabet
+            // symbols are present and base RLE on that ordering instead.
+            if (i && T[i-1]) {
+                for(rle_i=i+1; rle_i<256 && T[rle_i]; rle_i++)
+                    ;
+                rle_i -= i+1;
+                *cp++ = rle_i;
+            }
+        }
+
+        int *F_i_ = F[i];
+        x = 0;
+        rle_j = 0;
+        for (j = 0; j < 256; j++) {
+            if (F_i_[j]) {
+                //fprintf(stderr, "F[%d][%d]=%d, x=%d\n", i, j, F_i_[j], x);
+
+                // j
+                if (rle_j) {
+                    rle_j--;
+                } else {
+                    *cp++ = j;
+                    if (!rle_j && j && F_i_[j-1]) {
+                        for(rle_j=j+1; rle_j<256 && F_i_[rle_j]; rle_j++)
+                            ;
+                        rle_j -= j+1;
+                        *cp++ = rle_j;
+                    }
+                }
+
+                // F_i_[j]
+                if (F_i_[j]<128) {
+                    *cp++ = F_i_[j];
+                } else {
+                    *cp++ = 128 | (F_i_[j]>>8);
+                    *cp++ = F_i_[j]&0xff;
+                }
+
+                RansEncSymbolInit(&syms[i][j], x, F_i_[j], TF_SHIFT);
+                x += F_i_[j];
+            }
+        }
+        *cp++ = 0;
+    }
+    *cp++ = 0;
+
+    //write(2, out_buf+4, cp-(out_buf+4));
+    tab_size = cp - out_buf;
+    assert(tab_size < 257*257*3);
+    
+    RansState rans0, rans1, rans2, rans3;
+    RansEncInit(&rans0);
+    RansEncInit(&rans1);
+    RansEncInit(&rans2);
+    RansEncInit(&rans3);
+
+    uint8_t* ptr = out_end;
+
+    int isz4 = in_size>>2;
+    int i0 = 1*isz4-2;
+    int i1 = 2*isz4-2;
+    int i2 = 3*isz4-2;
+    int i3 = 4*isz4-2;
+
+    unsigned char l0 = in[i0+1];
+    unsigned char l1 = in[i1+1];
+    unsigned char l2 = in[i2+1];
+    unsigned char l3 = in[i3+1];
+
+    // Deal with the remainder
+    l3 = in[in_size-1];
+    for (i3 = in_size-2; i3 > 4*isz4-2; i3--) {
+        unsigned char c3 = in[i3];
+        RansEncPutSymbol(&rans3, &ptr, &syms[c3][l3]);
+        l3 = c3;
+    }
+
+    for (; likely(i0 >= 0); i0--, i1--, i2--, i3--) {
+        unsigned char c3 = in[i3];
+        unsigned char c2 = in[i2];
+        unsigned char c1 = in[i1];
+        unsigned char c0 = in[i0];
+
+        RansEncSymbol *s3 = &syms[c3][l3];
+        RansEncSymbol *s2 = &syms[c2][l2];
+        RansEncSymbol *s1 = &syms[c1][l1];
+        RansEncSymbol *s0 = &syms[c0][l0];
+
+        RansEncPutSymbol4(&rans3, &rans2, &rans1, &rans0, &ptr,
+                          s3, s2, s1, s0);
+
+        l3 = c3;
+        l2 = c2;
+        l1 = c1;
+        l0 = c0;
+    }
+
+    RansEncPutSymbol(&rans3, &ptr, &syms[0][l3]);
+    RansEncPutSymbol(&rans2, &ptr, &syms[0][l2]);
+    RansEncPutSymbol(&rans1, &ptr, &syms[0][l1]);
+    RansEncPutSymbol(&rans0, &ptr, &syms[0][l0]);
+
+    RansEncFlush(&rans3, &ptr);
+    RansEncFlush(&rans2, &ptr);
+    RansEncFlush(&rans1, &ptr);
+    RansEncFlush(&rans0, &ptr);
+
+    *out_size = (out_end - ptr) + tab_size;
+
+    cp = out_buf;
+    *cp++ = 1; // order
+
+    *cp++ = ((*out_size-9)>> 0) & 0xff;
+    *cp++ = ((*out_size-9)>> 8) & 0xff;
+    *cp++ = ((*out_size-9)>>16) & 0xff;
+    *cp++ = ((*out_size-9)>>24) & 0xff;
+
+    *cp++ = (in_size>> 0) & 0xff;
+    *cp++ = (in_size>> 8) & 0xff;
+    *cp++ = (in_size>>16) & 0xff;
+    *cp++ = (in_size>>24) & 0xff;
+
+    memmove(out_buf + tab_size, ptr, out_end-ptr);
+
+ cleanup:
+    htscodecs_tls_free(syms);
+
+    return out_buf;
+}
+
+static
+unsigned char *rans_uncompress_O1(unsigned char *in, unsigned int in_size,
+                                  unsigned int *out_size) {
+    /* Load in the static tables */
+    unsigned char *cp = in + 9;
+    unsigned char *ptr_end = in + in_size;
+    int i, j = -999, rle_i, rle_j;
+    unsigned int x;
+    unsigned int out_sz, in_sz;
+    char *out_buf = NULL;
+
+    // Sanity checking
+    if (in_size < 27) // Need at least this many bytes to start
+        return NULL;
+
+    if (*in++ != 1) // Order-1 check
+        return NULL;
+
+    in_sz  = ((in[0])<<0) | ((in[1])<<8) | ((in[2])<<16) | (((uint32_t)in[3])<<24);
+    out_sz = ((in[4])<<0) | ((in[5])<<8) | ((in[6])<<16) | (((uint32_t)in[7])<<24);
+    if (in_sz != in_size-9)
+        return NULL;
+
+    if (out_sz >= INT_MAX)
+        return NULL; // protect against some overflow cases
+
+    // For speeding up the fuzzer only.
+    // Small input can lead to large uncompressed data.
+    // We reject this as it just slows things up instead of testing more code
+    // paths (once we've verified a few times for large data).
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    if (out_sz > 100000)
+        return NULL;
+#endif
+
+    // Allocate decoding lookup tables
+    RansDecSymbol32 (*syms)[256];
+    uint8_t *mem = htscodecs_tls_calloc(256, sizeof(ari_decoder)
+                                        + sizeof(*syms));
+    if (!mem)
+        return NULL;
+    ari_decoder *const D = (ari_decoder *)mem;
+    syms = (RansDecSymbol32 (*)[256])(mem + 256*sizeof(ari_decoder));
+    int16_t map[256], map_i = 0;
+    
+    memset(map, -1, 256*sizeof(*map));
+
+    if (!D) goto cleanup;
+    /* These memsets prevent illegal memory access in syms due to
+       broken compressed data.  As D is calloc'd, all illegal transitions
+       will end up in either row or column 0 of syms. */
+    memset(&syms[0], 0, sizeof(syms[0]));
+    for (i = 0; i < 256; i++)
+        memset(&syms[i][0], 0, sizeof(syms[0][0]));
+
+    //fprintf(stderr, "out_sz=%d\n", out_sz);
+
+    //i = *cp++;
+    rle_i = 0;
+    i = *cp++;
+    do {
+        // Map arbitrary a,b,c to 0,1,2 to improve cache locality.
+        if (map[i] == -1)
+            map[i] = map_i++;
+        int m_i = map[i];
+
+        rle_j = x = 0;
+        j = *cp++;
+        do {
+            if (map[j] == -1)
+                map[j] = map_i++;
+
+            int F, C;
+            if (cp > ptr_end - 16) goto cleanup; // Not enough input bytes left
+            if ((F = *cp++) >= 128) {
+                F &= ~128;
+                F = ((F & 127) << 8) | *cp++;
+            }
+            C = x;
+
+            //fprintf(stderr, "i=%d j=%d F=%d C=%d\n", i, j, F, C);
+
+            if (unlikely(!F))
+                F = TOTFREQ;
+
+            RansDecSymbolInit32(&syms[m_i][j], C, F);
+
+            /* Build reverse lookup table */
+            //if (!D[i].R) D[i].R = (unsigned char *)malloc(TOTFREQ);
+            if (x + F > TOTFREQ)
+                goto cleanup;
+
+            memset(&D[m_i].R[x], j, F);
+            x += F;
+
+            if (!rle_j && j+1 == *cp) {
+                j = *cp++;
+                rle_j = *cp++;
+            } else if (rle_j) {
+                rle_j--;
+                j++;
+                if (j > 255)
+                    goto cleanup;
+            } else {
+                j = *cp++;
+            }
+        } while(j);
+
+        if (x < TOTFREQ-1 || x > TOTFREQ)
+            goto cleanup;
+        if (x < TOTFREQ) // historically we fill 4095, not 4096
+            D[i].R[x] = D[i].R[x-1];
+
+        if (!rle_i && i+1 == *cp) {
+            i = *cp++;
+            rle_i = *cp++;
+        } else if (rle_i) {
+            rle_i--;
+            i++;
+            if (i > 255)
+                goto cleanup;
+        } else {
+            i = *cp++;
+        }
+    } while (i);
+    for (i = 0; i < 256; i++)
+        if (map[i] == -1)
+            map[i] = 0;
+
+    RansState rans0, rans1, rans2, rans3;
+    uint8_t *ptr = cp;
+    if (cp > ptr_end - 16) goto cleanup; // Not enough input bytes left
+    RansDecInit(&rans0, &ptr); if (rans0 < RANS_BYTE_L) goto cleanup;
+    RansDecInit(&rans1, &ptr); if (rans1 < RANS_BYTE_L) goto cleanup;
+    RansDecInit(&rans2, &ptr); if (rans2 < RANS_BYTE_L) goto cleanup;
+    RansDecInit(&rans3, &ptr); if (rans3 < RANS_BYTE_L) goto cleanup;
+
+    RansState R[4];
+    R[0] = rans0;
+    R[1] = rans1;
+    R[2] = rans2;
+    R[3] = rans3;
+
+    unsigned int isz4 = out_sz>>2;
+    uint32_t l0 = 0;
+    uint32_t l1 = 0;
+    uint32_t l2 = 0;
+    uint32_t l3 = 0;
+    
+    unsigned int i4[] = {0*isz4, 1*isz4, 2*isz4, 3*isz4};
+
+    /* Allocate output buffer */
+    out_buf = malloc(out_sz);
+    if (!out_buf) goto cleanup;
+
+    uint8_t cc0 = D[map[l0]].R[R[0] & ((1u << TF_SHIFT)-1)];
+    uint8_t cc1 = D[map[l1]].R[R[1] & ((1u << TF_SHIFT)-1)];
+    uint8_t cc2 = D[map[l2]].R[R[2] & ((1u << TF_SHIFT)-1)];
+    uint8_t cc3 = D[map[l3]].R[R[3] & ((1u << TF_SHIFT)-1)];
+
+    ptr_end -= 8;
+    for (; likely(i4[0] < isz4); i4[0]++, i4[1]++, i4[2]++, i4[3]++) {
+        // seq4-head2: file q40b
+        //          O3      O2
+        // gcc7     296/291 290/260
+        // gcc10    292/292 290/261
+        // gcc11    293/293 290/265
+        // gcc12    293/290 291/266
+        // clang10  293/290 296/272
+        // clang13  300/290 290/266
+        out_buf[i4[0]] = cc0;
+        out_buf[i4[1]] = cc1;
+        out_buf[i4[2]] = cc2;
+        out_buf[i4[3]] = cc3;
+
+        RansDecSymbol32 s[4] = {
+            syms[l0][cc0],
+            syms[l1][cc1],
+            syms[l2][cc2],
+            syms[l3][cc3],
+        };
+        RansDecAdvanceStep(&R[0], s[0].start, s[0].freq, TF_SHIFT);
+        RansDecAdvanceStep(&R[1], s[1].start, s[1].freq, TF_SHIFT);
+        RansDecAdvanceStep(&R[2], s[2].start, s[2].freq, TF_SHIFT);
+        RansDecAdvanceStep(&R[3], s[3].start, s[3].freq, TF_SHIFT);
+
+        // Likely here helps speed of high-entropy data by 10-11%,
+        // but harms low entropy-data speed by 3-4%.
+        if ((ptr < ptr_end)) {
+            RansDecRenorm2(&R[0], &R[1], &ptr);
+            RansDecRenorm2(&R[2], &R[3], &ptr);
+        } else {
+            RansDecRenormSafe(&R[0], &ptr, ptr_end+8);
+            RansDecRenormSafe(&R[1], &ptr, ptr_end+8);
+            RansDecRenormSafe(&R[2], &ptr, ptr_end+8);
+            RansDecRenormSafe(&R[3], &ptr, ptr_end+8);
+        }
+
+        l0 = map[cc0];
+        l1 = map[cc1];
+        l2 = map[cc2];
+        l3 = map[cc3];
+
+        cc0 = D[l0].R[R[0] & ((1u << TF_SHIFT)-1)];
+        cc1 = D[l1].R[R[1] & ((1u << TF_SHIFT)-1)];
+        cc2 = D[l2].R[R[2] & ((1u << TF_SHIFT)-1)];
+        cc3 = D[l3].R[R[3] & ((1u << TF_SHIFT)-1)];
+    }
+
+    // Remainder
+    for (; i4[3] < out_sz; i4[3]++) {
+        unsigned char c3 = D[l3].R[RansDecGet(&R[3], TF_SHIFT)];
+        out_buf[i4[3]] = c3;
+
+        uint32_t m = R[3] & ((1u << TF_SHIFT)-1);
+        R[3] = syms[l3][c3].freq * (R[3]>>TF_SHIFT) + m - syms[l3][c3].start;
+        RansDecRenormSafe(&R[3], &ptr, ptr_end+8);
+        l3 = map[c3];
+    }
+    
+    *out_size = out_sz;
+
+ cleanup:
+    htscodecs_tls_free(D);
+
+    return (unsigned char *)out_buf;
+}
+
+/*-----------------------------------------------------------------------------
+ * Simple interface to the order-0 vs order-1 encoders and decoders.
+ */
+unsigned char *rans_compress(unsigned char *in, unsigned int in_size,
+                             unsigned int *out_size, int order) {
+    if (in_size > INT_MAX) {
+        *out_size = 0;
+        return NULL;
+    }
+
+    return order
+        ? rans_compress_O1(in, in_size, out_size)
+        : rans_compress_O0(in, in_size, out_size);
+}
+
+unsigned char *rans_uncompress(unsigned char *in, unsigned int in_size,
+                               unsigned int *out_size) {
+    /* Both rans_uncompress functions need to be able to read at least 9
+       bytes. */
+    if (in_size < 9)
+        return NULL;
+    return in[0]
+        ? rans_uncompress_O1(in, in_size, out_size)
+        : rans_uncompress_O0(in, in_size, out_size);
+}
diff --git a/ext/htslib/htscodecs/htscodecs/rANS_static.h b/ext/htslib/htscodecs/htscodecs/rANS_static.h
new file mode 100644
index 0000000..357f46e
--- /dev/null
+++ b/ext/htslib/htscodecs/htscodecs/rANS_static.h
@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2014-2019 Genome Research Ltd.
+ * Author(s): James Bonfield
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above
+ *       copyright notice, this list of conditions and the following
+ *       disclaimer in the documentation and/or other materials provided
+ *       with the distribution.
+ *
+ *    3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+ *       Institute nor the names of its contributors may be used to endorse
+ *       or promote products derived from this software without specific
+ *       prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH
+ * LTD OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef RANS_STATIC_H
+#define RANS_STATIC_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+unsigned char *rans_compress(unsigned char *in, unsigned int in_size,
+                             unsigned int *out_size, int order);
+unsigned char *rans_uncompress(unsigned char *in, unsigned int in_size,
+                               unsigned int *out_size);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RANS_STATIC_H */
diff --git a/ext/htslib/htscodecs/htscodecs/rANS_static16_int.h b/ext/htslib/htscodecs/htscodecs/rANS_static16_int.h
new file mode 100644
index 0000000..340df88
--- /dev/null
+++ b/ext/htslib/htscodecs/htscodecs/rANS_static16_int.h
@@ -0,0 +1,636 @@
+#ifndef RANS_INTERNAL_H
+#define RANS_INTERNAL_H
+
+#include "config.h"
+#include "varint.h"
+#include "utils.h"
+
+/*
+ * Copyright (c) 2017-2022 Genome Research Ltd.
+ * Author(s): James Bonfield
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above
+ *       copyright notice, this list of conditions and the following
+ *       disclaimer in the documentation and/or other materials provided
+ *       with the distribution.
+ *
+ *    3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+ *       Institute nor the names of its contributors may be used to endorse
+ *       or promote products derived from this software without specific
+ *       prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH
+ * LTD OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+// Internal: common parts to all the rANSNx16pr implementations.
+
+// As per standard rANS_static but using optional RLE or bit-packing
+// techniques prior to entropy encoding.  This is a significant
+// reduction in some data sets.
+
+// top bits in order byte
+#define X_PACK   0x80    // Pack 2,4,8 or infinite symbols into a byte.
+#define X_RLE    0x40    // Run length encoding with runs & lits encoded separately
+#define X_CAT    0x20    // Nop; for tiny segments where rANS overhead is too big
+#define X_NOSZ   0x10    // Don't store the original size; used by STRIPE mode
+#define X_STRIPE 0x08    // For N-byte integer data; rotate & encode N streams.
+#define X_32     0x04    // 32-way unrolling instead of 4-way
+
+// Not part of the file format, but used to direct the encoder
+#define X_SIMD_AUTO 0x100 // automatically enable X_32 if we deem it worthy
+#define X_SW32_ENC  0x200 // forcibly use the software version of X_32
+#define X_SW32_DEC  0x400 // forcibly use the software version of X_32
+#define X_NO_AVX512 0x800 // turn off avx512, but permits AVX2
+
+#define TF_SHIFT 12
+#define TOTFREQ (1<<TF_SHIFT)
+
+
+// 9-11 is considerably faster in the O1 variant due to reduced table size.
+// We auto-tune between 10 and 12 though.  Anywhere from 9 to 14 are viable.
+#ifndef TF_SHIFT_O1
+#define TF_SHIFT_O1 12
+#endif
+#ifndef TF_SHIFT_O1_FAST
+#define TF_SHIFT_O1_FAST 10
+#endif
+#define TOTFREQ_O1 (1<<TF_SHIFT_O1)
+#define TOTFREQ_O1_FAST (1<<TF_SHIFT_O1_FAST)
+
+unsigned char *rans_compress_O0_4x16(unsigned char *in, unsigned int in_size,
+                                     unsigned char *out, unsigned int *out_size);
+unsigned char *rans_uncompress_O0_4x16(unsigned char *in, unsigned int in_size,
+                                       unsigned char *out, unsigned int out_sz);
+
+int rans_compute_shift(uint32_t *F0, uint32_t (*F)[256], uint32_t *T,
+                       uint32_t *S);
+
+// Rounds to next power of 2.
+// credit to http://graphics.stanford.edu/~seander/bithacks.html
+static inline uint32_t round2(uint32_t v) {
+    v--;
+    v |= v >> 1;
+    v |= v >> 2;
+    v |= v >> 4;
+    v |= v >> 8;
+    v |= v >> 16;
+    v++;
+    return v;
+}
+
+static inline int normalise_freq(uint32_t *F, int size, uint32_t tot) {
+    int m, M, j, loop = 0;
+    uint64_t tr;
+    if (!size)
+        return 0;
+
+ again:
+    tr = ((uint64_t)tot<<31)/size + (1<<30)/size;
+
+    for (size = m = M = j = 0; j < 256; j++) {
+        if (!F[j])
+            continue;
+
+        if (m < F[j])
+            m = F[j], M = j;
+
+        if ((F[j] = (F[j]*tr)>>31) == 0)
+            F[j] = 1;
+        size += F[j];
+//      if (F[j] == tot)
+//          F[j]--;
+    }
+
+    int adjust = tot - size;
+    if (adjust > 0) {
+        F[M] += adjust;
+    } else if (adjust < 0) {
+        if (F[M] > -adjust && (loop == 1 || F[M]/2 >= -adjust)) {
+            F[M] += adjust;
+        } else {
+            if (loop < 1) {
+                loop++;
+                goto again;
+            }
+            adjust += F[M]-1;
+            F[M] = 1;
+            for (j = 0; adjust && j < 256; j++) {
+                if (F[j] < 2) continue;
+
+                int d = F[j] > -adjust;
+                int m = d ? adjust : 1-F[j];
+                F[j]   += m;
+                adjust -= m;
+            }
+        }
+    }
+
+    //printf("F[%d]=%d\n", M, F[M]);
+    return F[M]>0 ? 0 : -1;
+}
+
+// A specialised version of normalise_freq_shift where the input size
+// is already normalised to a power of 2, meaning we can just perform
+// shifts instead of hard to define multiplications and adjustments.
+static inline void normalise_freq_shift(uint32_t *F, uint32_t size,
+                                        uint32_t max_tot) {
+    if (size == 0 || size == max_tot)
+        return;
+
+    int shift = 0, i;
+    while (size < max_tot)
+        size*=2, shift++;
+
+    for (i = 0; i < 256; i++)
+        F[i] <<= shift;
+}
+
+// symbols only
+static inline int encode_alphabet(uint8_t *cp, uint32_t *F) {
+    uint8_t *op = cp;
+    int rle, j;
+
+    for (rle = j = 0; j < 256; j++) {
+        if (F[j]) {
+            // j
+            if (rle) {
+                rle--;
+            } else {
+                *cp++ = j;
+                if (!rle && j && F[j-1])  {
+                    for(rle=j+1; rle<256 && F[rle]; rle++)
+                        ;
+                    rle -= j+1;
+                    *cp++ = rle;
+                }
+                //fprintf(stderr, "%d: %d %d\n", j, rle, N[j]);
+            }
+        }
+    }
+    *cp++ = 0;
+    
+    return cp - op;
+}
+
+static inline int decode_alphabet(uint8_t *cp, uint8_t *cp_end, uint32_t *F) {
+    if (cp == cp_end)
+        return 0;
+
+    uint8_t *op = cp;
+    int rle = 0;
+    int j = *cp++;
+    if (cp+2 >= cp_end)
+        goto carefully;
+
+    do {
+        F[j] = 1;
+        if (!rle && j+1 == *cp) {
+            j = *cp++;
+            rle = *cp++;
+        } else if (rle) {
+            rle--;
+            j++;
+            if (j > 255)
+                return 0;
+        } else {
+            j = *cp++;
+        }
+    } while(j && cp+2 < cp_end);
+
+ carefully:
+    if (j) {
+        do {
+            F[j] = 1;
+            if(cp >= cp_end) return 0;
+            if (!rle && j+1 == *cp) {
+                if (cp+1 >= cp_end) return 0;
+                j = *cp++;
+                rle = *cp++;
+            } else if (rle) {
+                rle--;
+                j++;
+                if (j > 255)
+                    return 0;
+            } else {
+                if (cp >= cp_end) return 0;
+                j = *cp++;
+            }
+        } while(j && cp < cp_end);
+    }
+
+    return cp - op;
+}
+
+static inline int encode_freq(uint8_t *cp, uint32_t *F) {
+    uint8_t *op = cp;
+    int j;
+
+    cp += encode_alphabet(cp, F);
+
+    for (j = 0; j < 256; j++) {
+        if (F[j])
+            cp += var_put_u32(cp, NULL, F[j]);
+    }
+
+    return cp - op;
+}
+
+static inline int decode_freq(uint8_t *cp, uint8_t *cp_end, uint32_t *F,
+                              uint32_t *fsum) {
+    if (cp == cp_end)
+        return 0;
+
+    uint8_t *op = cp;
+    cp += decode_alphabet(cp, cp_end, F);
+
+    int j, tot = 0;
+    for (j = 0; j < 256; j++) {
+        if (F[j]) {
+            cp += var_get_u32(cp, cp_end, (unsigned int *)&F[j]);
+            tot += F[j];
+        }
+    }
+
+    *fsum = tot;
+    return cp - op;
+}
+
+
+// Use the order-0 freqs in F0 to encode the order-1 stats in F.
+// All symbols present in F are present in F0, but some in F0 will
+// be empty in F.  Thus we run-length encode the 0 frequencies.
+static inline int encode_freq_d(uint8_t *cp, uint32_t *F0, uint32_t *F) {
+    uint8_t *op = cp;
+    int j, dz;
+
+    for (dz = j = 0; j < 256; j++) {
+        if (F0[j]) {
+            if (F[j] != 0) {
+                if (dz) {
+                    // Replace dz zeros with zero + dz-1 run length
+                    cp -= dz-1;
+                    *cp++ = dz-1;
+                }
+                dz = 0;
+                cp += var_put_u32(cp, NULL, F[j]);
+            } else {
+                //fprintf(stderr, "2: j=%d F0[j]=%d, F[j]=%d, dz=%d\n", j, F0[j], F[j], dz);
+                dz++;
+                *cp++ = 0;
+            }
+        }
+    }
+    
+    if (dz) {
+        cp -= dz-1;
+        *cp++ = dz-1;
+    }
+
+    return cp - op;
+}
+
+// Normalise frequency total T[i] to match TOTFREQ_O1 and encode.
+// Also initialises the RansEncSymbol structs.
+//
+// Returns the desired TF_SHIFT; 10 or 12 bit, or -1 on error.
+static inline int encode_freq1(uint8_t *in, uint32_t in_size, int Nway,
+                               RansEncSymbol syms[256][256], uint8_t **cp_p) {
+    int i, j, z;
+    uint8_t *out = *cp_p, *cp = out;
+
+    // Compute O1 frequency statistics
+    uint32_t (*F)[256] = htscodecs_tls_calloc(256, (sizeof(*F)));
+    if (!F)
+        return -1;
+    uint32_t T[256+MAGIC] = {0};
+    int isz4 = in_size/Nway;
+    if (hist1_4(in, in_size, F, T) < 0)
+        goto err;
+    for (z = 1; z < Nway; z++)
+        F[0][in[z*isz4]]++;
+    T[0]+=Nway-1;
+
+    // Potential fix for the wrap-around bug in AVX2 O1 encoder with shift=12.
+    // This occurs when we have one single symbol, giving freq=4096.
+    // We fix it elsewhere for now by looking for the wrap-around.
+    // See "if (1)" statements in the AVX2 code, which is an alternative
+    // to the "if (0)" here.
+//    if (0) {
+//      int x = -1, y = -1;
+//      int n1, n2;
+//      for (x = 0; x < 256; x++) {
+//          n1 = n2 = -1;
+//          for (y = 0; y < 256; y++) {
+//              if (F[x][y])
+//                  n2 = n1, n1 = y;
+//          }
+//          if (n2!=-1 || n1 == -1)
+//              continue;
+//
+//          for (y = 0; y < 256; y++)
+//              if (!F[x][y])
+//                  break;
+//          assert(y<256);
+//          F[x][y]++;
+//          F[0][y]++; T[y]++; F0[y]=1;
+//          F[0][x]++; T[x]++; F0[x]=1;
+//      }
+//    }
+
+    // Encode the order-0 stats
+    int tmp_T0 = T[0];
+    T[0] = 1;
+    *cp++ = 0; // marker for uncompressed (may change)
+    cp += encode_alphabet(cp, T);
+    T[0] = tmp_T0;
+
+    // Decide between 10-bit and 12-bit freqs.
+    // Fills out S[] to hold the new scaled maximum value.
+    uint32_t S[256] = {0};
+    int shift = rans_compute_shift(T, F, T, S);
+
+    // Normalise so T[i] == TOTFREQ_O1
+    for (i = 0; i < 256; i++) {
+        unsigned int x;
+
+        if (T[i] == 0)
+            continue;
+
+        uint32_t max_val = S[i];
+        if (shift == TF_SHIFT_O1_FAST && max_val > TOTFREQ_O1_FAST)
+            max_val = TOTFREQ_O1_FAST;
+
+        if (normalise_freq(F[i], T[i], max_val) < 0)
+            goto err;
+        T[i]=max_val;
+
+        // Encode our frequency array
+        cp += encode_freq_d(cp, T, F[i]);
+
+        normalise_freq_shift(F[i], T[i], 1<<shift); T[i]=1<<shift;
+
+        // Initialise Rans Symbol struct too.
+        uint32_t *F_i_ = F[i];
+        for (x = j = 0; j < 256; j++) {
+            RansEncSymbolInit(&syms[i][j], x, F_i_[j], shift);
+            x += F_i_[j];
+        }
+    }
+
+    *out = shift<<4;
+    if (cp - out > 1000) {
+        uint8_t *op = out;
+        // try rans0 compression of header
+        unsigned int u_freq_sz = cp-(op+1);
+        unsigned int c_freq_sz;
+        unsigned char *c_freq = rans_compress_O0_4x16(op+1, u_freq_sz, NULL,
+                                                      &c_freq_sz);
+        if (c_freq && c_freq_sz + 6 < cp-op) {
+            *op++ |= 1; // compressed
+            op += var_put_u32(op, NULL, u_freq_sz);
+            op += var_put_u32(op, NULL, c_freq_sz);
+            memcpy(op, c_freq, c_freq_sz);
+            cp = op+c_freq_sz;
+        }
+        free(c_freq);
+    }
+
+    *cp_p = cp;
+    htscodecs_tls_free(F);
+    return shift;
+
+ err:
+    htscodecs_tls_free(F);
+    return -1;
+}
+
+// Part of decode_freq1 below.  This decodes an order-1 frequency table
+// using an order-0 table to determine which stats may be stored.
+static inline int decode_freq_d(uint8_t *cp, uint8_t *cp_end, uint32_t *F0,
+                                uint32_t *F, uint32_t *total) {
+    if (cp == cp_end)
+        return 0;
+
+    uint8_t *op = cp;
+    int j, dz, T = 0;
+
+    for (j = dz = 0; j < 256 && cp < cp_end; j++) {
+        //if (F0[j]) fprintf(stderr, "F0[%d]=%d\n", j, F0[j]);
+        if (!F0[j])
+            continue;
+
+        uint32_t f;
+        if (dz) {
+            f = 0;
+            dz--;
+        } else {
+            if (cp >= cp_end) return 0;
+            cp += var_get_u32(cp, cp_end, &f);
+            if (f == 0) {
+                if (cp >= cp_end) return 0;
+                dz = *cp++;
+            }
+        }
+        F[j] = f;
+        T += f;
+    }
+
+    if (total) *total = T;
+    return cp - op;
+}
+
+typedef struct {
+    uint16_t f;
+    uint16_t b;
+} fb_t;
+
+// Decode order-1 frequency table, filling out various lookup tables
+// in the process. (Which will depend on shift and which values have
+// been passed in.)
+//
+// Returns the number of bytes decoded.
+static inline int decode_freq1(uint8_t *cp, uint8_t *cp_end, int shift,
+                               uint32_t s3 [256][TOTFREQ_O1],
+                               uint32_t s3F[256][TOTFREQ_O1_FAST],
+                               uint8_t *sfb[256], fb_t fb[256][256]) {
+    uint8_t *cp_start = cp;
+    int i, j, x;
+    uint32_t F0[256] = {0};
+    int fsz = decode_alphabet(cp, cp_end, F0);
+    if (!fsz)
+        goto err;
+    cp += fsz;
+
+    if (cp >= cp_end)
+        goto err;
+
+    // silence false gcc warnings
+    if (fb) {fb [0][0].b= 0;}
+    if (s3) {s3 [0][0]  = 0;}
+    if (s3F){s3F[0][0]  = 0;}
+
+    for (i = 0; i < 256; i++) {
+        if (F0[i] == 0)
+            continue;
+
+        uint32_t F[256] = {0}, T = 0;
+        fsz = decode_freq_d(cp, cp_end, F0, F, &T);
+        if (!fsz)
+            goto err;
+        cp += fsz;
+
+        if (!T) {
+            //fprintf(stderr, "No freq for F_%d\n", i);
+            continue;
+        }
+
+        normalise_freq_shift(F, T, 1<<shift);
+
+        // Build symbols; fixme, do as part of decode, see the _d variant
+        for (j = x = 0; j < 256; j++) {
+            if (F[j]) {
+                if (F[j] > (1<<shift) - x)
+                    goto err;
+
+                if (sfb && shift == TF_SHIFT_O1) {
+                    memset(&sfb[i][x], j, F[j]);
+                    fb[i][j].f = F[j];
+                    fb[i][j].b = x;
+                } else if (s3 && shift == TF_SHIFT_O1) {
+                    int y;
+                    for (y = 0; y < F[j]; y++)
+                        s3[i][y+x] = (((uint32_t)F[j])<<(shift+8)) |(y<<8) |j;
+                } else if (s3F && shift == TF_SHIFT_O1_FAST) {
+                    int y;
+                    for (y = 0; y < F[j]; y++)
+                        s3F[i][y+x] = (((uint32_t)F[j])<<(shift+8)) |(y<<8) |j;
+                }
+
+                x += F[j];
+            }
+        }
+        if (x != (1<<shift))
+            goto err;
+    }
+
+    return cp - cp_start;
+
+ err:
+    return 0;
+}
+
+// Build s3 symbol lookup table.
+// This is 12 bit freq, 12 bit bias and 8 bit symbol.
+static inline int rans_F_to_s3(const uint32_t *F, int shift, uint32_t *s3) {
+    int j, x;
+    for (j = x = 0; j < 256; j++) {
+        if (F[j] && F[j] <= (1<<shift) - x) {
+            uint32_t base = (((uint32_t)F[j])<<(shift+8))|j, y;
+            for (y = 0; y < F[j]; y++, x++)
+                s3[x] = base + (y<<8);
+        }
+    }
+
+    return x == (1<<shift) ? 0 : 1;
+}
+
+#ifdef ROT32_SIMD
+#include <x86intrin.h>
+
+// Our own implementation of _mm256_set_m128i as it's not there on older
+// gcc implementations.  This is basically the same thing.
+static inline __m256i _mm256_set_m128ix(__m128i H, __m128i L) {
+    return _mm256_insertf128_si256(_mm256_castsi128_si256(L), H, 1);
+}
+
+static inline void rot32_simd(uint8_t t[32][32], uint8_t *out, int iN[32]) {
+    int z;
+
+    __m256i lh8[32];
+    for (z = 0; z < 32/2; z+=2) {
+        __m256i a, b, c, d;
+        a = _mm256_loadu_si256((__m256i *)&t[z*2+0]);
+        b = _mm256_loadu_si256((__m256i *)&t[z*2+1]);
+        c = _mm256_loadu_si256((__m256i *)&t[z*2+2]);
+        d = _mm256_loadu_si256((__m256i *)&t[z*2+3]);
+
+        lh8[z+0]  = _mm256_unpacklo_epi8(a, b);
+        lh8[z+16] = _mm256_unpackhi_epi8(a, b);
+        lh8[z+1]  = _mm256_unpacklo_epi8(c, d);
+        lh8[z+17] = _mm256_unpackhi_epi8(c, d);
+    }
+
+    __m256i lh32[32];
+    for (z = 0; z < 32/4; z+=2) {
+        __m256i a, b, c, d;
+        a = _mm256_unpacklo_epi16(lh8[z*4+0], lh8[z*4+1]);
+        b = _mm256_unpacklo_epi16(lh8[z*4+2], lh8[z*4+3]);
+        c = _mm256_unpackhi_epi16(lh8[z*4+0], lh8[z*4+1]);
+        d = _mm256_unpackhi_epi16(lh8[z*4+2], lh8[z*4+3]);
+
+        __m256i e, f, g, h;
+        e = _mm256_unpacklo_epi16(lh8[(z+1)*4+0], lh8[(z+1)*4+1]);
+        f = _mm256_unpacklo_epi16(lh8[(z+1)*4+2], lh8[(z+1)*4+3]);
+        g = _mm256_unpackhi_epi16(lh8[(z+1)*4+0], lh8[(z+1)*4+1]);
+        h = _mm256_unpackhi_epi16(lh8[(z+1)*4+2], lh8[(z+1)*4+3]);
+
+        lh32[z+0]  = _mm256_unpacklo_epi32(a,b);
+        lh32[z+8]  = _mm256_unpacklo_epi32(c,d);
+        lh32[z+16] = _mm256_unpackhi_epi32(a,b);
+        lh32[z+24] = _mm256_unpackhi_epi32(c,d);
+
+        lh32[z+1+0]  = _mm256_unpacklo_epi32(e,f);
+        lh32[z+1+8]  = _mm256_unpacklo_epi32(g,h);
+        lh32[z+1+16] = _mm256_unpackhi_epi32(e,f);
+        lh32[z+1+24] = _mm256_unpackhi_epi32(g,h);
+    }
+
+    // Final unpack 64 and store
+    int idx[] = {0, 8, 4, 12, 2, 10, 6, 14};
+    for (z = 0; z < 8; z++) {
+        int i = idx[z];
+
+        // Putting this here doesn't soeed things up
+        __m256i a = _mm256_unpacklo_epi64(lh32[i*2+0], lh32[i*2+1]);
+        __m256i b = _mm256_unpacklo_epi64(lh32[i*2+2], lh32[i*2+3]);
+        __m256i c = _mm256_unpackhi_epi64(lh32[i*2+0], lh32[i*2+1]);
+        __m256i d = _mm256_unpackhi_epi64(lh32[i*2+2], lh32[i*2+3]);
+
+        __m256i p = _mm256_set_m128ix(_mm256_extracti128_si256(b,0),
+                                      _mm256_extracti128_si256(a,0));
+        __m256i q = _mm256_set_m128ix(_mm256_extracti128_si256(d,0),
+                                      _mm256_extracti128_si256(c,0));
+        __m256i r = _mm256_set_m128ix(_mm256_extracti128_si256(b,1),
+                                      _mm256_extracti128_si256(a,1));
+        __m256i s = _mm256_set_m128ix(_mm256_extracti128_si256(d,1),
+                                      _mm256_extracti128_si256(c,1));
+
+        _mm256_storeu_si256((__m256i *)(&out[iN[z*2+0]]),  p);
+        _mm256_storeu_si256((__m256i *)(&out[iN[z*2+1]]),  q);
+        _mm256_storeu_si256((__m256i *)(&out[iN[z*2+16]]), r);
+        _mm256_storeu_si256((__m256i *)(&out[iN[z*2+17]]), s);
+    }
+
+    // Store
+    for (z = 0; z < 32; z++)
+        iN[z] += 32;
+}
+#endif
+
+#endif // RANS_INTERNAL_H
diff --git a/ext/htslib/htscodecs/htscodecs/rANS_static32x16pr.c b/ext/htslib/htscodecs/htscodecs/rANS_static32x16pr.c
new file mode 100644
index 0000000..51ea554
--- /dev/null
+++ b/ext/htslib/htscodecs/htscodecs/rANS_static32x16pr.c
@@ -0,0 +1,758 @@
+/*
+ * Copyright (c) 2017-2023 Genome Research Ltd.
+ * Author(s): James Bonfield
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above
+ *       copyright notice, this list of conditions and the following
+ *       disclaimer in the documentation and/or other materials provided
+ *       with the distribution.
+ *
+ *    3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+ *       Institute nor the names of its contributors may be used to endorse
+ *       or promote products derived from this software without specific
+ *       prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH
+ * LTD OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <assert.h>
+#include <string.h>
+#include <limits.h>
+
+#include "rANS_word.h"
+#include "rANS_static4x16.h"
+#include "rANS_static16_int.h"
+#include "varint.h"
+#include "utils.h"
+
+#define TF_SHIFT 12
+#define TOTFREQ (1<<TF_SHIFT)
+
+
+// 9-11 is considerably faster in the O1 variant due to reduced table size.
+// We auto-tune between 10 and 12 though.  Anywhere from 9 to 14 are viable.
+#ifndef TF_SHIFT_O1
+#define TF_SHIFT_O1 12
+#endif
+#ifndef TF_SHIFT_O1_FAST
+#define TF_SHIFT_O1_FAST 10
+#endif
+#define TOTFREQ_O1 (1<<TF_SHIFT_O1)
+#define TOTFREQ_O1_FAST (1<<TF_SHIFT_O1_FAST)
+
+
+#define NX 32
+
+unsigned char *rans_compress_O0_32x16(unsigned char *in,
+                                      unsigned int in_size,
+                                      unsigned char *out,
+                                      unsigned int *out_size) {
+    unsigned char *cp, *out_end, *out_free = NULL;
+    RansEncSymbol syms[256];
+    RansState ransN[NX];
+    uint8_t* ptr;
+    uint32_t F[256+MAGIC] = {0};
+    int i, j, tab_size = 0, x, z;
+    // -20 for order/size/meta
+    unsigned int bound = rans_compress_bound_4x16(in_size,0)-20;
+
+    if (!out) {
+        *out_size = bound;
+        out = out_free = malloc(*out_size);
+    }
+    if (!out || bound > *out_size)
+        return NULL;
+
+    // If "out" isn't word aligned, tweak out_end/ptr to ensure it is.
+    // We already added more round in bound to allow for this.
+    if (((size_t)out)&1)
+        bound--;
+    ptr = out_end = out + bound;
+
+    if (in_size == 0)
+        goto empty;
+
+    // Compute statistics
+    double e = hist8e(in, in_size, F);
+    int low_ent = e < 2;
+
+    // Normalise so frequences sum to power of 2
+    uint32_t fsum = in_size;
+    uint32_t max_val = round2(fsum);
+    if (max_val > TOTFREQ)
+        max_val = TOTFREQ;
+
+    if (normalise_freq(F, fsum, max_val) < 0) {
+        free(out_free);
+        return NULL;
+    }
+    fsum=max_val;
+
+    cp = out;
+    cp += encode_freq(cp, F);
+    tab_size = cp-out;
+    //write(2, out+4, cp-(out+4));
+
+    if (normalise_freq(F, fsum, TOTFREQ) < 0) {
+        free(out_free);
+        return NULL;
+    }
+
+    // Encode statistics.
+    for (x = j = 0; j < 256; j++) {
+        if (F[j]) {
+            RansEncSymbolInit(&syms[j], x, F[j], TF_SHIFT);
+            x += F[j];
+        }
+    }
+
+    for (z = 0; z < NX; z++)
+      RansEncInit(&ransN[z]);
+
+    z = i = in_size&(NX-1);
+    while (z-- > 0)
+      RansEncPutSymbol(&ransN[z], &ptr, &syms[in[in_size-(i-z)]]);
+
+    if (low_ent) {
+        // orig
+        // gcc   446
+        // clang 427
+        for (i=(in_size &~(NX-1)); likely(i>0); i-=NX) {
+            for (z = NX-1; z >= 0; z-=4) {
+                RansEncSymbol *s0 = &syms[in[i-(NX-z+0)]];
+                RansEncSymbol *s1 = &syms[in[i-(NX-z+1)]];
+                RansEncSymbol *s2 = &syms[in[i-(NX-z+2)]];
+                RansEncSymbol *s3 = &syms[in[i-(NX-z+3)]];
+                RansEncPutSymbol_branched(&ransN[z-0], &ptr, s0);
+                RansEncPutSymbol_branched(&ransN[z-1], &ptr, s1);
+                RansEncPutSymbol_branched(&ransN[z-2], &ptr, s2);
+                RansEncPutSymbol_branched(&ransN[z-3], &ptr, s3);
+                if (NX%8 == 0) {
+                    z -= 4;
+                    RansEncSymbol *s0 = &syms[in[i-(NX-z+0)]];
+                    RansEncSymbol *s1 = &syms[in[i-(NX-z+1)]];
+                    RansEncSymbol *s2 = &syms[in[i-(NX-z+2)]];
+                    RansEncSymbol *s3 = &syms[in[i-(NX-z+3)]];
+                    RansEncPutSymbol_branched(&ransN[z-0], &ptr, s0);
+                    RansEncPutSymbol_branched(&ransN[z-1], &ptr, s1);
+                    RansEncPutSymbol_branched(&ransN[z-2], &ptr, s2);
+                    RansEncPutSymbol_branched(&ransN[z-3], &ptr, s3);
+                }
+            }
+            if (z < -1) abort();
+        }
+    } else {
+        // Branchless version optimises poorly with gcc unless we have
+        // AVX2 capability, so have a custom rewrite of it.
+        uint16_t* ptr16 = (uint16_t *)ptr;
+        for (i=(in_size &~(NX-1)); likely(i>0); i-=NX) {
+            // Unrolled copy of below, because gcc doesn't optimise this
+            // well in the original form.
+            //
+            // Gcc11:   328 MB/s (this) vs 208 MB/s (orig)
+            // Clang10: 352 MB/s (this) vs 340 MB/s (orig)
+            //
+            // for (z = NX-1; z >= 0; z-=4) {
+            //  RansEncSymbol *s0 = &syms[in[i-(NX-z+0)]];
+            //  RansEncSymbol *s1 = &syms[in[i-(NX-z+1)]];
+            //  RansEncSymbol *s2 = &syms[in[i-(NX-z+2)]];
+            //  RansEncSymbol *s3 = &syms[in[i-(NX-z+3)]];
+            //  RansEncPutSymbol(&ransN[z-0], &ptr, s0);
+            //  RansEncPutSymbol(&ransN[z-1], &ptr, s1);
+            //  RansEncPutSymbol(&ransN[z-2], &ptr, s2);
+            //  RansEncPutSymbol(&ransN[z-3], &ptr, s3);
+            // }
+
+            for (z = NX-1; z >= 0; z-=4) {
+                // RansEncPutSymbol added in-situ
+                RansState *rp = &ransN[z]-3;
+                RansEncSymbol *sy[4];
+                uint8_t *C = &in[i-(NX-z)]-3;
+
+                sy[0] = &syms[C[3]];
+                sy[1] = &syms[C[2]];
+
+                int c0  = rp[3-0] > sy[0]->x_max;
+                int c1  = rp[3-1] > sy[1]->x_max;
+
+#ifdef HTSCODECS_LITTLE_ENDIAN
+                ptr16[-1] = rp[3-0]; ptr16 -= c0;
+                ptr16[-1] = rp[3-1]; ptr16 -= c1;
+#else
+                ((uint8_t *)&ptr16[-1])[0] = rp[3-0];
+                ((uint8_t *)&ptr16[-1])[1] = rp[3-0]>>8;
+                ptr16 -= c0;
+                ((uint8_t *)&ptr16[-1])[0] = rp[3-1];
+                ((uint8_t *)&ptr16[-1])[1] = rp[3-1]>>8;
+                ptr16 -= c1;
+#endif
+
+                rp[3-0] = c0 ? rp[3-0]>>16 : rp[3-0];
+                rp[3-1] = c1 ? rp[3-1]>>16 : rp[3-1];
+
+                sy[2] = &syms[C[1]];
+                sy[3] = &syms[C[0]];
+
+                int c2  = rp[3-2] > sy[2]->x_max;
+                int c3  = rp[3-3] > sy[3]->x_max;
+#ifdef HTSCODECS_LITTLE_ENDIAN
+                ptr16[-1] = rp[3-2]; ptr16 -= c2;
+                ptr16[-1] = rp[3-3]; ptr16 -= c3;
+#else
+                ((uint8_t *)&ptr16[-1])[0] = rp[3-2];
+                ((uint8_t *)&ptr16[-1])[1] = rp[3-2]>>8;
+                ptr16 -= c2;
+                ((uint8_t *)&ptr16[-1])[0] = rp[3-3];
+                ((uint8_t *)&ptr16[-1])[1] = rp[3-3]>>8;
+                ptr16 -= c3;
+#endif
+                rp[3-2] = c2 ? rp[3-2]>>16 : rp[3-2];
+                rp[3-3] = c3 ? rp[3-3]>>16 : rp[3-3];
+
+                int k;
+                for (k = 0; k < 4; k++) {
+                    uint64_t r64 = (uint64_t)rp[3-k];
+                    uint32_t q = (r64 * sy[k]->rcp_freq) >> sy[k]->rcp_shift;
+                    rp[3-k] += sy[k]->bias + q*sy[k]->cmpl_freq;
+                }
+            }
+            if (z < -1) abort();
+        }
+        ptr = (uint8_t *)ptr16;
+    }
+    for (z = NX-1; z >= 0; z--)
+        RansEncFlush(&ransN[z], &ptr);
+
+ empty:
+    // Finalise block size and return it
+    *out_size = (out_end - ptr) + tab_size;
+
+    memmove(out + tab_size, ptr, out_end-ptr);
+
+    return out;
+}
+
+unsigned char *rans_uncompress_O0_32x16(unsigned char *in,
+                                        unsigned int in_size,
+                                        unsigned char *out,
+                                        unsigned int out_sz) {
+    if (in_size < 16) // 4-states at least
+        return NULL;
+
+    if (out_sz >= INT_MAX)
+        return NULL; // protect against some overflow cases
+
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    if (out_sz > 100000)
+        return NULL;
+#endif
+
+    /* Load in the static tables */
+    unsigned char *cp = in, *out_free = NULL;
+    unsigned char *cp_end = in + in_size;
+    int i;
+    uint32_t s3[TOTFREQ]; // For TF_SHIFT <= 12
+
+    if (!out)
+        out_free = out = malloc(out_sz);
+    if (!out)
+        return NULL;
+
+    // Precompute reverse lookup of frequency.
+    uint32_t F[256] = {0}, fsum;
+    int fsz = decode_freq(cp, cp_end, F, &fsum);
+    if (!fsz)
+        goto err;
+    cp += fsz;
+
+    normalise_freq_shift(F, fsum, TOTFREQ);
+
+    // Build symbols; fixme, do as part of decode, see the _d variant
+    if (rans_F_to_s3(F, TF_SHIFT, s3))
+        goto err;
+
+    if (cp_end - cp < NX * 4)
+        goto err;
+
+    int z;
+    RansState R[NX];
+    for (z = 0; z < NX; z++) {
+        RansDecInit(&R[z], &cp);
+        if (R[z] < RANS_BYTE_L)
+            goto err;
+    }
+
+    int out_end = (out_sz&~(NX-1));
+    const uint32_t mask = (1u << TF_SHIFT)-1;
+    cp_end -= NX*2; // worst case for renorm bytes
+
+    // assume NX is divisible by 4
+    assert(NX%4==0);
+
+    // Unsafe loop with no ptr overflow checking within loop itself
+    for (i=0; likely(i < out_end && cp < cp_end); i+=NX) {
+        for (z = 0; z < NX; z+=4) {
+            uint32_t S[4];
+            S[0] = s3[R[z+0] & mask];
+            S[1] = s3[R[z+1] & mask];
+            S[2] = s3[R[z+2] & mask];
+            S[3] = s3[R[z+3] & mask];
+
+            R[z+0] = (S[0]>>(TF_SHIFT+8)) * (R[z+0] >> TF_SHIFT)
+                + ((S[0]>>8) & mask);
+            R[z+1] = (S[1]>>(TF_SHIFT+8)) * (R[z+1] >> TF_SHIFT)
+                + ((S[1]>>8) & mask);
+            R[z+2] = (S[2]>>(TF_SHIFT+8)) * (R[z+2] >> TF_SHIFT)
+                + ((S[2]>>8) & mask);
+            R[z+3] = (S[3]>>(TF_SHIFT+8)) * (R[z+3] >> TF_SHIFT)
+                + ((S[3]>>8) & mask);
+
+            out[i+z+0] = S[0];
+            out[i+z+1] = S[1];
+            out[i+z+2] = S[2];
+            out[i+z+3] = S[3];
+
+            RansDecRenorm(&R[z+0], &cp);
+            RansDecRenorm(&R[z+1], &cp);
+            RansDecRenorm(&R[z+2], &cp);
+            RansDecRenorm(&R[z+3], &cp);
+
+            if (NX%8==0) {
+                z += 4;
+                S[0] = s3[R[z+0] & mask];
+                S[1] = s3[R[z+1] & mask];
+                S[2] = s3[R[z+2] & mask];
+                S[3] = s3[R[z+3] & mask];
+
+                R[z+0] = (S[0]>>(TF_SHIFT+8)) * (R[z+0] >> TF_SHIFT)
+                    + ((S[0]>>8) & mask);
+                R[z+1] = (S[1]>>(TF_SHIFT+8)) * (R[z+1] >> TF_SHIFT)
+                    + ((S[1]>>8) & mask);
+                R[z+2] = (S[2]>>(TF_SHIFT+8)) * (R[z+2] >> TF_SHIFT)
+                    + ((S[2]>>8) & mask);
+                R[z+3] = (S[3]>>(TF_SHIFT+8)) * (R[z+3] >> TF_SHIFT)
+                    + ((S[3]>>8) & mask);
+
+                out[i+z+0] = S[0];
+                out[i+z+1] = S[1];
+                out[i+z+2] = S[2];
+                out[i+z+3] = S[3];
+
+                RansDecRenorm(&R[z+0], &cp);
+                RansDecRenorm(&R[z+1], &cp);
+                RansDecRenorm(&R[z+2], &cp);
+                RansDecRenorm(&R[z+3], &cp);
+            }
+        }
+    }
+
+    // Safe loop
+    for (; i < out_end; i+=NX) {
+        for (z = 0; z < NX; z+=4) {
+            uint32_t S[4];
+            S[0] = s3[R[z+0] & mask];
+            S[1] = s3[R[z+1] & mask];
+            S[2] = s3[R[z+2] & mask];
+            S[3] = s3[R[z+3] & mask];
+
+            R[z+0] = (S[0]>>(TF_SHIFT+8)) * (R[z+0] >> TF_SHIFT)
+                + ((S[0]>>8) & mask);
+            R[z+1] = (S[1]>>(TF_SHIFT+8)) * (R[z+1] >> TF_SHIFT)
+                + ((S[1]>>8) & mask);
+            R[z+2] = (S[2]>>(TF_SHIFT+8)) * (R[z+2] >> TF_SHIFT)
+                + ((S[2]>>8) & mask);
+            R[z+3] = (S[3]>>(TF_SHIFT+8)) * (R[z+3] >> TF_SHIFT)
+                + ((S[3]>>8) & mask);
+
+            out[i+z+0] = S[0];
+            out[i+z+1] = S[1];
+            out[i+z+2] = S[2];
+            out[i+z+3] = S[3];
+
+            RansDecRenormSafe(&R[z+0], &cp, cp_end+NX*2);
+            RansDecRenormSafe(&R[z+1], &cp, cp_end+NX*2);
+            RansDecRenormSafe(&R[z+2], &cp, cp_end+NX*2);
+            RansDecRenormSafe(&R[z+3], &cp, cp_end+NX*2);
+        }
+    }
+
+    for (z = out_sz & (NX-1); z-- > 0; )
+        out[out_end + z] = s3[R[z] & mask];
+
+    //fprintf(stderr, "    0 Decoded %d bytes\n", (int)(cp-in)); //c-size
+
+    return out;
+
+ err:
+    free(out_free);
+    return NULL;
+}
+
+
+//-----------------------------------------------------------------------------
+unsigned char *rans_compress_O1_32x16(unsigned char *in,
+                                      unsigned int in_size,
+                                      unsigned char *out,
+                                      unsigned int *out_size) {
+    unsigned char *cp, *out_end, *out_free = NULL;
+    unsigned int tab_size;
+    int bound = rans_compress_bound_4x16(in_size,1)-20, z;
+    RansState ransN[NX];
+
+    if (in_size < NX) // force O0 instead
+        return NULL;
+
+    if (!out) {
+        *out_size = bound;
+        out_free = out = malloc(*out_size);
+    }
+    if (!out || bound > *out_size)
+        return NULL;
+
+    if (((size_t)out)&1)
+        bound--;
+    out_end = out + bound;
+
+    RansEncSymbol (*syms)[256] = htscodecs_tls_alloc(256 * (sizeof(*syms)));
+    if (!syms) {
+        free(out_free);
+        return NULL;
+    }
+
+    cp = out;
+    int shift = encode_freq1(in, in_size, 32, syms, &cp); 
+    if (shift < 0) {
+        free(out_free);
+        htscodecs_tls_free(syms);
+        return NULL;
+    }
+    tab_size = cp - out;
+
+    for (z = 0; z < NX; z++)
+      RansEncInit(&ransN[z]);
+
+    uint8_t* ptr = out_end;
+
+    int iN[NX], isz4 = in_size/NX, i;
+    for (z = 0; z < NX; z++)
+        iN[z] = (z+1)*isz4-2;
+
+    unsigned char lN[NX];
+    for (z = 0; z < NX; z++)
+        lN[z] = in[iN[z]+1];
+
+    // Deal with the remainder
+    z = NX-1;
+    lN[z] = in[in_size-1];
+    for (iN[z] = in_size-2; iN[z] > NX*isz4-2; iN[z]--) {
+        unsigned char c = in[iN[z]];
+        RansEncPutSymbol(&ransN[z], &ptr, &syms[c][lN[z]]);
+        lN[z] = c;
+    }
+
+    unsigned char *i32[NX];
+    for (i = 0; i < NX; i++)
+        i32[i] = &in[iN[i]];
+
+    for (; likely(i32[0] >= in); ) {
+        uint16_t *ptr16 = (uint16_t *)ptr;
+        for (z = NX-1; z >= 0; z-=4) {
+            RansEncSymbol *sy[4];
+            int k;
+
+            for (k = 0; k < 4; k++) {
+                sy[k] = &syms[*i32[z-k]][lN[z-k]];
+                lN[z-k] = *i32[z-k]--;
+            }
+
+            // RansEncPutSymbol added in-situ
+            for (k = 0; k < 4; k++) {
+                int c = ransN[z-k] > sy[k]->x_max;
+#ifdef HTSCODECS_LITTLE_ENDIAN
+                ptr16[-1] = ransN[z-k];
+#else
+                ((uint8_t *)&ptr16[-1])[0] = ransN[z-k];
+                ((uint8_t *)&ptr16[-1])[1] = ransN[z-k]>>8;
+#endif
+                ptr16 -= c;
+                //ransN[z-k] >>= c<<4;
+                ransN[z-k] = c ? ransN[z-k]>>16 : ransN[z-k];
+            }
+
+            for (k = 0; k < 4; k++) {
+                uint64_t r64 = ransN[z-k];
+                uint32_t q = (r64 * sy[k]->rcp_freq) >> sy[k]->rcp_shift;
+                ransN[z-k] += sy[k]->bias + q*sy[k]->cmpl_freq;
+            }
+        }
+        ptr = (uint8_t *)ptr16;
+    }
+
+    for (z = NX-1; z>=0; z--)
+        RansEncPutSymbol(&ransN[z], &ptr, &syms[0][lN[z]]);
+
+    for (z = NX-1; z>=0; z--)
+        RansEncFlush(&ransN[z], &ptr);
+
+    *out_size = (out_end - ptr) + tab_size;
+
+    cp = out;
+    memmove(out + tab_size, ptr, out_end-ptr);
+
+    htscodecs_tls_free(syms);
+    return out;
+}
+
+//#define MAGIC2 111
+#define MAGIC2 179
+//#define MAGIC2 0
+
+unsigned char *rans_uncompress_O1_32x16(unsigned char *in,
+                                        unsigned int in_size,
+                                        unsigned char *out,
+                                        unsigned int out_sz) {
+    if (in_size < NX*4) // 4-states at least
+        return NULL;
+
+    if (out_sz >= INT_MAX)
+        return NULL; // protect against some overflow cases
+
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    if (out_sz > 100000)
+        return NULL;
+#endif
+
+    /* Load in the static tables */
+    unsigned char *cp = in, *cp_end = in+in_size, *out_free = NULL;
+    unsigned char *c_freq = NULL;
+    int i;
+
+    /*
+     * Somewhat complex memory layout.
+     * With shift==12 (TF_SHIFT_O1) we fill out use both sfb and fb.
+     * With shift==10 (...O1_FAST)  we fill out and use s3 only.
+     *
+     * sfb+fb is larger, therefore we allocate this much memory.
+     */
+    uint8_t *sfb_ = htscodecs_tls_alloc(256*
+                                        ((TOTFREQ_O1+MAGIC2)*sizeof(*sfb_)
+                                         +256 * sizeof(fb_t)));
+    if (!sfb_)
+        return NULL;
+
+    // sfb and fb are consecutive
+    uint8_t *sfb[257];
+    if ((*cp >> 4) == TF_SHIFT_O1) {
+        for (i = 0; i <= 256; i++)
+            sfb[i]=  sfb_ + i*(TOTFREQ_O1+MAGIC2);
+    } else {
+        for (i = 0; i <= 256; i++)
+            sfb[i]=  sfb_ + i*(TOTFREQ_O1_FAST+MAGIC2);
+    }
+    fb_t (*fb)[256] = (fb_t (*)[256]) sfb[256];
+
+    // NOTE: s3 overlaps sfb/fb
+    uint32_t (*s3)[TOTFREQ_O1_FAST] = (uint32_t (*)[TOTFREQ_O1_FAST])sfb_;
+
+    if (!out)
+        out_free = out = malloc(out_sz);
+
+    if (!out)
+        goto err;
+
+    //fprintf(stderr, "out_sz=%d\n", out_sz);
+
+    // compressed header? If so uncompress it
+    unsigned char *tab_end = NULL;
+    unsigned char *c_freq_end = cp_end;
+    unsigned int shift = *cp >> 4;
+    if (*cp++ & 1) {
+        uint32_t u_freq_sz, c_freq_sz;
+        cp += var_get_u32(cp, cp_end, &u_freq_sz);
+        cp += var_get_u32(cp, cp_end, &c_freq_sz);
+        if (c_freq_sz > cp_end - cp)
+            goto err;
+        tab_end = cp + c_freq_sz;
+        if (!(c_freq = rans_uncompress_O0_4x16(cp, c_freq_sz, NULL,u_freq_sz)))
+            goto err;
+        cp = c_freq;
+        c_freq_end = c_freq + u_freq_sz;
+    }
+
+    // Decode order-0 symbol list; avoids needing in order-1 tables
+    cp += decode_freq1(cp, c_freq_end, shift, NULL, s3, sfb, fb);
+
+    if (tab_end)
+        cp = tab_end;
+    free(c_freq);
+    c_freq = NULL;
+
+    if (cp_end - cp < NX * 4)
+        goto err;
+
+    RansState R[NX];
+    uint8_t *ptr = cp, *ptr_end = in + in_size - 2*NX;
+    int z;
+    for (z = 0; z < NX; z++) {
+        RansDecInit(&R[z], &ptr);
+        if (R[z] < RANS_BYTE_L)
+            goto err;
+    }
+
+    int isz4 = out_sz/NX;
+    int i4[NX], l[NX] = {0};
+    for (z = 0; z < NX; z++)
+        i4[z] = z*isz4;
+
+    const int low_ent = in_size < 0.2 * out_sz;
+
+    // Around 15% faster to specialise for 10/12 than to have one
+    // loop with shift as a variable.
+    if (shift == TF_SHIFT_O1) {
+        // TF_SHIFT_O1 = 12
+        const uint32_t mask = ((1u << TF_SHIFT_O1)-1);
+        for (; likely(i4[0] < isz4);) {
+            for (z = 0; z < NX; z+=4) {
+                uint16_t m[4], c[4];
+
+                c[0] = sfb[l[z+0]][m[0] = R[z+0] & mask];
+                c[1] = sfb[l[z+1]][m[1] = R[z+1] & mask];
+                c[2] = sfb[l[z+2]][m[2] = R[z+2] & mask];
+                c[3] = sfb[l[z+3]][m[3] = R[z+3] & mask];
+
+                R[z+0] = fb[l[z+0]][c[0]].f * (R[z+0]>>TF_SHIFT_O1);
+                R[z+0] += m[0] - fb[l[z+0]][c[0]].b;
+
+                R[z+1] = fb[l[z+1]][c[1]].f * (R[z+1]>>TF_SHIFT_O1);
+                R[z+1] += m[1] - fb[l[z+1]][c[1]].b;
+
+                R[z+2] = fb[l[z+2]][c[2]].f * (R[z+2]>>TF_SHIFT_O1);
+                R[z+2] += m[2] - fb[l[z+2]][c[2]].b;
+
+                R[z+3] = fb[l[z+3]][c[3]].f * (R[z+3]>>TF_SHIFT_O1);
+                R[z+3] += m[3] - fb[l[z+3]][c[3]].b;
+
+                out[i4[z+0]++] = l[z+0] = c[0];
+                out[i4[z+1]++] = l[z+1] = c[1];
+                out[i4[z+2]++] = l[z+2] = c[2];
+                out[i4[z+3]++] = l[z+3] = c[3];
+
+                if (!low_ent && likely(ptr < ptr_end)) {
+                    RansDecRenorm(&R[z+0], &ptr);
+                    RansDecRenorm(&R[z+1], &ptr);
+                    RansDecRenorm(&R[z+2], &ptr);
+                    RansDecRenorm(&R[z+3], &ptr);
+                } else {
+                    RansDecRenormSafe(&R[z+0], &ptr, ptr_end+2*NX);
+                    RansDecRenormSafe(&R[z+1], &ptr, ptr_end+2*NX);
+                    RansDecRenormSafe(&R[z+2], &ptr, ptr_end+2*NX);
+                    RansDecRenormSafe(&R[z+3], &ptr, ptr_end+2*NX);
+                }
+            }
+        }
+
+        // Remainder
+        for (; i4[NX-1] < out_sz; i4[NX-1]++) {
+            uint32_t m = R[NX-1] & ((1u<<TF_SHIFT_O1)-1);
+            unsigned char c = sfb[l[NX-1]][m];
+            out[i4[NX-1]] = c;
+            R[NX-1] = fb[l[NX-1]][c].f * (R[NX-1]>>TF_SHIFT_O1) +
+                m - fb[l[NX-1]][c].b;
+            RansDecRenormSafe(&R[NX-1], &ptr, ptr_end + 2*NX);
+            l[NX-1] = c;
+        }
+    } else {
+        // TF_SHIFT_O1 = 10
+        const uint32_t mask = ((1u << TF_SHIFT_O1_FAST)-1);
+        for (; likely(i4[0] < isz4);) {
+            for (z = 0; z < NX; z+=4) {
+                // Merged sfb and fb into single s3 lookup.
+                // The m[4] array completely vanishes in this method.
+                uint32_t S[4] = {
+                    s3[l[z+0]][R[z+0] & mask],
+                    s3[l[z+1]][R[z+1] & mask],
+                    s3[l[z+2]][R[z+2] & mask],
+                    s3[l[z+3]][R[z+3] & mask],
+                };
+
+                l[z+0] = out[i4[z+0]++] = S[0];
+                l[z+1] = out[i4[z+1]++] = S[1];
+                l[z+2] = out[i4[z+2]++] = S[2];
+                l[z+3] = out[i4[z+3]++] = S[3];
+
+                uint32_t F[4] = {
+                    S[0]>>(TF_SHIFT_O1_FAST+8),
+                    S[1]>>(TF_SHIFT_O1_FAST+8),
+                    S[2]>>(TF_SHIFT_O1_FAST+8),
+                    S[3]>>(TF_SHIFT_O1_FAST+8),
+                };
+                uint32_t B[4] = {
+                    (S[0]>>8) & mask,
+                    (S[1]>>8) & mask,
+                    (S[2]>>8) & mask,
+                    (S[3]>>8) & mask,
+                };
+
+                R[z+0] = F[0] * (R[z+0]>>TF_SHIFT_O1_FAST) + B[0];
+                R[z+1] = F[1] * (R[z+1]>>TF_SHIFT_O1_FAST) + B[1];
+                R[z+2] = F[2] * (R[z+2]>>TF_SHIFT_O1_FAST) + B[2];
+                R[z+3] = F[3] * (R[z+3]>>TF_SHIFT_O1_FAST) + B[3];
+
+                if (!low_ent && (ptr < ptr_end)) {
+                    // branchless & asm
+                    RansDecRenorm(&R[z+0], &ptr);
+                    RansDecRenorm(&R[z+1], &ptr);
+                    RansDecRenorm(&R[z+2], &ptr);
+                    RansDecRenorm(&R[z+3], &ptr);
+                } else {
+                    // branched, but better when predictable
+                    RansDecRenormSafe(&R[z+0], &ptr, ptr_end+2*NX);
+                    RansDecRenormSafe(&R[z+1], &ptr, ptr_end+2*NX);
+                    RansDecRenormSafe(&R[z+2], &ptr, ptr_end+2*NX);
+                    RansDecRenormSafe(&R[z+3], &ptr, ptr_end+2*NX);
+                }
+            }
+        }
+
+        // Remainder
+        for (; i4[NX-1] < out_sz; i4[NX-1]++) {
+            uint32_t S = s3[l[NX-1]][R[NX-1] & ((1u<<TF_SHIFT_O1_FAST)-1)];
+            out[i4[NX-1]] = l[NX-1] = S&0xff;
+            R[NX-1] = (S>>(TF_SHIFT_O1_FAST+8)) * (R[NX-1]>>TF_SHIFT_O1_FAST)
+                + ((S>>8) & ((1u<<TF_SHIFT_O1_FAST)-1));
+            RansDecRenormSafe(&R[NX-1], &ptr, ptr_end + 2*NX);
+        }
+    }
+    //fprintf(stderr, "    1 Decoded %d bytes\n", (int)(ptr-in)); //c-size
+
+    htscodecs_tls_free(sfb_);
+    return out;
+
+ err:
+    htscodecs_tls_free(sfb_);
+    free(out_free);
+    free(c_freq);
+
+    return NULL;
+}
diff --git a/ext/htslib/htscodecs/htscodecs/rANS_static32x16pr.h b/ext/htslib/htscodecs/htscodecs/rANS_static32x16pr.h
new file mode 100644
index 0000000..0aa2dcd
--- /dev/null
+++ b/ext/htslib/htscodecs/htscodecs/rANS_static32x16pr.h
@@ -0,0 +1,175 @@
+/*
+ * Copyright (c) 2017-2019 Genome Research Ltd.
+ * Author(s): James Bonfield
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above
+ *       copyright notice, this list of conditions and the following
+ *       disclaimer in the documentation and/or other materials provided
+ *       with the distribution.
+ *
+ *    3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+ *       Institute nor the names of its contributors may be used to endorse
+ *       or promote products derived from this software without specific
+ *       prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH
+ * LTD OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef RANS_STATIC32x16PR_H
+#define RANS_STATIC32x16PR_H
+
+/*
+ * This header contains standard scalar implementations of the 32-way
+ * unrolled rANS codec as well as declarations for the custom SIMD
+ * implementations of x86_64 and Arm Aarch64 CPUs.
+ *
+ * The AVX2 and AVX512 source files need to be compiled separately as
+ * we have per-file -march= compiler options and we don't wish to
+ * accidentally get AVX instructions in the scalar variant.  The x86-64
+ * binary then contains all 3 variants at the same time and selected
+ * automatically at run time.
+ *
+ * The ARM Neon version is currently different, as we don't have any test
+ * machines without this capability. I think it's default on all 64-bit
+ * CPUs, so it's not something we're concerned with.  For simplicity
+ * therefore the ARM code is simply #included into the scalar file.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+//----------------------------------------------------------------------
+// Standard scalar versions
+unsigned char *rans_compress_O0_32x16(unsigned char *in,
+                                      unsigned int in_size,
+                                      unsigned char *out,
+                                      unsigned int *out_size);
+
+unsigned char *rans_uncompress_O0_32x16(unsigned char *in,
+                                        unsigned int in_size,
+                                        unsigned char *out,
+                                        unsigned int out_sz);
+
+unsigned char *rans_compress_O1_32x16(unsigned char *in,
+                                      unsigned int in_size,
+                                      unsigned char *out,
+                                      unsigned int *out_size);
+
+unsigned char *rans_uncompress_O1_32x16(unsigned char *in,
+                                        unsigned int in_size,
+                                        unsigned char *out,
+                                        unsigned int out_sz);
+
+//----------------------------------------------------------------------
+// Intel SSE4 implementation.  Only the O0 decoder for now
+#if defined(HAVE_SSE4_1) && defined(HAVE_SSSE3) && defined(HAVE_POPCNT)
+unsigned char *rans_compress_O0_32x16_sse4(unsigned char *in,
+                                           unsigned int in_size,
+                                           unsigned char *out,
+                                           unsigned int *out_size);
+
+unsigned char *rans_uncompress_O0_32x16_sse4(unsigned char *in,
+                                             unsigned int in_size,
+                                             unsigned char *out,
+                                             unsigned int out_sz);
+
+unsigned char *rans_uncompress_O1_32x16_sse4(unsigned char *in,
+                                             unsigned int in_size,
+                                             unsigned char *out,
+                                             unsigned int out_sz);
+#endif
+
+//----------------------------------------------------------------------
+// Intel AVX2 implementation
+#ifdef HAVE_AVX2
+unsigned char *rans_compress_O0_32x16_avx2(unsigned char *in,
+                                           unsigned int in_size,
+                                           unsigned char *out,
+                                           unsigned int *out_size);
+
+unsigned char *rans_uncompress_O0_32x16_avx2(unsigned char *in,
+                                             unsigned int in_size,
+                                             unsigned char *out,
+                                             unsigned int out_sz);
+
+unsigned char *rans_compress_O1_32x16_avx2(unsigned char *in,
+                                           unsigned int in_size,
+                                           unsigned char *out,
+                                           unsigned int *out_size);
+
+unsigned char *rans_uncompress_O1_32x16_avx2(unsigned char *in,
+                                             unsigned int in_size,
+                                             unsigned char *out,
+                                             unsigned int out_sz);
+#endif // HAVE_AVX2
+
+//----------------------------------------------------------------------
+// Intel AVX512 implementation
+#ifdef HAVE_AVX512
+unsigned char *rans_compress_O0_32x16_avx512(unsigned char *in,
+                                             unsigned int in_size,
+                                             unsigned char *out,
+                                             unsigned int *out_size);
+
+unsigned char *rans_uncompress_O0_32x16_avx512(unsigned char *in,
+                                               unsigned int in_size,
+                                               unsigned char *out,
+                                               unsigned int out_sz);
+
+unsigned char *rans_compress_O1_32x16_avx512(unsigned char *in,
+                                             unsigned int in_size,
+                                             unsigned char *out,
+                                             unsigned int *out_size);
+
+unsigned char *rans_uncompress_O1_32x16_avx512(unsigned char *in,
+                                               unsigned int in_size,
+                                               unsigned char *out,
+                                               unsigned int out_sz);
+#endif // HAVE_AVX512
+
+//----------------------------------------------------------------------
+// Arm Neon implementation
+#if defined(__ARM_NEON) && defined(__aarch64__)
+unsigned char *rans_compress_O0_32x16_neon(unsigned char *in,
+                                           unsigned int in_size,
+                                           unsigned char *out,
+                                           unsigned int *out_size);
+
+unsigned char *rans_uncompress_O0_32x16_neon(unsigned char *in,
+                                             unsigned int in_size,
+                                             unsigned char *out,
+                                             unsigned int out_sz);
+
+unsigned char *rans_compress_O1_32x16_neon(unsigned char *in,
+                                           unsigned int in_size,
+                                           unsigned char *out,
+                                           unsigned int *out_size);
+
+unsigned char *rans_uncompress_O1_32x16_neon(unsigned char *in,
+                                             unsigned int in_size,
+                                             unsigned char *out,
+                                             unsigned int out_sz);
+#endif // ARM_NEON
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RANS_STATIC32x16PR_H */
diff --git a/ext/htslib/htscodecs/htscodecs/rANS_static32x16pr_avx2.c b/ext/htslib/htscodecs/htscodecs/rANS_static32x16pr_avx2.c
new file mode 100644
index 0000000..cf04578
--- /dev/null
+++ b/ext/htslib/htscodecs/htscodecs/rANS_static32x16pr_avx2.c
@@ -0,0 +1,1626 @@
+/*
+ * Copyright (c) 2017-2023 Genome Research Ltd.
+ * Author(s): James Bonfield
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above
+ *       copyright notice, this list of conditions and the following
+ *       disclaimer in the documentation and/or other materials provided
+ *       with the distribution.
+ *
+ *    3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+ *       Institute nor the names of its contributors may be used to endorse
+ *       or promote products derived from this software without specific
+ *       prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH
+ * LTD OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#if defined(__x86_64__) && defined(HAVE_AVX2)
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <limits.h>
+#include <x86intrin.h>
+
+#include "rANS_word.h"
+#include "rANS_static4x16.h"
+#define ROT32_SIMD
+#include "rANS_static16_int.h"
+#include "varint.h"
+#include "utils.h"
+#include "permute.h"
+
+#define NX 32
+
+#define LOAD1(a,b) __m256i a##1 = _mm256_load_si256((__m256i *)&b[0]);
+#define LOAD2(a,b) __m256i a##2 = _mm256_load_si256((__m256i *)&b[8]);
+#define LOAD3(a,b) __m256i a##3 = _mm256_load_si256((__m256i *)&b[16]);
+#define LOAD4(a,b) __m256i a##4 = _mm256_load_si256((__m256i *)&b[24]);
+#define LOAD(a,b) LOAD1(a,b);LOAD2(a,b);LOAD3(a,b);LOAD4(a,b)
+
+#define STORE1(a,b) _mm256_store_si256((__m256i *)&b[0],  a##1);
+#define STORE2(a,b) _mm256_store_si256((__m256i *)&b[8],  a##2);
+#define STORE3(a,b) _mm256_store_si256((__m256i *)&b[16], a##3);
+#define STORE4(a,b) _mm256_store_si256((__m256i *)&b[24], a##4);
+#define STORE(a,b) STORE1(a,b);STORE2(a,b);STORE3(a,b);STORE4(a,b)
+
+// _mm256__mul_epu32 is:
+//  -b -d -f -h
+//* -q -s -u -w
+//= BQ DS FU HW where BQ=b*q etc
+//
+// We want
+//  abcd efgh  (a)
+// *pqrs tuvw  (b)
+// =ABCD EFGH
+//
+// a    mul b      => BQ DS FU HW
+// >>= 8           => -B QD SF UH
+// &               => -B -D -F -H (1)
+// a>>8 mul b>>8   => AP CR ET GV
+// &               => A- C- E- G-
+// | with (1)      => AB CD EF GH
+#if 0
+static __m256i _mm256_mulhi_epu32(__m256i a, __m256i b) {
+    __m256i ab_lm = _mm256_mul_epu32(a, b);
+    ab_lm = _mm256_srli_epi64(ab_lm, 32);
+    a = _mm256_srli_epi64(a, 32);
+
+    ab_lm = _mm256_and_si256(ab_lm, _mm256_set1_epi64x(0xffffffff));
+    b = _mm256_srli_epi64(b, 32);
+
+    __m256i ab_hm = _mm256_mul_epu32(a, b);
+
+    //return _mm256_blend_epi32(ab_lm, ab_hm, 0xaa);
+    ab_hm = _mm256_and_si256(ab_hm,
+                         _mm256_set1_epi64x((uint64_t)0xffffffff00000000));
+    ab_hm = _mm256_or_si256(ab_hm, ab_lm);
+
+    return ab_hm;
+}
+#else
+static inline __m256i _mm256_mulhi_epu32(__m256i a, __m256i b) {
+    // Multiply bottom 4 items and top 4 items together.
+    __m256i ab_hm = _mm256_mul_epu32(_mm256_srli_epi64(a, 32),
+                                     _mm256_srli_epi64(b, 32));
+    __m256i ab_lm = _mm256_srli_epi64(_mm256_mul_epu32(a, b), 32);
+
+    return _mm256_blend_epi32(ab_lm, ab_hm, 0xaa);
+//
+//    // Shift to get hi 32-bit of each 64-bit product
+//    ab_hm = _mm256_and_si256(ab_hm,
+//                       _mm256_set1_epi64x((uint64_t)0xffffffff00000000));
+//
+//    return _mm256_or_si256(ab_lm, ab_hm);
+}
+#endif
+
+#ifndef USE_GATHER
+// Simulated gather.  This is sometimes faster if gathers are slow, either
+// due to the particular implementation (maybe on Zen4) or because of
+// a microcode patch such as Intel's Downfall fix.
+static inline __m256i _mm256_i32gather_epi32x(int *b, __m256i idx, int size) {
+    volatile // force the store to happen, hence forcing scalar loads
+    int c[8] __attribute__((aligned(32)));
+    _mm256_store_si256((__m256i *)c, idx);
+
+    // Fast with modern gccs, and no change with clang.
+    // Equivalent to:
+    //     return _mm256_set_epi32(b[c[7]], b[c[6]], b[c[5]], b[c[4]],
+    //                             b[c[3]], b[c[2]], b[c[1]], b[c[0]]);
+    register int bc1 = b[c[1]];
+    register int bc3 = b[c[3]];
+    register int bc5 = b[c[5]];
+    register int bc7 = b[c[7]];
+
+    __m128i x0a = _mm_cvtsi32_si128(b[c[0]]);
+    __m128i x1a = _mm_cvtsi32_si128(b[c[2]]);
+    __m128i x2a = _mm_cvtsi32_si128(b[c[4]]);
+    __m128i x3a = _mm_cvtsi32_si128(b[c[6]]);
+
+    __m128i x0 = _mm_insert_epi32(x0a, bc1, 1);
+    __m128i x1 = _mm_insert_epi32(x1a, bc3, 1);
+    __m128i x2 = _mm_insert_epi32(x2a, bc5, 1);
+    __m128i x3 = _mm_insert_epi32(x3a, bc7, 1);
+
+    __m128i x01 = _mm_unpacklo_epi64(x0, x1);
+    __m128i x23 = _mm_unpacklo_epi64(x2, x3);
+
+    __m256i y =_mm256_castsi128_si256(x01);
+    return _mm256_inserti128_si256(y, x23, 1);
+}
+
+#else
+#define _mm256_i32gather_epi32x _mm256_i32gather_epi32
+#endif
+
+unsigned char *rans_compress_O0_32x16_avx2(unsigned char *in,
+                                           unsigned int in_size,
+                                           unsigned char *out,
+                                           unsigned int *out_size) {
+    unsigned char *cp, *out_end;
+    RansEncSymbol syms[256];
+    RansState ransN[NX] __attribute__((aligned(32)));
+    uint8_t* ptr;
+    uint32_t F[256+MAGIC] = {0};
+    int i, j, tab_size = 0, x, z;
+    // -20 for order/size/meta
+    uint32_t bound = rans_compress_bound_4x16(in_size,0)-20;
+
+    if (!out) {
+        *out_size = bound;
+        out = malloc(*out_size);
+    }
+    if (!out || bound > *out_size)
+        return NULL;
+
+    // If "out" isn't word aligned, tweak out_end/ptr to ensure it is.
+    // We already added more round in bound to allow for this.
+    if (((size_t)out)&1)
+        bound--;
+    ptr = out_end = out + bound;
+
+    if (in_size == 0)
+        goto empty;
+
+    // Compute statistics
+    if (hist8(in, in_size, F) < 0)
+        return NULL;
+
+    // Normalise so frequences sum to power of 2
+    uint32_t fsum = in_size;
+    uint32_t max_val = round2(fsum);
+    if (max_val > TOTFREQ)
+        max_val = TOTFREQ;
+
+    if (normalise_freq(F, fsum, max_val) < 0)
+        return NULL;
+    fsum=max_val;
+
+    cp = out;
+    cp += encode_freq(cp, F);
+    tab_size = cp-out;
+    //write(2, out+4, cp-(out+4));
+
+    if (normalise_freq(F, fsum, TOTFREQ) < 0)
+        return NULL;
+
+    // Encode statistics.
+    for (x = j = 0; j < 256; j++) {
+        if (F[j]) {
+            RansEncSymbolInit(&syms[j], x, F[j], TF_SHIFT);
+            x += F[j];
+        }
+    }
+
+    for (z = 0; z < NX; z++)
+      RansEncInit(&ransN[z]);
+
+    z = i = in_size&(NX-1);
+    while (z-- > 0)
+      RansEncPutSymbol(&ransN[z], &ptr, &syms[in[in_size-(i-z)]]);
+
+    uint16_t *ptr16 = (uint16_t *)ptr;
+
+    LOAD(Rv, ransN);
+
+    for (i=(in_size &~(NX-1)); i>0; i-=NX) {
+        // We need to gather sym[curr_char][last_char] structs.
+        // These hold 4 32-bit values, so are 128 bit each, and
+        // are loaded from 32 distinct addresses.
+        //
+        // We load them into 32 128-bit lanes and then combine to get
+        // 16 avx-256 registers.
+        // These are now ABCD ABCD ABCD ABCD... orientation
+        // We can then "gather" from these registers via a combination
+        // of shuffle / permutes / and / or operations.  This is less
+        // IO than repeating 4 sets of gathers/loads 32-times over.
+
+        // DCBA holding 4 elements in a syms[] array
+        // -> 4-way rotate via shuffles
+        // [0] DCBA  11 10 01 00  E4
+        // [1] CBAD  10 01 00 11  93
+        // [2] BADC  01 00 11 10  4E
+        // [3] ADCB  00 11 10 01  39
+        //
+        // Then AND to select relevant lanes and OR
+        // [0] ......A0
+        // [1] ....A1..
+        // [2] ..A2....
+        // [3] A3......  OR to get A3A2A1A0
+        //
+        // or:
+        // [0] ....B0..
+        // [1] ..B1....
+        // [2] B2......
+        // [3] ......B3  OR to get B2B1B0B3 and shuffle to B3B2B1B0
+
+        __m256i sh[16];
+        for (z = 0; z < 16; z+=4) {
+            int Z = i - NX + z*2;
+
+#define m128_to_256 _mm256_castsi128_si256
+            __m256i t0, t1, t2, t3;
+            __m128i *s0, *s1, *s2, *s3;
+            s0 = (__m128i *)(&syms[in[Z+0]]);
+            s1 = (__m128i *)(&syms[in[Z+4]]);
+            s2 = (__m128i *)(&syms[in[Z+1]]);
+            s3 = (__m128i *)(&syms[in[Z+5]]);
+
+            // FIXME: try load instead of loadu, as 128-bit aligned.
+            t0 = _mm256_shuffle_epi32(m128_to_256(_mm_loadu_si128(s0)), 0xE4);
+            t1 = _mm256_shuffle_epi32(m128_to_256(_mm_loadu_si128(s1)), 0xE4);
+            t2 = _mm256_shuffle_epi32(m128_to_256(_mm_loadu_si128(s2)), 0x93);
+            t3 = _mm256_shuffle_epi32(m128_to_256(_mm_loadu_si128(s3)), 0x93);
+
+            sh[z+0] = _mm256_permute2x128_si256(t0, t1, 0x20);
+            sh[z+1] = _mm256_permute2x128_si256(t2, t3, 0x20);
+
+            s0 = (__m128i *)(&syms[in[Z+2]]);
+            s1 = (__m128i *)(&syms[in[Z+6]]);
+            s2 = (__m128i *)(&syms[in[Z+3]]);
+            s3 = (__m128i *)(&syms[in[Z+7]]);
+
+            t0 = _mm256_shuffle_epi32(m128_to_256(_mm_loadu_si128(s0)), 0x4E);
+            t1 = _mm256_shuffle_epi32(m128_to_256(_mm_loadu_si128(s1)), 0x4E);
+            t2 = _mm256_shuffle_epi32(m128_to_256(_mm_loadu_si128(s2)), 0x39);
+            t3 = _mm256_shuffle_epi32(m128_to_256(_mm_loadu_si128(s3)), 0x39);
+
+            sh[z+2] = _mm256_permute2x128_si256(t0, t1, 0x20);
+            sh[z+3] = _mm256_permute2x128_si256(t2, t3, 0x20);
+
+            // potential to set xmax, rf, bias, and SD in-situ here, removing
+            // the need to hold sh[] in regs.  Doing so doesn't seem to speed
+            // things up though.
+        }
+
+        __m256i xA = _mm256_set_epi32(0,0,0,-1, 0,0,0,-1);
+        __m256i xB = _mm256_set_epi32(0,0,-1,0, 0,0,-1,0);
+        __m256i xC = _mm256_set_epi32(0,-1,0,0, 0,-1,0,0);
+        __m256i xD = _mm256_set_epi32(-1,0,0,0, -1,0,0,0);
+
+#define SYM_LOAD(x, A, B, C, D)                                         \
+        _mm256_or_si256(_mm256_or_si256(_mm256_and_si256(sh[x+0], A),   \
+                                        _mm256_and_si256(sh[x+1], B)),  \
+                        _mm256_or_si256(_mm256_and_si256(sh[x+2], C),   \
+                                        _mm256_and_si256(sh[x+3], D)))
+
+        // Renorm:
+        // if (x > x_max) {*--ptr16 = x & 0xffff; x >>= 16;}
+        __m256i xmax1 = SYM_LOAD( 0, xA, xB, xC, xD);
+        __m256i xmax2 = SYM_LOAD( 4, xA, xB, xC, xD);
+        __m256i xmax3 = SYM_LOAD( 8, xA, xB, xC, xD);
+        __m256i xmax4 = SYM_LOAD(12, xA, xB, xC, xD);
+
+        __m256i cv1 = _mm256_cmpgt_epi32(Rv1, xmax1);
+        __m256i cv2 = _mm256_cmpgt_epi32(Rv2, xmax2);
+        __m256i cv3 = _mm256_cmpgt_epi32(Rv3, xmax3);
+        __m256i cv4 = _mm256_cmpgt_epi32(Rv4, xmax4);
+
+        // Store bottom 16-bits at ptr16
+        unsigned int imask1 = _mm256_movemask_ps((__m256)cv1);
+        unsigned int imask2 = _mm256_movemask_ps((__m256)cv2);
+        unsigned int imask3 = _mm256_movemask_ps((__m256)cv3);
+        unsigned int imask4 = _mm256_movemask_ps((__m256)cv4);
+
+        __m256i idx1 = _mm256_load_si256((const __m256i*)permutec[imask1]);
+        __m256i idx2 = _mm256_load_si256((const __m256i*)permutec[imask2]);
+        __m256i idx3 = _mm256_load_si256((const __m256i*)permutec[imask3]);
+        __m256i idx4 = _mm256_load_si256((const __m256i*)permutec[imask4]);
+
+        // Permute; to gather together the rans states that need flushing
+        __m256i V1, V2, V3, V4;
+        V1 = _mm256_permutevar8x32_epi32(_mm256_and_si256(Rv1, cv1), idx1);
+        V2 = _mm256_permutevar8x32_epi32(_mm256_and_si256(Rv2, cv2), idx2);
+        V3 = _mm256_permutevar8x32_epi32(_mm256_and_si256(Rv3, cv3), idx3);
+        V4 = _mm256_permutevar8x32_epi32(_mm256_and_si256(Rv4, cv4), idx4);
+        
+        // We only flush bottom 16 bits, to squash 32-bit states into 16 bit.
+        V1 = _mm256_and_si256(V1, _mm256_set1_epi32(0xffff));
+        V2 = _mm256_and_si256(V2, _mm256_set1_epi32(0xffff));
+        V3 = _mm256_and_si256(V3, _mm256_set1_epi32(0xffff));
+        V4 = _mm256_and_si256(V4, _mm256_set1_epi32(0xffff));
+        __m256i V12 = _mm256_packus_epi32(V1, V2);
+        __m256i V34 = _mm256_packus_epi32(V3, V4);
+
+        // It's BAba order, want BbAa so shuffle.
+        V12 = _mm256_permute4x64_epi64(V12, 0xd8);
+        V34 = _mm256_permute4x64_epi64(V34, 0xd8);
+
+        // Now we have bottom N 16-bit values in each V12/V34 to flush
+        __m128i f =  _mm256_extractf128_si256(V34, 1);
+        _mm_storeu_si128((__m128i *)(ptr16-8), f);
+        ptr16 -= _mm_popcnt_u32(imask4);
+
+        f =  _mm256_extractf128_si256(V34, 0);
+        _mm_storeu_si128((__m128i *)(ptr16-8), f);
+        ptr16 -= _mm_popcnt_u32(imask3);
+
+        f =  _mm256_extractf128_si256(V12, 1);
+        _mm_storeu_si128((__m128i *)(ptr16-8), f);
+        ptr16 -= _mm_popcnt_u32(imask2);
+
+        f =  _mm256_extractf128_si256(V12, 0);
+        _mm_storeu_si128((__m128i *)(ptr16-8), f);
+        ptr16 -= _mm_popcnt_u32(imask1);
+
+        __m256i Rs;
+        Rs = _mm256_srli_epi32(Rv1,16); Rv1 = _mm256_blendv_epi8(Rv1, Rs, cv1);
+        Rs = _mm256_srli_epi32(Rv2,16); Rv2 = _mm256_blendv_epi8(Rv2, Rs, cv2);
+        Rs = _mm256_srli_epi32(Rv3,16); Rv3 = _mm256_blendv_epi8(Rv3, Rs, cv3);
+        Rs = _mm256_srli_epi32(Rv4,16); Rv4 = _mm256_blendv_epi8(Rv4, Rs, cv4);
+
+        // Cannot trivially replace the multiply as mulhi_epu32 doesn't
+        // exist (only mullo).
+        // However we can use _mm256_mul_epu32 twice to get 64bit results
+        // (half our lanes) and shift/or to get the answer.
+        //
+        // (AVX512 allows us to hold it all in 64-bit lanes and use mullo_epi64
+        // plus a shift.  KNC has mulhi_epi32, but not sure if this is
+        // available.)
+        __m256i rfv1 = _mm256_shuffle_epi32(SYM_LOAD( 0, xB, xC, xD, xA),0x39);
+        __m256i rfv2 = _mm256_shuffle_epi32(SYM_LOAD( 4, xB, xC, xD, xA),0x39);
+        __m256i rfv3 = _mm256_shuffle_epi32(SYM_LOAD( 8, xB, xC, xD, xA),0x39);
+        __m256i rfv4 = _mm256_shuffle_epi32(SYM_LOAD(12, xB, xC, xD, xA),0x39);
+
+        rfv1 = _mm256_mulhi_epu32(Rv1, rfv1);
+        rfv2 = _mm256_mulhi_epu32(Rv2, rfv2);
+        rfv3 = _mm256_mulhi_epu32(Rv3, rfv3);
+        rfv4 = _mm256_mulhi_epu32(Rv4, rfv4);
+
+        __m256i SDv1 = _mm256_shuffle_epi32(SYM_LOAD( 0, xD, xA, xB, xC),0x93);
+        __m256i SDv2 = _mm256_shuffle_epi32(SYM_LOAD( 4, xD, xA, xB, xC),0x93);
+        __m256i SDv3 = _mm256_shuffle_epi32(SYM_LOAD( 8, xD, xA, xB, xC),0x93);
+        __m256i SDv4 = _mm256_shuffle_epi32(SYM_LOAD(12, xD, xA, xB, xC),0x93);
+
+        __m256i shiftv1 = _mm256_srli_epi32(SDv1, 16);
+        __m256i shiftv2 = _mm256_srli_epi32(SDv2, 16);
+        __m256i shiftv3 = _mm256_srli_epi32(SDv3, 16);
+        __m256i shiftv4 = _mm256_srli_epi32(SDv4, 16);
+
+        shiftv1 = _mm256_sub_epi32(shiftv1, _mm256_set1_epi32(32));
+        shiftv2 = _mm256_sub_epi32(shiftv2, _mm256_set1_epi32(32));
+        shiftv3 = _mm256_sub_epi32(shiftv3, _mm256_set1_epi32(32));
+        shiftv4 = _mm256_sub_epi32(shiftv4, _mm256_set1_epi32(32));
+
+        __m256i qv1 = _mm256_srlv_epi32(rfv1, shiftv1);
+        __m256i qv2 = _mm256_srlv_epi32(rfv2, shiftv2);
+
+        __m256i freqv1 = _mm256_and_si256(SDv1, _mm256_set1_epi32(0xffff));
+        __m256i freqv2 = _mm256_and_si256(SDv2, _mm256_set1_epi32(0xffff));
+        qv1 = _mm256_mullo_epi32(qv1, freqv1);
+        qv2 = _mm256_mullo_epi32(qv2, freqv2);
+
+        __m256i qv3 = _mm256_srlv_epi32(rfv3, shiftv3);
+        __m256i qv4 = _mm256_srlv_epi32(rfv4, shiftv4);
+
+        __m256i freqv3 = _mm256_and_si256(SDv3, _mm256_set1_epi32(0xffff));
+        __m256i freqv4 = _mm256_and_si256(SDv4, _mm256_set1_epi32(0xffff));
+        qv3 = _mm256_mullo_epi32(qv3, freqv3);
+        qv4 = _mm256_mullo_epi32(qv4, freqv4);
+
+        __m256i biasv1=_mm256_shuffle_epi32(SYM_LOAD( 0, xC, xD, xA, xB),0x4E);
+        __m256i biasv2=_mm256_shuffle_epi32(SYM_LOAD( 4, xC, xD, xA, xB),0x4E);
+        __m256i biasv3=_mm256_shuffle_epi32(SYM_LOAD( 8, xC, xD, xA, xB),0x4E);
+        __m256i biasv4=_mm256_shuffle_epi32(SYM_LOAD(12, xC, xD, xA, xB),0x4E);
+
+        qv1 = _mm256_add_epi32(qv1, biasv1);
+        qv2 = _mm256_add_epi32(qv2, biasv2);
+        qv3 = _mm256_add_epi32(qv3, biasv3);
+        qv4 = _mm256_add_epi32(qv4, biasv4);
+
+        Rv1 = _mm256_add_epi32(Rv1, qv1);
+        Rv2 = _mm256_add_epi32(Rv2, qv2);
+        Rv3 = _mm256_add_epi32(Rv3, qv3);
+        Rv4 = _mm256_add_epi32(Rv4, qv4);
+    }
+
+    STORE(Rv, ransN);
+
+    ptr = (uint8_t *)ptr16;
+    for (z = NX-1; z >= 0; z--)
+      RansEncFlush(&ransN[z], &ptr);
+
+ empty:
+    // Finalise block size and return it
+    *out_size = (out_end - ptr) + tab_size;
+
+//    cp = out;
+//    *cp++ = (in_size>> 0) & 0xff;
+//    *cp++ = (in_size>> 8) & 0xff;
+//    *cp++ = (in_size>>16) & 0xff;
+//    *cp++ = (in_size>>24) & 0xff;
+
+    memmove(out + tab_size, ptr, out_end-ptr);
+
+    return out;
+}
+
+unsigned char *rans_uncompress_O0_32x16_avx2(unsigned char *in,
+                                             unsigned int in_size,
+                                             unsigned char *out,
+                                             unsigned int out_sz) {
+    if (in_size < 16) // 4-states at least
+        return NULL;
+
+    if (out_sz >= INT_MAX)
+        return NULL; // protect against some overflow cases
+
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    if (out_sz > 100000)
+        return NULL;
+#endif
+
+    /* Load in the static tables */
+    unsigned char *cp = in, *out_free = NULL;
+    unsigned char *cp_end = in + in_size;
+    int i;
+    uint32_t s3[TOTFREQ] __attribute__((aligned(32))); // For TF_SHIFT <= 12
+
+    if (!out)
+        out_free = out = malloc(out_sz);
+    if (!out)
+        return NULL;
+
+    // Precompute reverse lookup of frequency.
+    uint32_t F[256] = {0}, fsum;
+    int fsz = decode_freq(cp, cp_end, F, &fsum);
+    if (!fsz)
+        goto err;
+    cp += fsz;
+
+    normalise_freq_shift(F, fsum, TOTFREQ);
+
+    // Build symbols; fixme, do as part of decode, see the _d variant
+    if (rans_F_to_s3(F, TF_SHIFT, s3))
+        goto err;
+
+    if (cp_end - cp < NX * 4)
+        goto err;
+
+    int z;
+    RansState R[NX] __attribute__((aligned(32)));
+    for (z = 0; z < NX; z++) {
+        RansDecInit(&R[z], &cp);
+        if (R[z] < RANS_BYTE_L)
+            goto err;
+    }
+
+    uint16_t *sp = (uint16_t *)cp;
+    uint8_t overflow[64+64] = {0};
+    cp_end -= 64;
+
+    // Protect against running off the end of in buffer.
+    // We copy it to a worst-case local buffer when near the end.
+    if ((uint8_t *)sp > cp_end) {
+        memmove(overflow, sp, cp_end+64 - (uint8_t *)sp);
+        sp = (uint16_t *)overflow;
+        cp_end = overflow + sizeof(overflow) - 64;
+    }
+
+    int out_end = (out_sz&~(NX-1));
+    const uint32_t mask = (1u << TF_SHIFT)-1;
+
+    __m256i maskv  = _mm256_set1_epi32(mask); // set mask in all lanes
+    LOAD(Rv, R);
+
+    for (i=0; i < out_end; i+=NX) {
+        //for (z = 0; z < NX; z++)
+        //  m[z] = R[z] & mask;
+        __m256i masked1 = _mm256_and_si256(Rv1, maskv);
+	__m256i masked2 = _mm256_and_si256(Rv2, maskv);
+        __m256i masked3 = _mm256_and_si256(Rv3, maskv);
+        __m256i masked4 = _mm256_and_si256(Rv4, maskv);
+
+	//  S[z] = s3[m[z]];
+	__m256i Sv1 = _mm256_i32gather_epi32x((int *)s3, masked1, sizeof(*s3));
+        __m256i Sv2 = _mm256_i32gather_epi32x((int *)s3, masked2, sizeof(*s3));
+        __m256i Sv3 = _mm256_i32gather_epi32x((int *)s3, masked3, sizeof(*s3));
+        __m256i Sv4 = _mm256_i32gather_epi32x((int *)s3, masked4, sizeof(*s3));
+
+        //  f[z] = S[z]>>(TF_SHIFT+8);
+        __m256i fv1 = _mm256_srli_epi32(Sv1, TF_SHIFT+8);
+        __m256i fv2 = _mm256_srli_epi32(Sv2, TF_SHIFT+8);
+
+        //  b[z] = (S[z]>>8) & mask;
+        __m256i bv1 = _mm256_and_si256(_mm256_srli_epi32(Sv1, 8), maskv);
+        __m256i bv2 = _mm256_and_si256(_mm256_srli_epi32(Sv2, 8), maskv);
+
+        //  s[z] = S[z] & 0xff;
+        __m256i sv1 = _mm256_and_si256(Sv1, _mm256_set1_epi32(0xff));
+        __m256i sv2 = _mm256_and_si256(Sv2, _mm256_set1_epi32(0xff));
+
+        //  R[z] = f[z] * (R[z] >> TF_SHIFT) + b[z];
+        Rv1 = _mm256_add_epi32(
+                  _mm256_mullo_epi32(
+                      _mm256_srli_epi32(Rv1,TF_SHIFT), fv1), bv1);
+        Rv2 = _mm256_add_epi32(
+                  _mm256_mullo_epi32(
+                      _mm256_srli_epi32(Rv2,TF_SHIFT), fv2), bv2);
+
+        // Tricky one:  out[i+z] = s[z];
+        //             ---h---g ---f---e  ---d---c ---b---a
+        //             ---p---o ---n---m  ---l---k ---j---i
+        // packs_epi32 -p-o-n-m -h-g-f-e  -l-k-j-i -d-c-b-a
+        // permute4x64 -p-o-n-m -l-k-j-i  -h-g-f-e -d-c-b-a
+        // packs_epi16 ponmlkji ponmlkji  hgfedcba hgfedcba
+        sv1 = _mm256_packus_epi32(sv1, sv2);
+        sv1 = _mm256_permute4x64_epi64(sv1, 0xd8);
+
+        // Protect against running off the end of in buffer.
+        // We copy it to a worst-case local buffer when near the end.
+        if ((uint8_t *)sp > cp_end) {
+            memmove(overflow, sp, cp_end+64 - (uint8_t *)sp);
+            sp = (uint16_t *)overflow;
+            cp_end = overflow + sizeof(overflow) - 64;
+        }
+
+        __m256i Vv1 = _mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i *)sp));
+        sv1 = _mm256_packus_epi16(sv1, sv1);
+
+        // c =  R[z] < RANS_BYTE_L;
+
+// The lack of unsigned comparisons means we have to jump through hoops.
+// in AVX2 land the second version comes out best (and first in SSE land).
+
+//#define _mm256_cmplt_epu32_imm(a,b) _mm256_andnot_si256(_mm256_cmpeq_epi32(_mm256_max_epu32((a),_mm256_set1_epi32(b)), (a)), _mm256_set1_epi32(-1));
+
+#define _mm256_cmplt_epu32_imm(a,b) _mm256_cmpgt_epi32(_mm256_set1_epi32((b)-0x80000000), _mm256_xor_si256((a), _mm256_set1_epi32(0x80000000)))
+
+        __m256i renorm_mask1, renorm_mask2;
+        renorm_mask1 = _mm256_cmplt_epu32_imm(Rv1, RANS_BYTE_L);
+        renorm_mask2 = _mm256_cmplt_epu32_imm(Rv2, RANS_BYTE_L);
+
+        // y = (R[z] << 16) | V[z];
+        unsigned int imask1 = _mm256_movemask_ps((__m256)renorm_mask1);
+        __m256i idx1 = _mm256_load_si256((const __m256i*)permute[imask1]);
+        __m256i Yv1 = _mm256_slli_epi32(Rv1, 16);
+        Vv1 = _mm256_permutevar8x32_epi32(Vv1, idx1);
+        __m256i Yv2 = _mm256_slli_epi32(Rv2, 16);
+
+        // Shuffle the renorm values to correct lanes and incr sp pointer
+        unsigned int imask2 = _mm256_movemask_ps((__m256)renorm_mask2);
+        sp += _mm_popcnt_u32(imask1);
+
+        __m256i idx2 = _mm256_load_si256((const __m256i*)permute[imask2]);
+        __m256i Vv2 = _mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i *)sp));
+        sp += _mm_popcnt_u32(imask2);
+
+        Yv1 = _mm256_or_si256(Yv1, Vv1);
+        Vv2 = _mm256_permutevar8x32_epi32(Vv2, idx2);
+        Yv2 = _mm256_or_si256(Yv2, Vv2);
+
+        // R[z] = c ? Y[z] : R[z];
+        Rv1 = _mm256_blendv_epi8(Rv1, Yv1, renorm_mask1);
+        Rv2 = _mm256_blendv_epi8(Rv2, Yv2, renorm_mask2);
+
+        // ------------------------------------------------------------
+        //  f[z] = S[z]>>(TF_SHIFT+8);
+        __m256i fv3 = _mm256_srli_epi32(Sv3, TF_SHIFT+8);
+        __m256i fv4 = _mm256_srli_epi32(Sv4, TF_SHIFT+8);
+
+        //  b[z] = (S[z]>>8) & mask;
+        __m256i bv3 = _mm256_and_si256(_mm256_srli_epi32(Sv3, 8), maskv);
+        __m256i bv4 = _mm256_and_si256(_mm256_srli_epi32(Sv4, 8), maskv);
+
+        //  s[z] = S[z] & 0xff;
+        __m256i sv3 = _mm256_and_si256(Sv3, _mm256_set1_epi32(0xff));
+        __m256i sv4 = _mm256_and_si256(Sv4, _mm256_set1_epi32(0xff));
+
+        //  R[z] = f[z] * (R[z] >> TF_SHIFT) + b[z];
+        Rv3 = _mm256_add_epi32(_mm256_mullo_epi32(_mm256_srli_epi32(Rv3,TF_SHIFT),fv3),bv3);
+        Rv4 = _mm256_add_epi32(_mm256_mullo_epi32(_mm256_srli_epi32(Rv4,TF_SHIFT),fv4),bv4);
+
+        // Tricky one:  out[i+z] = s[z];
+        //             ---h---g ---f---e  ---d---c ---b---a
+        //             ---p---o ---n---m  ---l---k ---j---i
+        // packs_epi32 -p-o-n-m -h-g-f-e  -l-k-j-i -d-c-b-a
+        // permute4x64 -p-o-n-m -l-k-j-i  -h-g-f-e -d-c-b-a
+        // packs_epi16 ponmlkji ponmlkji  hgfedcba hgfedcba
+        sv3 = _mm256_packus_epi32(sv3, sv4);
+        sv3 = _mm256_permute4x64_epi64(sv3, 0xd8);
+
+        // c =  R[z] < RANS_BYTE_L;
+        __m256i renorm_mask3, renorm_mask4;
+        renorm_mask3 = _mm256_cmplt_epu32_imm(Rv3, RANS_BYTE_L);
+        sv3 = _mm256_packus_epi16(sv3, sv3);
+        renorm_mask4 = _mm256_cmplt_epu32_imm(Rv4, RANS_BYTE_L);
+
+        // y = (R[z] << 16) | V[z];
+        __m256i Vv3 = _mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i *)sp));
+
+        *(uint64_t *)&out[i+0]  = _mm256_extract_epi64(sv1, 0);
+        *(uint64_t *)&out[i+8]  = _mm256_extract_epi64(sv1, 2);
+        *(uint64_t *)&out[i+16] = _mm256_extract_epi64(sv3, 0);
+        *(uint64_t *)&out[i+24] = _mm256_extract_epi64(sv3, 2);
+
+        __m256i Yv3 = _mm256_slli_epi32(Rv3, 16);
+        unsigned int imask3 = _mm256_movemask_ps((__m256)renorm_mask3);
+        __m256i idx3 = _mm256_load_si256((const __m256i*)permute[imask3]);
+
+        // Shuffle the renorm values to correct lanes and incr sp pointer
+        Vv3 = _mm256_permutevar8x32_epi32(Vv3, idx3);
+        __m256i Yv4 = _mm256_slli_epi32(Rv4, 16);
+        unsigned int imask4 = _mm256_movemask_ps((__m256)renorm_mask4);
+        sp += _mm_popcnt_u32(imask3);
+
+        __m256i idx4 = _mm256_load_si256((const __m256i*)permute[imask4]);
+        __m256i Vv4 = _mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i *)sp));
+
+        //Vv = _mm256_and_si256(Vv, renorm_mask);  (blend does the AND anyway)
+        Yv3 = _mm256_or_si256(Yv3, Vv3);
+        Vv4 = _mm256_permutevar8x32_epi32(Vv4, idx4);
+        Yv4 = _mm256_or_si256(Yv4, Vv4);
+        sp += _mm_popcnt_u32(imask4);
+
+        // R[z] = c ? Y[z] : R[z];
+        Rv3 = _mm256_blendv_epi8(Rv3, Yv3, renorm_mask3);
+        Rv4 = _mm256_blendv_epi8(Rv4, Yv4, renorm_mask4);
+    }
+
+    STORE(Rv, R);
+    //_mm256_store_si256((__m256i *)&R[0], Rv1);
+    //_mm256_store_si256((__m256i *)&R[8], Rv2);
+    //_mm256_store_si256((__m256i *)&R[16], Rv3);
+    //_mm256_store_si256((__m256i *)&R[24], Rv4);
+
+    for (z = out_sz & (NX-1); z-- > 0; )
+      out[out_end + z] = s3[R[z] & mask];
+
+    //fprintf(stderr, "    0 Decoded %d bytes\n", (int)(cp-in)); //c-size
+
+    return out;
+
+ err:
+    free(out_free);
+    return NULL;
+}
+
+//-----------------------------------------------------------------------------
+
+unsigned char *rans_compress_O1_32x16_avx2(unsigned char *in, unsigned int in_size,
+                                           unsigned char *out, unsigned int *out_size) {
+    unsigned char *cp, *out_end, *out_free = NULL;
+    unsigned int tab_size;
+    uint32_t bound = rans_compress_bound_4x16(in_size,1)-20;
+    int z;
+    RansState ransN[NX] __attribute__((aligned(32)));
+
+    if (in_size < NX) // force O0 instead
+        return NULL;
+
+    if (!out) {
+        *out_size = bound;
+        out = malloc(*out_size);
+    }
+    if (!out || bound > *out_size)
+        return NULL;
+
+    if (((size_t)out)&1)
+        bound--;
+    out_end = out + bound;
+
+    RansEncSymbol (*syms)[256] = htscodecs_tls_alloc(256 * (sizeof(*syms)));
+    if (!syms) {
+        free(out_free);
+        return NULL;
+    }
+
+    cp = out;
+    int shift = encode_freq1(in, in_size, 32, syms, &cp); 
+    if (shift < 0) {
+        free(out_free);
+        htscodecs_tls_free(syms);
+        return NULL;
+    }
+    tab_size = cp - out;
+
+    for (z = 0; z < NX; z++)
+      RansEncInit(&ransN[z]);
+
+    uint8_t* ptr = out_end;
+
+    int iN[NX], isz4 = in_size/NX;
+    for (z = 0; z < NX; z++)
+        iN[z] = (z+1)*isz4-2;
+
+    unsigned char lN[NX];
+    for (z = 0; z < NX; z++)
+        lN[z] = in[iN[z]+1];
+
+    // Deal with the remainder
+    z = NX-1;
+    lN[z] = in[in_size-1];
+    for (iN[z] = in_size-2; iN[z] > NX*isz4-2; iN[z]--) {
+        unsigned char c = in[iN[z]];
+        RansEncPutSymbol(&ransN[z], &ptr, &syms[c][lN[z]]);
+        lN[z] = c;
+    }
+
+    uint16_t *ptr16 = (uint16_t *)ptr;
+
+//       clang16      clang10      gcc7         gcc13
+//       587 435 381  588 438 403  504 386 415  527 381 394
+// simT  611 432 402  475 401 367  472 422 386  486 353 324
+
+    LOAD(Rv, ransN);
+
+    for (; iN[0] >= 0; ) {
+        // We need to gather sym[curr_char][last_char] structs.
+        // These hold 4 32-bit values, so are 128 bit each, and
+        // are loaded from 32 distinct addresses.
+        //
+        // We load them into 32 128-bit lanes and then combine to get
+        // 16 avx-256 registers.
+        // These are now ABCD ABCD ABCD ABCD... orientation
+        // Code we can then "gather" from these registers via a combination
+        // of shuffle / permutes / and / or operations.  This is less
+        // IO than repeating 4 sets of gathers/loads 32-times over.
+
+        // DCBA holding 4 elements in a syms[] array
+        // -> 4-way rotate via shuffles
+        // [0] DCBA  11 10 01 00  E4
+        // [1] CBAD  10 01 00 11  93
+        // [2] BADC  01 00 11 10  4E
+        // [3] ADCB  00 11 10 01  39
+        //
+        // Then AND to select relevant lanes and OR
+        // [0] ......A0
+        // [1] ....A1..
+        // [2] ..A2....
+        // [3] A3......  OR to get A3A2A1A0
+        //
+        // or:
+        // [0] ....B0..
+        // [1] ..B1....
+        // [2] B2......
+        // [3] ......B3  OR to get B2B1B0B3 and shuffle to B3B2B1B0
+
+        __m256i xmaxv[4];
+        __m256i rfv[4];
+        __m256i SDv[4];
+        __m256i biasv[4];
+
+        const __m256i xA = _mm256_set_epi32(0,0,0,-1, 0,0,0,-1);
+        const __m256i xB = _mm256_set_epi32(0,0,-1,0, 0,0,-1,0);
+        const __m256i xC = _mm256_set_epi32(0,-1,0,0, 0,-1,0,0);
+        const __m256i xD = _mm256_set_epi32(-1,0,0,0, -1,0,0,0);
+
+        for (z = 0; z < 32; z += 8) {
+#define m128_to_256 _mm256_castsi128_si256
+            __m128i *s0 = (__m128i *)(&syms[in[iN[z+0]]][lN[z+0]]);
+            __m128i *s1 = (__m128i *)(&syms[in[iN[z+4]]][lN[z+4]]);
+            __m128i *s2 = (__m128i *)(&syms[in[iN[z+1]]][lN[z+1]]);
+            __m128i *s3 = (__m128i *)(&syms[in[iN[z+5]]][lN[z+5]]);
+
+            __m256i t0, t1, t2, t3;
+            t0 = _mm256_shuffle_epi32(m128_to_256(_mm_loadu_si128(s0)), 0xE4);
+            t1 = _mm256_shuffle_epi32(m128_to_256(_mm_loadu_si128(s1)), 0xE4);
+            t2 = _mm256_shuffle_epi32(m128_to_256(_mm_loadu_si128(s2)), 0x93);
+            t3 = _mm256_shuffle_epi32(m128_to_256(_mm_loadu_si128(s3)), 0x93);
+
+            __m256i sh0 = _mm256_permute2x128_si256(t0, t1, 0x20);
+            __m256i sh1 = _mm256_permute2x128_si256(t2, t3, 0x20);
+
+            lN[z+0] = in[iN[z+0]];
+            lN[z+4] = in[iN[z+4]];
+            lN[z+1] = in[iN[z+1]];
+            lN[z+5] = in[iN[z+5]];
+
+            // Initialise first half of xmax, rf, SD and bias vectors
+            __m128i *s4 = (__m128i *)(&syms[in[iN[z+2]]][lN[z+2]]);
+            __m128i *s5 = (__m128i *)(&syms[in[iN[z+6]]][lN[z+6]]);
+            __m128i *s6 = (__m128i *)(&syms[in[iN[z+3]]][lN[z+3]]);
+            __m128i *s7 = (__m128i *)(&syms[in[iN[z+7]]][lN[z+7]]);
+
+            __m256i t4, t5, t6, t7;
+            t4 = _mm256_shuffle_epi32(m128_to_256(_mm_loadu_si128(s4)), 0x4E);
+            t5 = _mm256_shuffle_epi32(m128_to_256(_mm_loadu_si128(s5)), 0x4E);
+            t6 = _mm256_shuffle_epi32(m128_to_256(_mm_loadu_si128(s6)), 0x39);
+            t7 = _mm256_shuffle_epi32(m128_to_256(_mm_loadu_si128(s7)), 0x39);
+
+            __m256i sh2 = _mm256_permute2x128_si256(t4, t5, 0x20);
+            __m256i sh3 = _mm256_permute2x128_si256(t6, t7, 0x20);
+
+            lN[z+2] = in[iN[z+2]];
+            lN[z+6] = in[iN[z+6]];
+            lN[z+3] = in[iN[z+3]];
+            lN[z+7] = in[iN[z+7]];
+
+#define SH_LOAD(A, B, C, D)                                        \
+            _mm256_or_si256(_mm256_or_si256(_mm256_and_si256(sh0, A), \
+                                            _mm256_and_si256(sh1, B)),\
+                            _mm256_or_si256(_mm256_and_si256(sh2, C), \
+                                            _mm256_and_si256(sh3, D)))
+            xmaxv[z/8] = SH_LOAD(xA, xB, xC, xD);
+            rfv  [z/8] = SH_LOAD(xB, xC, xD, xA);
+            SDv  [z/8] = SH_LOAD(xD, xA, xB, xC);
+            biasv[z/8] = SH_LOAD(xC, xD, xA, xB);
+
+            rfv  [z/8] = _mm256_shuffle_epi32(rfv  [z/8],  0x39);
+            SDv  [z/8] = _mm256_shuffle_epi32(SDv  [z/8],  0x93);
+            biasv[z/8] = _mm256_shuffle_epi32(biasv[z/8],0x4E);
+        }
+
+        // ------------------------------------------------------------
+        //      for (z = NX-1; z >= 0; z--) {
+        //          if (ransN[z] >= x_max[z]) {
+        //              *--ptr16 = ransN[z] & 0xffff;
+        //              ransN[z] >>= 16;
+        //          }
+        //      }
+        __m256i cv1 = _mm256_cmpgt_epi32(Rv1, xmaxv[0]);
+        __m256i cv2 = _mm256_cmpgt_epi32(Rv2, xmaxv[1]);
+        __m256i cv3 = _mm256_cmpgt_epi32(Rv3, xmaxv[2]);
+        __m256i cv4 = _mm256_cmpgt_epi32(Rv4, xmaxv[3]);
+
+        // Store bottom 16-bits at ptr16
+        //
+        // for (z = NX-1; z >= 0; z--) {
+        //     if (cond[z]) *--ptr16 = (uint16_t )(ransN[z] & 0xffff);
+        // }
+        unsigned int imask1 = _mm256_movemask_ps((__m256)cv1);
+        unsigned int imask2 = _mm256_movemask_ps((__m256)cv2);
+        unsigned int imask3 = _mm256_movemask_ps((__m256)cv3);
+        unsigned int imask4 = _mm256_movemask_ps((__m256)cv4);
+
+        __m256i idx1 = _mm256_load_si256((const __m256i*)permutec[imask1]);
+        __m256i idx2 = _mm256_load_si256((const __m256i*)permutec[imask2]);
+        __m256i idx3 = _mm256_load_si256((const __m256i*)permutec[imask3]);
+        __m256i idx4 = _mm256_load_si256((const __m256i*)permutec[imask4]);
+
+        // Permute; to gather together the rans states that need flushing
+        __m256i V1, V2, V3, V4;
+        V1 = _mm256_permutevar8x32_epi32(_mm256_and_si256(Rv1, cv1), idx1);
+        V2 = _mm256_permutevar8x32_epi32(_mm256_and_si256(Rv2, cv2), idx2);
+        V3 = _mm256_permutevar8x32_epi32(_mm256_and_si256(Rv3, cv3), idx3);
+        V4 = _mm256_permutevar8x32_epi32(_mm256_and_si256(Rv4, cv4), idx4);
+
+        // We only flush bottom 16 bits, to squash 32-bit states into 16 bit.
+        V1 = _mm256_and_si256(V1, _mm256_set1_epi32(0xffff));
+        V2 = _mm256_and_si256(V2, _mm256_set1_epi32(0xffff));
+        V3 = _mm256_and_si256(V3, _mm256_set1_epi32(0xffff));
+        V4 = _mm256_and_si256(V4, _mm256_set1_epi32(0xffff));
+        __m256i V12 = _mm256_packus_epi32(V1, V2);
+        __m256i V34 = _mm256_packus_epi32(V3, V4);
+
+        // It's BAba order, want BbAa so shuffle.
+        V12 = _mm256_permute4x64_epi64(V12, 0xd8);
+        V34 = _mm256_permute4x64_epi64(V34, 0xd8);
+
+        // Now we have bottom N 16-bit values in each V12/V34 to flush
+        __m128i f =  _mm256_extractf128_si256(V34, 1);
+        _mm_storeu_si128((__m128i *)(ptr16-8), f);
+        ptr16 -= _mm_popcnt_u32(imask4);
+
+        f =  _mm256_extractf128_si256(V34, 0);
+        _mm_storeu_si128((__m128i *)(ptr16-8), f);
+        ptr16 -= _mm_popcnt_u32(imask3);
+
+        f =  _mm256_extractf128_si256(V12, 1);
+        _mm_storeu_si128((__m128i *)(ptr16-8), f);
+        ptr16 -= _mm_popcnt_u32(imask2);
+
+        f =  _mm256_extractf128_si256(V12, 0);
+        _mm_storeu_si128((__m128i *)(ptr16-8), f);
+        ptr16 -= _mm_popcnt_u32(imask1);
+
+        __m256i Rs1, Rs2, Rs3, Rs4;
+        Rs1 = _mm256_srli_epi32(Rv1,16);
+        Rs2 = _mm256_srli_epi32(Rv2,16);
+        Rs3 = _mm256_srli_epi32(Rv3,16);
+        Rs4 = _mm256_srli_epi32(Rv4,16);
+
+        Rv1 = _mm256_blendv_epi8(Rv1, Rs1, cv1);
+        Rv2 = _mm256_blendv_epi8(Rv2, Rs2, cv2);
+        Rv3 = _mm256_blendv_epi8(Rv3, Rs3, cv3);
+        Rv4 = _mm256_blendv_epi8(Rv4, Rs4, cv4);
+
+        // ------------------------------------------------------------
+        // uint32_t q = (uint32_t) (((uint64_t)ransN[z] *
+        //                           rcp_freq[z]) >> rcp_shift[z]);
+        // ransN[z] = ransN[z] + bias[z] + q * cmpl_freq[z];
+
+        // Cannot trivially replace the multiply as mulhi_epu32 doesn't exist
+        // (only mullo).  However we can use _mm256_mul_epu32 twice to get
+        // 64bit results (half our lanes) and shift/or to get the answer.
+        //
+        // (AVX512 allows us to hold it all in 64-bit lanes and use mullo_epi64
+        // plus a shift.  KNC has mulhi_epi32, but not sure if this is
+        // available.)
+        rfv[0] = _mm256_mulhi_epu32(Rv1, rfv[0]);
+        rfv[1] = _mm256_mulhi_epu32(Rv2, rfv[1]);
+        rfv[2] = _mm256_mulhi_epu32(Rv3, rfv[2]);
+        rfv[3] = _mm256_mulhi_epu32(Rv4, rfv[3]);
+
+        __m256i shiftv1 = _mm256_srli_epi32(SDv[0], 16);
+        __m256i shiftv2 = _mm256_srli_epi32(SDv[1], 16);
+        __m256i shiftv3 = _mm256_srli_epi32(SDv[2], 16);
+        __m256i shiftv4 = _mm256_srli_epi32(SDv[3], 16);
+
+        shiftv1 = _mm256_sub_epi32(shiftv1, _mm256_set1_epi32(32));
+        shiftv2 = _mm256_sub_epi32(shiftv2, _mm256_set1_epi32(32));
+        shiftv3 = _mm256_sub_epi32(shiftv3, _mm256_set1_epi32(32));
+        shiftv4 = _mm256_sub_epi32(shiftv4, _mm256_set1_epi32(32));
+
+        __m256i qv1 = _mm256_srlv_epi32(rfv[0], shiftv1);
+        __m256i qv2 = _mm256_srlv_epi32(rfv[1], shiftv2);
+
+        __m256i freqv1 = _mm256_and_si256(SDv[0], _mm256_set1_epi32(0xffff));
+        __m256i freqv2 = _mm256_and_si256(SDv[1], _mm256_set1_epi32(0xffff));
+        qv1 = _mm256_mullo_epi32(qv1, freqv1);
+        qv2 = _mm256_mullo_epi32(qv2, freqv2);
+
+        __m256i qv3 = _mm256_srlv_epi32(rfv[2], shiftv3);
+        __m256i qv4 = _mm256_srlv_epi32(rfv[3], shiftv4);
+
+        __m256i freqv3 = _mm256_and_si256(SDv[2], _mm256_set1_epi32(0xffff));
+        __m256i freqv4 = _mm256_and_si256(SDv[3], _mm256_set1_epi32(0xffff));
+        qv3 = _mm256_mullo_epi32(qv3, freqv3);
+        qv4 = _mm256_mullo_epi32(qv4, freqv4);
+
+        qv1 = _mm256_add_epi32(qv1, biasv[0]);
+        qv2 = _mm256_add_epi32(qv2, biasv[1]);
+        qv3 = _mm256_add_epi32(qv3, biasv[2]);
+        qv4 = _mm256_add_epi32(qv4, biasv[3]);
+
+        for (z = 0; z < NX; z++)
+            iN[z]--;
+
+        Rv1 = _mm256_add_epi32(Rv1, qv1);
+        Rv2 = _mm256_add_epi32(Rv2, qv2);
+        Rv3 = _mm256_add_epi32(Rv3, qv3);
+        Rv4 = _mm256_add_epi32(Rv4, qv4);
+    }
+
+    STORE(Rv, ransN);
+
+    ptr = (uint8_t *)ptr16;
+
+    for (z = NX-1; z>=0; z--)
+        RansEncPutSymbol(&ransN[z], &ptr, &syms[0][lN[z]]);
+
+    for (z = NX-1; z>=0; z--)
+        RansEncFlush(&ransN[z], &ptr);
+
+    *out_size = (out_end - ptr) + tab_size;
+
+    cp = out;
+    memmove(out + tab_size, ptr, out_end-ptr);
+
+    htscodecs_tls_free(syms);
+    return out;
+}
+
+/*
+ * A 32 x 32 matrix transpose and serialise from t[][] to out.
+ * Storing in the other orientation speeds up the decoder, and we
+ * can then flush to out in 1KB blocks.
+ */
+static inline void transpose_and_copy(uint8_t *out, int iN[32],
+                                      uint8_t t[32][32]) {
+//  int z;
+//  for (z = 0; z < NX; z++) {
+//      int k;
+//      for (k = 0; k < 32; k++)
+//      out[iN[z]+k] = t[k][z];
+//      iN[z] += 32;
+//  }
+
+    rot32_simd(t, out, iN);
+}
+
+unsigned char *rans_uncompress_O1_32x16_avx2(unsigned char *in,
+                                             unsigned int in_size,
+                                             unsigned char *out,
+                                             unsigned int out_sz) {
+    if (in_size < NX*4) // 4-states at least
+        return NULL;
+
+    if (out_sz >= INT_MAX)
+        return NULL; // protect against some overflow cases
+
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    if (out_sz > 100000)
+        return NULL;
+#endif
+
+    /* Load in the static tables */
+    unsigned char *cp = in, *cp_end = in+in_size, *out_free = NULL;
+    unsigned char *c_freq = NULL;
+
+    uint32_t (*s3)[TOTFREQ_O1] = htscodecs_tls_alloc(256*TOTFREQ_O1*4);
+    if (!s3)
+        return NULL;
+    //uint32_t s3[256][TOTFREQ_O1] __attribute__((aligned(32)));
+    uint32_t (*s3F)[TOTFREQ_O1_FAST] = (uint32_t (*)[TOTFREQ_O1_FAST])s3;
+
+    if (!out)
+        out_free = out = malloc(out_sz);
+
+    if (!out)
+        goto err;
+
+    //fprintf(stderr, "out_sz=%d\n", out_sz);
+
+    // compressed header? If so uncompress it
+    unsigned char *tab_end = NULL;
+    unsigned char *c_freq_end = cp_end;
+    unsigned int shift = *cp >> 4;
+    if (*cp++ & 1) {
+        uint32_t u_freq_sz, c_freq_sz;
+        cp += var_get_u32(cp, cp_end, &u_freq_sz);
+        cp += var_get_u32(cp, cp_end, &c_freq_sz);
+        if (c_freq_sz > cp_end - cp)
+            goto err;
+        tab_end = cp + c_freq_sz;
+        if (!(c_freq = rans_uncompress_O0_4x16(cp, c_freq_sz, NULL,
+                                               u_freq_sz)))
+            goto err;
+        cp = c_freq;
+        c_freq_end = c_freq + u_freq_sz;
+    }
+
+    // Decode order-0 symbol list; avoids needing in order-1 tables
+    cp += decode_freq1(cp, c_freq_end, shift, s3, s3F, NULL, NULL);
+
+    if (tab_end)
+        cp = tab_end;
+    free(c_freq);
+    c_freq = NULL;
+
+    if (cp_end - cp < NX * 4)
+        goto err;
+
+    RansState R[NX] __attribute__((aligned(32)));
+    uint8_t *ptr = cp, *ptr_end = in + in_size;
+    int z;
+    for (z = 0; z < NX; z++) {
+        RansDecInit(&R[z], &ptr);
+        if (R[z] < RANS_BYTE_L)
+            goto err;
+    }
+
+    int isz4 = out_sz/NX;
+    int iN[NX], lN[NX] __attribute__((aligned(32))) = {0};
+    for (z = 0; z < NX; z++)
+        iN[z] = z*isz4;
+
+    uint16_t *sp = (uint16_t *)ptr;
+    const uint32_t mask = (1u << shift)-1;
+
+    __m256i maskv  = _mm256_set1_epi32(mask);
+    LOAD(Rv, R);
+    LOAD(Lv, lN);
+
+    union {
+        unsigned char tbuf[32][32];
+        uint64_t tbuf64[32][4];
+    } u  __attribute__((aligned(32)));
+    unsigned int tidx = 0;
+
+    if (0) {
+        int z;
+        for (z = 0; z < 32; z++)
+            iN[z] = iN[z] & ~31;
+    }
+
+    if (shift == TF_SHIFT_O1) {
+        isz4 -= 64;
+        for (; iN[0] < isz4 && (uint8_t *)sp+64 < ptr_end; ) {
+            // m[z] = R[z] & mask;
+            __m256i masked1 = _mm256_and_si256(Rv1, maskv);
+            __m256i masked2 = _mm256_and_si256(Rv2, maskv);
+
+            //  S[z] = s3[lN[z]][m[z]];
+            Lv1 = _mm256_slli_epi32(Lv1, TF_SHIFT_O1);
+            masked1 = _mm256_add_epi32(masked1, Lv1);
+
+            Lv2 = _mm256_slli_epi32(Lv2, TF_SHIFT_O1);
+            masked2 = _mm256_add_epi32(masked2, Lv2);
+
+            __m256i masked3 = _mm256_and_si256(Rv3, maskv);
+            __m256i masked4 = _mm256_and_si256(Rv4, maskv);
+
+            Lv3 = _mm256_slli_epi32(Lv3, TF_SHIFT_O1);
+            masked3 = _mm256_add_epi32(masked3, Lv3);
+
+            Lv4 = _mm256_slli_epi32(Lv4, TF_SHIFT_O1);
+            masked4 = _mm256_add_epi32(masked4, Lv4);
+
+            __m256i Sv1 = _mm256_i32gather_epi32x((int *)&s3[0][0], masked1,
+                                                  sizeof(s3[0][0]));
+            __m256i Sv2 = _mm256_i32gather_epi32x((int *)&s3[0][0], masked2,
+                                                  sizeof(s3[0][0]));
+
+            //  f[z] = S[z]>>(TF_SHIFT_O1+8);
+            __m256i fv1 = _mm256_srli_epi32(Sv1, TF_SHIFT_O1+8);
+            __m256i fv2 = _mm256_srli_epi32(Sv2, TF_SHIFT_O1+8);
+
+            __m256i Sv3 = _mm256_i32gather_epi32x((int *)&s3[0][0], masked3,
+                                                  sizeof(s3[0][0]));
+            __m256i Sv4 = _mm256_i32gather_epi32x((int *)&s3[0][0], masked4,
+                                                  sizeof(s3[0][0]));
+
+            __m256i fv3 = _mm256_srli_epi32(Sv3, TF_SHIFT_O1+8);
+            __m256i fv4 = _mm256_srli_epi32(Sv4, TF_SHIFT_O1+8);
+
+            //  b[z] = (S[z]>>8) & mask;
+            __m256i bv1 = _mm256_and_si256(_mm256_srli_epi32(Sv1, 8), maskv);
+            __m256i bv2 = _mm256_and_si256(_mm256_srli_epi32(Sv2, 8), maskv);
+            __m256i bv3 = _mm256_and_si256(_mm256_srli_epi32(Sv3, 8), maskv);
+            __m256i bv4 = _mm256_and_si256(_mm256_srli_epi32(Sv4, 8), maskv);
+
+            //  s[z] = S[z] & 0xff;
+            __m256i sv1 = _mm256_and_si256(Sv1, _mm256_set1_epi32(0xff));
+            __m256i sv2 = _mm256_and_si256(Sv2, _mm256_set1_epi32(0xff));
+            __m256i sv3 = _mm256_and_si256(Sv3, _mm256_set1_epi32(0xff));
+            __m256i sv4 = _mm256_and_si256(Sv4, _mm256_set1_epi32(0xff));
+
+            if (1) {
+                // A maximum frequency of 4096 doesn't fit in our s3 array.
+                // as it's 12 bit + 12 bit + 8 bit.  It wraps around to zero.
+                // (We don't have this issue for TOTFREQ_O1_FAST.)
+                //
+                // Solution 1 is to change to spec to forbid freq of 4096.
+                // Easy hack is to add an extra symbol so it sums correctly.
+                // => 572 MB/s on q40 (deskpro).
+                //
+                // Solution 2 implemented here is to look for the wrap around
+                // and fix it.
+                // => 556 MB/s on q40
+                // cope with max freq of 4096.  Only 3% hit
+                __m256i max_freq = _mm256_set1_epi32(TOTFREQ_O1);
+                __m256i zero = _mm256_setzero_si256();
+                __m256i cmp1 = _mm256_cmpeq_epi32(fv1, zero);
+                fv1 = _mm256_blendv_epi8(fv1, max_freq, cmp1);
+                __m256i cmp2 = _mm256_cmpeq_epi32(fv2, zero);
+                fv2 = _mm256_blendv_epi8(fv2, max_freq, cmp2);
+            }
+
+            //  R[z] = f[z] * (R[z] >> TF_SHIFT_O1) + b[z];
+            Rv1 = _mm256_add_epi32(
+                      _mm256_mullo_epi32(
+                          _mm256_srli_epi32(Rv1,TF_SHIFT_O1), fv1), bv1);
+            Rv2 = _mm256_add_epi32(
+                      _mm256_mullo_epi32(
+                          _mm256_srli_epi32(Rv2,TF_SHIFT_O1), fv2), bv2);
+
+
+            //for (z = 0; z < NX; z++) lN[z] = c[z];
+            Lv1 = sv1;
+            Lv2 = sv2;
+
+            sv1 = _mm256_packus_epi32(sv1, sv2);
+            sv1 = _mm256_permute4x64_epi64(sv1, 0xd8);
+
+            // Start loading next batch of normalised states
+            __m256i Vv1 = _mm256_cvtepu16_epi32(
+                              _mm_loadu_si128((__m128i *)sp));
+
+            sv1 = _mm256_packus_epi16(sv1, sv1);
+
+            // out[iN[z]] = c[z];  // simulate scatter
+            // RansDecRenorm(&R[z], &ptr);      
+            __m256i renorm_mask1, renorm_mask2;
+            renorm_mask1 = _mm256_xor_si256(Rv1,_mm256_set1_epi32(0x80000000));
+            renorm_mask2 = _mm256_xor_si256(Rv2,_mm256_set1_epi32(0x80000000));
+
+            renorm_mask1 = _mm256_cmpgt_epi32(
+                               _mm256_set1_epi32(RANS_BYTE_L-0x80000000),
+                               renorm_mask1);
+            renorm_mask2 = _mm256_cmpgt_epi32(
+                               _mm256_set1_epi32(RANS_BYTE_L-0x80000000),
+                               renorm_mask2);
+
+            unsigned int imask1 = _mm256_movemask_ps((__m256)renorm_mask1);
+            __m256i idx1 = _mm256_load_si256((const __m256i*)permute[imask1]);
+            __m256i Yv1 = _mm256_slli_epi32(Rv1, 16);
+            __m256i Yv2 = _mm256_slli_epi32(Rv2, 16);
+
+            unsigned int imask2 = _mm256_movemask_ps((__m256)renorm_mask2);
+            Vv1 = _mm256_permutevar8x32_epi32(Vv1, idx1);
+            sp += _mm_popcnt_u32(imask1);
+
+            __m256i idx2 = _mm256_load_si256((const __m256i*)permute[imask2]);
+            __m256i Vv2 = _mm256_cvtepu16_epi32(
+                              _mm_loadu_si128((__m128i *)sp));
+            sp += _mm_popcnt_u32(imask2);
+            Vv2 = _mm256_permutevar8x32_epi32(Vv2, idx2);
+
+            Yv1 = _mm256_or_si256(Yv1, Vv1);
+            Yv2 = _mm256_or_si256(Yv2, Vv2);
+
+            Rv1 = _mm256_blendv_epi8(Rv1, Yv1, renorm_mask1);
+            Rv2 = _mm256_blendv_epi8(Rv2, Yv2, renorm_mask2);
+
+            //////////////////////////////////////////////////////////////////
+            // Start loading next batch of normalised states
+            __m256i Vv3 = _mm256_cvtepu16_epi32(
+                              _mm_loadu_si128((__m128i *)sp));
+
+            if (1) {
+                // cope with max freq of 4096
+                __m256i max_freq = _mm256_set1_epi32(TOTFREQ_O1);
+                __m256i zero = _mm256_setzero_si256();
+                __m256i cmp3 = _mm256_cmpeq_epi32(fv3, zero);
+                fv3 = _mm256_blendv_epi8(fv3, max_freq, cmp3);
+                __m256i cmp4 = _mm256_cmpeq_epi32(fv4, zero);
+                fv4 = _mm256_blendv_epi8(fv4, max_freq, cmp4);
+            }
+
+            //  R[z] = f[z] * (R[z] >> TF_SHIFT_O1) + b[z];
+            Rv3 = _mm256_add_epi32(
+                      _mm256_mullo_epi32(
+                          _mm256_srli_epi32(Rv3,TF_SHIFT_O1), fv3), bv3);
+            Rv4 = _mm256_add_epi32(
+                      _mm256_mullo_epi32(
+                          _mm256_srli_epi32(Rv4,TF_SHIFT_O1), fv4), bv4);
+
+            //for (z = 0; z < NX; z++) lN[z] = c[z];
+            Lv3 = sv3;
+            Lv4 = sv4;
+
+            // out[iN[z]] = c[z];  // simulate scatter
+            // RansDecRenorm(&R[z], &ptr);      
+            __m256i renorm_mask3, renorm_mask4;
+            renorm_mask3 = _mm256_xor_si256(Rv3,_mm256_set1_epi32(0x80000000));
+            renorm_mask4 = _mm256_xor_si256(Rv4,_mm256_set1_epi32(0x80000000));
+
+            renorm_mask3 = _mm256_cmpgt_epi32(
+                               _mm256_set1_epi32(RANS_BYTE_L-0x80000000),
+                               renorm_mask3);
+            renorm_mask4 = _mm256_cmpgt_epi32(
+                               _mm256_set1_epi32(RANS_BYTE_L-0x80000000),
+                               renorm_mask4);
+
+            __m256i Yv3 = _mm256_slli_epi32(Rv3, 16);
+            __m256i Yv4 = _mm256_slli_epi32(Rv4, 16);
+
+            unsigned int imask3 = _mm256_movemask_ps((__m256)renorm_mask3);
+            unsigned int imask4 = _mm256_movemask_ps((__m256)renorm_mask4);
+            __m256i idx3 = _mm256_load_si256((const __m256i*)permute[imask3]);
+            sp += _mm_popcnt_u32(imask3);
+            Vv3 = _mm256_permutevar8x32_epi32(Vv3, idx3);
+
+            sv3 = _mm256_packus_epi32(sv3, sv4);
+            sv3 = _mm256_permute4x64_epi64(sv3, 0xd8);
+            sv3 = _mm256_packus_epi16(sv3, sv3);
+
+            u.tbuf64[tidx][0] = _mm256_extract_epi64(sv1, 0);
+            u.tbuf64[tidx][1] = _mm256_extract_epi64(sv1, 2);
+            u.tbuf64[tidx][2] = _mm256_extract_epi64(sv3, 0);
+            u.tbuf64[tidx][3] = _mm256_extract_epi64(sv3, 2);
+
+            iN[0]++;
+            if (++tidx == 32) {
+                iN[0]-=32;
+
+                transpose_and_copy(out, iN, u.tbuf);
+                tidx = 0;
+            }
+
+            __m256i idx4 = _mm256_load_si256((const __m256i*)permute[imask4]);
+            __m256i Vv4 = _mm256_cvtepu16_epi32(_mm_loadu_si128((__m128i *)sp));
+
+            //Vv = _mm256_and_si256(Vv, renorm_mask);  (blend does the AND anyway)
+            Yv3 = _mm256_or_si256(Yv3, Vv3);
+            Vv4 = _mm256_permutevar8x32_epi32(Vv4, idx4);
+            Yv4 = _mm256_or_si256(Yv4, Vv4);
+
+            sp += _mm_popcnt_u32(imask4);
+
+            Rv3 = _mm256_blendv_epi8(Rv3, Yv3, renorm_mask3);
+            Rv4 = _mm256_blendv_epi8(Rv4, Yv4, renorm_mask4);
+
+        }
+        isz4 += 64;
+
+        STORE(Rv, R);
+        STORE(Lv, lN);
+        ptr = (uint8_t *)sp;
+
+        if (1) {
+            iN[0]-=tidx;
+            int T;
+            for (z = 0; z < NX; z++)
+                for (T = 0; T < tidx; T++)
+                    out[iN[z]++] = u.tbuf[T][z];
+        }
+
+        // Scalar version for close to the end of in[] array so we don't
+        // do SIMD loads beyond the end of the buffer
+        for (; iN[0] < isz4;) {
+            for (z = 0; z < NX; z++) {
+                uint32_t m = R[z] & ((1u<<TF_SHIFT_O1)-1);
+                uint32_t S = s3[lN[z]][m];
+                unsigned char c = S & 0xff;
+                out[iN[z]++] = c;
+                uint32_t F = S>>(TF_SHIFT_O1+8);
+                R[z] = (F?F:4096) * (R[z]>>TF_SHIFT_O1) +
+                    ((S>>8) & ((1u<<TF_SHIFT_O1)-1));
+                RansDecRenormSafe(&R[z], &ptr, ptr_end);
+                lN[z] = c;
+            }
+        }
+
+        // Remainder
+        z = NX-1;
+        for (; iN[z] < out_sz; ) {
+            uint32_t m = R[z] & ((1u<<TF_SHIFT_O1)-1);
+            uint32_t S = s3[lN[z]][m];
+            unsigned char c = S & 0xff;
+            out[iN[z]++] = c;
+            uint32_t F = S>>(TF_SHIFT_O1+8);
+            R[z] = (F?F:4096) * (R[z]>>TF_SHIFT_O1) +
+                ((S>>8) & ((1u<<TF_SHIFT_O1)-1));
+            RansDecRenormSafe(&R[z], &ptr, ptr_end);
+            lN[z] = c;
+        }
+    } else {
+        // TF_SHIFT_O1_FAST.  This is the most commonly used variant.
+
+        // SIMD version ends decoding early as it reads at most 64 bytes
+        // from input via 4 vectorised loads.
+        isz4 -= 64;
+        for (; iN[0] < isz4 && (uint8_t *)sp+64 < ptr_end; ) {
+            // m[z] = R[z] & mask;
+            __m256i masked1 = _mm256_and_si256(Rv1, maskv);
+            __m256i masked2 = _mm256_and_si256(Rv2, maskv);
+
+            //  S[z] = s3[lN[z]][m[z]];
+            Lv1 = _mm256_slli_epi32(Lv1, TF_SHIFT_O1_FAST);
+            masked1 = _mm256_add_epi32(masked1, Lv1);
+
+            Lv2 = _mm256_slli_epi32(Lv2, TF_SHIFT_O1_FAST);
+            masked2 = _mm256_add_epi32(masked2, Lv2);
+
+            __m256i masked3 = _mm256_and_si256(Rv3, maskv);
+            __m256i masked4 = _mm256_and_si256(Rv4, maskv);
+
+            Lv3 = _mm256_slli_epi32(Lv3, TF_SHIFT_O1_FAST);
+            masked3 = _mm256_add_epi32(masked3, Lv3);
+
+            Lv4 = _mm256_slli_epi32(Lv4, TF_SHIFT_O1_FAST);
+            masked4 = _mm256_add_epi32(masked4, Lv4);
+
+            __m256i Sv1 = _mm256_i32gather_epi32x((int *)&s3F[0][0], masked1,
+                                                  sizeof(s3F[0][0]));
+            __m256i Sv2 = _mm256_i32gather_epi32x((int *)&s3F[0][0], masked2,
+                                                  sizeof(s3F[0][0]));
+
+            //  f[z] = S[z]>>(TF_SHIFT_O1+8);
+            __m256i fv1 = _mm256_srli_epi32(Sv1, TF_SHIFT_O1_FAST+8);
+            __m256i fv2 = _mm256_srli_epi32(Sv2, TF_SHIFT_O1_FAST+8);
+
+            __m256i Sv3 = _mm256_i32gather_epi32x((int *)&s3F[0][0], masked3,
+                                                  sizeof(s3F[0][0]));
+            __m256i Sv4 = _mm256_i32gather_epi32x((int *)&s3F[0][0], masked4,
+                                                  sizeof(s3F[0][0]));
+
+            __m256i fv3 = _mm256_srli_epi32(Sv3, TF_SHIFT_O1_FAST+8);
+            __m256i fv4 = _mm256_srli_epi32(Sv4, TF_SHIFT_O1_FAST+8);
+
+            //  b[z] = (S[z]>>8) & mask;
+            __m256i bv1 = _mm256_and_si256(_mm256_srli_epi32(Sv1, 8), maskv);
+            __m256i bv2 = _mm256_and_si256(_mm256_srli_epi32(Sv2, 8), maskv);
+            __m256i bv3 = _mm256_and_si256(_mm256_srli_epi32(Sv3, 8), maskv);
+            __m256i bv4 = _mm256_and_si256(_mm256_srli_epi32(Sv4, 8), maskv);
+
+            //  s[z] = S[z] & 0xff;
+            __m256i sv1 = _mm256_and_si256(Sv1, _mm256_set1_epi32(0xff));
+            __m256i sv2 = _mm256_and_si256(Sv2, _mm256_set1_epi32(0xff));
+            __m256i sv3 = _mm256_and_si256(Sv3, _mm256_set1_epi32(0xff));
+            __m256i sv4 = _mm256_and_si256(Sv4, _mm256_set1_epi32(0xff));
+
+            //  R[z] = f[z] * (R[z] >> TF_SHIFT_O1) + b[z];
+            Rv1 = _mm256_add_epi32(
+                      _mm256_mullo_epi32(
+                          _mm256_srli_epi32(Rv1,TF_SHIFT_O1_FAST), fv1), bv1);
+            Rv2 = _mm256_add_epi32(
+                      _mm256_mullo_epi32(
+                          _mm256_srli_epi32(Rv2,TF_SHIFT_O1_FAST), fv2), bv2);
+
+
+            //for (z = 0; z < NX; z++) lN[z] = c[z];
+            Lv1 = sv1;
+            Lv2 = sv2;
+
+            sv1 = _mm256_packus_epi32(sv1, sv2);
+            sv1 = _mm256_permute4x64_epi64(sv1, 0xd8);
+
+            // Start loading next batch of normalised states
+            __m256i Vv1 = _mm256_cvtepu16_epi32(
+                              _mm_loadu_si128((__m128i *)sp));
+
+            sv1 = _mm256_packus_epi16(sv1, sv1);
+
+            // out[iN[z]] = c[z];  // simulate scatter
+            // RansDecRenorm(&R[z], &ptr);      
+            __m256i renorm_mask1, renorm_mask2;
+            renorm_mask1 = _mm256_xor_si256(Rv1,_mm256_set1_epi32(0x80000000));
+            renorm_mask2 = _mm256_xor_si256(Rv2,_mm256_set1_epi32(0x80000000));
+
+            renorm_mask1 = _mm256_cmpgt_epi32(
+                               _mm256_set1_epi32(RANS_BYTE_L-0x80000000),
+                               renorm_mask1);
+            renorm_mask2 = _mm256_cmpgt_epi32(
+                               _mm256_set1_epi32(RANS_BYTE_L-0x80000000),
+                               renorm_mask2);
+
+            unsigned int imask1 = _mm256_movemask_ps((__m256)renorm_mask1);
+            __m256i idx1 = _mm256_load_si256((const __m256i*)permute[imask1]);
+            __m256i Yv1 = _mm256_slli_epi32(Rv1, 16);
+            __m256i Yv2 = _mm256_slli_epi32(Rv2, 16);
+
+            unsigned int imask2 = _mm256_movemask_ps((__m256)renorm_mask2);
+            Vv1 = _mm256_permutevar8x32_epi32(Vv1, idx1);
+            sp += _mm_popcnt_u32(imask1);
+
+            __m256i idx2 = _mm256_load_si256((const __m256i*)permute[imask2]);
+            __m256i Vv2 = _mm256_cvtepu16_epi32(
+                              _mm_loadu_si128((__m128i *)sp));
+            sp += _mm_popcnt_u32(imask2);
+            Vv2 = _mm256_permutevar8x32_epi32(Vv2, idx2);
+
+            Yv1 = _mm256_or_si256(Yv1, Vv1);
+            Yv2 = _mm256_or_si256(Yv2, Vv2);
+
+            Rv1 = _mm256_blendv_epi8(Rv1, Yv1, renorm_mask1);
+            Rv2 = _mm256_blendv_epi8(Rv2, Yv2, renorm_mask2);
+
+            /////////////////////////////////////////////////////////////////
+            // Start loading next batch of normalised states
+            __m256i Vv3 = _mm256_cvtepu16_epi32(
+                              _mm_loadu_si128((__m128i *)sp));
+
+            //  R[z] = f[z] * (R[z] >> TF_SHIFT_O1) + b[z];
+            Rv3 = _mm256_add_epi32(
+                      _mm256_mullo_epi32(
+                          _mm256_srli_epi32(Rv3,TF_SHIFT_O1_FAST), fv3), bv3);
+            Rv4 = _mm256_add_epi32(
+                      _mm256_mullo_epi32(
+                          _mm256_srli_epi32(Rv4,TF_SHIFT_O1_FAST), fv4), bv4);
+
+            //for (z = 0; z < NX; z++) lN[z] = c[z];
+            Lv3 = sv3;
+            Lv4 = sv4;
+
+            // out[iN[z]] = c[z];  // simulate scatter
+            // RansDecRenorm(&R[z], &ptr);      
+            __m256i renorm_mask3, renorm_mask4;
+            renorm_mask3 = _mm256_xor_si256(Rv3,_mm256_set1_epi32(0x80000000));
+            renorm_mask4 = _mm256_xor_si256(Rv4,_mm256_set1_epi32(0x80000000));
+
+            renorm_mask3 = _mm256_cmpgt_epi32(
+                               _mm256_set1_epi32(RANS_BYTE_L-0x80000000),
+                               renorm_mask3);
+            renorm_mask4 = _mm256_cmpgt_epi32(
+                               _mm256_set1_epi32(RANS_BYTE_L-0x80000000),
+                               renorm_mask4);
+
+            __m256i Yv3 = _mm256_slli_epi32(Rv3, 16);
+            __m256i Yv4 = _mm256_slli_epi32(Rv4, 16);
+
+            unsigned int imask3 = _mm256_movemask_ps((__m256)renorm_mask3);
+            unsigned int imask4 = _mm256_movemask_ps((__m256)renorm_mask4);
+            __m256i idx3 = _mm256_load_si256((const __m256i*)permute[imask3]);
+            sp += _mm_popcnt_u32(imask3);
+            Vv3 = _mm256_permutevar8x32_epi32(Vv3, idx3);
+
+            // sv3 sv4 are 32-bit ints with lowest bit being char
+            sv3 = _mm256_packus_epi32(sv3, sv4);       // 32 to 16; ABab
+            sv3 = _mm256_permute4x64_epi64(sv3, 0xd8); // shuffle;  AaBb
+            sv3 = _mm256_packus_epi16(sv3, sv3);       // 16 to 8
+
+            u.tbuf64[tidx][0] = _mm256_extract_epi64(sv1, 0);
+            u.tbuf64[tidx][1] = _mm256_extract_epi64(sv1, 2);
+            u.tbuf64[tidx][2] = _mm256_extract_epi64(sv3, 0);
+            u.tbuf64[tidx][3] = _mm256_extract_epi64(sv3, 2);
+
+            iN[0]++;
+            if (++tidx == 32) {
+                iN[0]-=32;
+
+                // We have tidx[x][y] which we want to store in
+                // memory in out[y][z] instead.  This is an unrolled
+                // transposition.
+                //
+                // A straight memcpy (obviously wrong) decodes my test
+                // data in around 1030MB/s vs 930MB/s for this transpose,
+                // giving an idea of the time spent in this portion.
+                transpose_and_copy(out, iN, u.tbuf);
+
+                tidx = 0;
+            }
+
+            __m256i idx4 = _mm256_load_si256((const __m256i*)permute[imask4]);
+            __m256i Vv4 = _mm256_cvtepu16_epi32(
+                              _mm_loadu_si128((__m128i *)sp));
+
+            Yv3 = _mm256_or_si256(Yv3, Vv3);
+            Vv4 = _mm256_permutevar8x32_epi32(Vv4, idx4);
+            Yv4 = _mm256_or_si256(Yv4, Vv4);
+
+            sp += _mm_popcnt_u32(imask4);
+
+            Rv3 = _mm256_blendv_epi8(Rv3, Yv3, renorm_mask3);
+            Rv4 = _mm256_blendv_epi8(Rv4, Yv4, renorm_mask4);
+        }
+        isz4 += 64;
+
+        STORE(Rv, R);
+        STORE(Lv, lN);
+        ptr = (uint8_t *)sp;
+
+        if (1) {
+            iN[0]-=tidx;
+            int T;
+            for (z = 0; z < NX; z++)
+                for (T = 0; T < tidx; T++)
+                    out[iN[z]++] = u.tbuf[T][z];
+        }
+
+        // Scalar version for close to the end of in[] array so we don't
+        // do SIMD loads beyond the end of the buffer
+        for (; iN[0] < isz4;) {
+            for (z = 0; z < NX; z++) {
+                uint32_t m = R[z] & ((1u<<TF_SHIFT_O1_FAST)-1);
+                uint32_t S = s3F[lN[z]][m];
+                unsigned char c = S & 0xff;
+                out[iN[z]++] = c;
+                R[z] = (S>>(TF_SHIFT_O1_FAST+8)) * (R[z]>>TF_SHIFT_O1_FAST) +
+                    ((S>>8) & ((1u<<TF_SHIFT_O1_FAST)-1));
+                RansDecRenormSafe(&R[z], &ptr, ptr_end);
+                lN[z] = c;
+            }
+        }
+
+        // Remainder
+        z = NX-1;
+        for (; iN[z] < out_sz; ) {
+            uint32_t m = R[z] & ((1u<<TF_SHIFT_O1_FAST)-1);
+            uint32_t S = s3F[lN[z]][m];
+            unsigned char c = S & 0xff;
+            out[iN[z]++] = c;
+            R[z] = (S>>(TF_SHIFT_O1_FAST+8)) * (R[z]>>TF_SHIFT_O1_FAST) +
+                ((S>>8) & ((1u<<TF_SHIFT_O1_FAST)-1));
+            RansDecRenormSafe(&R[z], &ptr, ptr_end);
+            lN[z] = c;
+        }
+    }
+
+    htscodecs_tls_free(s3);
+    return out;
+
+ err:
+    htscodecs_tls_free(s3);
+    free(out_free);
+    free(c_freq);
+
+    return NULL;
+}
+#else  // HAVE_AVX2
+// Prevent "empty translation unit" errors when building without AVX2
+const char *rANS_static32x16pr_avx2_disabled = "No AVX2";
+#endif // HAVE_AVX2
diff --git a/ext/htslib/htscodecs/htscodecs/rANS_static32x16pr_avx512.c b/ext/htslib/htscodecs/htscodecs/rANS_static32x16pr_avx512.c
new file mode 100644
index 0000000..3563f83
--- /dev/null
+++ b/ext/htslib/htscodecs/htscodecs/rANS_static32x16pr_avx512.c
@@ -0,0 +1,1192 @@
+/*
+ * Copyright (c) 2017-2023 Genome Research Ltd.
+ * Author(s): James Bonfield
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above
+ *       copyright notice, this list of conditions and the following
+ *       disclaimer in the documentation and/or other materials provided
+ *       with the distribution.
+ *
+ *    3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+ *       Institute nor the names of its contributors may be used to endorse
+ *       or promote products derived from this software without specific
+ *       prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH
+ * LTD OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * This is an AVX512 implementation of the 32-way interleaved 16-bit rANS.
+ * For now it only contains an order-0 implementation.  The AVX2 code may
+ * be used for order-1.
+ */
+
+#include "config.h"
+
+#if defined(__x86_64__) && defined(HAVE_AVX512)
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <limits.h>
+#include <x86intrin.h>
+
+#include "rANS_word.h"
+#include "rANS_static4x16.h"
+#define ROT32_SIMD
+#include "rANS_static16_int.h"
+#include "varint.h"
+#include "utils.h"
+
+#ifndef USE_GATHER
+
+// Speds with Downfall mitigation Off/On and Zen4.
+//         <------ AVX512 --------->   <------- AVX2 -------->
+// -o4:    IntelOff IntelOn  AMDZen4   
+// gcc7    544/1673 562/1711 448/1818  649/1515 645/1525 875/1624
+// gcc13   557/1672 576/1711 582/1761  630/1623 629/1652 866/1762
+// clang10 541/1547 564/1630 807/1912  620/1456 637/1481 837/1606
+// clang16 533/1431 555/1510 890/1611  629/1370 627/1406 996/1432
+//
+// Zen4 encode is particularly slow with gcc, but AVX2 encode is
+// faster and we use that already.
+static inline __m512i _mm512_i32gather_epi32x(__m512i idx, void *v, int size) {
+    uint32_t *b = (uint32_t *)v;
+
+#ifndef __clang__
+    volatile
+#endif
+    int c[16] __attribute__((aligned(32)));
+
+    //_mm512_store_si512((__m512i *)c, idx); // equivalent, but slower
+    _mm256_store_si256((__m256i *)(c),   _mm512_castsi512_si256(idx));
+    _mm256_store_si256((__m256i *)(c+8), _mm512_extracti64x4_epi64(idx, 1));
+
+    __m128i x0 = _mm_insert_epi32(_mm_cvtsi32_si128(b[c[0]]), b[c[1]], 1);
+    __m128i x1 = _mm_insert_epi32(_mm_cvtsi32_si128(b[c[2]]), b[c[3]], 1);
+    __m128i x2 = _mm_insert_epi32(_mm_cvtsi32_si128(b[c[4]]), b[c[5]], 1);
+    __m128i x3 = _mm_insert_epi32(_mm_cvtsi32_si128(b[c[6]]), b[c[7]], 1);
+
+    __m128i x01 = _mm_unpacklo_epi64(x0, x1);
+    __m128i x23 = _mm_unpacklo_epi64(x2, x3);
+    __m256i y0 =_mm256_castsi128_si256(x01);
+
+    __m128i x4 = _mm_insert_epi32(_mm_cvtsi32_si128(b[c[ 8]]), b[c[ 9]], 1);
+    __m128i x5 = _mm_insert_epi32(_mm_cvtsi32_si128(b[c[10]]), b[c[11]], 1);
+    __m128i x6 = _mm_insert_epi32(_mm_cvtsi32_si128(b[c[12]]), b[c[13]], 1);
+    __m128i x7 = _mm_insert_epi32(_mm_cvtsi32_si128(b[c[14]]), b[c[15]], 1);
+
+    __m128i x45 = _mm_unpacklo_epi64(x4, x5);
+    __m128i x67 = _mm_unpacklo_epi64(x6, x7);
+    __m256i y1 =_mm256_castsi128_si256(x45);
+
+    y0 = _mm256_inserti128_si256(y0, x23, 1);
+    y1 = _mm256_inserti128_si256(y1, x67, 1);
+
+    return _mm512_inserti64x4(_mm512_castsi256_si512(y0), y1, 1);
+}
+
+// 32-bit indices, 8-bit quantities into 32-bit lanes
+static inline __m512i _mm512_i32gather_epi32x1(__m512i idx, void *v) {
+    uint8_t *b = (uint8_t *)v;
+    volatile int c[16] __attribute__((aligned(32)));
+
+    //_mm512_store_si512((__m512i *)c, idx); // equivalent, but slower
+    _mm256_store_si256((__m256i *)(c),   _mm512_castsi512_si256(idx));
+    _mm256_store_si256((__m256i *)(c+8), _mm512_extracti64x4_epi64(idx, 1));
+
+    return _mm512_set_epi32(b[c[15]], b[c[14]], b[c[13]], b[c[12]],
+                            b[c[11]], b[c[10]], b[c[9]], b[c[8]],
+                            b[c[7]], b[c[6]], b[c[5]], b[c[4]],
+                            b[c[3]], b[c[2]], b[c[1]], b[c[0]]);
+}
+
+#else
+// real gathers
+#define _mm512_i32gather_epi32x _mm512_i32gather_epi32
+#endif
+
+unsigned char *rans_compress_O0_32x16_avx512(unsigned char *in,
+                                             unsigned int in_size,
+                                             unsigned char *out,
+                                             unsigned int *out_size) {
+    unsigned char *cp, *out_end;
+    RansEncSymbol syms[256];
+    RansState ransN[32] __attribute__((aligned(64)));
+    uint8_t* ptr;
+    uint32_t F[256+MAGIC] = {0};
+    int i, j, tab_size = 0, x, z;
+    // -20 for order/size/meta
+    uint32_t bound = rans_compress_bound_4x16(in_size,0)-20;
+
+    if (!out) {
+        *out_size = bound;
+        out = malloc(*out_size);
+    }
+    if (!out || bound > *out_size)
+        return NULL;
+
+    // If "out" isn't word aligned, tweak out_end/ptr to ensure it is.
+    // We already added more round in bound to allow for this.
+    if (((size_t)out)&1)
+        bound--;
+    ptr = out_end = out + bound;
+
+    if (in_size == 0)
+        goto empty;
+
+    // Compute statistics
+    if (hist8(in, in_size, F) < 0)
+        return NULL;
+
+    // Normalise so frequences sum to power of 2
+    uint32_t fsum = in_size;
+    uint32_t max_val = round2(fsum);
+    if (max_val > TOTFREQ)
+        max_val = TOTFREQ;
+
+    if (normalise_freq(F, fsum, max_val) < 0)
+        return NULL;
+    fsum=max_val;
+
+    cp = out;
+    cp += encode_freq(cp, F);
+    tab_size = cp-out;
+    //write(2, out+4, cp-(out+4));
+
+    if (normalise_freq(F, fsum, TOTFREQ) < 0)
+        return NULL;
+
+    // Encode statistics and build lookup tables for SIMD encoding.
+    uint32_t SB[256], SA[256], SD[256], SC[256];
+    for (x = j = 0; j < 256; j++) {
+        if (F[j]) {
+            RansEncSymbolInit(&syms[j], x, F[j], TF_SHIFT);
+            SB[j] = syms[j].x_max;
+            SA[j] = syms[j].rcp_freq;
+            SD[j] = (syms[j].cmpl_freq<<0) | ((syms[j].rcp_shift-32)<<16);
+            SC[j] = syms[j].bias;
+            x += F[j];
+        }
+    }
+
+    for (z = 0; z < 32; z++)
+      RansEncInit(&ransN[z]);
+
+    z = i = in_size&(32-1);
+    while (z-- > 0)
+      RansEncPutSymbol(&ransN[z], &ptr, &syms[in[in_size-(i-z)]]);
+
+#define LOAD512(a,b)                                     \
+    __m512i a##1 = _mm512_load_si512((__m512i *)&b[0]); \
+    __m512i a##2 = _mm512_load_si512((__m512i *)&b[16]);
+
+#define STORE512(a,b)                           \
+    _mm512_store_si512((__m256i *)&b[0], a##1); \
+    _mm512_store_si512((__m256i *)&b[16], a##2);
+
+    LOAD512(Rv, ransN);
+
+    uint16_t *ptr16 = (uint16_t *)ptr;
+    for (i=(in_size &~(32-1)); i>0; i-=32) {
+        uint8_t *c = &in[i-32];
+
+        // GATHER versions
+        // Much faster now we have an efficient loadu mechanism in place,
+        // BUT...
+        // Try this for avx2 variant too?  Better way to populate the mm256
+        // regs for mix of avx2 and avx512 opcodes.
+        __m256i c12 = _mm256_loadu_si256((__m256i const *)c);
+        __m512i c1 = _mm512_cvtepu8_epi32(_mm256_extracti128_si256(c12,0));
+        __m512i c2 = _mm512_cvtepu8_epi32(_mm256_extracti128_si256(c12,1));
+#define SET512(a,b) \
+        __m512i a##1 = _mm512_i32gather_epi32x(c1, b, 4); \
+        __m512i a##2 = _mm512_i32gather_epi32x(c2, b, 4)
+
+        SET512(xmax, SB);
+
+        uint16_t gt_mask1 = _mm512_cmpgt_epi32_mask(Rv1, xmax1);
+        int pc1 = _mm_popcnt_u32(gt_mask1);
+        __m512i Rp1 = _mm512_and_si512(Rv1, _mm512_set1_epi32(0xffff));
+        __m512i Rp2 = _mm512_and_si512(Rv2, _mm512_set1_epi32(0xffff));
+        uint16_t gt_mask2 = _mm512_cmpgt_epi32_mask(Rv2, xmax2);
+        SET512(SDv,  SD);
+        int pc2 = _mm_popcnt_u32(gt_mask2);
+
+        Rp1 = _mm512_maskz_compress_epi32(gt_mask1, Rp1);
+        Rp2 = _mm512_maskz_compress_epi32(gt_mask2, Rp2);
+
+        _mm512_mask_cvtepi32_storeu_epi16(ptr16-pc2, (1<<pc2)-1, Rp2);
+        ptr16 -= pc2;
+        _mm512_mask_cvtepi32_storeu_epi16(ptr16-pc1, (1<<pc1)-1, Rp1);
+        ptr16 -= pc1;
+
+        SET512(rfv,  SA);
+        Rv1 = _mm512_mask_srli_epi32(Rv1, gt_mask1, Rv1, 16);
+        Rv2 = _mm512_mask_srli_epi32(Rv2, gt_mask2, Rv2, 16);
+
+        // interleaved form of this, helps on icc a bit
+        //rfv1 = _mm512_mulhi_epu32(Rv1, rfv1);
+        //rfv2 = _mm512_mulhi_epu32(Rv2, rfv2);
+
+        // Alternatives here:
+        // SHIFT right/left instead of AND: (very marginally slower)
+        //   rf1_hm = _mm512_and_epi32(
+        //           rf1_hm, _mm512_set1_epi64((uint64_t)0xffffffff00000000));
+        // vs
+        //   rf1_hm = _mm512_srli_epi64(rf1_hm, 32);
+        //   rf1_hm = _mm512_slli_epi64(rf1_hm, 32);
+        __m512i rf1_hm = _mm512_mul_epu32(_mm512_srli_epi64(Rv1,  32),
+                                          _mm512_srli_epi64(rfv1, 32));
+        __m512i rf2_hm = _mm512_mul_epu32(_mm512_srli_epi64(Rv2,  32),
+                                          _mm512_srli_epi64(rfv2, 32));
+        __m512i rf1_lm = _mm512_srli_epi64(_mm512_mul_epu32(Rv1, rfv1), 32);
+        __m512i rf2_lm = _mm512_srli_epi64(_mm512_mul_epu32(Rv2, rfv2), 32);
+        rf1_hm = _mm512_and_epi32(rf1_hm,
+                                  _mm512_set1_epi64((uint64_t)0xffffffff<<32));
+        rf2_hm = _mm512_and_epi32(rf2_hm,
+                                  _mm512_set1_epi64((uint64_t)0xffffffff<<32));
+        rfv1 = _mm512_or_epi32(rf1_lm, rf1_hm);
+        rfv2 = _mm512_or_epi32(rf2_lm, rf2_hm);
+
+        // Or a pure masked blend approach, sadly slower.
+        // rfv1 = _mm512_mask_blend_epi32(0x5555,
+        //            _mm512_mul_epu32(
+        //                _mm512_srli_epi64(Rv1, 32),
+        //                _mm512_srli_epi64(rfv1, 32)),
+        //            _mm512_srli_epi64(
+        //                _mm512_mul_epu32(Rv1, rfv1),
+        //                32));
+        // rfv2 = _mm512_mask_blend_epi32(0xaaaa,
+        //            _mm512_srli_epi64(
+        //                _mm512_mul_epu32(Rv2, rfv2),
+        //                32),
+        //            _mm512_mul_epu32(
+        //                _mm512_srli_epi64(Rv2, 32),
+        //                _mm512_srli_epi64(rfv2, 32)));
+
+        SET512(biasv, SC);
+        __m512i shiftv1 = _mm512_srli_epi32(SDv1, 16);
+        __m512i shiftv2 = _mm512_srli_epi32(SDv2, 16);
+
+        __m512i qv1 = _mm512_srlv_epi32(rfv1, shiftv1);
+        __m512i qv2 = _mm512_srlv_epi32(rfv2, shiftv2);
+
+        qv1 = _mm512_mullo_epi32(qv1,
+                  _mm512_and_si512(SDv1,_mm512_set1_epi32(0xffff)));
+        qv1 = _mm512_add_epi32(qv1, biasv1);
+        Rv1 = _mm512_add_epi32(Rv1, qv1);
+
+        qv2 = _mm512_mullo_epi32(qv2,
+                  _mm512_and_si512(SDv2, _mm512_set1_epi32(0xffff)));
+        qv2 = _mm512_add_epi32(qv2, biasv2);
+        Rv2 = _mm512_add_epi32(Rv2, qv2);
+    }
+    ptr = (uint8_t *)ptr16;
+    STORE512(Rv, ransN);
+
+    for (z = 32-1; z >= 0; z--)
+        RansEncFlush(&ransN[z], &ptr);
+
+ empty:
+    // Finalise block size and return it
+    *out_size = (out_end - ptr) + tab_size;
+
+    memmove(out + tab_size, ptr, out_end-ptr);
+
+    return out;
+}
+
+unsigned char *rans_uncompress_O0_32x16_avx512(unsigned char *in,
+                                               unsigned int in_size,
+                                               unsigned char *out,
+                                               unsigned int out_sz) {
+    if (in_size < 32*4) // 32-states at least
+        return NULL;
+
+    if (out_sz >= INT_MAX)
+        return NULL; // protect against some overflow cases
+
+    /* Load in the static tables */
+    unsigned char *cp = in, *out_free = NULL;
+    unsigned char *cp_end = in + in_size;
+    int i;
+    uint32_t s3[TOTFREQ]  __attribute__((aligned(64))); // For TF_SHIFT <= 12
+
+    if (!out)
+        out_free = out = malloc(out_sz);
+    if (!out)
+        return NULL;
+
+    // Precompute reverse lookup of frequency.
+    uint32_t F[256] = {0}, fsum;
+    int fsz = decode_freq(cp, cp_end, F, &fsum);
+    if (!fsz)
+        goto err;
+    cp += fsz;
+
+    normalise_freq_shift(F, fsum, TOTFREQ);
+
+    // Build symbols; fixme, do as part of decode, see the _d variant
+    if (rans_F_to_s3(F, TF_SHIFT, s3))
+        goto err;
+
+    if (cp_end - cp < 32 * 4)
+        goto err;
+
+    int z;
+    RansState Rv[32] __attribute__((aligned(64)));
+    for (z = 0; z < 32; z++) {
+        RansDecInit(&Rv[z], &cp);
+        if (Rv[z] < RANS_BYTE_L)
+            goto err;
+    }
+
+    uint16_t *sp = (uint16_t *)cp;
+
+    int out_end = (out_sz&~(32-1));
+    const uint32_t mask = (1u << TF_SHIFT)-1;
+
+    __m512i maskv = _mm512_set1_epi32(mask); // set mask in all lanes
+    __m512i R1 = _mm512_load_epi32(&Rv[0]);
+    __m512i R2 = _mm512_load_epi32(&Rv[16]);
+
+    // Start of the first loop iteration, which we do move to the end of the
+    // loop for the next cycle so we can remove some of the instr. latency.
+    __m512i masked1 = _mm512_and_epi32(R1, maskv);
+    __m512i masked2 = _mm512_and_epi32(R2, maskv);
+    __m512i S1 = _mm512_i32gather_epi32x(masked1, (int *)s3, sizeof(*s3));
+    __m512i S2 = _mm512_i32gather_epi32x(masked2, (int *)s3, sizeof(*s3));
+
+    uint8_t overflow[64+64] = {0};
+    for (i=0; i < out_end; i+=32) {
+      //for (z = 0; z < 16; z++) {
+
+      // Protect against running off the end of in buffer.
+      // We copy it to a worst-case local buffer when near the end.
+      if ((uint8_t *)sp+64 > cp_end) {
+        memmove(overflow, sp, cp_end - (uint8_t *)sp);
+        sp = (uint16_t *)overflow;
+        cp_end = overflow + sizeof(overflow);
+      }
+
+      //uint32_t S = s3[R[z] & mask];
+      __m512i renorm_words1 = _mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i *)sp)); // next 16 words
+
+      //uint16_t f = S>>(TF_SHIFT+8), b = (S>>8) & mask;
+      __m512i f1 = _mm512_srli_epi32(S1, TF_SHIFT+8);
+      __m512i f2 = _mm512_srli_epi32(S2, TF_SHIFT+8);
+      __m512i b1 = _mm512_and_epi32(_mm512_srli_epi32(S1, 8), maskv);
+      __m512i b2 = _mm512_and_epi32(_mm512_srli_epi32(S2, 8), maskv);
+
+      //R[z] = f * (R[z] >> TF_SHIFT) + b;
+      // approx 10 cycle latency on mullo.
+      R1 = _mm512_add_epi32(
+               _mm512_mullo_epi32(
+                   _mm512_srli_epi32(R1, TF_SHIFT), f1), b1);
+      __mmask16 renorm_mask1, renorm_mask2;
+      renorm_mask1=_mm512_cmplt_epu32_mask(R1, _mm512_set1_epi32(RANS_BYTE_L));
+      R2 = _mm512_add_epi32(
+               _mm512_mullo_epi32(
+                   _mm512_srli_epi32(R2, TF_SHIFT), f2), b2);
+
+      // renorm. this is the interesting part:
+      renorm_mask2=_mm512_cmplt_epu32_mask(R2, _mm512_set1_epi32(RANS_BYTE_L));
+      // advance by however many words we actually read
+      sp += _mm_popcnt_u32(renorm_mask1);
+      __m512i renorm_words2 = _mm512_cvtepu16_epi32(_mm256_loadu_si256(
+                                      (const __m256i *)sp));
+
+      // select masked only
+      __m512i renorm_vals1, renorm_vals2;
+
+      renorm_vals1 = _mm512_mask_expand_epi32(R1, renorm_mask1, renorm_words1);
+      renorm_vals2 = _mm512_mask_expand_epi32(R2, renorm_mask2, renorm_words2);
+
+      // For start of next loop iteration.  This has been moved here
+      // (and duplicated to before the loop starts) so we can do something
+      // with the latency period of gather, such as finishing up the
+      // renorm offset and writing the results.
+      __m512i S1_ = S1; // temporary copy for use in out[]=S later
+      __m512i S2_ = S2;
+
+      masked1 = _mm512_and_epi32(renorm_vals1, maskv);
+      S1 = _mm512_i32gather_epi32x(masked1, (int *)s3, sizeof(*s3));
+      masked2 = _mm512_and_epi32(renorm_vals2, maskv);
+      S2 = _mm512_i32gather_epi32x(masked2, (int *)s3, sizeof(*s3));
+
+      R1 = _mm512_mask_slli_epi32(R1, renorm_mask1, R1, 16);
+      R2 = _mm512_mask_slli_epi32(R2, renorm_mask2, R2, 16);
+
+      __m512i m16 = _mm512_set1_epi32(0xffff);
+      renorm_vals1 = _mm512_maskz_and_epi32(renorm_mask1, renorm_vals1, m16);
+      renorm_vals2 = _mm512_maskz_and_epi32(renorm_mask2, renorm_vals2, m16);
+
+      // advance by however many words we actually read
+      sp += _mm_popcnt_u32(renorm_mask2);
+
+      R1 = _mm512_add_epi32(R1, renorm_vals1);
+      R2 = _mm512_add_epi32(R2, renorm_vals2);
+
+      //out[i+z] = S;
+      _mm_storeu_si128((__m128i *)(out+i),    _mm512_cvtepi32_epi8(S1_));
+      _mm_storeu_si128((__m128i *)(out+i+16), _mm512_cvtepi32_epi8(S2_));
+    }
+
+    _mm512_store_epi32(&Rv[ 0], R1);
+    _mm512_store_epi32(&Rv[16], R2);
+
+    for (z = out_sz & (32-1); z-- > 0; )
+      out[out_end + z] = s3[Rv[z] & mask];
+
+    return out;
+
+ err:
+    free(out_free);
+    return NULL;
+}
+
+#define TBUF8
+#ifdef TBUF8
+// 15% quicker overall O1 decode now due to rot32_simd below.
+
+// NB: This uses AVX2 though and we could rewrite using AVX512 for
+// further speed gains.
+static inline void transpose_and_copy(uint8_t *out, int iN[32],
+                                      uint8_t t[32][32]) {
+//  int z;
+//  for (z = 0; z < 32; z++) {
+//      int k;
+//      for (k = 0; k < 32; k++)
+//          out[iN[z]+k] = t[k][z];
+//      iN[z] += 32;
+//  }
+
+    rot32_simd(t, out, iN);
+}
+
+#else
+// Implemented using AVX512 gathers.
+// This is faster than a naive scalar implementation, but doesn't beat the
+// AVX2 vectorised 32x32 transpose function.
+static inline void transpose_and_copy_avx512(uint8_t *out, int iN[32],
+                                             uint32_t t32[32][32]) {
+    int z;
+//  for (z = 0; z < 32; z++) {
+//      int k;
+//      for (k = 0; k < 32; k++)
+//          out[iN[z]+k] = t32[k][z];
+//      iN[z] += 32;
+//  }
+
+
+    __m512i v1 = _mm512_set_epi32(15, 14, 13, 12, 11, 10,  9,  8,
+                                   7,  6,  5,  4,  3,  2,  1,  0);
+    v1 = _mm512_slli_epi32(v1, 5);
+
+    for (z = 0; z < 32; z++) {
+        __m512i t1 = _mm512_i32gather_epi32x(v1, &t32[ 0][z], 4);
+        __m512i t2 = _mm512_i32gather_epi32x(v1, &t32[16][z], 4);
+        _mm_storeu_si128((__m128i*)(&out[iN[z]   ]), _mm512_cvtepi32_epi8(t1));
+        _mm_storeu_si128((__m128i*)(&out[iN[z]+16]), _mm512_cvtepi32_epi8(t2));
+        iN[z] += 32;
+    }
+}
+#endif // TBUF
+
+unsigned char *rans_compress_O1_32x16_avx512(unsigned char *in,
+                                             unsigned int in_size,
+                                             unsigned char *out,
+                                             unsigned int *out_size) {
+    unsigned char *cp, *out_end, *out_free = NULL;
+    unsigned int tab_size;
+    uint32_t bound = rans_compress_bound_4x16(in_size,1)-20;
+    int z;
+    RansState ransN[32] __attribute__((aligned(64)));
+
+    if (in_size < 32) // force O0 instead
+        return NULL;
+
+    if (!out) {
+        *out_size = bound;
+        out = malloc(*out_size);
+    }
+    if (!out || bound > *out_size)
+        return NULL;
+
+    if (((size_t)out)&1)
+        bound--;
+    out_end = out + bound;
+
+    RansEncSymbol (*syms)[256] = htscodecs_tls_alloc(256 * (sizeof(*syms)));
+    if (!syms) {
+        free(out_free);
+        return NULL;
+    }
+
+    cp = out;
+    int shift = encode_freq1(in, in_size, 32, syms, &cp);
+    if (shift < 0) {
+        free(out_free);
+        htscodecs_tls_free(syms);
+        return NULL;
+    }
+    tab_size = cp - out;
+
+    for (z = 0; z < 32; z++)
+      RansEncInit(&ransN[z]);
+
+    uint8_t* ptr = out_end;
+
+    int iN[32] __attribute__((aligned(64)));
+    int isz4 = in_size/32;
+    for (z = 0; z < 32; z++)
+        iN[z] = (z+1)*isz4-2;
+
+    uint32_t lN[32] __attribute__((aligned(64)));
+    for (z = 0; z < 32; z++)
+        lN[z] = in[iN[z]+1];
+
+    // Deal with the remainder
+    z = 32-1;
+    lN[z] = in[in_size-1];
+    for (iN[z] = in_size-2; iN[z] > 32*isz4-2; iN[z]--) {
+        unsigned char c = in[iN[z]];
+        RansEncPutSymbol(&ransN[z], &ptr, &syms[c][lN[z]]);
+        lN[z] = c;
+    }
+
+    LOAD512(Rv, ransN);
+
+    uint16_t *ptr16 = (uint16_t *)ptr;
+    LOAD512(iN, iN);
+    LOAD512(last, lN);
+
+    __m512i c1 = _mm512_i32gather_epi32x1(iN1, in);
+    __m512i c2 = _mm512_i32gather_epi32x1(iN2, in);
+
+    // We cache the next 64-bytes locally and transpose.
+    // This means we can load 32 ints from t32[x] with load instructions
+    // instead of gathers.  The copy, transpose and expand is easier done
+    // in scalar code.
+#define BATCH 64
+    uint8_t t32[BATCH][32] __attribute__((aligned(64)));
+    int next_batch;
+    if (iN[0] > BATCH) {
+        int i, j;
+        for (i = 0; i < BATCH; i++)
+            // memcpy(c[i], &in[iN[i]-32], 32); fast mode
+            for (j = 0; j < 32; j++)
+                t32[BATCH-1-i][j] = in[iN[j]-1-i];
+        next_batch = BATCH;
+    } else {
+        next_batch = -1;
+    }
+
+    for (; iN[0] >= 0; iN[0]--) {
+        // Note, consider doing the same approach for the AVX2 encoder.
+        // Maybe we can also get gather working well there?
+        // Gather here is still a major latency bottleneck, consuming
+        // around 40% of CPU cycles overall.
+
+        // FIXME: maybe we need to cope with in[31] read over-flow
+        // on loop cycles 0, 1, 2 where gather reads 32-bits instead of
+        // 8 bits.  Use set instead there on c2?
+
+        // index into syms[0][0] array, used for x_max, rcp_freq, and bias
+        __m512i vidx1 = _mm512_slli_epi32(c1, 8);
+        __m512i vidx2 = _mm512_slli_epi32(c2, 8);
+        vidx1 = _mm512_add_epi32(vidx1, last1);
+        vidx2 = _mm512_add_epi32(vidx2, last2);
+        vidx1 = _mm512_slli_epi32(vidx1, 2);
+        vidx2 = _mm512_slli_epi32(vidx2, 2);
+
+        // ------------------------------------------------------------
+        //      for (z = NX-1; z >= 0; z--) {
+        //          if (ransN[z] >= x_max[z]) {
+        //              *--ptr16 = ransN[z] & 0xffff;
+        //              ransN[z] >>= 16;
+        //          }
+        //      }
+
+#define SET512x(a,x) \
+        __m512i a##1 = _mm512_i32gather_epi32x(vidx1, &syms[0][0].x, 4); \
+        __m512i a##2 = _mm512_i32gather_epi32x(vidx2, &syms[0][0].x, 4)
+
+        // Start of next loop, moved here to remove latency.
+        // last[z] = c[z]
+        // iN[z]--
+        // c[z] = in[iN[z]]
+        last1 = c1;
+        last2 = c2;
+        iN1 = _mm512_sub_epi32(iN1, _mm512_set1_epi32(1));
+        iN2 = _mm512_sub_epi32(iN2, _mm512_set1_epi32(1));
+
+        // Code below is equivalent to this:
+        // c1 = _mm512_i32gather_epi32(iN1, in, 1);
+        // c2 = _mm512_i32gather_epi32(iN2, in, 1);
+
+        // Better when we have a power of 2
+        if (next_batch >= 0) {
+            if (--next_batch < 0 && iN[0] > BATCH) {
+                // Load 32 BATCH blocks of data.
+                // Executed once every BATCH cycles
+                int i, j;
+                uint8_t c[32][BATCH];
+                iN[0] += BATCH;
+                for (j = 0; j < 32; j++) {
+                    iN[j] -= BATCH;
+                    memcpy(c[j], &in[iN[j]-BATCH], BATCH);
+                }
+                // transpose matrix from 32xBATCH to BATCHx32
+                for (j = 0; j < 32; j++) {
+                    for (i = 0; i < BATCH; i+=16) {
+                        int z;
+                        for (z = 0; z < 16; z++)
+                            t32[i+z][j] = c[j][i+z];
+                    }
+                }
+                next_batch = BATCH-1;
+            }
+            if (next_batch >= 0) {
+                // Copy from our of our pre-loaded BATCHx32 tables
+                // Executed every cycles
+                __m128i c1_ = _mm_load_si128((__m128i *)&t32[next_batch][0]);
+                __m128i c2_ = _mm_load_si128((__m128i *)&t32[next_batch][16]);
+                c1 = _mm512_cvtepu8_epi32(c1_);
+                c2 = _mm512_cvtepu8_epi32(c2_);
+            }
+        }
+
+        if (next_batch < 0 && iN[0]) {
+            // no pre-loaded data as within BATCHx32 of input end
+            c1 = _mm512_i32gather_epi32x1(iN1, in);
+            c2 = _mm512_i32gather_epi32x1(iN2, in);
+
+            // Speeds up clang, even though not needed any more.
+            // Harmless to leave here.
+            c1 = _mm512_and_si512(c1, _mm512_set1_epi32(0xff));
+            c2 = _mm512_and_si512(c2, _mm512_set1_epi32(0xff));
+        }
+        // End of "equivalent to" code block
+
+        SET512x(xmax, x_max); // high latency
+
+        uint16_t gt_mask1 = _mm512_cmpgt_epi32_mask(Rv1, xmax1);
+        int pc1 = _mm_popcnt_u32(gt_mask1);
+        __m512i Rp1 = _mm512_and_si512(Rv1, _mm512_set1_epi32(0xffff));
+        __m512i Rp2 = _mm512_and_si512(Rv2, _mm512_set1_epi32(0xffff));
+        uint16_t gt_mask2 = _mm512_cmpgt_epi32_mask(Rv2, xmax2);
+        SET512x(SDv, cmpl_freq); // good
+        int pc2 = _mm_popcnt_u32(gt_mask2);
+
+        Rp1 = _mm512_maskz_compress_epi32(gt_mask1, Rp1);
+        Rp2 = _mm512_maskz_compress_epi32(gt_mask2, Rp2);
+
+        _mm512_mask_cvtepi32_storeu_epi16(ptr16-pc2, (1<<pc2)-1, Rp2);
+        ptr16 -= pc2;
+        _mm512_mask_cvtepi32_storeu_epi16(ptr16-pc1, (1<<pc1)-1, Rp1);
+        ptr16 -= pc1;
+
+        Rv1 = _mm512_mask_srli_epi32(Rv1, gt_mask1, Rv1, 16);
+        Rv2 = _mm512_mask_srli_epi32(Rv2, gt_mask2, Rv2, 16);
+
+        // ------------------------------------------------------------
+        // uint32_t q = (uint32_t) (((uint64_t)ransN[z] * rcp_freq[z])
+        //                          >> rcp_shift[z]);
+        // ransN[z] = ransN[z] + bias[z] + q * cmpl_freq[z];
+        SET512x(rfv, rcp_freq); // good-ish
+
+        __m512i rf1_hm = _mm512_mul_epu32(_mm512_srli_epi64(Rv1, 32),
+                                          _mm512_srli_epi64(rfv1, 32));
+        __m512i rf2_hm = _mm512_mul_epu32(_mm512_srli_epi64(Rv2, 32),
+                                          _mm512_srli_epi64(rfv2, 32));
+        __m512i rf1_lm = _mm512_srli_epi64(_mm512_mul_epu32(Rv1, rfv1), 32);
+        __m512i rf2_lm = _mm512_srli_epi64(_mm512_mul_epu32(Rv2, rfv2), 32);
+
+        const __m512i top32 = _mm512_set1_epi64((uint64_t)0xffffffff00000000);
+        rf1_hm = _mm512_and_epi32(rf1_hm, top32);
+        rf2_hm = _mm512_and_epi32(rf2_hm, top32);
+        rfv1 = _mm512_or_epi32(rf1_lm, rf1_hm);
+        rfv2 = _mm512_or_epi32(rf2_lm, rf2_hm);
+
+        SET512x(biasv, bias); // good
+        __m512i shiftv1 = _mm512_srli_epi32(SDv1, 16);
+        __m512i shiftv2 = _mm512_srli_epi32(SDv2, 16);
+
+        shiftv1 = _mm512_sub_epi32(shiftv1, _mm512_set1_epi32(32));
+        shiftv2 = _mm512_sub_epi32(shiftv2, _mm512_set1_epi32(32));
+
+        __m512i qv1 = _mm512_srlv_epi32(rfv1, shiftv1);
+        __m512i qv2 = _mm512_srlv_epi32(rfv2, shiftv2);
+
+        const __m512i bot16 = _mm512_set1_epi32(0xffff);
+        qv1 = _mm512_mullo_epi32(qv1, _mm512_and_si512(SDv1, bot16));
+        qv2 = _mm512_mullo_epi32(qv2, _mm512_and_si512(SDv2, bot16));
+
+        qv1 = _mm512_add_epi32(qv1, biasv1);
+        Rv1 = _mm512_add_epi32(Rv1, qv1);
+
+        qv2 = _mm512_add_epi32(qv2, biasv2);
+        Rv2 = _mm512_add_epi32(Rv2, qv2);
+    }
+
+    STORE512(Rv, ransN);
+    STORE512(last, lN);
+
+    ptr = (uint8_t *)ptr16;
+
+    for (z = 32-1; z>=0; z--)
+        RansEncPutSymbol(&ransN[z], &ptr, &syms[0][lN[z]]);
+
+    for (z = 32-1; z >= 0; z--)
+        RansEncFlush(&ransN[z], &ptr);
+
+    // Finalise block size and return it
+    *out_size = (out_end - ptr) + tab_size;
+
+//    cp = out;
+//    *cp++ = (in_size>> 0) & 0xff;
+//    *cp++ = (in_size>> 8) & 0xff;
+//    *cp++ = (in_size>>16) & 0xff;
+//    *cp++ = (in_size>>24) & 0xff;
+
+    memmove(out + tab_size, ptr, out_end-ptr);
+
+    htscodecs_tls_free(syms);
+    return out;
+}
+
+#define NX 32
+unsigned char *rans_uncompress_O1_32x16_avx512(unsigned char *in,
+                                               unsigned int in_size,
+                                               unsigned char *out,
+                                               unsigned int out_sz) {
+    if (in_size < NX*4) // 4-states at least
+        return NULL;
+
+    if (out_sz >= INT_MAX)
+        return NULL; // protect against some overflow cases
+
+    /* Load in the static tables */
+    unsigned char *cp = in, *cp_end = in+in_size, *out_free = NULL;
+    unsigned char *c_freq = NULL;
+
+    uint32_t (*s3)[TOTFREQ_O1] = htscodecs_tls_alloc(256*TOTFREQ_O1*4);
+    if (!s3)
+        return NULL;
+    uint32_t (*s3F)[TOTFREQ_O1_FAST] = (uint32_t (*)[TOTFREQ_O1_FAST])s3;
+
+    if (!out)
+        out_free = out = malloc(out_sz);
+
+    if (!out)
+        goto err;
+
+    //fprintf(stderr, "out_sz=%d\n", out_sz);
+
+    // compressed header? If so uncompress it
+    unsigned char *tab_end = NULL;
+    unsigned char *c_freq_end = cp_end;
+    unsigned int shift = *cp >> 4;
+    if (*cp++ & 1) {
+        uint32_t u_freq_sz, c_freq_sz;
+        cp += var_get_u32(cp, cp_end, &u_freq_sz);
+        cp += var_get_u32(cp, cp_end, &c_freq_sz);
+        if (c_freq_sz > cp_end - cp)
+            goto err;
+        tab_end = cp + c_freq_sz;
+        if (!(c_freq = rans_uncompress_O0_4x16(cp, c_freq_sz, NULL,
+                                               u_freq_sz)))
+            goto err;
+        cp = c_freq;
+        c_freq_end = c_freq + u_freq_sz;
+    }
+
+    // Decode order-0 symbol list; avoids needing in order-1 tables
+    cp += decode_freq1(cp, c_freq_end, shift, s3, s3F, NULL, NULL);
+
+    if (tab_end)
+        cp = tab_end;
+    free(c_freq);
+    c_freq = NULL;
+
+    if (cp_end - cp < NX * 4)
+        goto err;
+
+    RansState R[NX] __attribute__((aligned(64)));
+    uint8_t *ptr = cp, *ptr_end = in + in_size;
+    int z;
+    for (z = 0; z < NX; z++) {
+        RansDecInit(&R[z], &ptr);
+        if (R[z] < RANS_BYTE_L)
+            goto err;
+    }
+
+    int isz4 = out_sz/NX;
+    int iN[NX], lN[NX] __attribute__((aligned(64))) = {0};
+    for (z = 0; z < NX; z++)
+        iN[z] = z*isz4;
+
+    uint16_t *sp = (uint16_t *)ptr;
+    const uint32_t mask = (1u << shift)-1;
+
+    __m512i _maskv  = _mm512_set1_epi32(mask);
+    LOAD512(_Rv, R);
+    LOAD512(_Lv, lN);
+
+#ifdef TBUF8
+    union {
+        unsigned char tbuf[32][32];
+        uint64_t tbuf64[32][4];
+    } u __attribute__((aligned(32)));
+#else
+    uint32_t tbuf[32][32];
+#endif
+
+    unsigned int tidx = 0;
+
+    if (shift == TF_SHIFT_O1) {
+        isz4 -= 64;
+        for (; iN[0] < isz4 && (uint8_t *)sp+64 < ptr_end; ) {
+            // m[z] = R[z] & mask;
+            __m512i _masked1 = _mm512_and_si512(_Rv1, _maskv);
+            __m512i _masked2 = _mm512_and_si512(_Rv2, _maskv);
+
+            //  S[z] = s3[lN[z]][m[z]];
+            _Lv1 = _mm512_slli_epi32(_Lv1, TF_SHIFT_O1);
+            _Lv2 = _mm512_slli_epi32(_Lv2, TF_SHIFT_O1);
+
+            _masked1 = _mm512_add_epi32(_masked1, _Lv1);
+            _masked2 = _mm512_add_epi32(_masked2, _Lv2);
+
+            // This is the biggest bottleneck
+            __m512i _Sv1 = _mm512_i32gather_epi32x(_masked1, (int *)&s3F[0][0],
+                                                   sizeof(s3F[0][0]));
+            __m512i _Sv2 = _mm512_i32gather_epi32x(_masked2, (int *)&s3F[0][0],
+                                                   sizeof(s3F[0][0]));
+
+            //  f[z] = S[z]>>(TF_SHIFT_O1+8);
+            __m512i _fv1 = _mm512_srli_epi32(_Sv1, TF_SHIFT_O1+8);
+            __m512i _fv2 = _mm512_srli_epi32(_Sv2, TF_SHIFT_O1+8);
+
+            //  b[z] = (S[z]>>8) & mask;
+            __m512i _bv1 = _mm512_and_si512(_mm512_srli_epi32(_Sv1,8), _maskv);
+            __m512i _bv2 = _mm512_and_si512(_mm512_srli_epi32(_Sv2,8), _maskv);
+
+            //  s[z] = S[z] & 0xff;
+            __m512i _sv1 = _mm512_and_si512(_Sv1, _mm512_set1_epi32(0xff));
+            __m512i _sv2 = _mm512_and_si512(_Sv2, _mm512_set1_epi32(0xff));
+
+            // A maximum frequency of 4096 doesn't fit in our s3 array.
+            // as it's 12 bit + 12 bit + 8 bit.  It wraps around to zero.
+            // (We don't have this issue for TOTFREQ_O1_FAST.)
+            //
+            // Solution 1 is to change to spec to forbid freq of 4096.
+            // Easy hack is to add an extra symbol so it sums correctly.
+            // => 572 MB/s on q40 (deskpro).
+            //
+            // Solution 2 implemented here is to look for the wrap around
+            // and fix it.
+            // => 556 MB/s on q40
+            // cope with max freq of 4096.  Only 3% hit
+            __m512i max_freq = _mm512_set1_epi32(TOTFREQ_O1);
+            __m512i zero = _mm512_setzero_si512();
+            __mmask16 cmp1 = _mm512_cmpeq_epi32_mask(_fv1, zero);
+            __mmask16 cmp2 = _mm512_cmpeq_epi32_mask(_fv2, zero);
+            _fv1 = _mm512_mask_blend_epi32(cmp1, _fv1, max_freq);
+            _fv2 = _mm512_mask_blend_epi32(cmp2, _fv2, max_freq);
+
+            //  R[z] = f[z] * (R[z] >> TF_SHIFT_O1) + b[z];
+            _Rv1 = _mm512_add_epi32(
+                       _mm512_mullo_epi32(
+                           _mm512_srli_epi32(_Rv1,TF_SHIFT_O1), _fv1), _bv1);
+            _Rv2 = _mm512_add_epi32(
+                       _mm512_mullo_epi32(
+                           _mm512_srli_epi32(_Rv2,TF_SHIFT_O1), _fv2), _bv2);
+
+            //for (z = 0; z < NX; z++) lN[z] = c[z];
+            _Lv1 = _sv1;
+            _Lv2 = _sv2;
+
+            // RansDecRenorm(&R[z], &ptr);
+            __m512i _renorm_mask1 = _mm512_xor_si512(_Rv1,
+                                        _mm512_set1_epi32(0x80000000));
+            __m512i _renorm_mask2 = _mm512_xor_si512(_Rv2,
+                                        _mm512_set1_epi32(0x80000000));
+
+            int _imask1 =_mm512_cmpgt_epi32_mask
+                (_mm512_set1_epi32(RANS_BYTE_L-0x80000000), _renorm_mask1);
+            int _imask2 = _mm512_cmpgt_epi32_mask
+                (_mm512_set1_epi32(RANS_BYTE_L-0x80000000), _renorm_mask2);
+
+            __m512i renorm_words1 = _mm512_cvtepu16_epi32
+                (_mm256_loadu_si256((const __m256i *)sp));
+            sp += _mm_popcnt_u32(_imask1);
+
+            __m512i renorm_words2 = _mm512_cvtepu16_epi32
+                (_mm256_loadu_si256((const __m256i *)sp));
+            sp += _mm_popcnt_u32(_imask2);
+
+            __m512i _renorm_vals1 =
+                _mm512_maskz_expand_epi32(_imask1, renorm_words1);
+            __m512i _renorm_vals2 =
+                _mm512_maskz_expand_epi32(_imask2, renorm_words2);
+
+            _Rv1 = _mm512_mask_slli_epi32(_Rv1, _imask1, _Rv1, 16);
+            _Rv2 = _mm512_mask_slli_epi32(_Rv2, _imask2, _Rv2, 16);
+
+            _Rv1 = _mm512_add_epi32(_Rv1, _renorm_vals1);
+            _Rv2 = _mm512_add_epi32(_Rv2, _renorm_vals2);
+
+#ifdef TBUF8
+            _mm_storeu_si128((__m128i *)(&u.tbuf64[tidx][0]),
+                             _mm512_cvtepi32_epi8(_Sv1)); // or _sv1?
+            _mm_storeu_si128((__m128i *)(&u.tbuf64[tidx][2]),
+                             _mm512_cvtepi32_epi8(_Sv2));
+#else
+            _mm512_storeu_si512((__m512i *)(&tbuf[tidx][ 0]), _sv1);
+            _mm512_storeu_si512((__m512i *)(&tbuf[tidx][16]), _sv2);
+#endif
+
+            iN[0]++;
+            if (++tidx == 32) {
+                iN[0]-=32;
+
+                // We have tidx[x][y] which we want to store in
+                // memory in out[y][z] instead.  This is an unrolled
+                // transposition.
+#ifdef TBUF8
+                transpose_and_copy(out, iN, u.tbuf);
+#else
+                transpose_and_copy_avx512(out, iN, tbuf);
+#endif
+                tidx = 0;
+            }
+        }
+        isz4 += 64;
+
+        STORE512(_Rv, R);
+        STORE512(_Lv, lN);
+        ptr = (uint8_t *)sp;
+
+        if (1) {
+            iN[0]-=tidx;
+            int T;
+            for (z = 0; z < NX; z++)
+                for (T = 0; T < tidx; T++)
+#ifdef TBUF8
+                    out[iN[z]++] = u.tbuf[T][z];
+#else
+                    out[iN[z]++] = tbuf[T][z];
+#endif
+        }
+
+        // Scalar version for close to the end of in[] array so we don't
+        // do SIMD loads beyond the end of the buffer
+        for (; iN[0] < isz4;) {
+            for (z = 0; z < NX; z++) {
+                uint32_t m = R[z] & ((1u<<TF_SHIFT_O1)-1);
+                uint32_t S = s3[lN[z]][m];
+                unsigned char c = S & 0xff;
+                out[iN[z]++] = c;
+                uint32_t F = S>>(TF_SHIFT_O1+8);
+                R[z] = (F?F:4096) * (R[z]>>TF_SHIFT_O1) +
+                    ((S>>8) & ((1u<<TF_SHIFT_O1)-1));
+                RansDecRenormSafe(&R[z], &ptr, ptr_end);
+                lN[z] = c;
+            }
+        }
+
+        // Remainder
+        z = NX-1;
+        for (; iN[z] < out_sz; ) {
+            uint32_t m = R[z] & ((1u<<TF_SHIFT_O1)-1);
+            uint32_t S = s3[lN[z]][m];
+            unsigned char c = S & 0xff;
+            out[iN[z]++] = c;
+            uint32_t F = S>>(TF_SHIFT_O1+8);
+            R[z] = (F?F:4096) * (R[z]>>TF_SHIFT_O1) +
+                ((S>>8) & ((1u<<TF_SHIFT_O1)-1));
+            RansDecRenormSafe(&R[z], &ptr, ptr_end);
+            lN[z] = c;
+        }
+    } else {
+        // TF_SHIFT_O1_FAST.  This is the most commonly used variant.
+
+        // SIMD version ends decoding early as it reads at most 64 bytes
+        // from input via 4 vectorised loads.
+        isz4 -= 64;
+        for (; iN[0] < isz4 && (uint8_t *)sp+64 < ptr_end; ) {
+            // m[z] = R[z] & mask;
+            __m512i _masked1 = _mm512_and_si512(_Rv1, _maskv);
+            __m512i _masked2 = _mm512_and_si512(_Rv2, _maskv);
+
+            //  S[z] = s3[lN[z]][m[z]];
+            _Lv1 = _mm512_slli_epi32(_Lv1, TF_SHIFT_O1_FAST);
+            _Lv2 = _mm512_slli_epi32(_Lv2, TF_SHIFT_O1_FAST);
+
+            _masked1 = _mm512_add_epi32(_masked1, _Lv1);
+            _masked2 = _mm512_add_epi32(_masked2, _Lv2);
+
+            // This is the biggest bottleneck
+            __m512i _Sv1 = _mm512_i32gather_epi32x(_masked1, (int *)&s3F[0][0],
+                                                   sizeof(s3F[0][0]));
+            __m512i _Sv2 = _mm512_i32gather_epi32x(_masked2, (int *)&s3F[0][0],
+                                                   sizeof(s3F[0][0]));
+
+            //  f[z] = S[z]>>(TF_SHIFT_O1+8);
+            __m512i _fv1 = _mm512_srli_epi32(_Sv1, TF_SHIFT_O1_FAST+8);
+            __m512i _fv2 = _mm512_srli_epi32(_Sv2, TF_SHIFT_O1_FAST+8);
+
+            //  b[z] = (S[z]>>8) & mask;
+            __m512i _bv1 = _mm512_and_si512(_mm512_srli_epi32(_Sv1,8), _maskv);
+            __m512i _bv2 = _mm512_and_si512(_mm512_srli_epi32(_Sv2,8), _maskv);
+
+            //  s[z] = S[z] & 0xff;
+            __m512i _sv1 = _mm512_and_si512(_Sv1, _mm512_set1_epi32(0xff));
+            __m512i _sv2 = _mm512_and_si512(_Sv2, _mm512_set1_epi32(0xff));
+
+            //  R[z] = f[z] * (R[z] >> TF_SHIFT_O1) + b[z];
+            _Rv1 = _mm512_add_epi32(
+                       _mm512_mullo_epi32(
+                           _mm512_srli_epi32(_Rv1,TF_SHIFT_O1_FAST),
+                           _fv1), _bv1);
+            _Rv2 = _mm512_add_epi32(
+                       _mm512_mullo_epi32(
+                           _mm512_srli_epi32(_Rv2,TF_SHIFT_O1_FAST),
+                           _fv2), _bv2);
+
+            //for (z = 0; z < NX; z++) lN[z] = c[z];
+            _Lv1 = _sv1;
+            _Lv2 = _sv2;
+
+            // RansDecRenorm(&R[z], &ptr);
+            __m512i _renorm_mask1 = _mm512_xor_si512(_Rv1,
+                                        _mm512_set1_epi32(0x80000000));
+            __m512i _renorm_mask2 = _mm512_xor_si512(_Rv2,
+                                        _mm512_set1_epi32(0x80000000));
+
+            int _imask1 =_mm512_cmpgt_epi32_mask
+                (_mm512_set1_epi32(RANS_BYTE_L-0x80000000), _renorm_mask1);
+            int _imask2 = _mm512_cmpgt_epi32_mask
+                (_mm512_set1_epi32(RANS_BYTE_L-0x80000000), _renorm_mask2);
+
+            __m512i renorm_words1 = _mm512_cvtepu16_epi32
+                (_mm256_loadu_si256((const __m256i *)sp));
+            sp += _mm_popcnt_u32(_imask1);
+
+            __m512i renorm_words2 = _mm512_cvtepu16_epi32
+                (_mm256_loadu_si256((const __m256i *)sp));
+            sp += _mm_popcnt_u32(_imask2);
+
+            __m512i _renorm_vals1 =
+                _mm512_maskz_expand_epi32(_imask1, renorm_words1);
+            __m512i _renorm_vals2 =
+                _mm512_maskz_expand_epi32(_imask2, renorm_words2);
+
+            _Rv1 = _mm512_mask_slli_epi32(_Rv1, _imask1, _Rv1, 16);
+            _Rv2 = _mm512_mask_slli_epi32(_Rv2, _imask2, _Rv2, 16);
+
+            _Rv1 = _mm512_add_epi32(_Rv1, _renorm_vals1);
+            _Rv2 = _mm512_add_epi32(_Rv2, _renorm_vals2);
+
+#ifdef TBUF8
+            _mm_storeu_si128((__m128i *)(&u.tbuf64[tidx][0]),
+                             _mm512_cvtepi32_epi8(_Sv1)); // or _sv1?
+            _mm_storeu_si128((__m128i *)(&u.tbuf64[tidx][2]),
+                             _mm512_cvtepi32_epi8(_Sv2));
+#else
+            _mm512_storeu_si512((__m512i *)(&tbuf[tidx][ 0]), _sv1);
+            _mm512_storeu_si512((__m512i *)(&tbuf[tidx][16]), _sv2);
+#endif
+
+            iN[0]++;
+            if (++tidx == 32) {
+                iN[0]-=32;
+#ifdef TBUF8
+                transpose_and_copy(out, iN, u.tbuf);
+#else
+                transpose_and_copy_avx512(out, iN, tbuf);
+#endif
+                tidx = 0;
+            }
+        }
+        isz4 += 64;
+
+        STORE512(_Rv, R);
+        STORE512(_Lv, lN);
+        ptr = (uint8_t *)sp;
+
+        if (1) {
+            iN[0]-=tidx;
+            int T;
+            for (z = 0; z < NX; z++)
+                for (T = 0; T < tidx; T++)
+#ifdef TBUF8
+                    out[iN[z]++] = u.tbuf[T][z];
+#else
+                    out[iN[z]++] = tbuf[T][z];
+#endif
+        }
+
+        // Scalar version for close to the end of in[] array so we don't
+        // do SIMD loads beyond the end of the buffer
+        for (; iN[0] < isz4;) {
+            for (z = 0; z < NX; z++) {
+                uint32_t m = R[z] & ((1u<<TF_SHIFT_O1_FAST)-1);
+                uint32_t S = s3F[lN[z]][m];
+                unsigned char c = S & 0xff;
+                out[iN[z]++] = c;
+                R[z] = (S>>(TF_SHIFT_O1_FAST+8)) * (R[z]>>TF_SHIFT_O1_FAST) +
+                    ((S>>8) & ((1u<<TF_SHIFT_O1_FAST)-1));
+                RansDecRenormSafe(&R[z], &ptr, ptr_end);
+                lN[z] = c;
+            }
+        }
+
+        // Remainder
+        z = NX-1;
+        for (; iN[z] < out_sz; ) {
+            uint32_t m = R[z] & ((1u<<TF_SHIFT_O1_FAST)-1);
+            uint32_t S = s3F[lN[z]][m];
+            unsigned char c = S & 0xff;
+            out[iN[z]++] = c;
+            R[z] = (S>>(TF_SHIFT_O1_FAST+8)) * (R[z]>>TF_SHIFT_O1_FAST) +
+                ((S>>8) & ((1u<<TF_SHIFT_O1_FAST)-1));
+            RansDecRenormSafe(&R[z], &ptr, ptr_end);
+            lN[z] = c;
+        }
+    }
+
+    htscodecs_tls_free(s3);
+    return out;
+
+ err:
+    htscodecs_tls_free(s3);
+    free(out_free);
+    free(c_freq);
+
+    return NULL;
+}
+#else  // HAVE_AVX512
+// Prevent "empty translation unit" errors when building without AVX512
+const char *rANS_static32x16pr_avx512_disabled = "No AVX512";
+#endif // HAVE_AVX512
diff --git a/ext/htslib/htscodecs/htscodecs/rANS_static32x16pr_neon.c b/ext/htslib/htscodecs/htscodecs/rANS_static32x16pr_neon.c
new file mode 100644
index 0000000..bb6b268
--- /dev/null
+++ b/ext/htslib/htscodecs/htscodecs/rANS_static32x16pr_neon.c
@@ -0,0 +1,1962 @@
+/*
+ * Copyright (c) 2017-2023 Genome Research Ltd.
+ * Author(s): James Bonfield
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above
+ *       copyright notice, this list of conditions and the following
+ *       disclaimer in the documentation and/or other materials provided
+ *       with the distribution.
+ *
+ *    3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+ *       Institute nor the names of its contributors may be used to endorse
+ *       or promote products derived from this software without specific
+ *       prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH
+ * LTD OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+#if defined(__ARM_NEON) && defined(__aarch64__)
+#include <arm_neon.h>
+
+#include <limits.h>
+#include <assert.h>
+
+#include "rANS_word.h"
+#include "rANS_static4x16.h"
+#include "rANS_static16_int.h"
+#include "varint.h"
+#include "utils.h"
+
+#define NX 32
+
+// TODO: get access to MVE architecture so we can tune for the newer
+// SIMD instructions.
+//
+// #if __ARM_FEATURE_MVE & 1
+// #include <arm_mve.h> // Helium, eg for use of vcreateq_u32
+// #endif
+
+#define _ 99
+static uint8x8_t vtab[16] = {
+    {_,_,  _,_,  _,_,  _,_ },
+    {_,_,  _,_,  _,_, 12,13},
+    {_,_,  _,_,  _,_,  8,9 },
+    {_,_,  _,_,  8,9, 12,13},
+    {_,_,  _,_,  _,_,  4,5 },
+    {_,_,  _,_,  4,5, 12,13},
+    {_,_,  _,_,  4,5,  8,9 },
+    {_,_,  4,5,  8,9, 12,13},
+    {_,_,  _,_,  _,_,  0,1 },
+    {_,_,  _,_,  0,1, 12,13},
+    {_,_,  _,_,  0,1,  8,9 },
+    {_,_,  0,1 , 8,9, 12,13},
+    {_,_,  _,_,  0,1,  4,5 },
+    {_,_,  0,1,  4,5, 12,13},
+    {_,_,  0,1,  4,5,  8,9 },
+    {0,1,  4,5,  8,9, 12,13},
+};
+#undef _
+
+unsigned char *rans_compress_O0_32x16_neon(unsigned char *in,
+                                           unsigned int in_size,
+                                           unsigned char *out,
+                                           unsigned int *out_size) {
+    unsigned char *cp, *out_end;
+    RansEncSymbol syms[256];
+    RansState R[NX];
+    uint8_t* ptr;
+    uint32_t F[256+MAGIC] = {0};
+    int i, j, tab_size = 0, x, z;
+    // -20 for order/size/meta
+    uint32_t bound = rans_compress_bound_4x16(in_size,0)-20;
+
+    if (!out) {
+        *out_size = bound;
+        out = malloc(*out_size);
+    }
+    if (!out || bound > *out_size)
+        return NULL;
+
+    // If "out" isn't word aligned, tweak out_end/ptr to ensure it is.
+    // We already added more round in bound to allow for this.
+    if (((size_t)out)&1)
+        bound--;
+    ptr = out_end = out + bound;
+
+    if (in_size == 0)
+        goto empty;
+
+    // Compute statistics
+    if (hist8(in, in_size, F) < 0)
+        return NULL;
+
+    // Normalise so frequences sum to power of 2
+    uint32_t fsum = in_size;
+    uint32_t max_val = round2(fsum);
+    if (max_val > TOTFREQ)
+        max_val = TOTFREQ;
+
+    if (normalise_freq(F, fsum, max_val) < 0)
+        return NULL;
+    fsum=max_val;
+
+    cp = out;
+    cp += encode_freq(cp, F);
+    tab_size = cp-out;
+    //write(2, out+4, cp-(out+4));
+
+    if (normalise_freq(F, fsum, TOTFREQ) < 0)
+        return NULL;
+
+    // Encode statistics.
+    for (x = j = 0; j < 256; j++) {
+        if (F[j]) {
+            RansEncSymbolInit(&syms[j], x, F[j], TF_SHIFT);
+            x += F[j];
+        }
+    }
+
+    for (z = 0; z < NX; z++)
+      RansEncInit(&R[z]);
+
+    z = i = in_size&(NX-1);
+    while (z-- > 0)
+      RansEncPutSymbol(&R[z], &ptr, &syms[in[in_size-(i-z)]]);
+
+    for (i=(in_size &~(NX-1)); i>0; i-=NX) {
+//      // Scalar equivalent
+//      for (z = NX-1; z >= 0; z-=4) {
+//          // 327 / 272
+//          RansEncSymbol *s0 = &syms[in[i-(NX-z+0)]];
+//          RansEncSymbol *s1 = &syms[in[i-(NX-z+1)]];
+//          RansEncSymbol *s2 = &syms[in[i-(NX-z+2)]];
+//          RansEncSymbol *s3 = &syms[in[i-(NX-z+3)]];
+//
+//          RansEncPutSymbol(&R[z-0], &ptr, s0);
+//          RansEncPutSymbol(&R[z-1], &ptr, s1);
+//          RansEncPutSymbol(&R[z-2], &ptr, s2);
+//          RansEncPutSymbol(&R[z-3], &ptr, s3);
+//      }
+
+        // SIMD with 16-way unrolling
+        for (z = NX-1; z >= 0; z-=8) {
+            RansEncSymbol *s0 = &syms[in[i-(NX-z+0)]];
+            RansEncSymbol *s1 = &syms[in[i-(NX-z+1)]];
+            RansEncSymbol *s2 = &syms[in[i-(NX-z+2)]];
+            RansEncSymbol *s3 = &syms[in[i-(NX-z+3)]];
+
+            RansEncSymbol *s4 = &syms[in[i-(NX-z+4)]];
+            RansEncSymbol *s5 = &syms[in[i-(NX-z+5)]];
+            RansEncSymbol *s6 = &syms[in[i-(NX-z+6)]];
+            RansEncSymbol *s7 = &syms[in[i-(NX-z+7)]];
+
+            uint32x4_t Rv1 = vld1q_u32(&R[z-3]);
+            uint32x4_t Rv2 = vld1q_u32(&R[z-7]);
+
+            // Sym bit sizes = 128bits
+            // 32: x_max
+            // 32: rcp_freq
+            // 32: bias
+            // 16: cmpl_freq
+            // 16: rcp_shift
+
+            // Load and shuffle around
+            //   A <---Xmax---><---RFreq--><---Bias---><-cf-><-rs->
+            //   B <---Xmax---><---RFreq--><---Bias---><-cf-><-rs->
+            //   C <---Xmax---><---RFreq--><---Bias---><-cf-><-rs->
+            //   D <---Xmax---><---RFreq--><---Bias---><-cf-><-rs->
+            // vtrn1q_u32 vtrn2q_u32  (A1 = A+B)
+            //   A1  <---Xmax---><---Xmax---><---Bias---><---Bias--->
+            //   C1  <---Xmax---><---Xmax---><---Bias---><---Bias--->
+            //   A2  <---RFreq--><---RFreq--><-cf-><-rs-><-cf-><-rs->
+            //   C2  <---RFreq--><---RFreq--><-cf-><-rs-><-cf-><-rs->
+            // vtrn1q_u64 vtrn2q_u64  (A11 = A1+C1)
+            //   A11 <---Xmax---><---Xmax---><---Xmax---><---Xmax--->
+            //   A12 <---Bias---><---Bias---><---Bias---><---Bias--->
+            //   A21 <---RFreq--><---RFreq--><---RFreq--><---RFreq-->
+            //   A22 <-cf-><-rs-><-cf-><-rs-><-cf-><-rs-><-cf-><-rs->
+            uint32x4_t A_1 = vld1q_u32((void *)s3);
+            uint32x4_t B_1 = vld1q_u32((void *)s2);
+            uint32x4_t C_1 = vld1q_u32((void *)s1);
+            uint32x4_t D_1 = vld1q_u32((void *)s0);
+
+            uint32x4_t A1_1 = vtrn1q_u32(A_1, B_1);
+            uint32x4_t C1_1 = vtrn1q_u32(C_1, D_1);
+            uint32x4_t A2_1 = vtrn2q_u32(A_1, B_1);
+            uint32x4_t C2_1 = vtrn2q_u32(C_1, D_1);
+
+#define u32_u64(x) vreinterpretq_u32_u64((x))
+#define u64_u32(x) vreinterpretq_u64_u32((x))
+            uint32x4_t Xmaxv1=u32_u64(vtrn1q_u64(u64_u32(A1_1),u64_u32(C1_1)));
+            uint32x4_t Biasv1=u32_u64(vtrn2q_u64(u64_u32(A1_1),u64_u32(C1_1)));
+            uint32x4_t RFv1  =u32_u64(vtrn1q_u64(u64_u32(A2_1),u64_u32(C2_1)));
+            uint32x4_t FSv1  =u32_u64(vtrn2q_u64(u64_u32(A2_1),u64_u32(C2_1)));
+
+            uint32x4_t A_2 = vld1q_u32((void *)s7);
+            uint32x4_t B_2 = vld1q_u32((void *)s6);
+            uint32x4_t C_2 = vld1q_u32((void *)s5);
+            uint32x4_t D_2 = vld1q_u32((void *)s4);
+
+            uint32x4_t A1_2 = vtrn1q_u32(A_2, B_2);
+            uint32x4_t C1_2 = vtrn1q_u32(C_2, D_2);
+            uint32x4_t A2_2 = vtrn2q_u32(A_2, B_2);
+            uint32x4_t C2_2 = vtrn2q_u32(C_2, D_2);
+
+            uint32x4_t Xmaxv2=u32_u64(vtrn1q_u64(u64_u32(A1_2),u64_u32(C1_2)));
+            uint32x4_t Biasv2=u32_u64(vtrn2q_u64(u64_u32(A1_2),u64_u32(C1_2)));
+            uint32x4_t RFv2  =u32_u64(vtrn1q_u64(u64_u32(A2_2),u64_u32(C2_2)));
+            uint32x4_t FSv2  =u32_u64(vtrn2q_u64(u64_u32(A2_2),u64_u32(C2_2)));
+            
+            // Turn multi R<xmax checks into a bit-field (imask)
+            uint32x4_t Cv1 = vcgtq_u32(Rv1, Xmaxv1);
+            uint32x4_t Cv2 = vcgtq_u32(Rv2, Xmaxv2);
+
+            uint32x4_t bit = {8,4,2,1};
+            uint32_t imask1 = vaddvq_u32(vandq_u32(Cv1, bit));
+            uint32_t imask2 = vaddvq_u32(vandq_u32(Cv2, bit));
+
+            // Select low 16-bits from Rv based on imask, using tbl
+            uint8x8_t norm1, norm2;
+            norm1 = vqtbl1_u8(vreinterpretq_u8_u32(Rv1),vtab[imask1]);
+            norm2 = vqtbl1_u8(vreinterpretq_u8_u32(Rv2),vtab[imask2]);
+
+            static int nbits[16] = { 0,2,2,4, 2,4,4,6, 2,4,4,6, 4,6,6,8 };
+            vst1_u8(ptr-8, norm1);   ptr -= nbits[imask1];
+            vst1_u8(ptr-8, norm2);   ptr -= nbits[imask2];
+
+            // R' = R>>16
+            uint32x4_t Rv1_r = vshrq_n_u32(Rv1, 16);
+            uint32x4_t Rv2_r = vshrq_n_u32(Rv2, 16);
+
+            // Blend R and R' based on Cv.
+            Rv1 = vbslq_u32(Cv1, Rv1_r, Rv1);
+            Rv2 = vbslq_u32(Cv2, Rv2_r, Rv2);
+
+            // R -> R' update
+            //   q = (uint32_t) (((uint64_t)x * rcp_freq) >> rcp_shift);
+            //   R' = R + sym->bias + q * sym->cmpl_freq;
+
+            // Mix SIMD (mul) & scalar (shift). 365MB/s
+
+            // We do 32 x 32 mul to get 64-bit, but then extract this
+            // a 64-bit quantity and shift as scalar, before
+            // recreating the 32x4 result.  Despite SIMD-scalar-SIMD reg
+            // it's slightly quicker.
+
+            uint64x2_t qvl1 = vmull_u32(vget_low_u32(Rv1), vget_low_u32(RFv1));
+            uint64x2_t qvh1 = vmull_high_u32(Rv1, RFv1);
+
+            uint64x2_t qvl2 = vmull_u32(vget_low_u32(Rv2), vget_low_u32(RFv2));
+            uint64x2_t qvh2 = vmull_high_u32(Rv2, RFv2);
+
+            uint32x2_t qv1a =
+                 vcreate_u32(vgetq_lane_u64(qvl1, 1) >> s2->rcp_shift << 32 |
+                             vgetq_lane_u64(qvl1, 0) >> s3->rcp_shift);
+            uint32x2_t qv1b =
+                 vcreate_u32(vgetq_lane_u64(qvh1, 1) >> s0->rcp_shift << 32 |
+                             vgetq_lane_u64(qvh1, 0) >> s1->rcp_shift);
+
+            uint32x2_t qv2a =
+                 vcreate_u32(vgetq_lane_u64(qvl2, 1) >> s6->rcp_shift << 32 |
+                             vgetq_lane_u64(qvl2, 0) >> s7->rcp_shift);
+            uint32x2_t qv2b =
+                 vcreate_u32(vgetq_lane_u64(qvh2, 1) >> s4->rcp_shift << 32 |
+                             vgetq_lane_u64(qvh2, 0) >> s5->rcp_shift);
+
+            uint32x4_t qv1 = vcombine_u32(qv1a, qv1b);
+            uint32x4_t qv2 = vcombine_u32(qv2a, qv2b);
+                
+            FSv1 = vandq_u32(FSv1, vdupq_n_u32(0xffff)); // cmpl_freq
+            FSv2 = vandq_u32(FSv2, vdupq_n_u32(0xffff));
+        
+            qv1 = vmlaq_u32(Biasv1, qv1, FSv1);
+            qv2 = vmlaq_u32(Biasv2, qv2, FSv2);
+
+            Rv1 = vaddq_u32(Rv1, qv1);
+            Rv2 = vaddq_u32(Rv2, qv2);
+
+            vst1q_u32(&R[z-3], Rv1);
+            vst1q_u32(&R[z-7], Rv2);
+        }
+        if (z < -1) abort();
+    }
+    for (z = NX-1; z >= 0; z--)
+      RansEncFlush(&R[z], &ptr);
+
+ empty:
+    // Finalise block size and return it
+    *out_size = (out_end - ptr) + tab_size;
+
+    memmove(out + tab_size, ptr, out_end-ptr);
+
+    return out;
+}
+
+#define _ 99
+static uint8x8_t idx[16] = {
+    { _,_,_,_,_,_,_,_ }, // 0000
+    { _,_,_,_,_,_,0,1 }, // 0001
+    { _,_,_,_,0,1,_,_ }, // 0010
+    { _,_,_,_,0,1,2,3 }, // 0011
+
+    { _,_,0,1,_,_,_,_ }, // 0100
+    { _,_,0,1,_,_,2,3 }, // 0101
+    { _,_,0,1,2,3,_,_ }, // 0110
+    { _,_,0,1,2,3,4,5 }, // 0111
+
+    { 0,1,_,_,_,_,_,_ }, // 1000
+    { 0,1,_,_,_,_,2,3 }, // 1001
+    { 0,1,_,_,2,3,_,_ }, // 1010
+    { 0,1,_,_,2,3,4,5 }, // 1011
+
+    { 0,1,2,3,_,_,_,_ }, // 1100
+    { 0,1,2,3,_,_,4,5 }, // 1101
+    { 0,1,2,3,4,5,_,_ }, // 1110
+    { 0,1,2,3,4,5,6,7 }, // 1111
+};
+
+// norm2 with norm1 in top 4 bits already consumed
+static uint8x8_t idx2[256] = {
+    { _, _, _, _, _, _, _, _, },
+    { _, _, _, _, _, _, 0, 1, },
+    { _, _, _, _, 0, 1, _, _, },
+    { _, _, _, _, 0, 1, 2, 3, },
+    { _, _, 0, 1, _, _, _, _, },
+    { _, _, 0, 1, _, _, 2, 3, },
+    { _, _, 0, 1, 2, 3, _, _, },
+    { _, _, 0, 1, 2, 3, 4, 5, },
+    { 0, 1, _, _, _, _, _, _, },
+    { 0, 1, _, _, _, _, 2, 3, },
+    { 0, 1, _, _, 2, 3, _, _, },
+    { 0, 1, _, _, 2, 3, 4, 5, },
+    { 0, 1, 2, 3, _, _, _, _, },
+    { 0, 1, 2, 3, _, _, 4, 5, },
+    { 0, 1, 2, 3, 4, 5, _, _, },
+    { 0, 1, 2, 3, 4, 5, 6, 7, },
+    { _, _, _, _, _, _, _, _, },
+    { _, _, _, _, _, _, 2, 3, },
+    { _, _, _, _, 2, 3, _, _, },
+    { _, _, _, _, 2, 3, 4, 5, },
+    { _, _, 2, 3, _, _, _, _, },
+    { _, _, 2, 3, _, _, 4, 5, },
+    { _, _, 2, 3, 4, 5, _, _, },
+    { _, _, 2, 3, 4, 5, 6, 7, },
+    { 2, 3, _, _, _, _, _, _, },
+    { 2, 3, _, _, _, _, 4, 5, },
+    { 2, 3, _, _, 4, 5, _, _, },
+    { 2, 3, _, _, 4, 5, 6, 7, },
+    { 2, 3, 4, 5, _, _, _, _, },
+    { 2, 3, 4, 5, _, _, 6, 7, },
+    { 2, 3, 4, 5, 6, 7, _, _, },
+    { 2, 3, 4, 5, 6, 7, 8, 9, },
+    { _, _, _, _, _, _, _, _, },
+    { _, _, _, _, _, _, 2, 3, },
+    { _, _, _, _, 2, 3, _, _, },
+    { _, _, _, _, 2, 3, 4, 5, },
+    { _, _, 2, 3, _, _, _, _, },
+    { _, _, 2, 3, _, _, 4, 5, },
+    { _, _, 2, 3, 4, 5, _, _, },
+    { _, _, 2, 3, 4, 5, 6, 7, },
+    { 2, 3, _, _, _, _, _, _, },
+    { 2, 3, _, _, _, _, 4, 5, },
+    { 2, 3, _, _, 4, 5, _, _, },
+    { 2, 3, _, _, 4, 5, 6, 7, },
+    { 2, 3, 4, 5, _, _, _, _, },
+    { 2, 3, 4, 5, _, _, 6, 7, },
+    { 2, 3, 4, 5, 6, 7, _, _, },
+    { 2, 3, 4, 5, 6, 7, 8, 9, },
+    { _, _, _, _, _, _, _, _, },
+    { _, _, _, _, _, _, 4, 5, },
+    { _, _, _, _, 4, 5, _, _, },
+    { _, _, _, _, 4, 5, 6, 7, },
+    { _, _, 4, 5, _, _, _, _, },
+    { _, _, 4, 5, _, _, 6, 7, },
+    { _, _, 4, 5, 6, 7, _, _, },
+    { _, _, 4, 5, 6, 7, 8, 9, },
+    { 4, 5, _, _, _, _, _, _, },
+    { 4, 5, _, _, _, _, 6, 7, },
+    { 4, 5, _, _, 6, 7, _, _, },
+    { 4, 5, _, _, 6, 7, 8, 9, },
+    { 4, 5, 6, 7, _, _, _, _, },
+    { 4, 5, 6, 7, _, _, 8, 9, },
+    { 4, 5, 6, 7, 8, 9, _, _, },
+    { 4, 5, 6, 7, 8, 9,10,11, },
+    { _, _, _, _, _, _, _, _, },
+    { _, _, _, _, _, _, 2, 3, },
+    { _, _, _, _, 2, 3, _, _, },
+    { _, _, _, _, 2, 3, 4, 5, },
+    { _, _, 2, 3, _, _, _, _, },
+    { _, _, 2, 3, _, _, 4, 5, },
+    { _, _, 2, 3, 4, 5, _, _, },
+    { _, _, 2, 3, 4, 5, 6, 7, },
+    { 2, 3, _, _, _, _, _, _, },
+    { 2, 3, _, _, _, _, 4, 5, },
+    { 2, 3, _, _, 4, 5, _, _, },
+    { 2, 3, _, _, 4, 5, 6, 7, },
+    { 2, 3, 4, 5, _, _, _, _, },
+    { 2, 3, 4, 5, _, _, 6, 7, },
+    { 2, 3, 4, 5, 6, 7, _, _, },
+    { 2, 3, 4, 5, 6, 7, 8, 9, },
+    { _, _, _, _, _, _, _, _, },
+    { _, _, _, _, _, _, 4, 5, },
+    { _, _, _, _, 4, 5, _, _, },
+    { _, _, _, _, 4, 5, 6, 7, },
+    { _, _, 4, 5, _, _, _, _, },
+    { _, _, 4, 5, _, _, 6, 7, },
+    { _, _, 4, 5, 6, 7, _, _, },
+    { _, _, 4, 5, 6, 7, 8, 9, },
+    { 4, 5, _, _, _, _, _, _, },
+    { 4, 5, _, _, _, _, 6, 7, },
+    { 4, 5, _, _, 6, 7, _, _, },
+    { 4, 5, _, _, 6, 7, 8, 9, },
+    { 4, 5, 6, 7, _, _, _, _, },
+    { 4, 5, 6, 7, _, _, 8, 9, },
+    { 4, 5, 6, 7, 8, 9, _, _, },
+    { 4, 5, 6, 7, 8, 9,10,11, },
+    { _, _, _, _, _, _, _, _, },
+    { _, _, _, _, _, _, 4, 5, },
+    { _, _, _, _, 4, 5, _, _, },
+    { _, _, _, _, 4, 5, 6, 7, },
+    { _, _, 4, 5, _, _, _, _, },
+    { _, _, 4, 5, _, _, 6, 7, },
+    { _, _, 4, 5, 6, 7, _, _, },
+    { _, _, 4, 5, 6, 7, 8, 9, },
+    { 4, 5, _, _, _, _, _, _, },
+    { 4, 5, _, _, _, _, 6, 7, },
+    { 4, 5, _, _, 6, 7, _, _, },
+    { 4, 5, _, _, 6, 7, 8, 9, },
+    { 4, 5, 6, 7, _, _, _, _, },
+    { 4, 5, 6, 7, _, _, 8, 9, },
+    { 4, 5, 6, 7, 8, 9, _, _, },
+    { 4, 5, 6, 7, 8, 9,10,11, },
+    { _, _, _, _, _, _, _, _, },
+    { _, _, _, _, _, _, 6, 7, },
+    { _, _, _, _, 6, 7, _, _, },
+    { _, _, _, _, 6, 7, 8, 9, },
+    { _, _, 6, 7, _, _, _, _, },
+    { _, _, 6, 7, _, _, 8, 9, },
+    { _, _, 6, 7, 8, 9, _, _, },
+    { _, _, 6, 7, 8, 9,10,11, },
+    { 6, 7, _, _, _, _, _, _, },
+    { 6, 7, _, _, _, _, 8, 9, },
+    { 6, 7, _, _, 8, 9, _, _, },
+    { 6, 7, _, _, 8, 9,10,11, },
+    { 6, 7, 8, 9, _, _, _, _, },
+    { 6, 7, 8, 9, _, _,10,11, },
+    { 6, 7, 8, 9,10,11, _, _, },
+    { 6, 7, 8, 9,10,11,12,13, },
+    { _, _, _, _, _, _, _, _, },
+    { _, _, _, _, _, _, 2, 3, },
+    { _, _, _, _, 2, 3, _, _, },
+    { _, _, _, _, 2, 3, 4, 5, },
+    { _, _, 2, 3, _, _, _, _, },
+    { _, _, 2, 3, _, _, 4, 5, },
+    { _, _, 2, 3, 4, 5, _, _, },
+    { _, _, 2, 3, 4, 5, 6, 7, },
+    { 2, 3, _, _, _, _, _, _, },
+    { 2, 3, _, _, _, _, 4, 5, },
+    { 2, 3, _, _, 4, 5, _, _, },
+    { 2, 3, _, _, 4, 5, 6, 7, },
+    { 2, 3, 4, 5, _, _, _, _, },
+    { 2, 3, 4, 5, _, _, 6, 7, },
+    { 2, 3, 4, 5, 6, 7, _, _, },
+    { 2, 3, 4, 5, 6, 7, 8, 9, },
+    { _, _, _, _, _, _, _, _, },
+    { _, _, _, _, _, _, 4, 5, },
+    { _, _, _, _, 4, 5, _, _, },
+    { _, _, _, _, 4, 5, 6, 7, },
+    { _, _, 4, 5, _, _, _, _, },
+    { _, _, 4, 5, _, _, 6, 7, },
+    { _, _, 4, 5, 6, 7, _, _, },
+    { _, _, 4, 5, 6, 7, 8, 9, },
+    { 4, 5, _, _, _, _, _, _, },
+    { 4, 5, _, _, _, _, 6, 7, },
+    { 4, 5, _, _, 6, 7, _, _, },
+    { 4, 5, _, _, 6, 7, 8, 9, },
+    { 4, 5, 6, 7, _, _, _, _, },
+    { 4, 5, 6, 7, _, _, 8, 9, },
+    { 4, 5, 6, 7, 8, 9, _, _, },
+    { 4, 5, 6, 7, 8, 9,10,11, },
+    { _, _, _, _, _, _, _, _, },
+    { _, _, _, _, _, _, 4, 5, },
+    { _, _, _, _, 4, 5, _, _, },
+    { _, _, _, _, 4, 5, 6, 7, },
+    { _, _, 4, 5, _, _, _, _, },
+    { _, _, 4, 5, _, _, 6, 7, },
+    { _, _, 4, 5, 6, 7, _, _, },
+    { _, _, 4, 5, 6, 7, 8, 9, },
+    { 4, 5, _, _, _, _, _, _, },
+    { 4, 5, _, _, _, _, 6, 7, },
+    { 4, 5, _, _, 6, 7, _, _, },
+    { 4, 5, _, _, 6, 7, 8, 9, },
+    { 4, 5, 6, 7, _, _, _, _, },
+    { 4, 5, 6, 7, _, _, 8, 9, },
+    { 4, 5, 6, 7, 8, 9, _, _, },
+    { 4, 5, 6, 7, 8, 9,10,11, },
+    { _, _, _, _, _, _, _, _, },
+    { _, _, _, _, _, _, 6, 7, },
+    { _, _, _, _, 6, 7, _, _, },
+    { _, _, _, _, 6, 7, 8, 9, },
+    { _, _, 6, 7, _, _, _, _, },
+    { _, _, 6, 7, _, _, 8, 9, },
+    { _, _, 6, 7, 8, 9, _, _, },
+    { _, _, 6, 7, 8, 9,10,11, },
+    { 6, 7, _, _, _, _, _, _, },
+    { 6, 7, _, _, _, _, 8, 9, },
+    { 6, 7, _, _, 8, 9, _, _, },
+    { 6, 7, _, _, 8, 9,10,11, },
+    { 6, 7, 8, 9, _, _, _, _, },
+    { 6, 7, 8, 9, _, _,10,11, },
+    { 6, 7, 8, 9,10,11, _, _, },
+    { 6, 7, 8, 9,10,11,12,13, },
+    { _, _, _, _, _, _, _, _, },
+    { _, _, _, _, _, _, 4, 5, },
+    { _, _, _, _, 4, 5, _, _, },
+    { _, _, _, _, 4, 5, 6, 7, },
+    { _, _, 4, 5, _, _, _, _, },
+    { _, _, 4, 5, _, _, 6, 7, },
+    { _, _, 4, 5, 6, 7, _, _, },
+    { _, _, 4, 5, 6, 7, 8, 9, },
+    { 4, 5, _, _, _, _, _, _, },
+    { 4, 5, _, _, _, _, 6, 7, },
+    { 4, 5, _, _, 6, 7, _, _, },
+    { 4, 5, _, _, 6, 7, 8, 9, },
+    { 4, 5, 6, 7, _, _, _, _, },
+    { 4, 5, 6, 7, _, _, 8, 9, },
+    { 4, 5, 6, 7, 8, 9, _, _, },
+    { 4, 5, 6, 7, 8, 9,10,11, },
+    { _, _, _, _, _, _, _, _, },
+    { _, _, _, _, _, _, 6, 7, },
+    { _, _, _, _, 6, 7, _, _, },
+    { _, _, _, _, 6, 7, 8, 9, },
+    { _, _, 6, 7, _, _, _, _, },
+    { _, _, 6, 7, _, _, 8, 9, },
+    { _, _, 6, 7, 8, 9, _, _, },
+    { _, _, 6, 7, 8, 9,10,11, },
+    { 6, 7, _, _, _, _, _, _, },
+    { 6, 7, _, _, _, _, 8, 9, },
+    { 6, 7, _, _, 8, 9, _, _, },
+    { 6, 7, _, _, 8, 9,10,11, },
+    { 6, 7, 8, 9, _, _, _, _, },
+    { 6, 7, 8, 9, _, _,10,11, },
+    { 6, 7, 8, 9,10,11, _, _, },
+    { 6, 7, 8, 9,10,11,12,13, },
+    { _, _, _, _, _, _, _, _, },
+    { _, _, _, _, _, _, 6, 7, },
+    { _, _, _, _, 6, 7, _, _, },
+    { _, _, _, _, 6, 7, 8, 9, },
+    { _, _, 6, 7, _, _, _, _, },
+    { _, _, 6, 7, _, _, 8, 9, },
+    { _, _, 6, 7, 8, 9, _, _, },
+    { _, _, 6, 7, 8, 9,10,11, },
+    { 6, 7, _, _, _, _, _, _, },
+    { 6, 7, _, _, _, _, 8, 9, },
+    { 6, 7, _, _, 8, 9, _, _, },
+    { 6, 7, _, _, 8, 9,10,11, },
+    { 6, 7, 8, 9, _, _, _, _, },
+    { 6, 7, 8, 9, _, _,10,11, },
+    { 6, 7, 8, 9,10,11, _, _, },
+    { 6, 7, 8, 9,10,11,12,13, },
+    { _, _, _, _, _, _, _, _, },
+    { _, _, _, _, _, _, 8, 9, },
+    { _, _, _, _, 8, 9, _, _, },
+    { _, _, _, _, 8, 9,10,11, },
+    { _, _, 8, 9, _, _, _, _, },
+    { _, _, 8, 9, _, _,10,11, },
+    { _, _, 8, 9,10,11, _, _, },
+    { _, _, 8, 9,10,11,12,13, },
+    { 8, 9, _, _, _, _, _, _, },
+    { 8, 9, _, _, _, _,10,11, },
+    { 8, 9, _, _,10,11, _, _, },
+    { 8, 9, _, _,10,11,12,13, },
+    { 8, 9,10,11, _, _, _, _, },
+    { 8, 9,10,11, _, _,12,13, },
+    { 8, 9,10,11,12,13, _, _, },
+    { 8, 9,10,11,12,13,14,15, },
+};
+
+// SIMD: 650MB/s
+unsigned char *rans_uncompress_O0_32x16_neon(unsigned char *in,
+                                             unsigned int in_size,
+                                             unsigned char *out,
+                                             unsigned int out_sz) {
+    if (in_size < 16) // 4-states at least
+        return NULL;
+
+    if (out_sz >= INT_MAX)
+        return NULL; // protect against some overflow cases
+
+    /* Load in the static tables */
+    unsigned char *cp = in, *out_free = NULL;
+    unsigned char *cp_end = in + in_size;
+    int i;
+    uint32_t s3[TOTFREQ]; // For TF_SHIFT <= 12
+
+    if (!out)
+        out_free = out = malloc(out_sz);
+    if (!out)
+        return NULL;
+
+    // Precompute reverse lookup of frequency.
+    uint32_t F[256] = {0}, fsum;
+    int fsz = decode_freq(cp, cp_end, F, &fsum);
+    if (!fsz)
+        goto err;
+    cp += fsz;
+
+    normalise_freq_shift(F, fsum, TOTFREQ);
+
+    // Build symbols; fixme, do as part of decode, see the _d variant
+    if (rans_F_to_s3(F, TF_SHIFT, s3))
+        goto err;
+
+    if (cp_end - cp < NX * 4)
+        goto err;
+
+    int z;
+    RansState R[NX];
+    for (z = 0; z < NX; z++) {
+        RansDecInit(&R[z], &cp);
+        if (R[z] < RANS_BYTE_L)
+            goto err;
+    }
+
+    int out_end = (out_sz&~(NX-1));
+    const uint32_t mask = (1u << TF_SHIFT)-1;
+    uint32x4_t maskv = vdupq_n_u32((1u << TF_SHIFT)-1);
+
+    // assume NX is divisible by 4
+    assert(NX%4==0);
+
+    uint32x4_t Rv1 = vld1q_u32(&R[0]);
+    uint32x4_t Rv2 = vld1q_u32(&R[4]);
+    uint32x4_t Rv3 = vld1q_u32(&R[8]);
+    uint32x4_t Rv4 = vld1q_u32(&R[12]);
+    uint32x4_t Rv5 = vld1q_u32(&R[16]);
+    uint32x4_t Rv6 = vld1q_u32(&R[20]);
+    uint32x4_t Rv7 = vld1q_u32(&R[24]);
+    uint32x4_t Rv8 = vld1q_u32(&R[28]);
+
+    // Note this has a considerable amount of manual instruction reordering
+    // to avoid latency.  We have 8 lanes of 4 rans states, but process 4
+    // lanes at a time with the two sets of 4-lane steps interleaved to
+    // ensure best use of processor pipeline units and latency removal.
+    //
+    // Unfortunately this is still poor with Clang-10.  Gcc more or less
+    // honours this order and on my test set operates at ~675MB/s decode.
+    // Clang without the manual reordering was at 440MB/s and with it at
+    // 500MB/s.  Clang does a lot of reordering of this code, removing some
+    // of the manual tuning benefits.  Short of dropping to assembly, for now
+    // I would recommend using gcc to compile this file.
+    uint16_t *sp = (uint16_t *)cp;
+    uint8_t overflow[64+64] = {0};
+    for (i=0; i < out_end; i+=NX) {
+        // Decode freq, bias and symbol from s3 lookups
+        uint32x4_t Sv1, Sv2, Sv3, Sv4, Sv5, Sv6, Sv7, Sv8;
+        uint32x4_t Fv1, Fv2, Fv3, Fv4, Fv5, Fv6, Fv7, Fv8;
+        uint32x4_t Bv1, Bv2, Bv3, Bv4, Bv5, Bv6, Bv7, Bv8;
+
+        // Note we could check __ARM_FEATURE_MVE & 1 and use
+        // vcreateq_u32 here, but I don't have a system to test with
+        // so cannot validate if the code works.
+        uint32x2_t s1a, s1b, s2a, s2b, s3a, s3b, s4a, s4b;
+        s1a = vcreate_u32((uint64_t)(s3[R[ 1]&mask])<<32 | (s3[R[ 0]&mask]));
+        s2a = vcreate_u32((uint64_t)(s3[R[ 5]&mask])<<32 | (s3[R[ 4]&mask]));
+        s1b = vcreate_u32((uint64_t)(s3[R[ 3]&mask])<<32 | (s3[R[ 2]&mask]));
+        s2b = vcreate_u32((uint64_t)(s3[R[ 7]&mask])<<32 | (s3[R[ 6]&mask]));
+        s3a = vcreate_u32((uint64_t)(s3[R[ 9]&mask])<<32 | (s3[R[ 8]&mask]));
+        s3b = vcreate_u32((uint64_t)(s3[R[11]&mask])<<32 | (s3[R[10]&mask]));
+        s4a = vcreate_u32((uint64_t)(s3[R[13]&mask])<<32 | (s3[R[12]&mask]));
+        s4b = vcreate_u32((uint64_t)(s3[R[15]&mask])<<32 | (s3[R[14]&mask]));
+
+        Sv1 = vcombine_u32(s1a, s1b);
+        Sv2 = vcombine_u32(s2a, s2b);
+        Sv3 = vcombine_u32(s3a, s3b);
+        Sv4 = vcombine_u32(s4a, s4b);
+
+        Fv1 = vshrq_n_u32(Sv1, TF_SHIFT+8); // Freq = S >> TF_SHIFT+8
+        Fv2 = vshrq_n_u32(Sv2, TF_SHIFT+8);
+        Fv3 = vshrq_n_u32(Sv3, TF_SHIFT+8);
+        Fv4 = vshrq_n_u32(Sv4, TF_SHIFT+8);
+
+        uint32x2_t s5a, s5b, s6a, s6b, s7a, s7b, s8a, s8b;
+        s5a = vcreate_u32((uint64_t)(s3[R[17]&mask])<<32 | (s3[R[16]&mask]));
+        s5b = vcreate_u32((uint64_t)(s3[R[19]&mask])<<32 | (s3[R[18]&mask]));
+        s6a = vcreate_u32((uint64_t)(s3[R[21]&mask])<<32 | (s3[R[20]&mask]));
+        s6b = vcreate_u32((uint64_t)(s3[R[23]&mask])<<32 | (s3[R[22]&mask]));
+        s7a = vcreate_u32((uint64_t)(s3[R[25]&mask])<<32 | (s3[R[24]&mask]));
+        s7b = vcreate_u32((uint64_t)(s3[R[27]&mask])<<32 | (s3[R[26]&mask]));
+        s8a = vcreate_u32((uint64_t)(s3[R[29]&mask])<<32 | (s3[R[28]&mask]));
+        s8b = vcreate_u32((uint64_t)(s3[R[31]&mask])<<32 | (s3[R[30]&mask]));
+
+        Bv1 = vshrq_n_u32(Sv1, 8);          // Bias = (S >> 8)
+        Bv2 = vshrq_n_u32(Sv2, 8);
+        Bv3 = vshrq_n_u32(Sv3, 8);
+        Bv4 = vshrq_n_u32(Sv4, 8);
+
+        // R[0] = (freq * (R[0] >> TF_SHIFT) + bias;
+        Rv1 = vshrq_n_u32(Rv1, TF_SHIFT);   // R >> TF_SHIFT
+        Rv2 = vshrq_n_u32(Rv2, TF_SHIFT);
+        Rv3 = vshrq_n_u32(Rv3, TF_SHIFT);
+        Rv4 = vshrq_n_u32(Rv4, TF_SHIFT);
+
+        Sv5 = vcombine_u32(s5a, s5b);
+        Sv6 = vcombine_u32(s6a, s6b);
+        Sv7 = vcombine_u32(s7a, s7b);
+        Sv8 = vcombine_u32(s8a, s8b);
+
+        Bv1 = vandq_u32(Bv1, maskv);        //      & mask
+        Bv2 = vandq_u32(Bv2, maskv);
+        Bv3 = vandq_u32(Bv3, maskv);
+        Bv4 = vandq_u32(Bv4, maskv);
+
+        Fv5 = vshrq_n_u32(Sv5, TF_SHIFT+8);
+        Fv6 = vshrq_n_u32(Sv6, TF_SHIFT+8);
+        Fv7 = vshrq_n_u32(Sv7, TF_SHIFT+8);
+        Fv8 = vshrq_n_u32(Sv8, TF_SHIFT+8);
+
+        // A mix of mul+add and mla instructions seems to win.
+        //Rv1 = vmulq_u32(Fv1, Rv1); Rv1 = vaddq_u32(Rv1, Bv1);
+        Rv1 = vmlaq_u32(Bv1, Fv1, Rv1);     // R = R*Freq + Bias
+        Rv2 = vmulq_u32(Fv2, Rv2); Rv2 = vaddq_u32(Rv2, Bv2);
+        //Rv2 = vmlaq_u32(Bv2, Fv2, Rv2);
+        //Rv3 = vmulq_u32(Fv3, Rv3); Rv3 = vaddq_u32(Rv3, Bv3);
+        Rv3 = vmlaq_u32(Bv3, Fv3, Rv3);
+        Rv4 = vmulq_u32(Fv4, Rv4); Rv4 = vaddq_u32(Rv4, Bv4);
+        //Rv4 = vmlaq_u32(Bv4, Fv4, Rv4);
+
+        Bv5 = vshrq_n_u32(Sv5, 8);
+        Bv6 = vshrq_n_u32(Sv6, 8);
+        Bv7 = vshrq_n_u32(Sv7, 8);
+        Bv8 = vshrq_n_u32(Sv8, 8);
+
+        // Renorm
+        uint32x4_t Rlt1 = vcltq_u32(Rv1, vdupq_n_u32(RANS_BYTE_L)); // R<L
+        uint32x4_t Rlt2 = vcltq_u32(Rv2, vdupq_n_u32(RANS_BYTE_L));
+        uint32x4_t Rlt3 = vcltq_u32(Rv3, vdupq_n_u32(RANS_BYTE_L));
+        uint32x4_t Rlt4 = vcltq_u32(Rv4, vdupq_n_u32(RANS_BYTE_L));
+
+        Rv5 = vshrq_n_u32(Rv5, TF_SHIFT);
+        Rv6 = vshrq_n_u32(Rv6, TF_SHIFT);
+        Rv7 = vshrq_n_u32(Rv7, TF_SHIFT);
+        Rv8 = vshrq_n_u32(Rv8, TF_SHIFT);
+
+        static int nbits[16] = { 0,1,1,2, 1,2,2,3, 1,2,2,3, 2,3,3,4 };
+
+        // Combine with Rlt & {8,4,2,1} to get single bits.
+        // Sum lanes to git a bit-field indicating which R < L
+        uint32x4_t bit = {8,4,2,1};
+        uint32_t imask1 = vaddvq_u32(vandq_u32(Rlt1, bit));
+        uint32_t imask2 = vaddvq_u32(vandq_u32(Rlt2, bit));
+        uint32_t imask3 = vaddvq_u32(vandq_u32(Rlt3, bit));
+        uint32_t imask4 = vaddvq_u32(vandq_u32(Rlt4, bit));
+
+        // Protect against running off the end of in buffer.
+        // We copy it to a worst-case local buffer when near the end.
+        if ((uint8_t *)sp+64 > cp_end) {
+            memmove(overflow, sp, cp_end - (uint8_t *)sp);
+            sp = (uint16_t *)overflow;
+            cp_end = overflow + sizeof(overflow);
+        }
+
+        uint16x8_t norm12 =  vld1q_u16(sp);
+        sp += nbits[imask1] + nbits[imask2];
+        uint16x8_t norm34 =  vld1q_u16(sp);
+        sp += nbits[imask3] + nbits[imask4];
+
+        Bv5 = vandq_u32(Bv5, maskv);
+        Bv6 = vandq_u32(Bv6, maskv);
+        Bv7 = vandq_u32(Bv7, maskv);
+        Bv8 = vandq_u32(Bv8, maskv);
+
+        // Shuffle norm to the corresponding R lanes, via imask
+        //Rv5 = vmulq_u32(Fv5, Rv5); Rv5 = vaddq_u32(Rv5, Bv5);
+        Rv5 = vmlaq_u32(Bv5, Fv5, Rv5);
+        Rv6 = vmulq_u32(Fv6, Rv6); Rv6 = vaddq_u32(Rv6, Bv6);
+        //Rv6 = vmlaq_u32(Bv6, Fv6, Rv6);
+        //Rv7 = vmulq_u32(Fv7, Rv7); Rv7 = vaddq_u32(Rv7, Bv7);
+        Rv7 = vmlaq_u32(Bv7, Fv7, Rv7);
+        Rv8 = vmulq_u32(Fv8, Rv8); Rv8 = vaddq_u32(Rv8, Bv8);
+        //Rv8 = vmlaq_u32(Bv8, Fv8, Rv8);
+        
+        uint32_t imask12 = (imask1<<4)|imask2;
+        uint32_t imask34 = (imask3<<4)|imask4;
+
+        // #define for brevity and formatting
+#define cast_u16_u8 vreinterpret_u16_u8
+#define cast_u8_u16 vreinterpretq_u8_u16
+        uint16x4_t norm1, norm2, norm3, norm4, norm5, norm6, norm7, norm8;
+        norm1 = cast_u16_u8(vqtbl1_u8(cast_u8_u16(norm12),idx [imask1]));
+        norm2 = cast_u16_u8(vqtbl1_u8(cast_u8_u16(norm12),idx2[imask12]));
+        norm3 = cast_u16_u8(vqtbl1_u8(cast_u8_u16(norm34),idx [imask3]));
+        norm4 = cast_u16_u8(vqtbl1_u8(cast_u8_u16(norm34),idx2[imask34]));
+
+        uint32x4_t Rlt5 = vcltq_u32(Rv5, vdupq_n_u32(RANS_BYTE_L));
+        uint32x4_t Rlt6 = vcltq_u32(Rv6, vdupq_n_u32(RANS_BYTE_L));
+        uint32x4_t Rlt7 = vcltq_u32(Rv7, vdupq_n_u32(RANS_BYTE_L));
+        uint32x4_t Rlt8 = vcltq_u32(Rv8, vdupq_n_u32(RANS_BYTE_L));
+
+        // Add norm to R<<16 (Rsl) and blend back in with R
+        uint32x4_t Rsl1 = vshlq_n_u32(Rv1, 16); // Rsl = R << 16
+        uint32x4_t Rsl2 = vshlq_n_u32(Rv2, 16);
+        uint32x4_t Rsl3 = vshlq_n_u32(Rv3, 16);
+        uint32x4_t Rsl4 = vshlq_n_u32(Rv4, 16);
+
+        uint16x8_t norm56 =  vld1q_u16(sp);
+        uint32_t imask5 = vaddvq_u32(vandq_u32(Rlt5, bit));
+        uint32_t imask6 = vaddvq_u32(vandq_u32(Rlt6, bit));
+        uint32_t imask7 = vaddvq_u32(vandq_u32(Rlt7, bit));
+        uint32_t imask8 = vaddvq_u32(vandq_u32(Rlt8, bit));
+
+        sp += nbits[imask5] + nbits[imask6];
+        uint16x8_t norm78 =  vld1q_u16(sp);
+        sp += nbits[imask7] + nbits[imask8];
+
+        Rsl1 = vaddw_u16(Rsl1, norm1);          // Rsl += norm
+        Rsl2 = vaddw_u16(Rsl2, norm2);
+        Rsl3 = vaddw_u16(Rsl3, norm3);
+        Rsl4 = vaddw_u16(Rsl4, norm4);
+
+        uint32_t imask56 = (imask5<<4)|imask6;
+        uint32_t imask78 = (imask7<<4)|imask8;
+
+        Rv1 = vbslq_u32(Rlt1, Rsl1, Rv1);       // R = R<L ? Rsl : R
+        Rv2 = vbslq_u32(Rlt2, Rsl2, Rv2);
+        Rv3 = vbslq_u32(Rlt3, Rsl3, Rv3);
+        Rv4 = vbslq_u32(Rlt4, Rsl4, Rv4);
+
+        norm5 = cast_u16_u8(vqtbl1_u8(cast_u8_u16(norm56),idx [imask5]));
+        norm6 = cast_u16_u8(vqtbl1_u8(cast_u8_u16(norm56),idx2[imask56]));
+        norm7 = cast_u16_u8(vqtbl1_u8(cast_u8_u16(norm78),idx [imask7]));
+        norm8 = cast_u16_u8(vqtbl1_u8(cast_u8_u16(norm78),idx2[imask78]));
+
+        uint32x4_t Rsl5 = vshlq_n_u32(Rv5, 16);
+        uint32x4_t Rsl6 = vshlq_n_u32(Rv6, 16);
+        uint32x4_t Rsl7 = vshlq_n_u32(Rv7, 16);
+        uint32x4_t Rsl8 = vshlq_n_u32(Rv8, 16);
+
+        Rsl5 = vaddw_u16(Rsl5, norm5);
+        Rsl6 = vaddw_u16(Rsl6, norm6);
+        Rsl7 = vaddw_u16(Rsl7, norm7);
+        Rsl8 = vaddw_u16(Rsl8, norm8);
+
+        // Can we avoid storing this in memory?
+        // It's tricky due to the need to do s3[R[z] & mask] and no
+        // vector gathers.  So we need to have something in memory.
+        vst1q_u32(&R[ 0], Rv1); // store R for Sv creation
+        vst1q_u32(&R[ 4], Rv2);
+        vst1q_u32(&R[ 8], Rv3);
+        vst1q_u32(&R[12], Rv4);
+
+        Rv5 = vbslq_u32(Rlt5, Rsl5, Rv5);
+        Rv6 = vbslq_u32(Rlt6, Rsl6, Rv6);
+        Rv7 = vbslq_u32(Rlt7, Rsl7, Rv7);
+        Rv8 = vbslq_u32(Rlt8, Rsl8, Rv8);
+
+        vst1q_u32(&R[16], Rv5);
+        vst1q_u32(&R[20], Rv6);
+        vst1q_u32(&R[24], Rv7);
+        vst1q_u32(&R[28], Rv8);
+
+        // out[i+z+?] = S[?]
+        uint16x4_t p16_1 = vmovn_u32(Sv1);
+        uint16x4_t p16_2 = vmovn_u32(Sv2);
+        uint16x4_t p16_3 = vmovn_u32(Sv3);
+        uint16x4_t p16_4 = vmovn_u32(Sv4);
+        uint16x4_t p16_5 = vmovn_u32(Sv5);
+        uint16x4_t p16_6 = vmovn_u32(Sv6);
+        uint16x4_t p16_7 = vmovn_u32(Sv7);
+        uint16x4_t p16_8 = vmovn_u32(Sv8);
+        uint8x8_t  p8_12  = vmovn_u16(vcombine_u16(p16_1,p16_2));
+        uint8x8_t  p8_34  = vmovn_u16(vcombine_u16(p16_3,p16_4));
+        uint8x8_t  p8_56  = vmovn_u16(vcombine_u16(p16_5,p16_6));
+        uint8x8_t  p8_78  = vmovn_u16(vcombine_u16(p16_7,p16_8));
+        uint8x16_t p8_a   = vcombine_u8(p8_12, p8_34);
+        uint8x16_t p8_b   = vcombine_u8(p8_56, p8_78);
+        vst1q_u8(&out[i   ], p8_a);
+        vst1q_u8(&out[i+16], p8_b);
+    }
+
+    for (z = out_sz & (NX-1); z-- > 0; )
+      out[out_end + z] = s3[R[z] & mask];
+
+    //fprintf(stderr, "    0 Decoded %d bytes\n", (int)(cp-in)); //c-size
+
+    return out;
+
+ err:
+    free(out_free);
+    return NULL;
+}
+
+//-----------------------------------------------------------------------------
+
+unsigned char *rans_compress_O1_32x16_neon(unsigned char *in,
+                                           unsigned int in_size,
+                                           unsigned char *out,
+                                           unsigned int *out_size) {
+    unsigned char *cp, *out_end, *out_free = NULL;
+    unsigned int tab_size;
+    uint32_t bound = rans_compress_bound_4x16(in_size,1)-20;
+    int z;
+    RansState ransN[NX];
+
+    if (in_size < NX) // force O0 instead
+        return NULL;
+
+    if (!out) {
+        *out_size = bound;
+        out_free = out = malloc(*out_size);
+    }
+    if (!out || bound > *out_size)
+        return NULL;
+
+    if (((size_t)out)&1)
+        bound--;
+    out_end = out + bound;
+
+    RansEncSymbol (*syms)[256] = htscodecs_tls_alloc(256 * (sizeof(*syms)));
+    if (!syms) {
+        free(out_free);
+        return NULL;
+    }
+
+    cp = out;
+    int shift = encode_freq1(in, in_size, 32, syms, &cp); 
+    if (shift < 0) {
+        free(out_free);
+        htscodecs_tls_free(syms);
+        return NULL;
+    }
+    tab_size = cp - out;
+
+    for (z = 0; z < NX; z++)
+      RansEncInit(&ransN[z]);
+
+    uint8_t* ptr = out_end;
+
+    int iN[NX], isz4 = in_size/NX;
+    for (z = 0; z < NX; z++)
+        iN[z] = (z+1)*isz4-2;
+
+    unsigned char lN[NX];
+    for (z = 0; z < NX; z++)
+        lN[z] = in[iN[z]+1];
+
+    // Deal with the remainder
+    z = NX-1;
+    lN[z] = in[in_size-1];
+    for (iN[z] = in_size-2; iN[z] > NX*isz4-2; iN[z]--) {
+        unsigned char c = in[iN[z]];
+        RansEncPutSymbol(&ransN[z], &ptr, &syms[c][lN[z]]);
+        lN[z] = c;
+    }
+
+#if 0
+    // Scalar code equivalent
+    for (; iN[0] >= 0; ) {
+        for (z = NX-1; z >= 0; z-=4) {
+            unsigned char c0;
+            unsigned char c1;
+            unsigned char c2;
+            unsigned char c3;
+
+            RansEncSymbol *s0 = &syms[c0=in[iN[z-0]--]][lN[z-0]]; lN[z-0] = c0;
+            RansEncSymbol *s1 = &syms[c1=in[iN[z-1]--]][lN[z-1]]; lN[z-1] = c1;
+            RansEncSymbol *s2 = &syms[c2=in[iN[z-2]--]][lN[z-2]]; lN[z-2] = c2;
+            RansEncSymbol *s3 = &syms[c3=in[iN[z-3]--]][lN[z-3]]; lN[z-3] = c3;
+
+            RansEncPutSymbol(&ransN[z-0], &ptr, s0);
+            RansEncPutSymbol(&ransN[z-1], &ptr, s1);
+            RansEncPutSymbol(&ransN[z-2], &ptr, s2);
+            RansEncPutSymbol(&ransN[z-3], &ptr, s3);
+        }
+    }
+#else
+    // SIMD code
+    for (; iN[0] >= 0; ) {
+        for (z = NX-1; z >= 0; z-=16) {
+            unsigned char c;
+
+            RansEncSymbol *s0 = &syms[c=in[iN[z- 0]--]][lN[z- 0]]; lN[z- 0]=c;
+            RansEncSymbol *s1 = &syms[c=in[iN[z- 1]--]][lN[z- 1]]; lN[z- 1]=c;
+            RansEncSymbol *s2 = &syms[c=in[iN[z- 2]--]][lN[z- 2]]; lN[z- 2]=c;
+            RansEncSymbol *s3 = &syms[c=in[iN[z- 3]--]][lN[z- 3]]; lN[z- 3]=c;
+
+            uint32x4_t Rv1 = vld1q_u32(&ransN[z-3]);
+            uint32x4_t Rv2 = vld1q_u32(&ransN[z-7]);
+            uint32x4_t Rv3 = vld1q_u32(&ransN[z-11]);
+            uint32x4_t Rv4 = vld1q_u32(&ransN[z-15]);
+        
+            RansEncSymbol *s4 = &syms[c=in[iN[z- 4]--]][lN[z- 4]]; lN[z- 4]=c;
+            RansEncSymbol *s5 = &syms[c=in[iN[z- 5]--]][lN[z- 5]]; lN[z- 5]=c;
+            RansEncSymbol *s6 = &syms[c=in[iN[z- 6]--]][lN[z- 6]]; lN[z- 6]=c;
+            RansEncSymbol *s7 = &syms[c=in[iN[z- 7]--]][lN[z- 7]]; lN[z- 7]=c;
+
+            uint32x4_t A_1 = vld1q_u32((void *)s3);
+            uint32x4_t B_1 = vld1q_u32((void *)s2);
+            uint32x4_t C_1 = vld1q_u32((void *)s1);
+            uint32x4_t D_1 = vld1q_u32((void *)s0);
+
+            uint32x4_t A1_1 = vtrn1q_u32(A_1, B_1);
+            uint32x4_t C1_1 = vtrn1q_u32(C_1, D_1);
+            uint32x4_t A2_1 = vtrn2q_u32(A_1, B_1);
+            uint32x4_t C2_1 = vtrn2q_u32(C_1, D_1);
+
+            uint32x4_t Xmaxv1=u32_u64(vtrn1q_u64(u64_u32(A1_1),u64_u32(C1_1)));
+            uint32x4_t Biasv1=u32_u64(vtrn2q_u64(u64_u32(A1_1),u64_u32(C1_1)));
+            uint32x4_t RFv1  =u32_u64(vtrn1q_u64(u64_u32(A2_1),u64_u32(C2_1)));
+            uint32x4_t FSv1  =u32_u64(vtrn2q_u64(u64_u32(A2_1),u64_u32(C2_1)));
+
+            uint32x4_t A_2 = vld1q_u32((void *)s7);
+            uint32x4_t B_2 = vld1q_u32((void *)s6);
+            uint32x4_t C_2 = vld1q_u32((void *)s5);
+            uint32x4_t D_2 = vld1q_u32((void *)s4);
+            uint32x4_t A1_2 = vtrn1q_u32(A_2, B_2);
+            uint32x4_t C1_2 = vtrn1q_u32(C_2, D_2);
+            uint32x4_t A2_2 = vtrn2q_u32(A_2, B_2);
+            uint32x4_t C2_2 = vtrn2q_u32(C_2, D_2);
+
+            uint32x4_t Xmaxv2=u32_u64(vtrn1q_u64(u64_u32(A1_2),u64_u32(C1_2)));
+            uint32x4_t Biasv2=u32_u64(vtrn2q_u64(u64_u32(A1_2),u64_u32(C1_2)));
+            uint32x4_t RFv2  =u32_u64(vtrn1q_u64(u64_u32(A2_2),u64_u32(C2_2)));
+            uint32x4_t FSv2  =u32_u64(vtrn2q_u64(u64_u32(A2_2),u64_u32(C2_2)));
+
+            uint32x4_t Cv1 = vcgtq_u32(Rv1, Xmaxv1);
+            uint32x4_t Cv2 = vcgtq_u32(Rv2, Xmaxv2);
+            uint32x4_t bit = {8,4,2,1};
+            uint32_t imask1 = vaddvq_u32(vandq_u32(Cv1, bit));
+            uint32_t imask2 = vaddvq_u32(vandq_u32(Cv2, bit));
+
+            RansEncSymbol *s8 = &syms[c=in[iN[z- 8]--]][lN[z- 8]]; lN[z- 8]=c;
+            RansEncSymbol *s9 = &syms[c=in[iN[z- 9]--]][lN[z- 9]]; lN[z- 9]=c;
+            RansEncSymbol *s10= &syms[c=in[iN[z-10]--]][lN[z-10]]; lN[z-10]=c;
+            RansEncSymbol *s11= &syms[c=in[iN[z-11]--]][lN[z-11]]; lN[z-11]=c;
+
+            RansEncSymbol *s12= &syms[c=in[iN[z-12]--]][lN[z-12]]; lN[z-12]=c;
+            RansEncSymbol *s13= &syms[c=in[iN[z-13]--]][lN[z-13]]; lN[z-13]=c;
+            RansEncSymbol *s14= &syms[c=in[iN[z-14]--]][lN[z-14]]; lN[z-14]=c;
+            RansEncSymbol *s15= &syms[c=in[iN[z-15]--]][lN[z-15]]; lN[z-15]=c;
+
+            uint32x4_t A_3 = vld1q_u32((void *)s11);
+            uint32x4_t B_3 = vld1q_u32((void *)s10);
+            uint32x4_t C_3 = vld1q_u32((void *)s9);
+            uint32x4_t D_3 = vld1q_u32((void *)s8);
+
+
+            uint32x4_t A1_3 = vtrn1q_u32(A_3, B_3);
+            uint32x4_t C1_3 = vtrn1q_u32(C_3, D_3);
+            uint32x4_t A2_3 = vtrn2q_u32(A_3, B_3);
+            uint32x4_t C2_3 = vtrn2q_u32(C_3, D_3);
+
+            uint32x4_t Xmaxv3=u32_u64(vtrn1q_u64(u64_u32(A1_3),u64_u32(C1_3)));
+            uint32x4_t Biasv3=u32_u64(vtrn2q_u64(u64_u32(A1_3),u64_u32(C1_3)));
+            uint32x4_t RFv3  =u32_u64(vtrn1q_u64(u64_u32(A2_3),u64_u32(C2_3)));
+            uint32x4_t FSv3  =u32_u64(vtrn2q_u64(u64_u32(A2_3),u64_u32(C2_3)));
+
+            uint32x4_t A_4 = vld1q_u32((void *)s15);
+            uint32x4_t B_4 = vld1q_u32((void *)s14);
+            uint32x4_t C_4 = vld1q_u32((void *)s13);
+            uint32x4_t D_4 = vld1q_u32((void *)s12);
+
+            uint32x4_t A1_4 = vtrn1q_u32(A_4, B_4);
+            uint32x4_t C1_4 = vtrn1q_u32(C_4, D_4);
+            uint32x4_t A2_4 = vtrn2q_u32(A_4, B_4);
+            uint32x4_t C2_4 = vtrn2q_u32(C_4, D_4);
+
+            uint32x4_t Xmaxv4=u32_u64(vtrn1q_u64(u64_u32(A1_4),u64_u32(C1_4)));
+            uint32x4_t Biasv4=u32_u64(vtrn2q_u64(u64_u32(A1_4),u64_u32(C1_4)));
+            uint32x4_t RFv4  =u32_u64(vtrn1q_u64(u64_u32(A2_4),u64_u32(C2_4)));
+            uint32x4_t FSv4  =u32_u64(vtrn2q_u64(u64_u32(A2_4),u64_u32(C2_4)));
+
+            uint32x4_t Cv3 = vcgtq_u32(Rv3, Xmaxv3);
+            uint32x4_t Cv4 = vcgtq_u32(Rv4, Xmaxv4);
+            uint32_t imask3 = vaddvq_u32(vandq_u32(Cv3, bit));
+            uint32_t imask4 = vaddvq_u32(vandq_u32(Cv4, bit));
+
+            // Select low 16-bits from Rv based on imask, using tbl
+            uint8x8_t norm1, norm2, norm3, norm4;
+            static int nbits[16] = { 0,2,2,4, 2,4,4,6, 2,4,4,6, 4,6,6,8 };
+            norm1 = vqtbl1_u8(vreinterpretq_u8_u32(Rv1),vtab[imask1]);
+            norm2 = vqtbl1_u8(vreinterpretq_u8_u32(Rv2),vtab[imask2]);
+
+            vst1_u8(ptr-8, norm1);   ptr -= nbits[imask1];
+            vst1_u8(ptr-8, norm2);   ptr -= nbits[imask2];
+
+            norm3 = vqtbl1_u8(vreinterpretq_u8_u32(Rv3),vtab[imask3]);
+            norm4 = vqtbl1_u8(vreinterpretq_u8_u32(Rv4),vtab[imask4]);
+
+            vst1_u8(ptr-8, norm3);   ptr -= nbits[imask3];
+            vst1_u8(ptr-8, norm4);   ptr -= nbits[imask4];
+
+            // R' = R>>16
+            uint32x4_t Rv1_r = vshrq_n_u32(Rv1, 16);
+            uint32x4_t Rv2_r = vshrq_n_u32(Rv2, 16);
+            uint32x4_t Rv3_r = vshrq_n_u32(Rv3, 16);
+            uint32x4_t Rv4_r = vshrq_n_u32(Rv4, 16);
+
+            // Blend R and R' based on Cv.
+            Rv1 = vbslq_u32(Cv1, Rv1_r, Rv1);
+            Rv2 = vbslq_u32(Cv2, Rv2_r, Rv2);
+            Rv3 = vbslq_u32(Cv3, Rv3_r, Rv3);
+            Rv4 = vbslq_u32(Cv4, Rv4_r, Rv4);
+
+            uint64x2_t qvl1 = vmull_u32(vget_low_u32(Rv1), vget_low_u32(RFv1));
+            uint64x2_t qvh1 = vmull_high_u32(Rv1, RFv1);
+            uint64x2_t qvl2 = vmull_u32(vget_low_u32(Rv2), vget_low_u32(RFv2));
+            uint64x2_t qvh2 = vmull_high_u32(Rv2, RFv2);
+
+            int32x4_t RSv1 = vnegq_s32(vreinterpretq_s32_u32(
+                                           vshrq_n_u32(FSv1, 16)));
+            int32x4_t RSv2 = vnegq_s32(vreinterpretq_s32_u32(
+                                           vshrq_n_u32(FSv2, 16)));
+
+            uint64x2_t qvl3 = vmull_u32(vget_low_u32(Rv3), vget_low_u32(RFv3));
+            uint64x2_t qvh3 = vmull_high_u32(Rv3, RFv3);
+            uint64x2_t qvl4 = vmull_u32(vget_low_u32(Rv4), vget_low_u32(RFv4));
+            uint64x2_t qvh4 = vmull_high_u32(Rv4, RFv4);
+
+            int32x4_t RSv3 = vnegq_s32(vreinterpretq_s32_u32(
+                                           vshrq_n_u32(FSv3, 16)));
+            int32x4_t RSv4 = vnegq_s32(vreinterpretq_s32_u32(
+                                           vshrq_n_u32(FSv4, 16)));
+
+            qvl1 = vreinterpretq_u64_s64(
+                       vshlq_s64(vreinterpretq_s64_u64(qvl1),
+                                 vmovl_s32(vget_low_s32(RSv1))));
+            qvh1 = vreinterpretq_u64_s64(
+                       vshlq_s64(vreinterpretq_s64_u64(qvh1),
+                                 vmovl_s32(vget_high_s32(RSv1))));
+
+            qvl2 = vreinterpretq_u64_s64(
+                       vshlq_s64(vreinterpretq_s64_u64(qvl2),
+                                 vmovl_s32(vget_low_s32(RSv2))));
+            qvh2 = vreinterpretq_u64_s64(
+                       vshlq_s64(vreinterpretq_s64_u64(qvh2),
+                                 vmovl_s32(vget_high_s32(RSv2))));
+
+            uint32x4_t qv1 = vcombine_u32(vmovn_u64(qvl1),
+                                          vmovn_u64(qvh1));
+            uint32x4_t qv2 = vcombine_u32(vmovn_u64(qvl2),
+                                          vmovn_u64(qvh2));
+
+            qvl3 = vreinterpretq_u64_s64(
+                       vshlq_s64(vreinterpretq_s64_u64(qvl3),
+                                 vmovl_s32(vget_low_s32(RSv3))));
+            qvh3 = vreinterpretq_u64_s64(
+                       vshlq_s64(vreinterpretq_s64_u64(qvh3),
+                                 vmovl_s32(vget_high_s32(RSv3))));
+
+            qvl4 = vreinterpretq_u64_s64(
+                       vshlq_s64(vreinterpretq_s64_u64(qvl4),
+                                 vmovl_s32(vget_low_s32(RSv4))));
+            qvh4 = vreinterpretq_u64_s64(
+                       vshlq_s64(vreinterpretq_s64_u64(qvh4),
+                                 vmovl_s32(vget_high_s32(RSv4))));
+
+            uint32x4_t qv3 = vcombine_u32(vmovn_u64(qvl3),
+                                          vmovn_u64(qvh3));
+            uint32x4_t qv4 = vcombine_u32(vmovn_u64(qvl4),
+                                          vmovn_u64(qvh4));
+
+            FSv1 = vandq_u32(FSv1, vdupq_n_u32(0xffff)); // cmpl_freq
+            FSv2 = vandq_u32(FSv2, vdupq_n_u32(0xffff));
+            FSv3 = vandq_u32(FSv3, vdupq_n_u32(0xffff));
+            FSv4 = vandq_u32(FSv4, vdupq_n_u32(0xffff));
+        
+            qv1 = vmlaq_u32(Biasv1, qv1, FSv1);
+            qv2 = vmlaq_u32(Biasv2, qv2, FSv2);
+            qv3 = vmlaq_u32(Biasv3, qv3, FSv3);
+            qv4 = vmlaq_u32(Biasv4, qv4, FSv4);
+
+            Rv1 = vaddq_u32(Rv1, qv1);
+            Rv2 = vaddq_u32(Rv2, qv2);
+            Rv3 = vaddq_u32(Rv3, qv3);
+            Rv4 = vaddq_u32(Rv4, qv4);
+
+            vst1q_u32(&ransN[z-3], Rv1);
+            vst1q_u32(&ransN[z-7], Rv2);
+            vst1q_u32(&ransN[z-11],Rv3);
+            vst1q_u32(&ransN[z-15],Rv4);
+        }
+    }
+#endif
+
+    for (z = NX-1; z>=0; z--)
+        RansEncPutSymbol(&ransN[z], &ptr, &syms[0][lN[z]]);
+
+    for (z = NX-1; z>=0; z--)
+        RansEncFlush(&ransN[z], &ptr);
+
+    *out_size = (out_end - ptr) + tab_size;
+
+    cp = out;
+    memmove(out + tab_size, ptr, out_end-ptr);
+
+    htscodecs_tls_free(syms);
+    return out;
+}
+
+//#define MAGIC2 111
+#define MAGIC2 179
+//#define MAGIC2 0
+typedef struct {
+  union {
+    struct {
+      uint16_t f;
+      uint16_t b;
+    } s;
+    uint32_t fb;
+  } u;
+} bf_t;
+
+static inline void transpose_and_copy(uint8_t *out, int iN[32],
+                                      uint8_t t[32][32]) {
+    int z;
+//  for (z = 0; z < NX; z++) {
+//      int k;
+//      for (k = 0; k < 32; k++)
+//      out[iN[z]+k] = t[k][z];
+//      iN[z] += 32;
+//  }
+
+    for (z = 0; z < NX; z+=4) {
+        *(uint64_t *)&out[iN[z]] =
+            ((uint64_t)(t[0][z])<< 0) +
+            ((uint64_t)(t[1][z])<< 8) +
+            ((uint64_t)(t[2][z])<<16) +
+            ((uint64_t)(t[3][z])<<24) +
+            ((uint64_t)(t[4][z])<<32) +
+            ((uint64_t)(t[5][z])<<40) +
+            ((uint64_t)(t[6][z])<<48) +
+            ((uint64_t)(t[7][z])<<56);
+        *(uint64_t *)&out[iN[z+1]] =
+            ((uint64_t)(t[0][z+1])<< 0) +
+            ((uint64_t)(t[1][z+1])<< 8) +
+            ((uint64_t)(t[2][z+1])<<16) +
+            ((uint64_t)(t[3][z+1])<<24) +
+            ((uint64_t)(t[4][z+1])<<32) +
+            ((uint64_t)(t[5][z+1])<<40) +
+            ((uint64_t)(t[6][z+1])<<48) +
+            ((uint64_t)(t[7][z+1])<<56);
+        *(uint64_t *)&out[iN[z+2]] =
+            ((uint64_t)(t[0][z+2])<< 0) +
+            ((uint64_t)(t[1][z+2])<< 8) +
+            ((uint64_t)(t[2][z+2])<<16) +
+            ((uint64_t)(t[3][z+2])<<24) +
+            ((uint64_t)(t[4][z+2])<<32) +
+            ((uint64_t)(t[5][z+2])<<40) +
+            ((uint64_t)(t[6][z+2])<<48) +
+            ((uint64_t)(t[7][z+2])<<56);
+        *(uint64_t *)&out[iN[z+3]] =
+            ((uint64_t)(t[0][z+3])<< 0) +
+            ((uint64_t)(t[1][z+3])<< 8) +
+            ((uint64_t)(t[2][z+3])<<16) +
+            ((uint64_t)(t[3][z+3])<<24) +
+            ((uint64_t)(t[4][z+3])<<32) +
+            ((uint64_t)(t[5][z+3])<<40) +
+            ((uint64_t)(t[6][z+3])<<48) +
+            ((uint64_t)(t[7][z+3])<<56);
+
+        *(uint64_t *)&out[iN[z]+8] =
+            ((uint64_t)(t[8+0][z])<< 0) +
+            ((uint64_t)(t[8+1][z])<< 8) +
+            ((uint64_t)(t[8+2][z])<<16) +
+            ((uint64_t)(t[8+3][z])<<24) +
+            ((uint64_t)(t[8+4][z])<<32) +
+            ((uint64_t)(t[8+5][z])<<40) +
+            ((uint64_t)(t[8+6][z])<<48) +
+            ((uint64_t)(t[8+7][z])<<56);
+        *(uint64_t *)&out[iN[z+1]+8] =
+            ((uint64_t)(t[8+0][z+1])<< 0) +
+            ((uint64_t)(t[8+1][z+1])<< 8) +
+            ((uint64_t)(t[8+2][z+1])<<16) +
+            ((uint64_t)(t[8+3][z+1])<<24) +
+            ((uint64_t)(t[8+4][z+1])<<32) +
+            ((uint64_t)(t[8+5][z+1])<<40) +
+            ((uint64_t)(t[8+6][z+1])<<48) +
+            ((uint64_t)(t[8+7][z+1])<<56);
+        *(uint64_t *)&out[iN[z+2]+8] =
+            ((uint64_t)(t[8+0][z+2])<< 0) +
+            ((uint64_t)(t[8+1][z+2])<< 8) +
+            ((uint64_t)(t[8+2][z+2])<<16) +
+            ((uint64_t)(t[8+3][z+2])<<24) +
+            ((uint64_t)(t[8+4][z+2])<<32) +
+            ((uint64_t)(t[8+5][z+2])<<40) +
+            ((uint64_t)(t[8+6][z+2])<<48) +
+            ((uint64_t)(t[8+7][z+2])<<56);
+        *(uint64_t *)&out[iN[z+3]+8] =
+            ((uint64_t)(t[8+0][z+3])<< 0) +
+            ((uint64_t)(t[8+1][z+3])<< 8) +
+            ((uint64_t)(t[8+2][z+3])<<16) +
+            ((uint64_t)(t[8+3][z+3])<<24) +
+            ((uint64_t)(t[8+4][z+3])<<32) +
+            ((uint64_t)(t[8+5][z+3])<<40) +
+            ((uint64_t)(t[8+6][z+3])<<48) +
+            ((uint64_t)(t[8+7][z+3])<<56);
+
+        *(uint64_t *)&out[iN[z]+16] =
+            ((uint64_t)(t[16+0][z])<< 0) +
+            ((uint64_t)(t[16+1][z])<< 8) +
+            ((uint64_t)(t[16+2][z])<<16) +
+            ((uint64_t)(t[16+3][z])<<24) +
+            ((uint64_t)(t[16+4][z])<<32) +
+            ((uint64_t)(t[16+5][z])<<40) +
+            ((uint64_t)(t[16+6][z])<<48) +
+            ((uint64_t)(t[16+7][z])<<56);
+        *(uint64_t *)&out[iN[z+1]+16] =
+            ((uint64_t)(t[16+0][z+1])<< 0) +
+            ((uint64_t)(t[16+1][z+1])<< 8) +
+            ((uint64_t)(t[16+2][z+1])<<16) +
+            ((uint64_t)(t[16+3][z+1])<<24) +
+            ((uint64_t)(t[16+4][z+1])<<32) +
+            ((uint64_t)(t[16+5][z+1])<<40) +
+            ((uint64_t)(t[16+6][z+1])<<48) +
+            ((uint64_t)(t[16+7][z+1])<<56);
+        *(uint64_t *)&out[iN[z+2]+16] =
+            ((uint64_t)(t[16+0][z+2])<< 0) +
+            ((uint64_t)(t[16+1][z+2])<< 8) +
+            ((uint64_t)(t[16+2][z+2])<<16) +
+            ((uint64_t)(t[16+3][z+2])<<24) +
+            ((uint64_t)(t[16+4][z+2])<<32) +
+            ((uint64_t)(t[16+5][z+2])<<40) +
+            ((uint64_t)(t[16+6][z+2])<<48) +
+            ((uint64_t)(t[16+7][z+2])<<56);
+        *(uint64_t *)&out[iN[z+3]+16] =
+            ((uint64_t)(t[16+0][z+3])<< 0) +
+            ((uint64_t)(t[16+1][z+3])<< 8) +
+            ((uint64_t)(t[16+2][z+3])<<16) +
+            ((uint64_t)(t[16+3][z+3])<<24) +
+            ((uint64_t)(t[16+4][z+3])<<32) +
+            ((uint64_t)(t[16+5][z+3])<<40) +
+            ((uint64_t)(t[16+6][z+3])<<48) +
+            ((uint64_t)(t[16+7][z+3])<<56);
+
+        *(uint64_t *)&out[iN[z]+24] =
+            ((uint64_t)(t[24+0][z])<< 0) +
+            ((uint64_t)(t[24+1][z])<< 8) +
+            ((uint64_t)(t[24+2][z])<<16) +
+            ((uint64_t)(t[24+3][z])<<24) +
+            ((uint64_t)(t[24+4][z])<<32) +
+            ((uint64_t)(t[24+5][z])<<40) +
+            ((uint64_t)(t[24+6][z])<<48) +
+            ((uint64_t)(t[24+7][z])<<56);
+        *(uint64_t *)&out[iN[z+1]+24] =
+            ((uint64_t)(t[24+0][z+1])<< 0) +
+            ((uint64_t)(t[24+1][z+1])<< 8) +
+            ((uint64_t)(t[24+2][z+1])<<16) +
+            ((uint64_t)(t[24+3][z+1])<<24) +
+            ((uint64_t)(t[24+4][z+1])<<32) +
+            ((uint64_t)(t[24+5][z+1])<<40) +
+            ((uint64_t)(t[24+6][z+1])<<48) +
+            ((uint64_t)(t[24+7][z+1])<<56);
+        *(uint64_t *)&out[iN[z+2]+24] =
+            ((uint64_t)(t[24+0][z+2])<< 0) +
+            ((uint64_t)(t[24+1][z+2])<< 8) +
+            ((uint64_t)(t[24+2][z+2])<<16) +
+            ((uint64_t)(t[24+3][z+2])<<24) +
+            ((uint64_t)(t[24+4][z+2])<<32) +
+            ((uint64_t)(t[24+5][z+2])<<40) +
+            ((uint64_t)(t[24+6][z+2])<<48) +
+            ((uint64_t)(t[24+7][z+2])<<56);
+        *(uint64_t *)&out[iN[z+3]+24] =
+            ((uint64_t)(t[24+0][z+3])<< 0) +
+            ((uint64_t)(t[24+1][z+3])<< 8) +
+            ((uint64_t)(t[24+2][z+3])<<16) +
+            ((uint64_t)(t[24+3][z+3])<<24) +
+            ((uint64_t)(t[24+4][z+3])<<32) +
+            ((uint64_t)(t[24+5][z+3])<<40) +
+            ((uint64_t)(t[24+6][z+3])<<48) +
+            ((uint64_t)(t[24+7][z+3])<<56);
+
+        iN[z+0] += 32;
+        iN[z+1] += 32;
+        iN[z+2] += 32;
+        iN[z+3] += 32;
+    }
+}
+
+unsigned char *rans_uncompress_O1_32x16_neon(unsigned char *in,
+                                             unsigned int in_size,
+                                             unsigned char *out,
+                                             unsigned int out_sz) {
+    if (in_size < NX*4) // 4-states at least
+        return NULL;
+
+    if (out_sz >= INT_MAX)
+        return NULL; // protect against some overflow cases
+
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    if (out_sz > 100000)
+        return NULL;
+#endif
+
+    /* Load in the static tables */
+    unsigned char *cp = in, *cp_end = in+in_size, *out_free = NULL;
+    unsigned char *c_freq = NULL;
+    int i, j = -999;
+    unsigned int x;
+
+    uint8_t *sfb_ = htscodecs_tls_alloc(256*(TOTFREQ_O1+MAGIC2)*sizeof(*sfb_));
+    uint32_t s3[256][TOTFREQ_O1_FAST];
+
+    if (!sfb_)
+        return NULL;
+    bf_t fb[256][256];
+    uint8_t *sfb[256];
+    if ((*cp >> 4) == TF_SHIFT_O1) {
+        for (i = 0; i < 256; i++)
+            sfb[i]=  sfb_ + i*(TOTFREQ_O1+MAGIC2);
+    } else {
+        for (i = 0; i < 256; i++)
+            sfb[i]=  sfb_ + i*(TOTFREQ_O1_FAST+MAGIC2);
+    }
+
+    if (!out)
+        out_free = out = malloc(out_sz);
+
+    if (!out)
+        goto err;
+
+    //fprintf(stderr, "out_sz=%d\n", out_sz);
+
+    // compressed header? If so uncompress it
+    unsigned char *tab_end = NULL;
+    unsigned char *c_freq_end = cp_end;
+    unsigned int shift = *cp >> 4;
+    if (*cp++ & 1) {
+        uint32_t u_freq_sz, c_freq_sz;
+        cp += var_get_u32(cp, cp_end, &u_freq_sz);
+        cp += var_get_u32(cp, cp_end, &c_freq_sz);
+        if (c_freq_sz > cp_end - cp)
+            goto err;
+        tab_end = cp + c_freq_sz;
+        if (!(c_freq = rans_uncompress_O0_4x16(cp, c_freq_sz, NULL, u_freq_sz)))
+            goto err;
+        cp = c_freq;
+        c_freq_end = c_freq + u_freq_sz;
+    }
+
+    // Decode order-0 symbol list; avoids needing in order-1 tables
+#if 0
+    // Disable inline for now as this is ~10% slower under gcc.  Why?
+    cp += decode_freq1(cp, c_freq_end, shift, NULL, s3, sfb, fb);
+#else
+    uint32_t F0[256] = {0};
+    int fsz = decode_alphabet(cp, c_freq_end, F0);
+    if (!fsz)
+        goto err;
+    cp += fsz;
+
+    if (cp >= c_freq_end)
+        goto err;
+
+    for (i = 0; i < 256; i++) {
+        if (F0[i] == 0)
+            continue;
+
+        uint32_t F[256] = {0}, T = 0;
+        fsz = decode_freq_d(cp, c_freq_end, F0, F, &T);
+        if (!fsz)
+            goto err;
+        cp += fsz;
+
+        if (!T) {
+            //fprintf(stderr, "No freq for F_%d\n", i);
+            continue;
+        }
+
+        normalise_freq_shift(F, T, 1<<shift);
+
+        // Build symbols; fixme, do as part of decode, see the _d variant
+        for (j = x = 0; j < 256; j++) {
+            if (F[j]) {
+                if (F[j] > (1<<shift) - x)
+                    goto err;
+
+                if (shift == TF_SHIFT_O1_FAST) {
+                    int y;
+                    for (y = 0; y < F[j]; y++)
+                        s3[i][y+x] = (((uint32_t)F[j])<<(shift+8)) |(y<<8) |j;
+                }
+                memset(&sfb[i][x], j, F[j]);
+                fb[i][j].u.s.f = F[j];
+                fb[i][j].u.s.b = x;
+
+                x += F[j];
+            }
+        }
+        if (x != (1<<shift))
+            goto err;
+    }
+#endif
+
+    if (tab_end)
+        cp = tab_end;
+    free(c_freq);
+    c_freq = NULL;
+
+    if (cp_end - cp < NX * 4)
+        goto err;
+
+    RansState R[NX];
+    uint8_t *ptr = cp, *ptr_end = in + in_size;
+    int z;
+    for (z = 0; z < NX; z++) {
+        RansDecInit(&R[z], &ptr);
+        if (R[z] < RANS_BYTE_L)
+            goto err;
+    }
+
+    int isz4 = out_sz/NX;
+    int i4[NX];
+    uint8_t l[NX] = {0};
+    for (z = 0; z < NX; z++)
+        i4[z] = z*isz4;
+
+    // Around 15% faster to specialise for 10/12 than to have one
+    // loop with shift as a variable.
+    if (shift == TF_SHIFT_O1) {
+        // TF_SHIFT_O1 = 12
+        const uint32_t mask = ((1u << TF_SHIFT_O1)-1);
+        uint32x4_t maskv = vdupq_n_u32((1u << TF_SHIFT_O1)-1);
+
+        // FIXME: plus room for "safe" renorm.
+        // Follow with 2nd copy doing scalar code instead?
+        unsigned char tbuf[32][32] = {0};
+        int tidx = 0;
+        for (; i4[0] < isz4 && ptr+64 < ptr_end;) {
+            for (z = 0; z < NX; z+=16) {
+                uint32x4_t Rv1 = vld1q_u32(&R[z+0]);
+                uint32x4_t Rv2 = vld1q_u32(&R[z+4]);
+                uint32x4_t Rv3 = vld1q_u32(&R[z+8]);
+                uint32x4_t Rv4 = vld1q_u32(&R[z+12]);
+
+                uint32x4_t Sv1, Sv2, Sv3, Sv4;
+                uint32x4_t Fv1, Fv2, Fv3, Fv4;
+                uint32x4_t Bv1, Bv2, Bv3, Bv4;
+
+                // Use sfb[256][256] + fb[256][256] instead of s3[256][4096].
+                // It's lower memory and thus less cache contention and also
+                // avoids the 12-bit issue of FREQ==0 vs 4096.
+                // However we need an extra step to store m = R&mask.
+                // Overall it's a big win for gcc (no difference to clang, but
+                // that's lagging behind gcc on this code).
+                //
+                // TODO: consider this for AVX2?
+                //
+                // sfb layout is char (sym)
+                // fb layout is 16-bit freq, 16-bit bias.
+                uint8_t *c = &tbuf[tidx][z];
+                uint32_t f32[16];
+                int i;
+                for (i = 0; i < 16; i++) {
+                  c[i] = sfb[l[z+i]][R[z+i] & mask];
+                  f32[i] = *(uint32_t *)&fb[l[z+i]][c[i]];
+                }
+                
+                // vcreate faster than vld1q_u32(&f32[0])
+                uint32x2_t s1a, s1b, s2a, s2b, s3a, s3b, s4a, s4b;
+                s1a = vcreate_u32((uint64_t)(f32[ 3])<<32 |
+                                  (uint64_t)(f32[ 2]));
+                s1b = vcreate_u32((uint64_t)(f32[ 1])<<32 |
+                                  (uint64_t)(f32[ 0]));
+                s2a = vcreate_u32((uint64_t)(f32[ 7])<<32 |
+                                  (uint64_t)(f32[ 6]));
+                s2b = vcreate_u32((uint64_t)(f32[ 5])<<32 |
+                                  (uint64_t)(f32[ 4]));
+                s3a = vcreate_u32((uint64_t)(f32[11])<<32 |
+                                  (uint64_t)(f32[10]));
+                s3b = vcreate_u32((uint64_t)(f32[ 9])<<32 |
+                                  (uint64_t)(f32[ 8]));
+                s4a = vcreate_u32((uint64_t)(f32[15])<<32 |
+                                  (uint64_t)(f32[14]));
+                s4b = vcreate_u32((uint64_t)(f32[13])<<32 |
+                                  (uint64_t)(f32[12]));
+
+                memcpy(l+z, c, 16);
+
+                // vcreate   = INS   = throughput 2, latency 2, pipe V
+                // vcombine  = DUP+INS = thr    2+2, lat   2+2, pipe V+V
+                // vandq     = AND   = throughput 2, latency 1, pipe V
+                // vshrq     = USHR  = throughput 2, latency 1, pipe V
+                Sv1 = vcombine_u32(s1b, s1a);
+                Sv2 = vcombine_u32(s2b, s2a);
+                Sv3 = vcombine_u32(s3b, s3a);
+                Sv4 = vcombine_u32(s4b, s4a);
+
+                Bv1 = vshrq_n_u32(Sv1, 16);
+                Bv2 = vshrq_n_u32(Sv2, 16);
+
+                Bv3 = vshrq_n_u32(Sv3, 16);
+                Bv4 = vshrq_n_u32(Sv4, 16);
+
+                Fv1 = vandq_u32(Sv1, vdupq_n_u32(0xffff));
+                Fv2 = vandq_u32(Sv2, vdupq_n_u32(0xffff));
+                Fv3 = vandq_u32(Sv3, vdupq_n_u32(0xffff));
+                Fv4 = vandq_u32(Sv4, vdupq_n_u32(0xffff));
+
+                Bv1 = vsubq_u32(vandq_u32(Rv1, maskv), Bv1);
+                Bv2 = vsubq_u32(vandq_u32(Rv2, maskv), Bv2);
+                Bv3 = vsubq_u32(vandq_u32(Rv3, maskv), Bv3);
+                Bv4 = vsubq_u32(vandq_u32(Rv4, maskv), Bv4);
+
+                Rv1 = vshrq_n_u32(Rv1, TF_SHIFT_O1);
+                Rv2 = vshrq_n_u32(Rv2, TF_SHIFT_O1);
+                Rv3 = vshrq_n_u32(Rv3, TF_SHIFT_O1);
+                Rv4 = vshrq_n_u32(Rv4, TF_SHIFT_O1);
+
+                Rv1 = vmlaq_u32(Bv1, Fv1, Rv1);
+                Rv2 = vmlaq_u32(Bv2, Fv2, Rv2);
+                Rv3 = vmlaq_u32(Bv3, Fv3, Rv3);
+                Rv4 = vmlaq_u32(Bv4, Fv4, Rv4);
+
+                // Renorm
+                uint32x4_t Rlt1 = vcltq_u32(Rv1, vdupq_n_u32(RANS_BYTE_L)); // R<L
+                uint32x4_t Rlt2 = vcltq_u32(Rv2, vdupq_n_u32(RANS_BYTE_L));
+                uint32x4_t Rlt3 = vcltq_u32(Rv3, vdupq_n_u32(RANS_BYTE_L));
+                uint32x4_t Rlt4 = vcltq_u32(Rv4, vdupq_n_u32(RANS_BYTE_L));
+                uint32x4_t all2 = {2,2,2,2};
+                // load 8 lanes of renorm data
+                uint16x8_t norm12 =  vld1q_u16((uint16_t *)ptr);
+                // move ptr by no. renorm lanes used
+                ptr += vaddvq_u32(vandq_u32(Rlt1, all2))
+                    +  vaddvq_u32(vandq_u32(Rlt2, all2));
+                uint16x8_t norm34 =  vld1q_u16((uint16_t *)ptr);
+                ptr += vaddvq_u32(vandq_u32(Rlt3, all2))
+                    +  vaddvq_u32(vandq_u32(Rlt4, all2));
+
+                // Compute lookup table index
+                uint32x4_t bit = {8,4,2,1};
+                uint32_t imask1 = vaddvq_u32(vandq_u32(Rlt1, bit));
+                uint32_t imask2 = vaddvq_u32(vandq_u32(Rlt2, bit));
+                uint32_t imask3 = vaddvq_u32(vandq_u32(Rlt3, bit));
+                uint32_t imask4 = vaddvq_u32(vandq_u32(Rlt4, bit));
+
+                uint32_t imask12 = (imask1<<4)|imask2;
+                uint32_t imask34 = (imask3<<4)|imask4;
+
+                // Shuffle norm to the corresponding R lanes, via imask
+                // #define for brevity and formatting
+                uint16x4_t norm1, norm2, norm3, norm4;
+                norm1 = cast_u16_u8(vqtbl1_u8(cast_u8_u16(norm12),
+                                              idx [imask1]));
+                norm2 = cast_u16_u8(vqtbl1_u8(cast_u8_u16(norm12),
+                                              idx2[imask12]));
+                norm3 = cast_u16_u8(vqtbl1_u8(cast_u8_u16(norm34),
+                                              idx [imask3]));
+                norm4 = cast_u16_u8(vqtbl1_u8(cast_u8_u16(norm34),
+                                              idx2[imask34]));
+
+                // Add norm to R<<16 and blend back in with R
+                uint32x4_t Rsl1 = vshlq_n_u32(Rv1, 16); // Rsl = R << 16
+                uint32x4_t Rsl2 = vshlq_n_u32(Rv2, 16);
+                uint32x4_t Rsl3 = vshlq_n_u32(Rv3, 16);
+                uint32x4_t Rsl4 = vshlq_n_u32(Rv4, 16);
+
+                Rsl1 = vaddw_u16(Rsl1, norm1);          // Rsl += norm
+                Rsl2 = vaddw_u16(Rsl2, norm2);
+                Rsl3 = vaddw_u16(Rsl3, norm3);
+                Rsl4 = vaddw_u16(Rsl4, norm4);
+//              Rsl1 = vaddq_u32(Rsl1, vmovl_u16(norm1));
+//              Rsl2 = vaddq_u32(Rsl2, vmovl_u16(norm2));
+//              Rsl3 = vaddq_u32(Rsl3, vmovl_u16(norm3));
+//              Rsl4 = vaddq_u32(Rsl4, vmovl_u16(norm4));
+
+                Rv1 = vbslq_u32(Rlt1, Rsl1, Rv1);       // R = R<L ? Rsl : R
+                Rv2 = vbslq_u32(Rlt2, Rsl2, Rv2);
+                Rv3 = vbslq_u32(Rlt3, Rsl3, Rv3);
+                Rv4 = vbslq_u32(Rlt4, Rsl4, Rv4);
+
+                vst1q_u32(&R[z+ 0], Rv1);
+                vst1q_u32(&R[z+ 4], Rv2);
+                vst1q_u32(&R[z+ 8], Rv3);
+                vst1q_u32(&R[z+12], Rv4);
+            }
+
+            i4[0]++;
+            if (++tidx == 32) {
+                i4[0] -= 32;
+
+                transpose_and_copy(out, i4, tbuf);
+                tidx = 0;
+            }
+        }
+
+        i4[0]-=tidx;
+        int T;
+        for (z = 0; z < NX; z++)
+            for (T = 0; T < tidx; T++)
+                out[i4[z]++] = tbuf[T][z];
+
+        // Scalar version for close to end of in[] array so we don't do
+        // SIMD loads beyond the end of the buffer
+        for (; i4[0] < isz4; ) {
+            for (z = 0; z < NX; z++) {
+                uint32_t m = R[z] & ((1u<<TF_SHIFT_O1)-1);
+                unsigned char c = sfb[l[z]][m];
+                out[i4[z]++] = c;
+                R[z] = fb[l[z]][c].u.s.f * (R[z]>>TF_SHIFT_O1) + m - fb[l[z]][c].u.s.b;
+                RansDecRenormSafe(&R[z], &ptr, ptr_end);
+                l[z] = c;
+            }
+        }
+
+
+        // Remainder
+        for (; i4[NX-1] < out_sz; i4[NX-1]++) {
+            uint32_t m = R[NX-1] & ((1u<<TF_SHIFT_O1)-1);
+            unsigned char c = sfb[l[NX-1]][m];
+            out[i4[NX-1]] = c;
+            R[NX-1] = fb[l[NX-1]][c].u.s.f * (R[NX-1]>>TF_SHIFT_O1) + m - fb[l[NX-1]][c].u.s.b;
+            RansDecRenormSafe(&R[NX-1], &ptr, ptr_end);
+            l[NX-1] = c;
+        }
+    } else {
+        // TF_SHIFT_O1 = 10
+        const uint32_t mask = ((1u << TF_SHIFT_O1_FAST)-1);
+        uint32x4_t maskv = vdupq_n_u32((1u << TF_SHIFT_O1_FAST)-1);
+
+        // FIXME: plus room for "safe" renorm.
+        // Follow with 2nd copy doing scalar code instead?
+        unsigned char tbuf[32][32];
+        int tidx = 0;
+
+        uint32x4_t RV[8] = {
+            vld1q_u32(&R[0]),
+            vld1q_u32(&R[4]),
+            vld1q_u32(&R[8]),
+            vld1q_u32(&R[12]),
+            vld1q_u32(&R[16]),
+            vld1q_u32(&R[20]),
+            vld1q_u32(&R[24]),
+            vld1q_u32(&R[28]),
+        };
+
+//      uint32x4_t MV[8] = {
+//            vandq_u32(RV[0], maskv),
+//            vandq_u32(RV[1], maskv),
+//            vandq_u32(RV[2], maskv),
+//            vandq_u32(RV[3], maskv),
+//            vandq_u32(RV[4], maskv),
+//            vandq_u32(RV[5], maskv),
+//            vandq_u32(RV[6], maskv),
+//            vandq_u32(RV[7], maskv),
+//      };
+
+        uint32_t m[NX];
+        for (z = 0; z < NX; z++)
+            m[z] = l[z]*TOTFREQ_O1_FAST + (R[z] & mask);
+
+        uint32_t *S3 = (uint32_t *)s3;
+        
+        for (; i4[0] < isz4 && ptr+64 < ptr_end;) {
+            int Z = 0;
+            for (z = 0; z < NX; z+=16, Z+=4) {
+                // streamline these.  Could swap between two banks and pre-load
+                uint32x4_t Sv1, Sv2, Sv3, Sv4;
+                uint32x4_t Fv1, Fv2, Fv3, Fv4;
+                uint32x4_t Bv1, Bv2, Bv3, Bv4;
+                uint32x2_t s1a, s1b, s2a, s2b, s3a, s3b, s4a, s4b;
+
+                s1a = vcreate_u32((uint64_t)(S3[m[z+1]])<<32  | (S3[m[z+0]]));
+                s1b = vcreate_u32((uint64_t)(S3[m[z+3]])<<32  | (S3[m[z+2]]));
+                s2a = vcreate_u32((uint64_t)(S3[m[z+5]])<<32  | (S3[m[z+4]]));
+                s2b = vcreate_u32((uint64_t)(S3[m[z+7]])<<32  | (S3[m[z+6]]));
+                s3a = vcreate_u32((uint64_t)(S3[m[z+9]])<<32  | (S3[m[z+8]]));
+                s3b = vcreate_u32((uint64_t)(S3[m[z+11]])<<32 | (S3[m[z+10]]));
+                s4a = vcreate_u32((uint64_t)(S3[m[z+13]])<<32 | (S3[m[z+12]]));
+                s4b = vcreate_u32((uint64_t)(S3[m[z+15]])<<32 | (S3[m[z+14]]));
+
+                Sv1 = vcombine_u32(s1a, s1b);
+                Sv2 = vcombine_u32(s2a, s2b);
+                Sv3 = vcombine_u32(s3a, s3b);
+                Sv4 = vcombine_u32(s4a, s4b);
+                
+                uint16x4_t p16_1 = vmovn_u32(Sv1);
+                uint16x4_t p16_2 = vmovn_u32(Sv2);
+                uint16x4_t p16_3 = vmovn_u32(Sv3);
+                uint16x4_t p16_4 = vmovn_u32(Sv4);
+
+                uint8x8_t  p8_12  = vmovn_u16(vcombine_u16(p16_1,p16_2));
+                uint8x8_t  p8_34  = vmovn_u16(vcombine_u16(p16_3,p16_4));
+                uint8x16_t p8_a   = vcombine_u8(p8_12, p8_34);
+                vst1q_u8(l+z, p8_a);
+
+                Fv1 = vshrq_n_u32(Sv1, TF_SHIFT_O1_FAST+8);
+                Fv2 = vshrq_n_u32(Sv2, TF_SHIFT_O1_FAST+8);
+                Fv3 = vshrq_n_u32(Sv3, TF_SHIFT_O1_FAST+8);
+                Fv4 = vshrq_n_u32(Sv4, TF_SHIFT_O1_FAST+8);
+
+                Bv1 = vandq_u32(vshrq_n_u32(Sv1, 8), maskv);
+                Bv2 = vandq_u32(vshrq_n_u32(Sv2, 8), maskv);
+                Bv3 = vandq_u32(vshrq_n_u32(Sv3, 8), maskv);
+                Bv4 = vandq_u32(vshrq_n_u32(Sv4, 8), maskv);
+
+                // Add in transpose here.
+                memcpy(&tbuf[tidx][z], &l[z], 16);
+                
+                RV[Z+0] = vshrq_n_u32(RV[Z+0], TF_SHIFT_O1_FAST);
+                RV[Z+1] = vshrq_n_u32(RV[Z+1], TF_SHIFT_O1_FAST);
+                RV[Z+2] = vshrq_n_u32(RV[Z+2], TF_SHIFT_O1_FAST);
+                RV[Z+3] = vshrq_n_u32(RV[Z+3], TF_SHIFT_O1_FAST);
+
+                // Ready for use in S3[] offset
+                Sv1 = vshlq_n_u32(vandq_u32(Sv1, vdupq_n_u32(0xff)), TF_SHIFT_O1_FAST);
+                Sv2 = vshlq_n_u32(vandq_u32(Sv2, vdupq_n_u32(0xff)), TF_SHIFT_O1_FAST);
+                Sv3 = vshlq_n_u32(vandq_u32(Sv3, vdupq_n_u32(0xff)), TF_SHIFT_O1_FAST);
+                Sv4 = vshlq_n_u32(vandq_u32(Sv4, vdupq_n_u32(0xff)), TF_SHIFT_O1_FAST);
+
+                RV[Z+0] = vmlaq_u32(Bv1, Fv1, RV[Z+0]);
+                RV[Z+1] = vmlaq_u32(Bv2, Fv2, RV[Z+1]);
+                RV[Z+2] = vmlaq_u32(Bv3, Fv3, RV[Z+2]);
+                RV[Z+3] = vmlaq_u32(Bv4, Fv4, RV[Z+3]);
+
+                // Renorm
+                uint32x4_t Rlt1 = vcltq_u32(RV[Z+0], vdupq_n_u32(RANS_BYTE_L));
+                uint32x4_t Rlt2 = vcltq_u32(RV[Z+1], vdupq_n_u32(RANS_BYTE_L));
+                uint32x4_t Rlt3 = vcltq_u32(RV[Z+2], vdupq_n_u32(RANS_BYTE_L));
+                uint32x4_t Rlt4 = vcltq_u32(RV[Z+3], vdupq_n_u32(RANS_BYTE_L));
+
+                // Compute lookup table index
+                static int nbits[16] = { 0,2,2,4, 2,4,4,6, 2,4,4,6, 4,6,6,8 };
+                uint32x4_t bit = {8,4,2,1};
+                uint32_t imask1 = vaddvq_u32(vandq_u32(Rlt1, bit));
+                uint32_t imask2 = vaddvq_u32(vandq_u32(Rlt2, bit));
+                uint32_t imask3 = vaddvq_u32(vandq_u32(Rlt3, bit));
+                uint32_t imask4 = vaddvq_u32(vandq_u32(Rlt4, bit));
+
+                // load 8 lanes of renorm data
+                uint16x8_t norm12 =  vld1q_u16((uint16_t *)ptr);
+                // move ptr by no. renorm lanes used
+                ptr += nbits[imask1] + nbits[imask2];
+                uint16x8_t norm34 =  vld1q_u16((uint16_t *)ptr);
+                ptr += nbits[imask3] + nbits[imask4];
+
+                uint32_t imask12 = (imask1<<4)|imask2;
+                uint32_t imask34 = (imask3<<4)|imask4;
+
+                // Shuffle norm to the corresponding R lanes, via imask
+                // #define for brevity and formatting
+                uint16x4_t norm1, norm2, norm3, norm4;
+                norm1 = cast_u16_u8(vqtbl1_u8(cast_u8_u16(norm12),idx [imask1]));
+                norm2 = cast_u16_u8(vqtbl1_u8(cast_u8_u16(norm12),idx2[imask12]));
+                norm3 = cast_u16_u8(vqtbl1_u8(cast_u8_u16(norm34),idx [imask3]));
+                norm4 = cast_u16_u8(vqtbl1_u8(cast_u8_u16(norm34),idx2[imask34]));
+
+                // Add norm to R<<16 and blend back in with R
+                uint32x4_t Rsl1 = vshlq_n_u32(RV[Z+0], 16); // Rsl = R << 16
+                uint32x4_t Rsl2 = vshlq_n_u32(RV[Z+1], 16);
+                uint32x4_t Rsl3 = vshlq_n_u32(RV[Z+2], 16);
+                uint32x4_t Rsl4 = vshlq_n_u32(RV[Z+3], 16);
+
+                Rsl1 = vaddw_u16(Rsl1, norm1);          // Rsl += norm
+                Rsl2 = vaddw_u16(Rsl2, norm2);
+                Rsl3 = vaddw_u16(Rsl3, norm3);
+                Rsl4 = vaddw_u16(Rsl4, norm4);
+
+                RV[Z+0] = vbslq_u32(Rlt1, Rsl1, RV[Z+0]);       // R = R<L ? Rsl : R
+                RV[Z+1] = vbslq_u32(Rlt2, Rsl2, RV[Z+1]);
+                RV[Z+2] = vbslq_u32(Rlt3, Rsl3, RV[Z+2]);
+                RV[Z+3] = vbslq_u32(Rlt4, Rsl4, RV[Z+3]);
+
+                // Offset into s3[l][c] => s3 + l*TOTFREQ_O1_FAST + c.
+                uint32x4_t off1 = vandq_u32(RV[Z+0], maskv);
+                uint32x4_t off2 = vandq_u32(RV[Z+1], maskv);
+                uint32x4_t off3 = vandq_u32(RV[Z+2], maskv);
+                uint32x4_t off4 = vandq_u32(RV[Z+3], maskv);
+
+                off1 = vaddq_u32(off1, Sv1);
+                off2 = vaddq_u32(off2, Sv2);
+                off3 = vaddq_u32(off3, Sv3);
+                off4 = vaddq_u32(off4, Sv4);
+                
+                vst1q_u32(&m[z+ 0], off1);
+                vst1q_u32(&m[z+ 4], off2);
+                vst1q_u32(&m[z+ 8], off3);
+                vst1q_u32(&m[z+12], off4);
+            }
+
+            i4[0]++;
+            if (++tidx == 32) {
+                i4[0] -= 32;
+
+                transpose_and_copy(out, i4, tbuf);
+                tidx = 0;
+            }
+        }
+
+        vst1q_u32(&R[ 0], RV[0]);
+        vst1q_u32(&R[ 4], RV[1]);
+        vst1q_u32(&R[ 8], RV[2]);
+        vst1q_u32(&R[12], RV[3]);
+        vst1q_u32(&R[16], RV[4]);
+        vst1q_u32(&R[20], RV[5]);
+        vst1q_u32(&R[24], RV[6]);
+        vst1q_u32(&R[28], RV[7]);
+
+        i4[0]-=tidx;
+        int T;
+        for (z = 0; z < NX; z++)
+            for (T = 0; T < tidx; T++)
+                out[i4[z]++] = tbuf[T][z];
+
+        // Scalar version for close to end of in[] array so we don't do
+        // SIMD loads beyond the end of the buffer
+        for (; i4[0] < isz4; ) {
+            for (z = 0; z < NX; z++) {
+                uint32_t m = R[z] & ((1u<<TF_SHIFT_O1_FAST)-1);
+                uint32_t S = s3[l[z]][m];
+                unsigned char c = S & 0xff;
+                out[i4[z]++] = c;
+                R[z] = (S>>(TF_SHIFT_O1_FAST+8)) * (R[z]>>TF_SHIFT_O1_FAST) +
+                    ((S>>8) & ((1u<<TF_SHIFT_O1_FAST)-1));
+                RansDecRenormSafe(&R[z], &ptr, ptr_end);
+                l[z] = c;
+            }
+        }
+
+        // Remainder
+        for (; i4[NX-1] < out_sz; i4[NX-1]++) {
+            uint32_t S = s3[l[NX-1]][R[NX-1] & ((1u<<TF_SHIFT_O1_FAST)-1)];
+            out[i4[NX-1]] = l[NX-1] = S&0xff;
+            R[NX-1] = (S>>(TF_SHIFT_O1_FAST+8)) * (R[NX-1]>>TF_SHIFT_O1_FAST)
+                + ((S>>8) & ((1u<<TF_SHIFT_O1_FAST)-1));
+            RansDecRenormSafe(&R[NX-1], &ptr, ptr_end);
+        }
+    }
+    //fprintf(stderr, "    1 Decoded %d bytes\n", (int)(ptr-in)); //c-size
+
+    htscodecs_tls_free(sfb_);
+    return out;
+
+ err:
+    htscodecs_tls_free(sfb_);
+    free(out_free);
+    free(c_freq);
+
+    return NULL;
+}
+
+#undef MAGIC2
+#else  /* __ARM_NEON */
+// Prevent "empty translation unit" errors when building without NEON
+const char *rANS_static32x16pr_neon_disabled = "No NEON";
+#endif /* __ARM_NEON */
diff --git a/ext/htslib/htscodecs/htscodecs/rANS_static32x16pr_sse4.c b/ext/htslib/htscodecs/htscodecs/rANS_static32x16pr_sse4.c
new file mode 100644
index 0000000..a9b9c4b
--- /dev/null
+++ b/ext/htslib/htscodecs/htscodecs/rANS_static32x16pr_sse4.c
@@ -0,0 +1,1831 @@
+/*
+ * Copyright (c) 2017-2023 Genome Research Ltd.
+ * Author(s): James Bonfield
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above
+ *       copyright notice, this list of conditions and the following
+ *       disclaimer in the documentation and/or other materials provided
+ *       with the distribution.
+ *
+ *    3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+ *       Institute nor the names of its contributors may be used to endorse
+ *       or promote products derived from this software without specific
+ *       prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH
+ * LTD OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#if defined(__x86_64__) && \
+    defined(HAVE_SSE4_1) && defined(HAVE_SSSE3) && defined(HAVE_POPCNT)
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <limits.h>
+#include <x86intrin.h>
+
+#include "rANS_word.h"
+#include "rANS_static4x16.h"
+#include "rANS_static16_int.h"
+#include "varint.h"
+#include "utils.h"
+
+/* Uses: SSE, SSE2, SSSE3, SSE4.1 and POPCNT
+SSE:
+_mm_movemask_ps
+
+SSE2:
+    _mm_load_si128 _mm_store_si128
+    _mm_set_epi32  _mm_set1_epi32
+    _mm_and_si128  _mm_or_si128
+    _mm_srli_epi32 _mm_slli_epi32 _mm_srli_epi64 _mm_set1_epi64x
+    _mm_add_epi32
+    _mm_packus_epi32
+    _mm_andnot_si128
+    _mm_cmpeq_epi32
+    _mm_mul_epu32
+    _mm_shuffle_epi32
+
+SSSE3:
+    _mm_shuffle_epi8
+
+SSE4.1:
+    _mm_mullo_epi32
+    _mm_packus_epi32
+    _mm_max_epu32
+    _mm_cvtepu16_epi32
+    _mm_blendv_epi8
+
+POPCNT:
+    _mm_popcnt_u32
+ */
+
+
+#define NX 32
+
+#define LOAD128(a,b)                                    \
+    __m128i a##1 = _mm_load_si128((__m128i *)&b[0]);    \
+    __m128i a##2 = _mm_load_si128((__m128i *)&b[4]);    \
+    __m128i a##3 = _mm_load_si128((__m128i *)&b[8]);    \
+    __m128i a##4 = _mm_load_si128((__m128i *)&b[12]);   \
+    __m128i a##5 = _mm_load_si128((__m128i *)&b[16]);   \
+    __m128i a##6 = _mm_load_si128((__m128i *)&b[20]);   \
+    __m128i a##7 = _mm_load_si128((__m128i *)&b[24]);   \
+    __m128i a##8 = _mm_load_si128((__m128i *)&b[28]);
+
+#define STORE128(a,b)                                   \
+    _mm_store_si128((__m128i *)&b[ 0], a##1);           \
+    _mm_store_si128((__m128i *)&b[ 4], a##2);           \
+    _mm_store_si128((__m128i *)&b[ 8], a##3);           \
+    _mm_store_si128((__m128i *)&b[12], a##4);           \
+    _mm_store_si128((__m128i *)&b[16], a##5);           \
+    _mm_store_si128((__m128i *)&b[20], a##6);           \
+    _mm_store_si128((__m128i *)&b[24], a##7);           \
+    _mm_store_si128((__m128i *)&b[28], a##8);
+
+static inline __m128i _mm_i32gather_epi32x(int *b, __m128i idx, int size) {
+    int c[4] __attribute__((aligned(32)));
+    _mm_store_si128((__m128i *)c, idx);
+    return _mm_set_epi32(b[c[3]], b[c[2]], b[c[1]], b[c[0]]);
+}
+
+// SSE4 implementation of the Order-0 encoder is poorly performing.
+// Disabled for now.
+#if 0
+#define LOAD128v(a,b)                                   \
+    __m128i a[8];                                       \
+    a[0] = _mm_load_si128((__m128i *)&b[0]);            \
+    a[1] = _mm_load_si128((__m128i *)&b[4]);            \
+    a[2] = _mm_load_si128((__m128i *)&b[8]);            \
+    a[3] = _mm_load_si128((__m128i *)&b[12]);           \
+    a[4] = _mm_load_si128((__m128i *)&b[16]);           \
+    a[5] = _mm_load_si128((__m128i *)&b[20]);           \
+    a[6] = _mm_load_si128((__m128i *)&b[24]);           \
+    a[7] = _mm_load_si128((__m128i *)&b[28]);
+
+#define STORE128v(a,b)                                  \
+    _mm_store_si128((__m128i *)&b[ 0], a[0]);           \
+    _mm_store_si128((__m128i *)&b[ 4], a[1]);           \
+    _mm_store_si128((__m128i *)&b[ 8], a[2]);           \
+    _mm_store_si128((__m128i *)&b[12], a[3]);           \
+    _mm_store_si128((__m128i *)&b[16], a[4]);           \
+    _mm_store_si128((__m128i *)&b[20], a[5]);           \
+    _mm_store_si128((__m128i *)&b[24], a[6]);           \
+    _mm_store_si128((__m128i *)&b[28], a[7]);
+
+static inline __m128i _mm_mulhi_epu32(__m128i a, __m128i b) {
+    // Multiply bottom 4 items and top 4 items together.
+    __m128i ab_hm = _mm_mul_epu32(_mm_srli_epi64(a, 32),_mm_srli_epi64(b, 32));
+    __m128i ab_lm = _mm_srli_epi64(_mm_mul_epu32(a, b), 32);
+
+    // Blend or and/or seems to make no difference.
+    return _mm_blend_epi16(ab_lm, ab_hm, 0xcc);
+
+//    // Shift to get hi 32-bit of each 64-bit product
+//    ab_hm = _mm_and_si128(ab_hm,_mm_set1_epi64x((uint64_t)0xffffffff00000000));
+//
+//    return _mm_or_si128(ab_lm, ab_hm);
+}
+
+// Shift A>>B for non-constant B exists in AVX2, but not SSE world.
+// We simulate this for now by store, shift, and load.  Ugly!
+static inline __m128i _mm_srlv_epi32x(__m128i a, __m128i b) {
+// Extract and inline shift.  Slowest clang, joint fastest gcc
+//    return _mm_set_epi32(_mm_extract_epi32(a,3)>>_mm_extract_epi32(b,3),
+//                       _mm_extract_epi32(a,2)>>_mm_extract_epi32(b,2),
+//                       _mm_extract_epi32(a,1)>>_mm_extract_epi32(b,1),
+//                       _mm_extract_epi32(a,0)>>_mm_extract_epi32(b,0));
+
+// Half store and inline shift; Fastest gcc, comparable to others below clang
+//    uint32_t A[4];
+//    _mm_storeu_si128((__m128i *)&A, a);
+//
+//    return _mm_set_epi32(A[3]>>_mm_extract_epi32(b,3),
+//                       A[2]>>_mm_extract_epi32(b,2),
+//                       A[1]>>_mm_extract_epi32(b,1),
+//                       A[0]>>_mm_extract_epi32(b,0));
+
+// Other half
+    uint32_t B[4];
+    _mm_storeu_si128((__m128i *)&B, b);
+    return _mm_set_epi32(_mm_extract_epi32(a,3)>>B[3],
+                         _mm_extract_epi32(a,2)>>B[2],
+                         _mm_extract_epi32(a,1)>>B[1],
+                         _mm_extract_epi32(a,0)>>B[0]);
+
+// Check if all b[] match, and constant shift if so.
+// Too costly, even on q4 where it's common for all shift to be identical.
+//    __m128i cmp = _mm_cmpeq_epi32(b, _mm_shuffle_epi32(b, 0x39));
+//    if (_mm_movemask_ps((__m128)cmp) == 15) {
+//      return _mm_srl_epi32(a, _mm_set1_epi64x(_mm_extract_epi32(b,0)));
+//      //_mm_storeu_si128((__m128i *)&B,_mm_set1_epi32(_mm_extract_epi32(b,0)));
+//    } else {
+//      uint32_t B[4];
+//      _mm_storeu_si128((__m128i *)&B, b);
+//      return _mm_set_epi32(_mm_extract_epi32(a,3)>>B[3],
+//                           _mm_extract_epi32(a,2)>>B[2],
+//                           _mm_extract_epi32(a,1)>>B[1],
+//                           _mm_extract_epi32(a,0)>>B[0]);
+//    }
+
+
+// Full store and inline shift
+//   uint32_t A[4], B[4] __attribute__((aligned(16)));
+//   _mm_storeu_si128((__m128i *)&A, a);
+//   _mm_storeu_si128((__m128i *)&B, b);
+//
+//   return _mm_set_epi32(A[3]>>B[3], A[2]>>B[2], A[1]>>B[1], A[0]>>B[0]);
+
+// Full store, shift and load
+//    uint32_t A[4], B[4] __attribute__((aligned(16)));
+//    _mm_storeu_si128((__m128i *)&A, a);
+//    _mm_storeu_si128((__m128i *)&B, b);
+//    A[0]>>=B[0];
+//    A[1]>>=B[1];
+//    A[2]>>=B[2];
+//    A[3]>>=B[3];
+//    return _mm_loadu_si128((__m128i *)A);
+}
+
+unsigned char *rans_compress_O0_32x16_sse4(unsigned char *in,
+                                           unsigned int in_size,
+                                           unsigned char *out,
+                                           unsigned int *out_size) {
+    unsigned char *cp, *out_end;
+    RansEncSymbol syms[256];
+    RansState ransN[NX];
+    uint8_t* ptr;
+    uint32_t F[256+MAGIC] = {0};
+    int i, j, tab_size = 0, x, z;
+    // -20 for order/size/meta
+    uint32_t bound = rans_compress_bound_4x16(in_size,0)-20;
+
+    if (!out) {
+        *out_size = bound;
+        out = malloc(*out_size);
+    }
+    if (!out || bound > *out_size)
+        return NULL;
+
+    // If "out" isn't word aligned, tweak out_end/ptr to ensure it is.
+    // We already added more round in bound to allow for this.
+    if (((size_t)out)&1)
+        bound--;
+    ptr = out_end = out + bound;
+
+    if (in_size == 0)
+        goto empty;
+
+    // Compute statistics
+    if (hist8(in, in_size, F) < 0)
+        return NULL;
+
+    // Normalise so frequences sum to power of 2
+    uint32_t fsum = in_size;
+    uint32_t max_val = round2(fsum);
+    if (max_val > TOTFREQ)
+        max_val = TOTFREQ;
+
+    if (normalise_freq(F, fsum, max_val) < 0)
+        return NULL;
+    fsum=max_val;
+
+    cp = out;
+    cp += encode_freq(cp, F);
+    tab_size = cp-out;
+    //write(2, out+4, cp-(out+4));
+
+    if (normalise_freq(F, fsum, TOTFREQ) < 0)
+        return NULL;
+
+    // Encode statistics.
+    for (x = j = 0; j < 256; j++) {
+        if (F[j]) {
+            RansEncSymbolInit(&syms[j], x, F[j], TF_SHIFT);
+            x += F[j];
+        }
+    }
+
+    for (z = 0; z < NX; z++)
+      RansEncInit(&ransN[z]);
+
+    z = i = in_size&(NX-1);
+    while (z-- > 0)
+      RansEncPutSymbol(&ransN[z], &ptr, &syms[in[in_size-(i-z)]]);
+
+    uint32_t SB[256], SA[256], SD[256], SC[256];
+
+    // Build lookup tables for SIMD encoding
+    uint16_t *ptr16 = (uint16_t *)ptr;
+    for (i = 0; i < 256; i++) {
+        SB[i] = syms[i].x_max;
+        SA[i] = syms[i].rcp_freq;
+        SD[i] = (syms[i].cmpl_freq<<0) | (syms[i].rcp_shift<<16);
+        SC[i] = syms[i].bias;
+    }
+
+    LOAD128v(Rv, ransN);
+
+    const __m128i shuf = _mm_set_epi8(0x80, 0x80, 0x80, 0x80,
+                                      0x80, 0x80, 0x80, 0x80,
+                                      0x0d, 0x0c, 0x09, 0x08,
+                                      0x05, 0x04, 0x01, 0x00);
+
+    // FIXME: slower!
+    // q40:  340 (scalar) vs 300 (this)
+    // q4:   430 (scalar) vs 304 (this)
+    for (i=(in_size &~(NX-1)); i>0; i-=NX) {
+      uint8_t *c = &in[i-NX];
+
+      int h;
+      // Slightly better in 4x8 instead of 2x16 cycles with clang,
+      // but the reverse with gcc.
+      //for (h=24; h >= 0; h -= 8) {
+      for (h=16; h >= 0; h -= 16) {
+        int H = h/4; // rans index
+        uint8_t *C = &in[i-NX+h];
+
+#define SET(i,a) _mm_set_epi32(a[C[i+3]],a[C[i+2]],a[C[i+1]],a[C[i+0]])
+        __m128i xmax8 = SET(12, SB);
+        __m128i xmax7 = SET( 8, SB);
+        __m128i xmax6 = SET( 4, SB);
+        __m128i xmax5 = SET( 0, SB);
+
+        __m128i cv8 = _mm_cmpgt_epi32(Rv[H+3], xmax8);
+        __m128i cv7 = _mm_cmpgt_epi32(Rv[H+2], xmax7);
+        __m128i cv6 = _mm_cmpgt_epi32(Rv[H+1], xmax6);
+        __m128i cv5 = _mm_cmpgt_epi32(Rv[H+0], xmax5);
+
+        // Store bottom 16-bits at ptr16
+        unsigned int imask8 = _mm_movemask_ps((__m128)cv8);
+        unsigned int imask7 = _mm_movemask_ps((__m128)cv7);
+        unsigned int imask6 = _mm_movemask_ps((__m128)cv6);
+        unsigned int imask5 = _mm_movemask_ps((__m128)cv5);
+
+#define X(A) 4*A,4*A+1,0x80,0x80
+#define _ 0x80,0x80,0x80,0x80
+        uint8_t permutec[16][16] __attribute__((aligned(16))) = {
+            {  _ ,  _ ,  _ ,  _ },
+            {  _ ,  _ ,  _ ,X(0)},
+            {  _ ,  _ ,  _ ,X(1)},
+            {  _ ,  _ ,X(0),X(1)},
+
+            {  _ ,  _ ,  _ ,X(2)},
+            {  _ ,  _ ,X(0),X(2)},
+            {  _ ,  _ ,X(1),X(2)},
+            {  _ ,X(0),X(1),X(2)},
+
+            {  _ ,  _ ,  _ ,X(3)},
+            {  _ ,  _ ,X(0),X(3)},
+            {  _ ,  _ ,X(1),X(3)},
+            {  _ ,X(0),X(1),X(3)},
+
+            {  _ ,  _ ,X(2),X(3)},
+            {  _ ,X(0),X(2),X(3)},
+            {  _ ,X(1),X(2),X(3)},
+            {X(0),X(1),X(2),X(3)},
+        };
+#undef X
+#undef _
+
+        __m128i idx8 = _mm_load_si128((__m128i *)permutec[imask8]);
+        __m128i idx7 = _mm_load_si128((__m128i *)permutec[imask7]);
+        __m128i idx6 = _mm_load_si128((__m128i *)permutec[imask6]);
+        __m128i idx5 = _mm_load_si128((__m128i *)permutec[imask5]);
+
+        // Permute; to gather together the rans states that need flushing
+        __m128i V1, V2, V3, V4, V5, V6, V7, V8;
+        V8 = _mm_shuffle_epi8(_mm_and_si128(Rv[H+3], cv8), idx8);
+        V7 = _mm_shuffle_epi8(_mm_and_si128(Rv[H+2], cv7), idx7);
+        V6 = _mm_shuffle_epi8(_mm_and_si128(Rv[H+1], cv6), idx6);
+        V5 = _mm_shuffle_epi8(_mm_and_si128(Rv[H+0], cv5), idx5);
+
+        // Shuffle alternating shorts together to collect low 16-bit
+        // elements together.  ... 9 8 5 4 1 0.
+        // Or as with avx2 code use packus instead.
+        V8 = _mm_shuffle_epi8(V8, shuf);
+        V7 = _mm_shuffle_epi8(V7, shuf);
+        V6 = _mm_shuffle_epi8(V6, shuf);
+        V5 = _mm_shuffle_epi8(V5, shuf);
+
+        _mm_storeu_si64(ptr16-4, V8); ptr16 -= _mm_popcnt_u32(imask8);
+        _mm_storeu_si64(ptr16-4, V7); ptr16 -= _mm_popcnt_u32(imask7);
+        _mm_storeu_si64(ptr16-4, V6); ptr16 -= _mm_popcnt_u32(imask6);
+        _mm_storeu_si64(ptr16-4, V5); ptr16 -= _mm_popcnt_u32(imask5);
+
+        Rv[H+3] = _mm_blendv_epi8(Rv[H+3], _mm_srli_epi32(Rv[H+3], 16), cv8);
+        Rv[H+2] = _mm_blendv_epi8(Rv[H+2], _mm_srli_epi32(Rv[H+2], 16), cv7);
+        Rv[H+1] = _mm_blendv_epi8(Rv[H+1], _mm_srli_epi32(Rv[H+1], 16), cv6);
+        Rv[H+0] = _mm_blendv_epi8(Rv[H+0], _mm_srli_epi32(Rv[H+0], 16), cv5);
+
+        // Cannot trivially replace the multiply as mulhi_epu32 doesn't
+        // exist (only mullo).
+        // However we can use _mm_mul_epu32 twice to get 64bit results
+        // (h our lanes) and shift/or to get the answer.
+        //
+        // (AVX512 allows us to hold it all in 64-bit lanes and use mullo_epi64
+        // plus a shift.  KNC has mulhi_epi32, but not sure if this is
+        // available.)
+        __m128i rfv8 = SET(12, SA);
+        __m128i rfv7 = SET( 8, SA);
+        __m128i rfv6 = SET( 4, SA);
+        __m128i rfv5 = SET( 0, SA);
+
+        rfv8 = _mm_mulhi_epu32(Rv[H+3], rfv8);
+        rfv7 = _mm_mulhi_epu32(Rv[H+2], rfv7);
+        rfv6 = _mm_mulhi_epu32(Rv[H+1], rfv6);
+        rfv5 = _mm_mulhi_epu32(Rv[H+0], rfv5);
+
+        __m128i SDv8 = SET(12, SD);
+        __m128i SDv7 = SET( 8, SD);
+        __m128i SDv6 = SET( 4, SD);
+        __m128i SDv5 = SET( 0, SD);
+
+        __m128i shiftv8 = _mm_srli_epi32(SDv8, 16);
+        __m128i shiftv7 = _mm_srli_epi32(SDv7, 16);
+        __m128i shiftv6 = _mm_srli_epi32(SDv6, 16);
+        __m128i shiftv5 = _mm_srli_epi32(SDv5, 16);
+
+        __m128i freqv8 = _mm_and_si128(SDv8, _mm_set1_epi32(0xffff));
+        __m128i freqv7 = _mm_and_si128(SDv7, _mm_set1_epi32(0xffff));
+        __m128i freqv6 = _mm_and_si128(SDv6, _mm_set1_epi32(0xffff));
+        __m128i freqv5 = _mm_and_si128(SDv5, _mm_set1_epi32(0xffff));
+
+        // Bake this into the tabel to start with?
+        shiftv8 = _mm_sub_epi32(shiftv8, _mm_set1_epi32(32));
+        shiftv7 = _mm_sub_epi32(shiftv7, _mm_set1_epi32(32));
+        shiftv6 = _mm_sub_epi32(shiftv6, _mm_set1_epi32(32));
+        shiftv5 = _mm_sub_epi32(shiftv5, _mm_set1_epi32(32));
+
+        // No way to shift by varying amounts.  Store, shift, load? Simulated
+        __m128i qv8 = _mm_srlv_epi32x(rfv8, shiftv8);
+        __m128i qv7 = _mm_srlv_epi32x(rfv7, shiftv7);
+        __m128i qv6 = _mm_srlv_epi32x(rfv6, shiftv6);
+        __m128i qv5 = _mm_srlv_epi32x(rfv5, shiftv5);
+
+        qv8 = _mm_mullo_epi32(qv8, freqv8);
+        qv7 = _mm_mullo_epi32(qv7, freqv7);
+        qv6 = _mm_mullo_epi32(qv6, freqv6);
+        qv5 = _mm_mullo_epi32(qv5, freqv5);
+
+        qv8 = _mm_add_epi32(qv8, SET(12, SC));
+        qv7 = _mm_add_epi32(qv7, SET( 8, SC));
+        qv6 = _mm_add_epi32(qv6, SET( 4, SC));
+        qv5 = _mm_add_epi32(qv5, SET( 0, SC));
+
+        Rv[H+3] = _mm_add_epi32(Rv[H+3], qv8);
+        Rv[H+2] = _mm_add_epi32(Rv[H+2], qv7);
+        Rv[H+1] = _mm_add_epi32(Rv[H+1], qv6);
+        Rv[H+0] = _mm_add_epi32(Rv[H+0], qv5);
+      }
+    }
+
+    STORE128v(Rv, ransN);
+
+    ptr = (uint8_t *)ptr16;
+
+    for (z = NX-1; z >= 0; z--)
+      RansEncFlush(&ransN[z], &ptr);
+
+ empty:
+    // Finalise block size and return it
+    *out_size = (out_end - ptr) + tab_size;
+
+//    cp = out;
+//    *cp++ = (in_size>> 0) & 0xff;
+//    *cp++ = (in_size>> 8) & 0xff;
+//    *cp++ = (in_size>>16) & 0xff;
+//    *cp++ = (in_size>>24) & 0xff;
+
+    memmove(out + tab_size, ptr, out_end-ptr);
+
+    return out;
+}
+#endif // disable SSE4 encoder
+
+unsigned char *rans_uncompress_O0_32x16_sse4(unsigned char *in,
+                                             unsigned int in_size,
+                                             unsigned char *out,
+                                             unsigned int out_sz) {
+    if (in_size < 16) // 4-states at least
+        return NULL;
+
+    if (out_sz >= INT_MAX)
+        return NULL; // protect against some overflow cases
+
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    if (out_sz > 100000)
+        return NULL;
+#endif
+
+    /* Load in the static tables */
+    unsigned char *cp = in, *out_free = NULL;
+    unsigned char *cp_end = in + in_size;
+    int i;
+    uint32_t s3[TOTFREQ] __attribute__((aligned(32))); // For TF_SHIFT <= 12
+
+    if (!out)
+        out_free = out = malloc(out_sz);
+    if (!out)
+        return NULL;
+
+    // Precompute reverse lookup of frequency.
+    uint32_t F[256] = {0}, fsum;
+    int fsz = decode_freq(cp, cp_end, F, &fsum);
+    if (!fsz)
+        goto err;
+    cp += fsz;
+
+    normalise_freq_shift(F, fsum, TOTFREQ);
+
+    // Build symbols; fixme, do as part of decode, see the _d variant
+    if (rans_F_to_s3(F, TF_SHIFT, s3))
+        goto err;
+
+    if (cp_end - cp < NX * 4)
+        goto err;
+
+    int z;
+    RansState R[NX] __attribute__((aligned(32)));
+    for (z = 0; z < NX; z++) {
+        RansDecInit(&R[z], &cp);
+        if (R[z] < RANS_BYTE_L)
+            goto err;
+    }
+
+    uint16_t *sp = (uint16_t *)cp;
+
+    int out_end = (out_sz&~(NX-1));
+    const uint32_t mask = (1u << TF_SHIFT)-1;
+
+    __m128i maskv  = _mm_set1_epi32(mask); // set mask in all lanes
+    LOAD128(Rv, R);
+
+    uint8_t overflow[72+64] = {0};
+    for (i=0; i < out_end; i+=NX) {
+        //for (z = 0; z < NX; z++)
+        //  m[z] = R[z] & mask;
+        __m128i masked1 = _mm_and_si128(Rv1, maskv);
+        __m128i masked2 = _mm_and_si128(Rv2, maskv);
+        __m128i masked3 = _mm_and_si128(Rv3, maskv);
+        __m128i masked4 = _mm_and_si128(Rv4, maskv);
+
+        //  S[z] = s3[m[z]];
+        __m128i Sv1 = _mm_i32gather_epi32x((int *)s3, masked1, sizeof(*s3));
+        __m128i Sv2 = _mm_i32gather_epi32x((int *)s3, masked2, sizeof(*s3));
+        __m128i Sv3 = _mm_i32gather_epi32x((int *)s3, masked3, sizeof(*s3));
+        __m128i Sv4 = _mm_i32gather_epi32x((int *)s3, masked4, sizeof(*s3));
+
+        //  f[z] = S[z]>>(TF_SHIFT+8);
+        __m128i fv1 = _mm_srli_epi32(Sv1, TF_SHIFT+8);
+        __m128i fv2 = _mm_srli_epi32(Sv2, TF_SHIFT+8);
+        __m128i fv3 = _mm_srli_epi32(Sv3, TF_SHIFT+8);
+        __m128i fv4 = _mm_srli_epi32(Sv4, TF_SHIFT+8);
+
+        //  b[z] = (S[z]>>8) & mask;
+        __m128i bv1 = _mm_and_si128(_mm_srli_epi32(Sv1, 8), maskv);
+        __m128i bv2 = _mm_and_si128(_mm_srli_epi32(Sv2, 8), maskv);
+        __m128i bv3 = _mm_and_si128(_mm_srli_epi32(Sv3, 8), maskv);
+        __m128i bv4 = _mm_and_si128(_mm_srli_epi32(Sv4, 8), maskv);
+
+        //  s[z] = S[z] & 0xff;
+        __m128i sv1 = _mm_and_si128(Sv1, _mm_set1_epi32(0xff));
+        __m128i sv2 = _mm_and_si128(Sv2, _mm_set1_epi32(0xff));
+        __m128i sv3 = _mm_and_si128(Sv3, _mm_set1_epi32(0xff));
+        __m128i sv4 = _mm_and_si128(Sv4, _mm_set1_epi32(0xff));
+
+        //  R[z] = f[z] * (R[z] >> TF_SHIFT) + b[z];
+        Rv1 = _mm_add_epi32(
+                  _mm_mullo_epi32(
+                      _mm_srli_epi32(Rv1,TF_SHIFT), fv1), bv1);
+        Rv2 = _mm_add_epi32(
+                  _mm_mullo_epi32(
+                      _mm_srli_epi32(Rv2,TF_SHIFT), fv2), bv2);
+        Rv3 = _mm_add_epi32(
+                  _mm_mullo_epi32(
+                      _mm_srli_epi32(Rv3,TF_SHIFT), fv3), bv3);
+        Rv4 = _mm_add_epi32(
+                  _mm_mullo_epi32(
+                      _mm_srli_epi32(Rv4,TF_SHIFT), fv4), bv4);
+
+        // Tricky one:  out[i+z] = s[z];
+        //             ---d---c ---b---a  sv1
+        //             ---h---g ---f---e  sv2
+        // packs_epi32 -h-g-f-e -d-c-b-a  sv1(2)
+        // packs_epi16 ponmlkji hgfedcba  sv1(2) / sv3(4)
+        sv1 = _mm_packus_epi32(sv1, sv2);
+        sv3 = _mm_packus_epi32(sv3, sv4);
+        sv1 = _mm_packus_epi16(sv1, sv3);
+
+        // c =  R[z] < RANS_BYTE_L;
+        // A little tricky as we only have signed comparisons.
+        // See https://stackoverflow.com/questions/32945410/sse2-intrinsics-comparing-unsigned-integers
+
+#define _mm_cmplt_epu32_imm(a,b) _mm_andnot_si128(_mm_cmpeq_epi32(_mm_max_epu32((a),_mm_set1_epi32(b)), (a)), _mm_set1_epi32(-1));
+
+//#define _mm_cmplt_epu32_imm(a,b) _mm_cmpgt_epi32(_mm_set1_epi32((b)-0x80000000), _mm_xor_si128((a), _mm_set1_epi32(0x80000000)))
+
+        __m128i renorm_mask1, renorm_mask2, renorm_mask3, renorm_mask4;
+        renorm_mask1 = _mm_cmplt_epu32_imm(Rv1, RANS_BYTE_L);
+        renorm_mask2 = _mm_cmplt_epu32_imm(Rv2, RANS_BYTE_L);
+        renorm_mask3 = _mm_cmplt_epu32_imm(Rv3, RANS_BYTE_L);
+        renorm_mask4 = _mm_cmplt_epu32_imm(Rv4, RANS_BYTE_L);
+
+//#define P(A,B,C,D) ((A)+((B)<<2) + ((C)<<4) + ((D)<<6))
+#define P(A,B,C,D)                              \
+        { A+0,A+1,A+2,A+3,                      \
+          B+0,B+1,B+2,B+3,                      \
+          C+0,C+1,C+2,C+3,                      \
+          D+0,D+1,D+2,D+3}
+#ifdef _
+#undef _
+#endif
+#define _ 0x80
+        uint8_t pidx[16][16] = {
+            P(_,_,_,_),
+            P(0,_,_,_),
+            P(_,0,_,_),
+            P(0,4,_,_),
+
+            P(_,_,0,_),
+            P(0,_,4,_),
+            P(_,0,4,_),
+            P(0,4,8,_),
+
+            P(_,_,_,0),
+            P(0,_,_,4),
+            P(_,0,_,4),
+            P(0,4,_,8),
+
+            P(_,_,0,4),
+            P(0,_,4,8),
+            P(_,0,4,8),
+            P(0,4,8,12),
+        };
+#undef _
+
+        // Protect against running off the end of in buffer.
+        // We copy it to a worst-case local buffer when near the end.
+        // 72 = 7*8(imask1..7) + 16;  worse case for 8th _mm_loadu_si128 call.
+        // An extra 64 bytes is to avoid triggering this multiple times
+        // after we swap sp/cp_end over.
+        if ((uint8_t *)sp+72 > cp_end) {
+            memmove(overflow, sp, cp_end - (uint8_t *)sp);
+            sp = (uint16_t *)overflow;
+            cp_end = (uint8_t *)overflow + sizeof(overflow);
+        }
+
+        // Shuffle the renorm values to correct lanes and incr sp pointer
+        __m128i Vv1 = _mm_cvtepu16_epi32(_mm_loadu_si128((__m128i *)sp));
+        unsigned int imask1 = _mm_movemask_ps((__m128)renorm_mask1);
+        Vv1 = _mm_shuffle_epi8(Vv1, _mm_load_si128((__m128i*)pidx[imask1]));
+        sp += _mm_popcnt_u32(imask1);
+
+        __m128i Vv2 = _mm_cvtepu16_epi32(_mm_loadu_si128((__m128i *)sp));
+        unsigned int imask2 = _mm_movemask_ps((__m128)renorm_mask2);
+        sp += _mm_popcnt_u32(imask2);
+        Vv2 = _mm_shuffle_epi8(Vv2, _mm_load_si128((__m128i*)pidx[imask2]));
+
+        __m128i Vv3 = _mm_cvtepu16_epi32(_mm_loadu_si128((__m128i *)sp));
+        unsigned int imask3 = _mm_movemask_ps((__m128)renorm_mask3);
+        Vv3 = _mm_shuffle_epi8(Vv3, _mm_load_si128((__m128i*)pidx[imask3]));
+        sp += _mm_popcnt_u32(imask3);
+
+        __m128i Vv4 = _mm_cvtepu16_epi32(_mm_loadu_si128((__m128i *)sp));
+        unsigned int imask4 = _mm_movemask_ps((__m128)renorm_mask4);
+        sp += _mm_popcnt_u32(imask4);
+        Vv4 = _mm_shuffle_epi8(Vv4, _mm_load_si128((__m128i*)pidx[imask4]));
+
+        __m128i Yv1 = _mm_slli_epi32(Rv1, 16);
+        __m128i Yv2 = _mm_slli_epi32(Rv2, 16);
+        __m128i Yv3 = _mm_slli_epi32(Rv3, 16);
+        __m128i Yv4 = _mm_slli_epi32(Rv4, 16);
+
+        // y = (R[z] << 16) | V[z];
+        Yv1 = _mm_or_si128(Yv1, Vv1);
+        Yv2 = _mm_or_si128(Yv2, Vv2);
+        Yv3 = _mm_or_si128(Yv3, Vv3);
+        Yv4 = _mm_or_si128(Yv4, Vv4);
+
+        // R[z] = c ? Y[z] : R[z];
+        Rv1 = _mm_blendv_epi8(Rv1, Yv1, renorm_mask1);
+        Rv2 = _mm_blendv_epi8(Rv2, Yv2, renorm_mask2);
+        Rv3 = _mm_blendv_epi8(Rv3, Yv3, renorm_mask3);
+        Rv4 = _mm_blendv_epi8(Rv4, Yv4, renorm_mask4);
+
+        // ------------------------------------------------------------
+
+        //  m[z] = R[z] & mask;
+        __m128i masked5 = _mm_and_si128(Rv5, maskv);
+        __m128i masked6 = _mm_and_si128(Rv6, maskv);
+        __m128i masked7 = _mm_and_si128(Rv7, maskv);
+        __m128i masked8 = _mm_and_si128(Rv8, maskv);
+
+        //  S[z] = s3[m[z]];
+        __m128i Sv5 = _mm_i32gather_epi32x((int *)s3, masked5, sizeof(*s3));
+        __m128i Sv6 = _mm_i32gather_epi32x((int *)s3, masked6, sizeof(*s3));
+        __m128i Sv7 = _mm_i32gather_epi32x((int *)s3, masked7, sizeof(*s3));
+        __m128i Sv8 = _mm_i32gather_epi32x((int *)s3, masked8, sizeof(*s3));
+
+        //  f[z] = S[z]>>(TF_SHIFT+8);
+        __m128i fv5 = _mm_srli_epi32(Sv5, TF_SHIFT+8);
+        __m128i fv6 = _mm_srli_epi32(Sv6, TF_SHIFT+8);
+        __m128i fv7 = _mm_srli_epi32(Sv7, TF_SHIFT+8);
+        __m128i fv8 = _mm_srli_epi32(Sv8, TF_SHIFT+8);
+
+        //  b[z] = (S[z]>>8) & mask;
+        __m128i bv5 = _mm_and_si128(_mm_srli_epi32(Sv5, 8), maskv);
+        __m128i bv6 = _mm_and_si128(_mm_srli_epi32(Sv6, 8), maskv);
+        __m128i bv7 = _mm_and_si128(_mm_srli_epi32(Sv7, 8), maskv);
+        __m128i bv8 = _mm_and_si128(_mm_srli_epi32(Sv8, 8), maskv);
+
+        //  s[z] = S[z] & 0xff;
+        __m128i sv5 = _mm_and_si128(Sv5, _mm_set1_epi32(0xff));
+        __m128i sv6 = _mm_and_si128(Sv6, _mm_set1_epi32(0xff));
+        __m128i sv7 = _mm_and_si128(Sv7, _mm_set1_epi32(0xff));
+        __m128i sv8 = _mm_and_si128(Sv8, _mm_set1_epi32(0xff));
+
+        //  R[z] = f[z] * (R[z] >> TF_SHIFT) + b[z];
+        Rv5 = _mm_add_epi32(
+                  _mm_mullo_epi32(
+                      _mm_srli_epi32(Rv5,TF_SHIFT), fv5), bv5);
+        Rv6 = _mm_add_epi32(
+                  _mm_mullo_epi32(
+                      _mm_srli_epi32(Rv6,TF_SHIFT), fv6), bv6);
+        Rv7 = _mm_add_epi32(
+                  _mm_mullo_epi32(
+                      _mm_srli_epi32(Rv7,TF_SHIFT), fv7), bv7);
+        Rv8 = _mm_add_epi32(
+                  _mm_mullo_epi32(
+                      _mm_srli_epi32(Rv8,TF_SHIFT), fv8), bv8);
+
+        // Tricky one:  out[i+z] = s[z];
+        //             ---d---c ---b---a  sv1
+        //             ---h---g ---f---e  sv2
+        // packs_epi32 -h-g-f-e -d-c-b-a  sv1(2)
+        // packs_epi16 ponmlkji hgfedcba  sv1(2) / sv3(4)
+        sv5 = _mm_packus_epi32(sv5, sv6);
+        sv7 = _mm_packus_epi32(sv7, sv8);
+        sv5 = _mm_packus_epi16(sv5, sv7);
+
+        // c =  R[z] < RANS_BYTE_L;
+        __m128i renorm_mask5, renorm_mask6, renorm_mask7, renorm_mask8;
+        renorm_mask5 = _mm_cmplt_epu32_imm(Rv5, RANS_BYTE_L);
+        renorm_mask6 = _mm_cmplt_epu32_imm(Rv6, RANS_BYTE_L);
+        renorm_mask7 = _mm_cmplt_epu32_imm(Rv7, RANS_BYTE_L);
+        renorm_mask8 = _mm_cmplt_epu32_imm(Rv8, RANS_BYTE_L);
+        
+        // Shuffle the renorm values to correct lanes and incr sp pointer
+        __m128i Vv5 = _mm_cvtepu16_epi32(_mm_loadu_si128((__m128i *)sp));
+        unsigned int imask5 = _mm_movemask_ps((__m128)renorm_mask5);
+        Vv5 = _mm_shuffle_epi8(Vv5, _mm_load_si128((__m128i*)pidx[imask5]));
+        sp += _mm_popcnt_u32(imask5);
+
+        __m128i Vv6 = _mm_cvtepu16_epi32(_mm_loadu_si128((__m128i *)sp));
+        unsigned int imask6 = _mm_movemask_ps((__m128)renorm_mask6);
+        sp += _mm_popcnt_u32(imask6);
+        Vv6 = _mm_shuffle_epi8(Vv6, _mm_load_si128((__m128i*)pidx[imask6]));
+
+        __m128i Vv7 = _mm_cvtepu16_epi32(_mm_loadu_si128((__m128i *)sp));
+        unsigned int imask7 = _mm_movemask_ps((__m128)renorm_mask7);
+        Vv7 = _mm_shuffle_epi8(Vv7, _mm_load_si128((__m128i*)pidx[imask7]));
+        sp += _mm_popcnt_u32(imask7);
+
+        __m128i Vv8 = _mm_cvtepu16_epi32(_mm_loadu_si128((__m128i *)sp));
+        unsigned int imask8 = _mm_movemask_ps((__m128)renorm_mask8);
+        sp += _mm_popcnt_u32(imask8);
+        Vv8 = _mm_shuffle_epi8(Vv8, _mm_load_si128((__m128i*)pidx[imask8]));
+
+        __m128i Yv5 = _mm_slli_epi32(Rv5, 16);
+        __m128i Yv6 = _mm_slli_epi32(Rv6, 16);
+        __m128i Yv7 = _mm_slli_epi32(Rv7, 16);
+        __m128i Yv8 = _mm_slli_epi32(Rv8, 16);
+
+        // y = (R[z] << 16) | V[z];
+        Yv5 = _mm_or_si128(Yv5, Vv5);
+        Yv6 = _mm_or_si128(Yv6, Vv6);
+        Yv7 = _mm_or_si128(Yv7, Vv7);
+        Yv8 = _mm_or_si128(Yv8, Vv8);
+
+        // R[z] = c ? Y[z] : R[z];
+        Rv5 = _mm_blendv_epi8(Rv5, Yv5, renorm_mask5);
+        Rv6 = _mm_blendv_epi8(Rv6, Yv6, renorm_mask6);
+        Rv7 = _mm_blendv_epi8(Rv7, Yv7, renorm_mask7);
+        Rv8 = _mm_blendv_epi8(Rv8, Yv8, renorm_mask8);
+
+        // Maybe just a store128 instead?
+        _mm_storeu_si128((__m128i *)&out[i+ 0], sv1);
+        _mm_storeu_si128((__m128i *)&out[i+16], sv5);
+//      *(uint64_t *)&out[i+ 0] = _mm_extract_epi64(sv1, 0);
+//      *(uint64_t *)&out[i+ 8] = _mm_extract_epi64(sv1, 1);
+//      *(uint64_t *)&out[i+16] = _mm_extract_epi64(sv5, 0);
+//      *(uint64_t *)&out[i+24] = _mm_extract_epi64(sv5, 1);
+    }
+
+    STORE128(Rv, R);
+
+    for (z = out_sz & (NX-1); z-- > 0; )
+      out[out_end + z] = s3[R[z] & mask];
+
+    //fprintf(stderr, "    0 Decoded %d bytes\n", (int)(cp-in)); //c-size
+
+    return out;
+
+ err:
+    free(out_free);
+    return NULL;
+}
+
+//#define MAGIC2 111
+#define MAGIC2 179
+//#define MAGIC2 0
+
+/*
+ * A 32 x 32 matrix transpose and serialise from t[][] to out.
+ * Storing in the other orientation speeds up the decoder, and we
+ * can then flush to out in 1KB blocks.
+ */
+static inline void transpose_and_copy(uint8_t *out, int iN[32],
+                                      uint8_t t[32][32]) {
+    int z;
+
+    // Simplified version from below.
+    // This is pretty good with zig cc, but slow on clang and very slow on
+    // gcc, even with -O3
+    /*
+    for (z = 0; z < NX; z++) {
+        int k;
+        for (k = 0; k < 32; k++)
+            out[iN[z]+k] = t[k][z];
+        iN[z] += 32;
+    }
+    */
+
+
+    // A better approach for clang and gcc can be had with some manual
+    // restructuring to attempt to do the two loops in explcit blocks.
+    // With gcc -O3 or -O2 -ftree-vectorize this is quite fast, as is clang
+    // and zig but neither beat the version below (or, for zig, the basic
+    // code above).
+    //
+    // It's left here incase we ever want to move to tidier code and
+    // to understand what auto-vectorises and what doesn't.
+    /*
+#define NZ 2
+#define NK 8
+    for (z = 0; z < 32; z+=NZ) {
+        for (int k = 0; k < 32; k+=NK) {
+            for (int z0 = z; z0 < z+NZ; z0++) {
+                uint8_t tmp[NK];// __attribute__((aligned(32)));
+                //uint8_t (*RESTRICT t0)[32] = &t[k];
+                for (int k0 = 0; k0 < NK; k0++)
+                    //tmp[k0] = t0[k0][z0];
+                    tmp[k0] = t[k+k0][z0];
+                memcpy(&out[iN[z0]+k], tmp, NK);
+            }
+        }
+        for (int z0 = z; z0 < z+NZ; z0++)
+            iN[z0] += 32;
+    }
+    */
+
+    // Manually unrolled code.
+    // This is fastest on gcc and clang and not far behind with zig cc.
+    // It also doesn't need aggressive gcc optimisation levels to be
+    // efficient.
+    //
+    // It works by constructing 64-bit ints and copying them with a single
+    // memory write.  The fixed size memcpys just boil down to a memory write,
+    // but unlike the earlier versions that did this direct, this isn't
+    // exploiting undefined behaviour.
+    for (z = 0; z < 32; z+=4) {
+        uint64_t i64;
+        i64 =
+            ((uint64_t)(t[0][z])<< 0) +
+            ((uint64_t)(t[1][z])<< 8) +
+            ((uint64_t)(t[2][z])<<16) +
+            ((uint64_t)(t[3][z])<<24) +
+            ((uint64_t)(t[4][z])<<32) +
+            ((uint64_t)(t[5][z])<<40) +
+            ((uint64_t)(t[6][z])<<48) +
+            ((uint64_t)(t[7][z])<<56);
+        memcpy(&out[iN[z]], &i64, 8);
+        i64 =
+            ((uint64_t)(t[0][z+1])<< 0) +
+            ((uint64_t)(t[1][z+1])<< 8) +
+            ((uint64_t)(t[2][z+1])<<16) +
+            ((uint64_t)(t[3][z+1])<<24) +
+            ((uint64_t)(t[4][z+1])<<32) +
+            ((uint64_t)(t[5][z+1])<<40) +
+            ((uint64_t)(t[6][z+1])<<48) +
+            ((uint64_t)(t[7][z+1])<<56);
+        memcpy(&out[iN[z+1]], &i64, 8);
+        i64 =
+            ((uint64_t)(t[0][z+2])<< 0) +
+            ((uint64_t)(t[1][z+2])<< 8) +
+            ((uint64_t)(t[2][z+2])<<16) +
+            ((uint64_t)(t[3][z+2])<<24) +
+            ((uint64_t)(t[4][z+2])<<32) +
+            ((uint64_t)(t[5][z+2])<<40) +
+            ((uint64_t)(t[6][z+2])<<48) +
+            ((uint64_t)(t[7][z+2])<<56);
+        memcpy(&out[iN[z+2]], &i64, 8);
+        i64 =
+            ((uint64_t)(t[0][z+3])<< 0) +
+            ((uint64_t)(t[1][z+3])<< 8) +
+            ((uint64_t)(t[2][z+3])<<16) +
+            ((uint64_t)(t[3][z+3])<<24) +
+            ((uint64_t)(t[4][z+3])<<32) +
+            ((uint64_t)(t[5][z+3])<<40) +
+            ((uint64_t)(t[6][z+3])<<48) +
+            ((uint64_t)(t[7][z+3])<<56);
+        memcpy(&out[iN[z+3]], &i64, 8);
+
+        i64 =
+            ((uint64_t)(t[8+0][z])<< 0) +
+            ((uint64_t)(t[8+1][z])<< 8) +
+            ((uint64_t)(t[8+2][z])<<16) +
+            ((uint64_t)(t[8+3][z])<<24) +
+            ((uint64_t)(t[8+4][z])<<32) +
+            ((uint64_t)(t[8+5][z])<<40) +
+            ((uint64_t)(t[8+6][z])<<48) +
+            ((uint64_t)(t[8+7][z])<<56);
+        memcpy(&out[iN[z]+8], &i64, 8);
+        i64 =
+            ((uint64_t)(t[8+0][z+1])<< 0) +
+            ((uint64_t)(t[8+1][z+1])<< 8) +
+            ((uint64_t)(t[8+2][z+1])<<16) +
+            ((uint64_t)(t[8+3][z+1])<<24) +
+            ((uint64_t)(t[8+4][z+1])<<32) +
+            ((uint64_t)(t[8+5][z+1])<<40) +
+            ((uint64_t)(t[8+6][z+1])<<48) +
+            ((uint64_t)(t[8+7][z+1])<<56);
+        memcpy(&out[iN[z+1]+8], &i64, 8);
+        i64 =
+            ((uint64_t)(t[8+0][z+2])<< 0) +
+            ((uint64_t)(t[8+1][z+2])<< 8) +
+            ((uint64_t)(t[8+2][z+2])<<16) +
+            ((uint64_t)(t[8+3][z+2])<<24) +
+            ((uint64_t)(t[8+4][z+2])<<32) +
+            ((uint64_t)(t[8+5][z+2])<<40) +
+            ((uint64_t)(t[8+6][z+2])<<48) +
+            ((uint64_t)(t[8+7][z+2])<<56);
+        memcpy(&out[iN[z+2]+8], &i64, 8);
+        i64 =
+            ((uint64_t)(t[8+0][z+3])<< 0) +
+            ((uint64_t)(t[8+1][z+3])<< 8) +
+            ((uint64_t)(t[8+2][z+3])<<16) +
+            ((uint64_t)(t[8+3][z+3])<<24) +
+            ((uint64_t)(t[8+4][z+3])<<32) +
+            ((uint64_t)(t[8+5][z+3])<<40) +
+            ((uint64_t)(t[8+6][z+3])<<48) +
+            ((uint64_t)(t[8+7][z+3])<<56);
+        memcpy(&out[iN[z+3]+8], &i64, 8);
+
+        i64 =
+            ((uint64_t)(t[16+0][z])<< 0) +
+            ((uint64_t)(t[16+1][z])<< 8) +
+            ((uint64_t)(t[16+2][z])<<16) +
+            ((uint64_t)(t[16+3][z])<<24) +
+            ((uint64_t)(t[16+4][z])<<32) +
+            ((uint64_t)(t[16+5][z])<<40) +
+            ((uint64_t)(t[16+6][z])<<48) +
+            ((uint64_t)(t[16+7][z])<<56);
+        memcpy(&out[iN[z]+16], &i64, 8);
+        i64 =
+            ((uint64_t)(t[16+0][z+1])<< 0) +
+            ((uint64_t)(t[16+1][z+1])<< 8) +
+            ((uint64_t)(t[16+2][z+1])<<16) +
+            ((uint64_t)(t[16+3][z+1])<<24) +
+            ((uint64_t)(t[16+4][z+1])<<32) +
+            ((uint64_t)(t[16+5][z+1])<<40) +
+            ((uint64_t)(t[16+6][z+1])<<48) +
+            ((uint64_t)(t[16+7][z+1])<<56);
+        memcpy(&out[iN[z+1]+16], &i64, 8);
+        i64 =
+            ((uint64_t)(t[16+0][z+2])<< 0) +
+            ((uint64_t)(t[16+1][z+2])<< 8) +
+            ((uint64_t)(t[16+2][z+2])<<16) +
+            ((uint64_t)(t[16+3][z+2])<<24) +
+            ((uint64_t)(t[16+4][z+2])<<32) +
+            ((uint64_t)(t[16+5][z+2])<<40) +
+            ((uint64_t)(t[16+6][z+2])<<48) +
+            ((uint64_t)(t[16+7][z+2])<<56);
+        memcpy(&out[iN[z+2]+16], &i64, 8);
+        i64 =
+            ((uint64_t)(t[16+0][z+3])<< 0) +
+            ((uint64_t)(t[16+1][z+3])<< 8) +
+            ((uint64_t)(t[16+2][z+3])<<16) +
+            ((uint64_t)(t[16+3][z+3])<<24) +
+            ((uint64_t)(t[16+4][z+3])<<32) +
+            ((uint64_t)(t[16+5][z+3])<<40) +
+            ((uint64_t)(t[16+6][z+3])<<48) +
+            ((uint64_t)(t[16+7][z+3])<<56);
+        memcpy(&out[iN[z+3]+16], &i64, 8);
+
+        i64 =
+            ((uint64_t)(t[24+0][z])<< 0) +
+            ((uint64_t)(t[24+1][z])<< 8) +
+            ((uint64_t)(t[24+2][z])<<16) +
+            ((uint64_t)(t[24+3][z])<<24) +
+            ((uint64_t)(t[24+4][z])<<32) +
+            ((uint64_t)(t[24+5][z])<<40) +
+            ((uint64_t)(t[24+6][z])<<48) +
+            ((uint64_t)(t[24+7][z])<<56);
+        memcpy(&out[iN[z]+24], &i64, 8);
+        i64 =
+            ((uint64_t)(t[24+0][z+1])<< 0) +
+            ((uint64_t)(t[24+1][z+1])<< 8) +
+            ((uint64_t)(t[24+2][z+1])<<16) +
+            ((uint64_t)(t[24+3][z+1])<<24) +
+            ((uint64_t)(t[24+4][z+1])<<32) +
+            ((uint64_t)(t[24+5][z+1])<<40) +
+            ((uint64_t)(t[24+6][z+1])<<48) +
+            ((uint64_t)(t[24+7][z+1])<<56);
+        memcpy(&out[iN[z+1]+24], &i64, 8);
+        i64 =
+            ((uint64_t)(t[24+0][z+2])<< 0) +
+            ((uint64_t)(t[24+1][z+2])<< 8) +
+            ((uint64_t)(t[24+2][z+2])<<16) +
+            ((uint64_t)(t[24+3][z+2])<<24) +
+            ((uint64_t)(t[24+4][z+2])<<32) +
+            ((uint64_t)(t[24+5][z+2])<<40) +
+            ((uint64_t)(t[24+6][z+2])<<48) +
+            ((uint64_t)(t[24+7][z+2])<<56);
+        memcpy(&out[iN[z+2]+24], &i64, 8);
+        i64 =
+            ((uint64_t)(t[24+0][z+3])<< 0) +
+            ((uint64_t)(t[24+1][z+3])<< 8) +
+            ((uint64_t)(t[24+2][z+3])<<16) +
+            ((uint64_t)(t[24+3][z+3])<<24) +
+            ((uint64_t)(t[24+4][z+3])<<32) +
+            ((uint64_t)(t[24+5][z+3])<<40) +
+            ((uint64_t)(t[24+6][z+3])<<48) +
+            ((uint64_t)(t[24+7][z+3])<<56);
+        memcpy(&out[iN[z+3]+24], &i64, 8);
+
+        iN[z+0] += 32;
+        iN[z+1] += 32;
+        iN[z+2] += 32;
+        iN[z+3] += 32;
+    }
+}
+
+unsigned char *rans_uncompress_O1_32x16_sse4(unsigned char *in,
+                                             unsigned int in_size,
+                                             unsigned char *out,
+                                             unsigned int out_sz) {
+    if (in_size < NX*4) // 4-states at least
+        return NULL;
+
+    if (out_sz >= INT_MAX)
+        return NULL; // protect against some overflow cases
+
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    if (out_sz > 100000)
+        return NULL;
+#endif
+
+    /* Load in the static tables */
+    unsigned char *cp = in, *cp_end = in+in_size, *out_free = NULL;
+    unsigned char *c_freq = NULL;
+
+    uint32_t (*s3)[TOTFREQ_O1] = htscodecs_tls_alloc(256*TOTFREQ_O1*4);
+    if (!s3)
+        return NULL;
+    uint32_t (*s3F)[TOTFREQ_O1_FAST] = (uint32_t (*)[TOTFREQ_O1_FAST])s3;
+
+    if (!out)
+        out_free = out = malloc(out_sz);
+
+    if (!out)
+        goto err;
+
+    //fprintf(stderr, "out_sz=%d\n", out_sz);
+
+    // compressed header? If so uncompress it
+    unsigned char *tab_end = NULL;
+    unsigned char *c_freq_end = cp_end;
+    unsigned int shift = *cp >> 4;
+    if (*cp++ & 1) {
+        uint32_t u_freq_sz, c_freq_sz;
+        cp += var_get_u32(cp, cp_end, &u_freq_sz);
+        cp += var_get_u32(cp, cp_end, &c_freq_sz);
+        if (c_freq_sz > cp_end - cp)
+            goto err;
+        tab_end = cp + c_freq_sz;
+        if (!(c_freq = rans_uncompress_O0_4x16(cp, c_freq_sz, NULL,u_freq_sz)))
+            goto err;
+        cp = c_freq;
+        c_freq_end = c_freq + u_freq_sz;
+    }
+
+    // Decode order-0 symbol list; avoids needing in order-1 tables
+    cp += decode_freq1(cp, c_freq_end, shift, s3, s3F, NULL, NULL);
+
+    if (tab_end)
+        cp = tab_end;
+    free(c_freq);
+    c_freq = NULL;
+
+    if (cp_end - cp < NX * 4)
+        goto err;
+
+    RansState R[NX];
+    uint8_t *ptr = cp, *ptr_end = in + in_size;
+    int z;
+    for (z = 0; z < NX; z++) {
+        RansDecInit(&R[z], &ptr);
+        if (R[z] < RANS_BYTE_L)
+            goto err;
+    }
+
+    int isz4 = out_sz/NX;
+    int i4[NX], l[NX] = {0};
+    for (z = 0; z < NX; z++)
+        i4[z] = z*isz4;
+
+    // Around 15% faster to specialise for 10/12 than to have one
+    // loop with shift as a variable.
+    if (shift == TF_SHIFT_O1) {
+        // TF_SHIFT_O1 = 12
+        uint16_t *sp = (uint16_t *)ptr;
+        const uint32_t mask = ((1u << TF_SHIFT_O1)-1);
+        __m128i maskv  = _mm_set1_epi32(mask); // set mask in all lanes
+        uint8_t tbuf[32][32];
+        int tidx = 0;
+        LOAD128(Rv, R);
+        LOAD128(Lv, l);
+
+        isz4 -= 64;
+        for (; i4[0] < isz4 && (uint8_t *)sp+72 < ptr_end; ) {
+            //for (z = 0; z < NX; z++)
+            //  m[z] = R[z] & mask;
+            __m128i masked1 = _mm_and_si128(Rv1, maskv);
+            __m128i masked2 = _mm_and_si128(Rv2, maskv);
+            __m128i masked3 = _mm_and_si128(Rv3, maskv);
+            __m128i masked4 = _mm_and_si128(Rv4, maskv);
+
+            Lv1 = _mm_slli_epi32(Lv1, TF_SHIFT_O1);
+            Lv2 = _mm_slli_epi32(Lv2, TF_SHIFT_O1);
+            Lv3 = _mm_slli_epi32(Lv3, TF_SHIFT_O1);
+            Lv4 = _mm_slli_epi32(Lv4, TF_SHIFT_O1);
+            masked1 = _mm_add_epi32(masked1, Lv1);
+            masked2 = _mm_add_epi32(masked2, Lv2);
+            masked3 = _mm_add_epi32(masked3, Lv3);
+            masked4 = _mm_add_epi32(masked4, Lv4);
+
+            //  S[z] = s3[l[z]][m[z]];
+            __m128i Sv1 = _mm_i32gather_epi32x((int *)s3, masked1, sizeof(*s3));
+            __m128i Sv2 = _mm_i32gather_epi32x((int *)s3, masked2, sizeof(*s3));
+            __m128i Sv3 = _mm_i32gather_epi32x((int *)s3, masked3, sizeof(*s3));
+            __m128i Sv4 = _mm_i32gather_epi32x((int *)s3, masked4, sizeof(*s3));
+
+            //  f[z] = S[z]>>(TF_SHIFT+8);
+            __m128i fv1 = _mm_srli_epi32(Sv1, TF_SHIFT_O1+8);
+            __m128i fv2 = _mm_srli_epi32(Sv2, TF_SHIFT_O1+8);
+            __m128i fv3 = _mm_srli_epi32(Sv3, TF_SHIFT_O1+8);
+            __m128i fv4 = _mm_srli_epi32(Sv4, TF_SHIFT_O1+8);
+
+            //  b[z] = (S[z]>>8) & mask;
+            __m128i bv1 = _mm_and_si128(_mm_srli_epi32(Sv1, 8), maskv);
+            __m128i bv2 = _mm_and_si128(_mm_srli_epi32(Sv2, 8), maskv);
+            __m128i bv3 = _mm_and_si128(_mm_srli_epi32(Sv3, 8), maskv);
+            __m128i bv4 = _mm_and_si128(_mm_srli_epi32(Sv4, 8), maskv);
+
+            //  s[z] = S[z] & 0xff;
+            __m128i sv1 = _mm_and_si128(Sv1, _mm_set1_epi32(0xff));
+            __m128i sv2 = _mm_and_si128(Sv2, _mm_set1_epi32(0xff));
+            __m128i sv3 = _mm_and_si128(Sv3, _mm_set1_epi32(0xff));
+            __m128i sv4 = _mm_and_si128(Sv4, _mm_set1_epi32(0xff));
+
+            // A maximum frequency of 4096 doesn't fit in our s3 array.
+            // as it's 12 bit + 12 bit + 8 bit.  It wraps around to zero.
+            // (We don't have this issue for TOTFREQ_O1_FAST.)
+            //
+            // Solution 1 is to change to spec to forbid freq of 4096.
+            // Easy hack is to add an extra symbol so it sums correctly.
+            //
+            // Solution 2 implemented here is to look for the wrap around
+            // and fix it.
+            __m128i max_freq = _mm_set1_epi32(TOTFREQ_O1);
+            __m128i zero = _mm_set1_epi32(0);
+            __m128i cmp1 = _mm_cmpeq_epi32(fv1, zero);
+            fv1 = _mm_blendv_epi8(fv1, max_freq, cmp1);
+            __m128i cmp2 = _mm_cmpeq_epi32(fv2, zero);
+            fv2 = _mm_blendv_epi8(fv2, max_freq, cmp2);
+            __m128i cmp3 = _mm_cmpeq_epi32(fv3, zero);
+            fv3 = _mm_blendv_epi8(fv3, max_freq, cmp3);
+            __m128i cmp4 = _mm_cmpeq_epi32(fv4, zero);
+            fv4 = _mm_blendv_epi8(fv4, max_freq, cmp4);
+
+            //  R[z] = f[z] * (R[z] >> TF_SHIFT_O1) + b[z];
+            Rv1 = _mm_add_epi32(
+                      _mm_mullo_epi32(
+                          _mm_srli_epi32(Rv1,TF_SHIFT_O1), fv1), bv1);
+            Rv2 = _mm_add_epi32(
+                      _mm_mullo_epi32(
+                          _mm_srli_epi32(Rv2,TF_SHIFT_O1), fv2), bv2);
+            Rv3 = _mm_add_epi32(
+                      _mm_mullo_epi32(
+                          _mm_srli_epi32(Rv3,TF_SHIFT_O1), fv3), bv3);
+            Rv4 = _mm_add_epi32(
+                      _mm_mullo_epi32(
+                          _mm_srli_epi32(Rv4,TF_SHIFT_O1), fv4), bv4);
+
+            Lv1 = sv1;
+            Lv2 = sv2;
+            Lv3 = sv3;
+            Lv4 = sv4;
+
+            // Tricky one:  out[i+z] = s[z];
+            //             ---d---c ---b---a  sv1
+            //             ---h---g ---f---e  sv2
+            // packs_epi32 -h-g-f-e -d-c-b-a  sv1(2)
+            // packs_epi16 ponmlkji hgfedcba  sv1(2) / sv3(4)
+            sv1 = _mm_packus_epi32(sv1, sv2);
+            sv3 = _mm_packus_epi32(sv3, sv4);
+            sv1 = _mm_packus_epi16(sv1, sv3);
+
+            // c =  R[z] < RANS_BYTE_L;
+            // A little tricky as we only have signed comparisons.
+            // See https://stackoverflow.com/questions/32945410/sse2-intrinsics-comparing-unsigned-integers
+
+//#define _mm_cmplt_epu32_imm(a,b) _mm_andnot_si128(_mm_cmpeq_epi32(_mm_max_epu32((a),_mm_set1_epi32(b)), (a)), _mm_set1_epi32(-1));
+
+            //#define _mm_cmplt_epu32_imm(a,b) _mm_cmpgt_epi32(_mm_set1_epi32((b)-0x80000000), _mm_xor_si128((a), _mm_set1_epi32(0x80000000)))
+
+            __m128i renorm_mask1, renorm_mask2, renorm_mask3, renorm_mask4;
+            renorm_mask1 = _mm_cmplt_epu32_imm(Rv1, RANS_BYTE_L);
+            renorm_mask2 = _mm_cmplt_epu32_imm(Rv2, RANS_BYTE_L);
+            renorm_mask3 = _mm_cmplt_epu32_imm(Rv3, RANS_BYTE_L);
+            renorm_mask4 = _mm_cmplt_epu32_imm(Rv4, RANS_BYTE_L);
+
+            //#define P(A,B,C,D) ((A)+((B)<<2) + ((C)<<4) + ((D)<<6))
+#define P(A,B,C,D)                              \
+            { A+0,A+1,A+2,A+3,                  \
+              B+0,B+1,B+2,B+3,                  \
+              C+0,C+1,C+2,C+3,                  \
+              D+0,D+1,D+2,D+3}
+#ifdef _
+#undef _
+#endif
+#define _ 0x80
+            uint8_t pidx[16][16] = {
+                P(_,_,_,_),
+                P(0,_,_,_),
+                P(_,0,_,_),
+                P(0,4,_,_),
+
+                P(_,_,0,_),
+                P(0,_,4,_),
+                P(_,0,4,_),
+                P(0,4,8,_),
+
+                P(_,_,_,0),
+                P(0,_,_,4),
+                P(_,0,_,4),
+                P(0,4,_,8),
+
+                P(_,_,0,4),
+                P(0,_,4,8),
+                P(_,0,4,8),
+                P(0,4,8,12),
+            };
+#undef _
+
+            // Shuffle the renorm values to correct lanes and incr sp pointer
+            __m128i Vv1 = _mm_cvtepu16_epi32(_mm_loadu_si128((__m128i *)sp));
+            unsigned int imask1 = _mm_movemask_ps((__m128)renorm_mask1);
+            Vv1 = _mm_shuffle_epi8(Vv1, _mm_load_si128((__m128i*)pidx[imask1]));
+            sp += _mm_popcnt_u32(imask1);
+
+            __m128i Vv2 = _mm_cvtepu16_epi32(_mm_loadu_si128((__m128i *)sp));
+            unsigned int imask2 = _mm_movemask_ps((__m128)renorm_mask2);
+            sp += _mm_popcnt_u32(imask2);
+            Vv2 = _mm_shuffle_epi8(Vv2, _mm_load_si128((__m128i*)pidx[imask2]));
+
+            __m128i Vv3 = _mm_cvtepu16_epi32(_mm_loadu_si128((__m128i *)sp));
+            unsigned int imask3 = _mm_movemask_ps((__m128)renorm_mask3);
+            Vv3 = _mm_shuffle_epi8(Vv3, _mm_load_si128((__m128i*)pidx[imask3]));
+            sp += _mm_popcnt_u32(imask3);
+
+            __m128i Vv4 = _mm_cvtepu16_epi32(_mm_loadu_si128((__m128i *)sp));
+            unsigned int imask4 = _mm_movemask_ps((__m128)renorm_mask4);
+            sp += _mm_popcnt_u32(imask4);
+            Vv4 = _mm_shuffle_epi8(Vv4, _mm_load_si128((__m128i*)pidx[imask4]));
+
+            __m128i Yv1 = _mm_slli_epi32(Rv1, 16);
+            __m128i Yv2 = _mm_slli_epi32(Rv2, 16);
+            __m128i Yv3 = _mm_slli_epi32(Rv3, 16);
+            __m128i Yv4 = _mm_slli_epi32(Rv4, 16);
+
+            // y = (R[z] << 16) | V[z];
+            Yv1 = _mm_or_si128(Yv1, Vv1);
+            Yv2 = _mm_or_si128(Yv2, Vv2);
+            Yv3 = _mm_or_si128(Yv3, Vv3);
+            Yv4 = _mm_or_si128(Yv4, Vv4);
+
+            // R[z] = c ? Y[z] : R[z];
+            Rv1 = _mm_blendv_epi8(Rv1, Yv1, renorm_mask1);
+            Rv2 = _mm_blendv_epi8(Rv2, Yv2, renorm_mask2);
+            Rv3 = _mm_blendv_epi8(Rv3, Yv3, renorm_mask3);
+            Rv4 = _mm_blendv_epi8(Rv4, Yv4, renorm_mask4);
+
+            // ------------------------------------------------------------
+
+            //  m[z] = R[z] & mask;
+            __m128i masked5 = _mm_and_si128(Rv5, maskv);
+            __m128i masked6 = _mm_and_si128(Rv6, maskv);
+            __m128i masked7 = _mm_and_si128(Rv7, maskv);
+            __m128i masked8 = _mm_and_si128(Rv8, maskv);
+
+
+            Lv5 = _mm_slli_epi32(Lv5, TF_SHIFT_O1);
+            Lv6 = _mm_slli_epi32(Lv6, TF_SHIFT_O1);
+            Lv7 = _mm_slli_epi32(Lv7, TF_SHIFT_O1);
+            Lv8 = _mm_slli_epi32(Lv8, TF_SHIFT_O1);
+            masked5 = _mm_add_epi32(masked5, Lv5);
+            masked6 = _mm_add_epi32(masked6, Lv6);
+            masked7 = _mm_add_epi32(masked7, Lv7);
+            masked8 = _mm_add_epi32(masked8, Lv8);
+
+            //  S[z] = s3[m[z]];
+            __m128i Sv5 = _mm_i32gather_epi32x((int *)s3, masked5, sizeof(*s3));
+            __m128i Sv6 = _mm_i32gather_epi32x((int *)s3, masked6, sizeof(*s3));
+            __m128i Sv7 = _mm_i32gather_epi32x((int *)s3, masked7, sizeof(*s3));
+            __m128i Sv8 = _mm_i32gather_epi32x((int *)s3, masked8, sizeof(*s3));
+
+            //  f[z] = S[z]>>(TF_SHIFT_O1+8);
+            __m128i fv5 = _mm_srli_epi32(Sv5, TF_SHIFT_O1+8);
+            __m128i fv6 = _mm_srli_epi32(Sv6, TF_SHIFT_O1+8);
+            __m128i fv7 = _mm_srli_epi32(Sv7, TF_SHIFT_O1+8);
+            __m128i fv8 = _mm_srli_epi32(Sv8, TF_SHIFT_O1+8);
+
+            //  b[z] = (S[z]>>8) & mask;
+            __m128i bv5 = _mm_and_si128(_mm_srli_epi32(Sv5, 8), maskv);
+            __m128i bv6 = _mm_and_si128(_mm_srli_epi32(Sv6, 8), maskv);
+            __m128i bv7 = _mm_and_si128(_mm_srli_epi32(Sv7, 8), maskv);
+            __m128i bv8 = _mm_and_si128(_mm_srli_epi32(Sv8, 8), maskv);
+
+            //  s[z] = S[z] & 0xff;
+            __m128i sv5 = _mm_and_si128(Sv5, _mm_set1_epi32(0xff));
+            __m128i sv6 = _mm_and_si128(Sv6, _mm_set1_epi32(0xff));
+            __m128i sv7 = _mm_and_si128(Sv7, _mm_set1_epi32(0xff));
+            __m128i sv8 = _mm_and_si128(Sv8, _mm_set1_epi32(0xff));
+
+            // A maximum frequency of 4096 doesn't fit in our s3 array. Fix
+            __m128i cmp5 = _mm_cmpeq_epi32(fv5, zero);
+            fv5 = _mm_blendv_epi8(fv5, max_freq, cmp5);
+            __m128i cmp6 = _mm_cmpeq_epi32(fv6, zero);
+            fv6 = _mm_blendv_epi8(fv6, max_freq, cmp6);
+            __m128i cmp7 = _mm_cmpeq_epi32(fv7, zero);
+            fv7 = _mm_blendv_epi8(fv7, max_freq, cmp7);
+            __m128i cmp8 = _mm_cmpeq_epi32(fv8, zero);
+            fv8 = _mm_blendv_epi8(fv8, max_freq, cmp8);
+
+            //  R[z] = f[z] * (R[z] >> TF_SHIFT_O1) + b[z];
+            Rv5 = _mm_add_epi32(
+                      _mm_mullo_epi32(
+                          _mm_srli_epi32(Rv5,TF_SHIFT_O1), fv5), bv5);
+            Rv6 = _mm_add_epi32(
+                      _mm_mullo_epi32(
+                          _mm_srli_epi32(Rv6,TF_SHIFT_O1), fv6), bv6);
+            Rv7 = _mm_add_epi32(
+                      _mm_mullo_epi32(
+                          _mm_srli_epi32(Rv7,TF_SHIFT_O1), fv7), bv7);
+            Rv8 = _mm_add_epi32(
+                      _mm_mullo_epi32(
+                          _mm_srli_epi32(Rv8,TF_SHIFT_O1), fv8), bv8);
+
+            Lv5 = sv5;
+            Lv6 = sv6;
+            Lv7 = sv7;
+            Lv8 = sv8;
+
+            // Tricky one:  out[i+z] = s[z];
+            //             ---d---c ---b---a  sv1
+            //             ---h---g ---f---e  sv2
+            // packs_epi32 -h-g-f-e -d-c-b-a  sv1(2)
+            // packs_epi16 ponmlkji hgfedcba  sv1(2) / sv3(4)
+            sv5 = _mm_packus_epi32(sv5, sv6);
+            sv7 = _mm_packus_epi32(sv7, sv8);
+            sv5 = _mm_packus_epi16(sv5, sv7);
+
+            // c =  R[z] < RANS_BYTE_L;
+            __m128i renorm_mask5, renorm_mask6, renorm_mask7, renorm_mask8;
+            renorm_mask5 = _mm_cmplt_epu32_imm(Rv5, RANS_BYTE_L);
+            renorm_mask6 = _mm_cmplt_epu32_imm(Rv6, RANS_BYTE_L);
+            renorm_mask7 = _mm_cmplt_epu32_imm(Rv7, RANS_BYTE_L);
+            renorm_mask8 = _mm_cmplt_epu32_imm(Rv8, RANS_BYTE_L);
+        
+            // Shuffle the renorm values to correct lanes and incr sp pointer
+            __m128i Vv5 = _mm_cvtepu16_epi32(_mm_loadu_si128((__m128i *)sp));
+            unsigned int imask5 = _mm_movemask_ps((__m128)renorm_mask5);
+            Vv5 = _mm_shuffle_epi8(Vv5, _mm_load_si128((__m128i*)pidx[imask5]));
+            sp += _mm_popcnt_u32(imask5);
+
+            __m128i Vv6 = _mm_cvtepu16_epi32(_mm_loadu_si128((__m128i *)sp));
+            unsigned int imask6 = _mm_movemask_ps((__m128)renorm_mask6);
+            sp += _mm_popcnt_u32(imask6);
+            Vv6 = _mm_shuffle_epi8(Vv6, _mm_load_si128((__m128i*)pidx[imask6]));
+
+            __m128i Vv7 = _mm_cvtepu16_epi32(_mm_loadu_si128((__m128i *)sp));
+            unsigned int imask7 = _mm_movemask_ps((__m128)renorm_mask7);
+            Vv7 = _mm_shuffle_epi8(Vv7, _mm_load_si128((__m128i*)pidx[imask7]));
+            sp += _mm_popcnt_u32(imask7);
+
+            __m128i Vv8 = _mm_cvtepu16_epi32(_mm_loadu_si128((__m128i *)sp));
+            unsigned int imask8 = _mm_movemask_ps((__m128)renorm_mask8);
+            sp += _mm_popcnt_u32(imask8);
+            Vv8 = _mm_shuffle_epi8(Vv8, _mm_load_si128((__m128i*)pidx[imask8]));
+
+            __m128i Yv5 = _mm_slli_epi32(Rv5, 16);
+            __m128i Yv6 = _mm_slli_epi32(Rv6, 16);
+            __m128i Yv7 = _mm_slli_epi32(Rv7, 16);
+            __m128i Yv8 = _mm_slli_epi32(Rv8, 16);
+
+            // y = (R[z] << 16) | V[z];
+            Yv5 = _mm_or_si128(Yv5, Vv5);
+            Yv6 = _mm_or_si128(Yv6, Vv6);
+            Yv7 = _mm_or_si128(Yv7, Vv7);
+            Yv8 = _mm_or_si128(Yv8, Vv8);
+
+            // R[z] = c ? Y[z] : R[z];
+            Rv5 = _mm_blendv_epi8(Rv5, Yv5, renorm_mask5);
+            Rv6 = _mm_blendv_epi8(Rv6, Yv6, renorm_mask6);
+            Rv7 = _mm_blendv_epi8(Rv7, Yv7, renorm_mask7);
+            Rv8 = _mm_blendv_epi8(Rv8, Yv8, renorm_mask8);
+
+            // Maybe just a store128 instead?
+            _mm_store_si128((__m128i *)&tbuf[tidx][ 0], sv1);
+            _mm_store_si128((__m128i *)&tbuf[tidx][16], sv5);
+            //  *(uint64_t *)&out[i+ 0] = _mm_extract_epi64(sv1, 0);
+            //  *(uint64_t *)&out[i+ 8] = _mm_extract_epi64(sv1, 1);
+            //  *(uint64_t *)&out[i+16] = _mm_extract_epi64(sv5, 0);
+            //  *(uint64_t *)&out[i+24] = _mm_extract_epi64(sv5, 1);
+
+            // WRONG - need to reorder these periodically.
+
+            i4[0]++;
+            if (++tidx == 32) {
+                i4[0]-=32;
+                transpose_and_copy(out, i4, tbuf);
+                tidx = 0;
+            }
+
+        }
+        isz4 += 64;
+
+        STORE128(Rv, R);
+        STORE128(Lv, l);
+        ptr = (uint8_t *)sp;
+
+        i4[0]-=tidx;
+        int T;
+        for (z = 0; z < NX; z++)
+            for (T = 0; T < tidx; T++)
+                out[i4[z]++] = tbuf[T][z];
+
+        // Scalar version for close to the end of in[] array so we don't
+        // do SIMD loads beyond the end of the buffer
+        for (; i4[0] < isz4;) {
+            for (z = 0; z < NX; z++) {
+                uint32_t m = R[z] & ((1u<<TF_SHIFT_O1)-1);
+                uint32_t S = s3[l[z]][m];
+                unsigned char c = S & 0xff;
+                out[i4[z]++] = c;
+                uint32_t F = S>>(TF_SHIFT_O1+8);
+                R[z] = (F?F:4096) * (R[z]>>TF_SHIFT_O1) +
+                    ((S>>8) & ((1u<<TF_SHIFT_O1)-1));
+                RansDecRenormSafe(&R[z], &ptr, ptr_end);
+                l[z] = c;
+            }
+        }
+
+        // Remainder
+        z = NX-1;
+        for (; i4[z] < out_sz; ) {
+            uint32_t m = R[z] & ((1u<<TF_SHIFT_O1)-1);
+            uint32_t S = s3[l[z]][m];
+            unsigned char c = S & 0xff;
+            out[i4[z]++] = c;
+            int f = (S>>(TF_SHIFT_O1+8));
+            if (f == 0)
+                f = 4096;
+            R[z] = f * (R[z]>>TF_SHIFT_O1) + ((S>>8) & ((1u<<TF_SHIFT_O1)-1));
+            RansDecRenormSafe(&R[z], &ptr, ptr_end);
+            l[z] = c;
+        }
+    } else {
+        // TF_SHIFT_O1 = 10
+        uint16_t *sp = (uint16_t *)ptr;
+        const uint32_t mask = ((1u << TF_SHIFT_O1_FAST)-1);
+        __m128i maskv  = _mm_set1_epi32(mask); // set mask in all lanes
+        uint8_t tbuf[32][32] __attribute__((aligned(32)));
+        int tidx = 0;
+        LOAD128(Rv, R);
+        LOAD128(Lv, l);
+
+        isz4 -= 64;
+        for (; i4[0] < isz4 && (uint8_t *)sp+72 < ptr_end; ) {
+            //for (z = 0; z < NX; z++)
+            //  m[z] = R[z] & mask;
+            __m128i masked1 = _mm_and_si128(Rv1, maskv);
+            __m128i masked2 = _mm_and_si128(Rv2, maskv);
+            __m128i masked3 = _mm_and_si128(Rv3, maskv);
+            __m128i masked4 = _mm_and_si128(Rv4, maskv);
+
+            Lv1 = _mm_slli_epi32(Lv1, TF_SHIFT_O1_FAST);
+            Lv2 = _mm_slli_epi32(Lv2, TF_SHIFT_O1_FAST);
+            Lv3 = _mm_slli_epi32(Lv3, TF_SHIFT_O1_FAST);
+            Lv4 = _mm_slli_epi32(Lv4, TF_SHIFT_O1_FAST);
+            masked1 = _mm_add_epi32(masked1, Lv1);
+            masked2 = _mm_add_epi32(masked2, Lv2);
+            masked3 = _mm_add_epi32(masked3, Lv3);
+            masked4 = _mm_add_epi32(masked4, Lv4);
+
+            //  S[z] = s3[l[z]][m[z]];
+            __m128i Sv1 = _mm_i32gather_epi32x((int *)s3F, masked1, sizeof(*s3));
+            __m128i Sv2 = _mm_i32gather_epi32x((int *)s3F, masked2, sizeof(*s3));
+            __m128i Sv3 = _mm_i32gather_epi32x((int *)s3F, masked3, sizeof(*s3));
+            __m128i Sv4 = _mm_i32gather_epi32x((int *)s3F, masked4, sizeof(*s3));
+
+            //  f[z] = S[z]>>(TF_SHIFT+8);
+            __m128i fv1 = _mm_srli_epi32(Sv1, TF_SHIFT_O1_FAST+8);
+            __m128i fv2 = _mm_srli_epi32(Sv2, TF_SHIFT_O1_FAST+8);
+            __m128i fv3 = _mm_srli_epi32(Sv3, TF_SHIFT_O1_FAST+8);
+            __m128i fv4 = _mm_srli_epi32(Sv4, TF_SHIFT_O1_FAST+8);
+
+            //  b[z] = (S[z]>>8) & mask;
+            __m128i bv1 = _mm_and_si128(_mm_srli_epi32(Sv1, 8), maskv);
+            __m128i bv2 = _mm_and_si128(_mm_srli_epi32(Sv2, 8), maskv);
+            __m128i bv3 = _mm_and_si128(_mm_srli_epi32(Sv3, 8), maskv);
+            __m128i bv4 = _mm_and_si128(_mm_srli_epi32(Sv4, 8), maskv);
+
+            //  s[z] = S[z] & 0xff;
+            __m128i sv1 = _mm_and_si128(Sv1, _mm_set1_epi32(0xff));
+            __m128i sv2 = _mm_and_si128(Sv2, _mm_set1_epi32(0xff));
+            __m128i sv3 = _mm_and_si128(Sv3, _mm_set1_epi32(0xff));
+            __m128i sv4 = _mm_and_si128(Sv4, _mm_set1_epi32(0xff));
+
+            //  R[z] = f[z] * (R[z] >> TF_SHIFT_O1_FAST) + b[z];
+            Rv1 = _mm_add_epi32(
+                      _mm_mullo_epi32(
+                          _mm_srli_epi32(Rv1,TF_SHIFT_O1_FAST), fv1), bv1);
+            Rv2 = _mm_add_epi32(
+                      _mm_mullo_epi32(
+                          _mm_srli_epi32(Rv2,TF_SHIFT_O1_FAST), fv2), bv2);
+            Rv3 = _mm_add_epi32(
+                      _mm_mullo_epi32(
+                          _mm_srli_epi32(Rv3,TF_SHIFT_O1_FAST), fv3), bv3);
+            Rv4 = _mm_add_epi32(
+                      _mm_mullo_epi32(
+                          _mm_srli_epi32(Rv4,TF_SHIFT_O1_FAST), fv4), bv4);
+
+            Lv1 = sv1;
+            Lv2 = sv2;
+            Lv3 = sv3;
+            Lv4 = sv4;
+
+            // Tricky one:  out[i+z] = s[z];
+            //             ---d---c ---b---a  sv1
+            //             ---h---g ---f---e  sv2
+            // packs_epi32 -h-g-f-e -d-c-b-a  sv1(2)
+            // packs_epi16 ponmlkji hgfedcba  sv1(2) / sv3(4)
+            sv1 = _mm_packus_epi32(sv1, sv2);
+            sv3 = _mm_packus_epi32(sv3, sv4);
+            sv1 = _mm_packus_epi16(sv1, sv3);
+
+            // c =  R[z] < RANS_BYTE_L;
+            // A little tricky as we only have signed comparisons.
+            // See https://stackoverflow.com/questions/32945410/sse2-intrinsics-comparing-unsigned-integers
+
+//#define _mm_cmplt_epu32_imm(a,b) _mm_andnot_si128(_mm_cmpeq_epi32(_mm_max_epu32((a),_mm_set1_epi32(b)), (a)), _mm_set1_epi32(-1));
+
+            //#define _mm_cmplt_epu32_imm(a,b) _mm_cmpgt_epi32(_mm_set1_epi32((b)-0x80000000), _mm_xor_si128((a), _mm_set1_epi32(0x80000000)))
+
+            __m128i renorm_mask1, renorm_mask2, renorm_mask3, renorm_mask4;
+            renorm_mask1 = _mm_cmplt_epu32_imm(Rv1, RANS_BYTE_L);
+            renorm_mask2 = _mm_cmplt_epu32_imm(Rv2, RANS_BYTE_L);
+            renorm_mask3 = _mm_cmplt_epu32_imm(Rv3, RANS_BYTE_L);
+            renorm_mask4 = _mm_cmplt_epu32_imm(Rv4, RANS_BYTE_L);
+
+            //#define P(A,B,C,D) ((A)+((B)<<2) + ((C)<<4) + ((D)<<6))
+#define P(A,B,C,D)                              \
+            { A+0,A+1,A+2,A+3,                  \
+              B+0,B+1,B+2,B+3,                  \
+              C+0,C+1,C+2,C+3,                  \
+              D+0,D+1,D+2,D+3}
+#ifdef _
+#undef _
+#endif
+#define _ 0x80
+            uint8_t pidx[16][16] = {
+                P(_,_,_,_),
+                P(0,_,_,_),
+                P(_,0,_,_),
+                P(0,4,_,_),
+
+                P(_,_,0,_),
+                P(0,_,4,_),
+                P(_,0,4,_),
+                P(0,4,8,_),
+
+                P(_,_,_,0),
+                P(0,_,_,4),
+                P(_,0,_,4),
+                P(0,4,_,8),
+
+                P(_,_,0,4),
+                P(0,_,4,8),
+                P(_,0,4,8),
+                P(0,4,8,12),
+            };
+#undef _
+
+            // Shuffle the renorm values to correct lanes and incr sp pointer
+            __m128i Vv1 = _mm_cvtepu16_epi32(_mm_loadu_si128((__m128i *)sp));
+            unsigned int imask1 = _mm_movemask_ps((__m128)renorm_mask1);
+            Vv1 = _mm_shuffle_epi8(Vv1, _mm_load_si128((__m128i*)pidx[imask1]));
+            sp += _mm_popcnt_u32(imask1);
+
+            __m128i Vv2 = _mm_cvtepu16_epi32(_mm_loadu_si128((__m128i *)sp));
+            unsigned int imask2 = _mm_movemask_ps((__m128)renorm_mask2);
+            sp += _mm_popcnt_u32(imask2);
+            Vv2 = _mm_shuffle_epi8(Vv2, _mm_load_si128((__m128i*)pidx[imask2]));
+
+            __m128i Vv3 = _mm_cvtepu16_epi32(_mm_loadu_si128((__m128i *)sp));
+            unsigned int imask3 = _mm_movemask_ps((__m128)renorm_mask3);
+            Vv3 = _mm_shuffle_epi8(Vv3, _mm_load_si128((__m128i*)pidx[imask3]));
+            sp += _mm_popcnt_u32(imask3);
+
+            __m128i Vv4 = _mm_cvtepu16_epi32(_mm_loadu_si128((__m128i *)sp));
+            unsigned int imask4 = _mm_movemask_ps((__m128)renorm_mask4);
+            sp += _mm_popcnt_u32(imask4);
+            Vv4 = _mm_shuffle_epi8(Vv4, _mm_load_si128((__m128i*)pidx[imask4]));
+
+            __m128i Yv1 = _mm_slli_epi32(Rv1, 16);
+            __m128i Yv2 = _mm_slli_epi32(Rv2, 16);
+            __m128i Yv3 = _mm_slli_epi32(Rv3, 16);
+            __m128i Yv4 = _mm_slli_epi32(Rv4, 16);
+
+            // y = (R[z] << 16) | V[z];
+            Yv1 = _mm_or_si128(Yv1, Vv1);
+            Yv2 = _mm_or_si128(Yv2, Vv2);
+            Yv3 = _mm_or_si128(Yv3, Vv3);
+            Yv4 = _mm_or_si128(Yv4, Vv4);
+
+            // R[z] = c ? Y[z] : R[z];
+            Rv1 = _mm_blendv_epi8(Rv1, Yv1, renorm_mask1);
+            Rv2 = _mm_blendv_epi8(Rv2, Yv2, renorm_mask2);
+            Rv3 = _mm_blendv_epi8(Rv3, Yv3, renorm_mask3);
+            Rv4 = _mm_blendv_epi8(Rv4, Yv4, renorm_mask4);
+
+            // ------------------------------------------------------------
+
+            //  m[z] = R[z] & mask;
+            __m128i masked5 = _mm_and_si128(Rv5, maskv);
+            __m128i masked6 = _mm_and_si128(Rv6, maskv);
+            __m128i masked7 = _mm_and_si128(Rv7, maskv);
+            __m128i masked8 = _mm_and_si128(Rv8, maskv);
+
+
+            Lv5 = _mm_slli_epi32(Lv5, TF_SHIFT_O1_FAST);
+            Lv6 = _mm_slli_epi32(Lv6, TF_SHIFT_O1_FAST);
+            Lv7 = _mm_slli_epi32(Lv7, TF_SHIFT_O1_FAST);
+            Lv8 = _mm_slli_epi32(Lv8, TF_SHIFT_O1_FAST);
+            masked5 = _mm_add_epi32(masked5, Lv5);
+            masked6 = _mm_add_epi32(masked6, Lv6);
+            masked7 = _mm_add_epi32(masked7, Lv7);
+            masked8 = _mm_add_epi32(masked8, Lv8);
+
+            //  S[z] = s3[m[z]];
+            __m128i Sv5 = _mm_i32gather_epi32x((int *)s3F, masked5, sizeof(*s3));
+            __m128i Sv6 = _mm_i32gather_epi32x((int *)s3F, masked6, sizeof(*s3));
+            __m128i Sv7 = _mm_i32gather_epi32x((int *)s3F, masked7, sizeof(*s3));
+            __m128i Sv8 = _mm_i32gather_epi32x((int *)s3F, masked8, sizeof(*s3));
+
+            //  f[z] = S[z]>>(TF_SHIFT_O1_FAST+8);
+            __m128i fv5 = _mm_srli_epi32(Sv5, TF_SHIFT_O1_FAST+8);
+            __m128i fv6 = _mm_srli_epi32(Sv6, TF_SHIFT_O1_FAST+8);
+            __m128i fv7 = _mm_srli_epi32(Sv7, TF_SHIFT_O1_FAST+8);
+            __m128i fv8 = _mm_srli_epi32(Sv8, TF_SHIFT_O1_FAST+8);
+
+            //  b[z] = (S[z]>>8) & mask;
+            __m128i bv5 = _mm_and_si128(_mm_srli_epi32(Sv5, 8), maskv);
+            __m128i bv6 = _mm_and_si128(_mm_srli_epi32(Sv6, 8), maskv);
+            __m128i bv7 = _mm_and_si128(_mm_srli_epi32(Sv7, 8), maskv);
+            __m128i bv8 = _mm_and_si128(_mm_srli_epi32(Sv8, 8), maskv);
+
+            //  s[z] = S[z] & 0xff;
+            __m128i sv5 = _mm_and_si128(Sv5, _mm_set1_epi32(0xff));
+            __m128i sv6 = _mm_and_si128(Sv6, _mm_set1_epi32(0xff));
+            __m128i sv7 = _mm_and_si128(Sv7, _mm_set1_epi32(0xff));
+            __m128i sv8 = _mm_and_si128(Sv8, _mm_set1_epi32(0xff));
+
+            //  R[z] = f[z] * (R[z] >> TF_SHIFT_O1_FAST) + b[z];
+            Rv5 = _mm_add_epi32(
+                      _mm_mullo_epi32(
+                          _mm_srli_epi32(Rv5,TF_SHIFT_O1_FAST), fv5), bv5);
+            Rv6 = _mm_add_epi32(
+                      _mm_mullo_epi32(
+                          _mm_srli_epi32(Rv6,TF_SHIFT_O1_FAST), fv6), bv6);
+            Rv7 = _mm_add_epi32(
+                      _mm_mullo_epi32(
+                          _mm_srli_epi32(Rv7,TF_SHIFT_O1_FAST), fv7), bv7);
+            Rv8 = _mm_add_epi32(
+                      _mm_mullo_epi32(
+                          _mm_srli_epi32(Rv8,TF_SHIFT_O1_FAST), fv8), bv8);
+
+            Lv5 = sv5;
+            Lv6 = sv6;
+            Lv7 = sv7;
+            Lv8 = sv8;
+
+            // Tricky one:  out[i+z] = s[z];
+            //             ---d---c ---b---a  sv1
+            //             ---h---g ---f---e  sv2
+            // packs_epi32 -h-g-f-e -d-c-b-a  sv1(2)
+            // packs_epi16 ponmlkji hgfedcba  sv1(2) / sv3(4)
+            sv5 = _mm_packus_epi32(sv5, sv6);
+            sv7 = _mm_packus_epi32(sv7, sv8);
+            sv5 = _mm_packus_epi16(sv5, sv7);
+
+            // c =  R[z] < RANS_BYTE_L;
+            __m128i renorm_mask5, renorm_mask6, renorm_mask7, renorm_mask8;
+            renorm_mask5 = _mm_cmplt_epu32_imm(Rv5, RANS_BYTE_L);
+            renorm_mask6 = _mm_cmplt_epu32_imm(Rv6, RANS_BYTE_L);
+            renorm_mask7 = _mm_cmplt_epu32_imm(Rv7, RANS_BYTE_L);
+            renorm_mask8 = _mm_cmplt_epu32_imm(Rv8, RANS_BYTE_L);
+        
+            // Shuffle the renorm values to correct lanes and incr sp pointer
+            __m128i Vv5 = _mm_cvtepu16_epi32(_mm_loadu_si128((__m128i *)sp));
+            unsigned int imask5 = _mm_movemask_ps((__m128)renorm_mask5);
+            Vv5 = _mm_shuffle_epi8(Vv5, _mm_load_si128((__m128i*)pidx[imask5]));
+            sp += _mm_popcnt_u32(imask5);
+
+            __m128i Vv6 = _mm_cvtepu16_epi32(_mm_loadu_si128((__m128i *)sp));
+            unsigned int imask6 = _mm_movemask_ps((__m128)renorm_mask6);
+            sp += _mm_popcnt_u32(imask6);
+            Vv6 = _mm_shuffle_epi8(Vv6, _mm_load_si128((__m128i*)pidx[imask6]));
+
+            __m128i Vv7 = _mm_cvtepu16_epi32(_mm_loadu_si128((__m128i *)sp));
+            unsigned int imask7 = _mm_movemask_ps((__m128)renorm_mask7);
+            Vv7 = _mm_shuffle_epi8(Vv7, _mm_load_si128((__m128i*)pidx[imask7]));
+            sp += _mm_popcnt_u32(imask7);
+
+            __m128i Vv8 = _mm_cvtepu16_epi32(_mm_loadu_si128((__m128i *)sp));
+            unsigned int imask8 = _mm_movemask_ps((__m128)renorm_mask8);
+            sp += _mm_popcnt_u32(imask8);
+            Vv8 = _mm_shuffle_epi8(Vv8, _mm_load_si128((__m128i*)pidx[imask8]));
+
+            __m128i Yv5 = _mm_slli_epi32(Rv5, 16);
+            __m128i Yv6 = _mm_slli_epi32(Rv6, 16);
+            __m128i Yv7 = _mm_slli_epi32(Rv7, 16);
+            __m128i Yv8 = _mm_slli_epi32(Rv8, 16);
+
+            // y = (R[z] << 16) | V[z];
+            Yv5 = _mm_or_si128(Yv5, Vv5);
+            Yv6 = _mm_or_si128(Yv6, Vv6);
+            Yv7 = _mm_or_si128(Yv7, Vv7);
+            Yv8 = _mm_or_si128(Yv8, Vv8);
+
+            // R[z] = c ? Y[z] : R[z];
+            Rv5 = _mm_blendv_epi8(Rv5, Yv5, renorm_mask5);
+            Rv6 = _mm_blendv_epi8(Rv6, Yv6, renorm_mask6);
+            Rv7 = _mm_blendv_epi8(Rv7, Yv7, renorm_mask7);
+            Rv8 = _mm_blendv_epi8(Rv8, Yv8, renorm_mask8);
+
+            // Maybe just a store128 instead?
+            _mm_store_si128((__m128i *)&tbuf[tidx][ 0], sv1);
+            _mm_store_si128((__m128i *)&tbuf[tidx][16], sv5);
+            //  *(uint64_t *)&out[i+ 0] = _mm_extract_epi64(sv1, 0);
+            //  *(uint64_t *)&out[i+ 8] = _mm_extract_epi64(sv1, 1);
+            //  *(uint64_t *)&out[i+16] = _mm_extract_epi64(sv5, 0);
+            //  *(uint64_t *)&out[i+24] = _mm_extract_epi64(sv5, 1);
+
+            // WRONG - need to reorder these periodically.
+
+            i4[0]++;
+            if (++tidx == 32) {
+                i4[0]-=32;
+                transpose_and_copy(out, i4, tbuf);
+                tidx = 0;
+            }
+        }
+        isz4 += 64;
+
+        STORE128(Rv, R);
+        STORE128(Lv, l);
+        ptr = (uint8_t *)sp;
+
+        i4[0]-=tidx;
+        int T;
+        for (z = 0; z < NX; z++)
+            for (T = 0; T < tidx; T++)
+                out[i4[z]++] = tbuf[T][z];
+
+        // Scalar version for close to the end of in[] array so we don't
+        // do SIMD loads beyond the end of the buffer
+        for (; i4[0] < isz4;) {
+            for (z = 0; z < NX; z++) {
+                uint32_t m = R[z] & ((1u<<TF_SHIFT_O1_FAST)-1);
+                uint32_t S = s3F[l[z]][m];
+                unsigned char c = S & 0xff;
+                out[i4[z]++] = c;
+                R[z] = (S>>(TF_SHIFT_O1_FAST+8)) * (R[z]>>TF_SHIFT_O1_FAST) +
+                    ((S>>8) & ((1u<<TF_SHIFT_O1_FAST)-1));
+                RansDecRenormSafe(&R[z], &ptr, ptr_end);
+                l[z] = c;
+            }
+        }
+
+        // Remainder
+        z = NX-1;
+        for (; i4[z] < out_sz; ) {
+            uint32_t m = R[z] & ((1u<<TF_SHIFT_O1_FAST)-1);
+            uint32_t S = s3F[l[z]][m];
+            unsigned char c = S & 0xff;
+            out[i4[z]++] = c;
+            R[z] = (S>>(TF_SHIFT_O1_FAST+8)) * (R[z]>>TF_SHIFT_O1_FAST) +
+                ((S>>8) & ((1u<<TF_SHIFT_O1_FAST)-1));
+            RansDecRenormSafe(&R[z], &ptr, ptr_end);
+            l[z] = c;
+        }
+    }
+    //fprintf(stderr, "    1 Decoded %d bytes\n", (int)(ptr-in)); //c-size
+
+    htscodecs_tls_free(s3);
+    return out;
+
+ err:
+    htscodecs_tls_free(s3);
+    free(out_free);
+    free(c_freq);
+
+    return NULL;
+}
+#else  // HAVE_SSE4_1 and HAVE_SSSE3
+// Prevent "empty translation unit" errors when building without SSE4 etc.
+const char *rANS_static32x16pr_sse4_disabled = "No SSE4";
+#endif // HAVE_SSE4_1 and HAVE_SSSE3
diff --git a/ext/htslib/htscodecs/htscodecs/rANS_static4x16.h b/ext/htslib/htscodecs/htscodecs/rANS_static4x16.h
new file mode 100644
index 0000000..361f1eb
--- /dev/null
+++ b/ext/htslib/htscodecs/htscodecs/rANS_static4x16.h
@@ -0,0 +1,109 @@
+/*
+ * Copyright (c) 2017-2019 Genome Research Ltd.
+ * Author(s): James Bonfield
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above
+ *       copyright notice, this list of conditions and the following
+ *       disclaimer in the documentation and/or other materials provided
+ *       with the distribution.
+ *
+ *    3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+ *       Institute nor the names of its contributors may be used to endorse
+ *       or promote products derived from this software without specific
+ *       prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH
+ * LTD OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef RANS_STATIC4x16_H
+#define RANS_STATIC4x16_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+unsigned int rans_compress_bound_4x16(unsigned int size, int order);
+unsigned char *rans_compress_to_4x16(unsigned char *in,  unsigned int in_size,
+                                     unsigned char *out, unsigned int *out_size,
+                                     int order);
+unsigned char *rans_compress_4x16(unsigned char *in, unsigned int in_size,
+                                  unsigned int *out_size, int order);
+unsigned char *rans_uncompress_to_4x16(unsigned char *in,  unsigned int in_size,
+                                       unsigned char *out, unsigned int *out_size);
+unsigned char *rans_uncompress_4x16(unsigned char *in, unsigned int in_size,
+                                    unsigned int *out_size);
+
+// CPU detection control.  Used for testing and benchmarking.
+// These bitfields control what methods are permitted to be used.
+#define RANS_CPU_ENC_SSE4     (1<<0)
+#define RANS_CPU_ENC_AVX2     (2<<0)
+#define RANS_CPU_ENC_AVX512   (4<<0)
+#define RANS_CPU_ENC_NEON     (8<<0)
+
+#define RANS_CPU_DEC_SSE4     (1<<8)
+#define RANS_CPU_DEC_AVX2     (2<<8)
+#define RANS_CPU_DEC_AVX512   (4<<8)
+#define RANS_CPU_DEC_NEON     (8<<8)
+
+void rans_set_cpu(int opts);
+
+// "Order" byte options. ORed into the order byte.
+// The bottom bits are the order itself, currently
+// supporting order-0 and order-1 but with expansion room
+// up to order-3 (unlikely).
+
+//--
+// The values below are stored in the file format
+
+// Pack 2,4,8 or infinite symbols into a byte.
+#define RANS_ORDER_PACK   0x80
+
+// Run length encoding with runs & lits encoded separately
+#define RANS_ORDER_RLE    0x40
+
+// Nop; for tiny segments where rANS overhead is too big
+#define RANS_ORDER_CAT    0x20
+
+// Don't store the original size; used by STRIPE mode
+#define RANS_ORDER_NOSZ   0x10
+
+// For N-byte integer data; rotate & encode N streams.
+#define RANS_ORDER_STRIPE 0x08
+
+// 32-way unrolling instead of 4-way
+#define RANS_ORDER_X32    0x04
+
+//--
+// order values below are not directly part of the file format, but control
+// the behaviour of the encoder.
+
+// Bit 8-15 of order hold the stripe size (N).
+// Note: N is stored separately after the order byte
+
+// Used to disable order-0 in the STRIPE sub-methods.
+#define RANS_ORDER_STRIPE_NO0 (1<<16)
+
+// Used to request automatic selection between 4-way and 32-way
+#define RANS_ORDER_SIMD_AUTO  (1<<17)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RANS_STATIC4x16_H */
diff --git a/ext/htslib/htscodecs/htscodecs/rANS_static4x16pr.c b/ext/htslib/htscodecs/htscodecs/rANS_static4x16pr.c
new file mode 100644
index 0000000..8c9a64a
--- /dev/null
+++ b/ext/htslib/htscodecs/htscodecs/rANS_static4x16pr.c
@@ -0,0 +1,1728 @@
+/*
+ * Copyright (c) 2017-2023 Genome Research Ltd.
+ * Author(s): James Bonfield
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above
+ *       copyright notice, this list of conditions and the following
+ *       disclaimer in the documentation and/or other materials provided
+ *       with the distribution.
+ *
+ *    3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+ *       Institute nor the names of its contributors may be used to endorse
+ *       or promote products derived from this software without specific
+ *       prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH
+ * LTD OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+// FIXME Can we get decoder to return the compressed sized read, avoiding
+// us needing to store it?  Yes we can.  See c-size comments.  If we added all these
+// together we could get rans_uncompress_to_4x16 to return the number of bytes
+// consumed, avoiding the calling code from needed to explicitly stored the size.
+// However the effect on name tokeniser is to save 0.1 to 0.2% so not worth it.
+
+/*-------------------------------------------------------------------------- */
+/*
+ * Example wrapper to use the rans_byte.h functions included above.
+ *
+ * This demonstrates how to use, and unroll, an order-0 and order-1 frequency
+ * model.
+ */
+
+#include "config.h"
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <assert.h>
+#include <string.h>
+#include <sys/time.h>
+#include <limits.h>
+#include <math.h>
+
+#ifndef NO_THREADS
+#include <pthread.h>
+#endif
+
+#include "rANS_word.h"
+#include "rANS_static4x16.h"
+#include "rANS_static16_int.h"
+#include "pack.h"
+#include "rle.h"
+#include "utils.h"
+
+#define TF_SHIFT 12
+#define TOTFREQ (1<<TF_SHIFT)
+
+// 9-11 is considerably faster in the O1 variant due to reduced table size.
+// We auto-tune between 10 and 12 though.  Anywhere from 9 to 14 are viable.
+#ifndef TF_SHIFT_O1
+#define TF_SHIFT_O1 12
+#endif
+#ifndef TF_SHIFT_O1_FAST
+#define TF_SHIFT_O1_FAST 10
+#endif
+#define TOTFREQ_O1 (1<<TF_SHIFT_O1)
+#define TOTFREQ_O1_FAST (1<<TF_SHIFT_O1_FAST)
+
+
+/*-----------------------------------------------------------------------------
+ * Memory to memory compression functions.
+ *
+ * These are original versions without any manual loop unrolling. They
+ * are easier to understand, but can be up to 2x slower.
+ */
+
+unsigned int rans_compress_bound_4x16(unsigned int size, int order) {
+    int N = (order>>8) & 0xff;
+    if (!N) N=4;
+
+    order &= 0xff;
+    unsigned int sz = (order == 0
+        ? 1.05*size + 257*3 + 4
+        : 1.05*size + 257*257*3 + 4 + 257*3+4) +
+        ((order & RANS_ORDER_PACK) ? 1 : 0) +
+        ((order & RANS_ORDER_RLE) ? 1 + 257*3+4: 0) + 20 +
+        ((order & RANS_ORDER_X32) ? (32-4)*4 : 0) +
+        ((order & RANS_ORDER_STRIPE) ? 7 + 5*N: 0);
+    return sz + (sz&1) + 2; // make this even so buffers are word aligned
+}
+
+// Compresses in_size bytes from 'in' to *out_size bytes in 'out'.
+//
+// NB: The output buffer does not hold the original size, so it is up to
+// the caller to store this.
+unsigned char *rans_compress_O0_4x16(unsigned char *in, unsigned int in_size,
+                                     unsigned char *out, unsigned int *out_size) {
+    unsigned char *cp, *out_end;
+    RansEncSymbol syms[256];
+    RansState rans0;
+    RansState rans2;
+    RansState rans1;
+    RansState rans3;
+    uint8_t* ptr;
+    uint32_t F[256+MAGIC] = {0};
+    int i, j, tab_size = 0, rle, x;
+    // -20 for order/size/meta
+    uint32_t bound = rans_compress_bound_4x16(in_size,0)-20;
+
+    if (!out) {
+        *out_size = bound;
+        out = malloc(*out_size);
+    }
+    if (!out || bound > *out_size)
+        return NULL;
+
+    // If "out" isn't word aligned, tweak out_end/ptr to ensure it is.
+    // We already added more round in bound to allow for this.
+    if (((size_t)out)&1)
+        bound--;
+    ptr = out_end = out + bound;
+
+    if (in_size == 0)
+        goto empty;
+
+    // Compute statistics
+    if (hist8(in, in_size, F) < 0)
+        return NULL;
+
+    // Normalise so frequences sum to power of 2
+    uint32_t fsum = in_size;
+    uint32_t max_val = round2(fsum);
+    if (max_val > TOTFREQ)
+        max_val = TOTFREQ;
+
+    if (normalise_freq(F, fsum, max_val) < 0)
+        return NULL;
+    fsum=max_val;
+
+    cp = out;
+    cp += encode_freq(cp, F);
+    tab_size = cp-out;
+    //write(2, out+4, cp-(out+4));
+
+    if (normalise_freq(F, fsum, TOTFREQ) < 0)
+        return NULL;
+
+    // Encode statistics.
+    for (x = rle = j = 0; j < 256; j++) {
+        if (F[j]) {
+            RansEncSymbolInit(&syms[j], x, F[j], TF_SHIFT);
+            x += F[j];
+        }
+    }
+
+    RansEncInit(&rans0);
+    RansEncInit(&rans1);
+    RansEncInit(&rans2);
+    RansEncInit(&rans3);
+
+    switch (i=(in_size&3)) {
+    case 3: RansEncPutSymbol(&rans2, &ptr, &syms[in[in_size-(i-2)]]);
+        // fall-through
+    case 2: RansEncPutSymbol(&rans1, &ptr, &syms[in[in_size-(i-1)]]);
+        // fall-through
+    case 1: RansEncPutSymbol(&rans0, &ptr, &syms[in[in_size-(i-0)]]);
+        // fall-through
+    case 0:
+        break;
+    }
+    for (i=(in_size &~3); i>0; i-=4) {
+        RansEncSymbol *s3 = &syms[in[i-1]];
+        RansEncSymbol *s2 = &syms[in[i-2]];
+        RansEncSymbol *s1 = &syms[in[i-3]];
+        RansEncSymbol *s0 = &syms[in[i-4]];
+
+#if 1
+        RansEncPutSymbol(&rans3, &ptr, s3);
+        RansEncPutSymbol(&rans2, &ptr, s2);
+        RansEncPutSymbol(&rans1, &ptr, s1);
+        RansEncPutSymbol(&rans0, &ptr, s0);
+#else
+        // Slightly beter on gcc, much better on clang
+        uint16_t *ptr16 = (uint16_t *)ptr;
+
+        if (rans3 >= s3->x_max) *--ptr16 = (uint16_t)rans3, rans3 >>= 16;
+        if (rans2 >= s2->x_max) *--ptr16 = (uint16_t)rans2, rans2 >>= 16;
+        uint32_t q3 = (uint32_t) (((uint64_t)rans3 * s3->rcp_freq) >> s3->rcp_shift);
+        uint32_t q2 = (uint32_t) (((uint64_t)rans2 * s2->rcp_freq) >> s2->rcp_shift);
+        rans3 += s3->bias + q3 * s3->cmpl_freq;
+        rans2 += s2->bias + q2 * s2->cmpl_freq;
+
+        if (rans1 >= s1->x_max) *--ptr16 = (uint16_t)rans1, rans1 >>= 16;
+        if (rans0 >= s0->x_max) *--ptr16 = (uint16_t)rans0, rans0 >>= 16;
+        uint32_t q1 = (uint32_t) (((uint64_t)rans1 * s1->rcp_freq) >> s1->rcp_shift);
+        uint32_t q0 = (uint32_t) (((uint64_t)rans0 * s0->rcp_freq) >> s0->rcp_shift);
+        rans1 += s1->bias + q1 * s1->cmpl_freq;
+        rans0 += s0->bias + q0 * s0->cmpl_freq;
+
+        ptr = (uint8_t *)ptr16;
+#endif
+    }
+
+    RansEncFlush(&rans3, &ptr);
+    RansEncFlush(&rans2, &ptr);
+    RansEncFlush(&rans1, &ptr);
+    RansEncFlush(&rans0, &ptr);
+
+ empty:
+    // Finalise block size and return it
+    *out_size = (out_end - ptr) + tab_size;
+
+    memmove(out + tab_size, ptr, out_end-ptr);
+
+    return out;
+}
+
+unsigned char *rans_uncompress_O0_4x16(unsigned char *in, unsigned int in_size,
+                                       unsigned char *out, unsigned int out_sz) {
+    if (in_size < 16) // 4-states at least
+        return NULL;
+
+    if (out_sz >= INT_MAX)
+        return NULL; // protect against some overflow cases
+
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    if (out_sz > 100000)
+        return NULL;
+#endif
+
+    /* Load in the static tables */
+    unsigned char *cp = in, *out_free = NULL;
+    unsigned char *cp_end = in + in_size - 8; // within 8 => be extra safe
+    int i, j;
+    unsigned int x, y;
+    uint16_t sfreq[TOTFREQ+32];
+    uint16_t sbase[TOTFREQ+32]; // faster to use 32-bit on clang
+    uint8_t  ssym [TOTFREQ+64]; // faster to use 16-bit on clang
+
+    if (!out)
+        out_free = out = malloc(out_sz);
+    if (!out)
+        return NULL;
+
+    // Precompute reverse lookup of frequency.
+    uint32_t F[256] = {0}, fsum;
+    int fsz = decode_freq(cp, cp_end, F, &fsum);
+    if (!fsz)
+        goto err;
+    cp += fsz;
+
+    normalise_freq_shift(F, fsum, TOTFREQ);
+
+    // Build symbols; fixme, do as part of decode, see the _d variant
+    for (j = x = 0; j < 256; j++) {
+        if (F[j]) {
+            if (F[j] > TOTFREQ - x)
+                goto err;
+            for (y = 0; y < F[j]; y++) {
+                ssym [y + x] = j;
+                sfreq[y + x] = F[j];
+                sbase[y + x] = y;
+            }
+            x += F[j];
+        }
+    }
+
+    if (x != TOTFREQ)
+        goto err;
+
+    if (cp+16 > cp_end+8)
+        goto err;
+
+    RansState R[4];
+    RansDecInit(&R[0], &cp); if (R[0] < RANS_BYTE_L) goto err;
+    RansDecInit(&R[1], &cp); if (R[1] < RANS_BYTE_L) goto err;
+    RansDecInit(&R[2], &cp); if (R[2] < RANS_BYTE_L) goto err;
+    RansDecInit(&R[3], &cp); if (R[3] < RANS_BYTE_L) goto err;
+
+// Simple version is comparable to below, but only with -O3
+//
+//    for (i = 0; cp < cp_end-8 && i < (out_sz&~7); i+=8) {
+//        for(j=0; j<8;j++) {
+//          RansState m = RansDecGet(&R[j%4], TF_SHIFT);
+//          R[j%4] = sfreq[m] * (R[j%4] >> TF_SHIFT) + sbase[m];
+//          out[i+j] = ssym[m];
+//          RansDecRenorm(&R[j%4], &cp);
+//        }
+//    }
+
+    for (i = 0; cp < cp_end-8 && i < (out_sz&~7); i+=8) {
+        for (j = 0; j < 8; j+=4) {
+            RansState m0 = RansDecGet(&R[0], TF_SHIFT);
+            RansState m1 = RansDecGet(&R[1], TF_SHIFT);
+            out[i+j+0] = ssym[m0];
+            out[i+j+1] = ssym[m1];
+
+            R[0] = sfreq[m0] * (R[0] >> TF_SHIFT) + sbase[m0];
+            R[1] = sfreq[m1] * (R[1] >> TF_SHIFT) + sbase[m1];
+
+            RansState m2 = RansDecGet(&R[2], TF_SHIFT);
+            RansState m3 = RansDecGet(&R[3], TF_SHIFT);
+
+            RansDecRenorm(&R[0], &cp);
+            RansDecRenorm(&R[1], &cp);
+
+            R[2] = sfreq[m2] * (R[2] >> TF_SHIFT) + sbase[m2];
+            R[3] = sfreq[m3] * (R[3] >> TF_SHIFT) + sbase[m3];
+
+            RansDecRenorm(&R[2], &cp);
+            RansDecRenorm(&R[3], &cp);
+
+            out[i+j+2] = ssym[m2];
+            out[i+j+3] = ssym[m3];
+        }
+    }
+
+    // remainder
+    for (; i < out_sz; i++) {
+        RansState m = RansDecGet(&R[i%4], TF_SHIFT);
+        R[i%4] = sfreq[m] * (R[i%4] >> TF_SHIFT) + sbase[m];
+        out[i] = ssym[m];
+        RansDecRenormSafe(&R[i%4], &cp, cp_end+8);
+    }
+
+    //fprintf(stderr, "    0 Decoded %d bytes\n", (int)(cp-in)); //c-size
+
+    return out;
+
+ err:
+    free(out_free);
+    return NULL;
+}
+
+//-----------------------------------------------------------------------------
+
+// Compute the entropy of 12-bit vs 10-bit frequency tables.
+// 10 bit means smaller memory footprint when decoding and
+// more speed due to cache hits, but it *may* be a poor
+// compression fit.
+int rans_compute_shift(uint32_t *F0, uint32_t (*F)[256], uint32_t *T,
+                       uint32_t *S) {
+    int i, j;
+
+    double e10 = 0, e12 = 0;
+    int max_tot = 0;
+    for (i = 0; i < 256; i++) {
+        if (F0[i] == 0)
+            continue;
+        unsigned int max_val = round2(T[i]);
+        int ns = 0;
+#define MAX(a,b) ((a)>(b)?(a):(b))
+
+        // Number of samples that get their freq bumped to 1
+        int sm10 = 0, sm12 = 0;
+        for (j = 0; j < 256; j++) {
+            if (F[i][j] && max_val / F[i][j] > TOTFREQ_O1_FAST)
+                sm10++;
+            if (F[i][j] && max_val / F[i][j] > TOTFREQ_O1)
+                sm12++;
+        }
+
+        double l10 = log(TOTFREQ_O1_FAST + sm10);
+        double l12 = log(TOTFREQ_O1      + sm12);
+        double T_slow = (double)TOTFREQ_O1/T[i];
+        double T_fast = (double)TOTFREQ_O1_FAST/T[i];
+
+        for (j = 0; j < 256; j++) {
+            if (F[i][j]) {
+                ns++;
+
+                e10 -= F[i][j] * (fast_log(MAX(F[i][j]*T_fast,1)) - l10);
+                e12 -= F[i][j] * (fast_log(MAX(F[i][j]*T_slow,1)) - l12);
+
+                // Estimation of compressed symbol freq table too.
+                e10 += 1.3;
+                e12 += 4.7;
+            }
+        }
+
+        // Order-1 frequencies often end up totalling under TOTFREQ.
+        // In this case it's smaller to output the real frequencies
+        // prior to normalisation and normalise after (with an extra
+        // normalisation step needed in the decoder too).
+        //
+        // Thus we normalise to a power of 2 only, store those,
+        // and renormalise later here (and in decoder) by bit-shift
+        // to get to the fixed size.
+        if (ns < 64 && max_val > 128) max_val /= 2;
+        if (max_val > 1024)           max_val /= 2;
+        if (max_val > TOTFREQ_O1)     max_val = TOTFREQ_O1;
+        S[i] = max_val; // scale to max this
+        if (max_tot < max_val)
+            max_tot = max_val;
+    }
+    int shift = e10/e12 < 1.01 || max_tot <= TOTFREQ_O1_FAST
+        ? TF_SHIFT_O1_FAST
+        : TF_SHIFT_O1;
+
+//    fprintf(stderr, "e10/12 = %f %f %f, shift %d\n",
+//          e10/log(256), e12/log(256), e10/e12, shift);
+
+    return shift;
+}
+
+static
+unsigned char *rans_compress_O1_4x16(unsigned char *in, unsigned int in_size,
+                                     unsigned char *out, unsigned int *out_size) {
+    unsigned char *cp, *out_end, *out_free = NULL;
+    unsigned int tab_size;
+    
+    // -20 for order/size/meta
+    uint32_t bound = rans_compress_bound_4x16(in_size,1)-20;
+
+    if (!out) {
+        *out_size = bound;
+        out_free = out = malloc(*out_size);
+    }
+    if (!out || bound > *out_size)
+        return NULL;
+
+    if (((size_t)out)&1)
+        bound--;
+    out_end = out + bound;
+
+    RansEncSymbol (*syms)[256] = htscodecs_tls_alloc(256 * (sizeof(*syms)));
+    if (!syms) {
+        free(out_free);
+        return NULL;
+    }
+
+    cp = out;
+    int shift = encode_freq1(in, in_size, 4, syms, &cp); 
+    if (shift < 0) {
+        htscodecs_tls_free(syms);
+        return NULL;
+    }
+    tab_size = cp - out;
+
+    RansState rans0, rans1, rans2, rans3;
+    RansEncInit(&rans0);
+    RansEncInit(&rans1);
+    RansEncInit(&rans2);
+    RansEncInit(&rans3);
+
+    uint8_t* ptr = out_end;
+
+    int isz4 = in_size>>2;
+    int i0 = 1*isz4-2;
+    int i1 = 2*isz4-2;
+    int i2 = 3*isz4-2;
+    int i3 = 4*isz4-2;
+
+    unsigned char l0 = in[i0+1];
+    unsigned char l1 = in[i1+1];
+    unsigned char l2 = in[i2+1];
+    unsigned char l3 = in[i3+1];
+
+    // Deal with the remainder
+    l3 = in[in_size-1];
+    for (i3 = in_size-2; i3 > 4*isz4-2; i3--) {
+        unsigned char c3 = in[i3];
+        RansEncPutSymbol(&rans3, &ptr, &syms[c3][l3]);
+        l3 = c3;
+    }
+
+    for (; i0 >= 0; i0--, i1--, i2--, i3--) {
+        unsigned char c0, c1, c2, c3;
+        RansEncSymbol *s3 = &syms[c3 = in[i3]][l3];
+        RansEncSymbol *s2 = &syms[c2 = in[i2]][l2];
+        RansEncSymbol *s1 = &syms[c1 = in[i1]][l1];
+        RansEncSymbol *s0 = &syms[c0 = in[i0]][l0];
+
+        RansEncPutSymbol(&rans3, &ptr, s3);
+        RansEncPutSymbol(&rans2, &ptr, s2);
+        RansEncPutSymbol(&rans1, &ptr, s1);
+        RansEncPutSymbol(&rans0, &ptr, s0);
+
+        l0 = c0;
+        l1 = c1;
+        l2 = c2;
+        l3 = c3;
+    }
+
+    RansEncPutSymbol(&rans3, &ptr, &syms[0][l3]);
+    RansEncPutSymbol(&rans2, &ptr, &syms[0][l2]);
+    RansEncPutSymbol(&rans1, &ptr, &syms[0][l1]);
+    RansEncPutSymbol(&rans0, &ptr, &syms[0][l0]);
+
+    RansEncFlush(&rans3, &ptr);
+    RansEncFlush(&rans2, &ptr);
+    RansEncFlush(&rans1, &ptr);
+    RansEncFlush(&rans0, &ptr);
+
+    *out_size = (out_end - ptr) + tab_size;
+
+    cp = out;
+    memmove(out + tab_size, ptr, out_end-ptr);
+
+    htscodecs_tls_free(syms);
+    return out;
+}
+
+//#define MAGIC2 111
+#define MAGIC2 179
+//#define MAGIC2 0
+
+static
+unsigned char *rans_uncompress_O1_4x16(unsigned char *in, unsigned int in_size,
+                                       unsigned char *out, unsigned int out_sz) {
+    if (in_size < 16) // 4-states at least
+        return NULL;
+
+    if (out_sz >= INT_MAX)
+        return NULL; // protect against some overflow cases
+
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    if (out_sz > 100000)
+        return NULL;
+#endif
+
+    /* Load in the static tables */
+    unsigned char *cp = in, *cp_end = in+in_size, *out_free = NULL;
+    unsigned char *c_freq = NULL;
+    int i, j = -999;
+    unsigned int x;
+
+    uint8_t *sfb_ = htscodecs_tls_alloc(256*(TOTFREQ_O1+MAGIC2)*sizeof(*sfb_));
+    uint32_t (*s3)[TOTFREQ_O1_FAST] = (uint32_t (*)[TOTFREQ_O1_FAST])sfb_;
+    // reuse the same memory for the fast mode lookup, but this only works
+    // if we're on e.g. 12-bit freqs vs 10-bit freqs as needs 4x larger array.
+    //uint32_t s3[256][TOTFREQ_O1_FAST];
+
+    if (!sfb_)
+        return NULL;
+    fb_t (*fb)[256] = htscodecs_tls_alloc(256 * sizeof(*fb));
+    if (!fb)
+        goto err;
+    uint8_t *sfb[256];
+    if ((*cp >> 4) == TF_SHIFT_O1) {
+        for (i = 0; i < 256; i++)
+            sfb[i]=  sfb_ + i*(TOTFREQ_O1+MAGIC2);
+    } else {
+        for (i = 0; i < 256; i++)
+            sfb[i]=  sfb_ + i*(TOTFREQ_O1_FAST+MAGIC2);
+    }
+
+    if (!out)
+        out_free = out = malloc(out_sz);
+
+    if (!out)
+        goto err;
+
+    //fprintf(stderr, "out_sz=%d\n", out_sz);
+
+    // compressed header? If so uncompress it
+    unsigned char *tab_end = NULL;
+    unsigned char *c_freq_end = cp_end;
+    unsigned int shift = *cp >> 4;
+    if (*cp++ & 1) {
+        uint32_t u_freq_sz, c_freq_sz;
+        cp += var_get_u32(cp, cp_end, &u_freq_sz);
+        cp += var_get_u32(cp, cp_end, &c_freq_sz);
+        if (c_freq_sz > cp_end - cp)
+            goto err;
+        tab_end = cp + c_freq_sz;
+        if (!(c_freq = rans_uncompress_O0_4x16(cp, c_freq_sz, NULL, u_freq_sz)))
+            goto err;
+        cp = c_freq;
+        c_freq_end = c_freq + u_freq_sz;
+    }
+
+    // Decode order-0 symbol list; avoids needing in order-1 tables
+    uint32_t F0[256] = {0};
+    int fsz = decode_alphabet(cp, c_freq_end, F0);
+    if (!fsz)
+        goto err;
+    cp += fsz;
+
+    if (cp >= c_freq_end)
+        goto err;
+
+    const int s3_fast_on = in_size >= 100000;
+
+    for (i = 0; i < 256; i++) {
+        if (F0[i] == 0)
+            continue;
+
+        uint32_t F[256] = {0}, T = 0;
+        fsz = decode_freq_d(cp, c_freq_end, F0, F, &T);
+        if (!fsz)
+            goto err;
+        cp += fsz;
+
+        if (!T) {
+            //fprintf(stderr, "No freq for F_%d\n", i);
+            continue;
+        }
+
+        normalise_freq_shift(F, T, 1<<shift);
+
+        // Build symbols; fixme, do as part of decode, see the _d variant
+        for (j = x = 0; j < 256; j++) {
+            if (F[j]) {
+                if (F[j] > (1<<shift) - x)
+                    goto err;
+
+                if (shift == TF_SHIFT_O1_FAST && s3_fast_on) {
+                    int y;
+                    for (y = 0; y < F[j]; y++)
+                        s3[i][y+x] = (((uint32_t)F[j])<<(shift+8)) |(y<<8) |j;
+                } else {
+                    memset(&sfb[i][x], j, F[j]);
+                    fb[i][j].f = F[j];
+                    fb[i][j].b = x;
+                }
+                x += F[j];
+            }
+        }
+        if (x != (1<<shift))
+            goto err;
+    }
+
+    if (tab_end)
+        cp = tab_end;
+    free(c_freq);
+    c_freq = NULL;
+
+    if (cp+16 > cp_end)
+        goto err;
+
+    RansState rans0, rans1, rans2, rans3;
+    uint8_t *ptr = cp, *ptr_end = in + in_size - 8;
+    RansDecInit(&rans0, &ptr); if (rans0 < RANS_BYTE_L) goto err;
+    RansDecInit(&rans1, &ptr); if (rans1 < RANS_BYTE_L) goto err;
+    RansDecInit(&rans2, &ptr); if (rans2 < RANS_BYTE_L) goto err;
+    RansDecInit(&rans3, &ptr); if (rans3 < RANS_BYTE_L) goto err;
+
+    unsigned int isz4 = out_sz>>2;
+    int l0 = 0, l1 = 0, l2 = 0, l3 = 0;
+    unsigned int i4[] = {0*isz4, 1*isz4, 2*isz4, 3*isz4};
+
+    RansState R[4];
+    R[0] = rans0;
+    R[1] = rans1;
+    R[2] = rans2;
+    R[3] = rans3;
+
+    // Around 15% faster to specialise for 10/12 than to have one
+    // loop with shift as a variable.
+    if (shift == TF_SHIFT_O1) {
+        // TF_SHIFT_O1 = 12
+
+        const uint32_t mask = ((1u << TF_SHIFT_O1)-1);
+        for (; i4[0] < isz4; i4[0]++, i4[1]++, i4[2]++, i4[3]++) {
+            uint16_t m, c;
+            c = sfb[l0][m = R[0] & mask];
+            R[0] = fb[l0][c].f * (R[0]>>TF_SHIFT_O1) + m - fb[l0][c].b;
+            out[i4[0]] = l0 = c;
+
+            c = sfb[l1][m = R[1] & mask];
+            R[1] = fb[l1][c].f * (R[1]>>TF_SHIFT_O1) + m - fb[l1][c].b;
+            out[i4[1]] = l1 = c;
+
+            c = sfb[l2][m = R[2] & mask];
+            R[2] = fb[l2][c].f * (R[2]>>TF_SHIFT_O1) + m - fb[l2][c].b;
+            out[i4[2]] = l2 = c;
+
+            c = sfb[l3][m = R[3] & mask];
+            R[3] = fb[l3][c].f * (R[3]>>TF_SHIFT_O1) + m - fb[l3][c].b;
+            out[i4[3]] = l3 = c;
+
+            if (ptr < ptr_end) {
+                RansDecRenorm(&R[0], &ptr);
+                RansDecRenorm(&R[1], &ptr);
+                RansDecRenorm(&R[2], &ptr);
+                RansDecRenorm(&R[3], &ptr);
+            } else {
+                RansDecRenormSafe(&R[0], &ptr, ptr_end+8);
+                RansDecRenormSafe(&R[1], &ptr, ptr_end+8);
+                RansDecRenormSafe(&R[2], &ptr, ptr_end+8);
+                RansDecRenormSafe(&R[3], &ptr, ptr_end+8);
+            }
+        }
+
+        // Remainder
+        for (; i4[3] < out_sz; i4[3]++) {
+            uint32_t m3 = R[3] & ((1u<<TF_SHIFT_O1)-1);
+            unsigned char c3 = sfb[l3][m3];
+            out[i4[3]] = c3;
+            R[3] = fb[l3][c3].f * (R[3]>>TF_SHIFT_O1) + m3 - fb[l3][c3].b;
+            RansDecRenormSafe(&R[3], &ptr, ptr_end + 8);
+            l3 = c3;
+        }
+    } else if (!s3_fast_on) {
+        // TF_SHIFT_O1 = 10 with sfb[256][1024] & fb[256]256] array lookup
+        // Slightly faster for -o193 on q4 (high comp), but also less
+        // initialisation cost for smaller data
+        const uint32_t mask = ((1u << TF_SHIFT_O1_FAST)-1);
+        for (; i4[0] < isz4; i4[0]++, i4[1]++, i4[2]++, i4[3]++) {
+            uint16_t m, c;
+            c = sfb[l0][m = R[0] & mask];
+            R[0] = fb[l0][c].f * (R[0]>>TF_SHIFT_O1_FAST) + m - fb[l0][c].b;
+            out[i4[0]] = l0 = c;
+
+            c = sfb[l1][m = R[1] & mask];
+            R[1] = fb[l1][c].f * (R[1]>>TF_SHIFT_O1_FAST) + m - fb[l1][c].b;
+            out[i4[1]] = l1 = c;
+
+            c = sfb[l2][m = R[2] & mask];
+            R[2] = fb[l2][c].f * (R[2]>>TF_SHIFT_O1_FAST) + m - fb[l2][c].b;
+            out[i4[2]] = l2 = c;
+
+            c = sfb[l3][m = R[3] & mask];
+            R[3] = fb[l3][c].f * (R[3]>>TF_SHIFT_O1_FAST) + m - fb[l3][c].b;
+            out[i4[3]] = l3 = c;
+
+            if (ptr < ptr_end) {
+                RansDecRenorm(&R[0], &ptr);
+                RansDecRenorm(&R[1], &ptr);
+                RansDecRenorm(&R[2], &ptr);
+                RansDecRenorm(&R[3], &ptr);
+            } else {
+                RansDecRenormSafe(&R[0], &ptr, ptr_end+8);
+                RansDecRenormSafe(&R[1], &ptr, ptr_end+8);
+                RansDecRenormSafe(&R[2], &ptr, ptr_end+8);
+                RansDecRenormSafe(&R[3], &ptr, ptr_end+8);
+            }
+        }
+
+        // Remainder
+        for (; i4[3] < out_sz; i4[3]++) {
+            uint32_t m3 = R[3] & ((1u<<TF_SHIFT_O1_FAST)-1);
+            unsigned char c3 = sfb[l3][m3];
+            out[i4[3]] = c3;
+            R[3] = fb[l3][c3].f * (R[3]>>TF_SHIFT_O1_FAST) + m3 - fb[l3][c3].b;
+            RansDecRenormSafe(&R[3], &ptr, ptr_end + 8);
+            l3 = c3;
+        }
+    } else {
+        // TF_SHIFT_O1_FAST.
+        // Significantly faster for -o1 on q40 (low comp).
+        // Higher initialisation cost, so only use if big blocks.
+        const uint32_t mask = ((1u << TF_SHIFT_O1_FAST)-1);
+        for (; i4[0] < isz4; i4[0]++, i4[1]++, i4[2]++, i4[3]++) {
+            uint32_t S0 = s3[l0][R[0] & mask];
+            uint32_t S1 = s3[l1][R[1] & mask];
+            l0 = out[i4[0]] = S0;
+            l1 = out[i4[1]] = S1;
+            uint16_t F0 = S0>>(TF_SHIFT_O1_FAST+8);
+            uint16_t F1 = S1>>(TF_SHIFT_O1_FAST+8);
+            uint16_t B0 = (S0>>8) & mask;
+            uint16_t B1 = (S1>>8) & mask;
+
+            R[0] = F0 * (R[0]>>TF_SHIFT_O1_FAST) + B0;
+            R[1] = F1 * (R[1]>>TF_SHIFT_O1_FAST) + B1;
+
+            uint32_t S2 = s3[l2][R[2] & mask];
+            uint32_t S3 = s3[l3][R[3] & mask];
+            l2 = out[i4[2]] = S2;
+            l3 = out[i4[3]] = S3;
+            uint16_t F2 = S2>>(TF_SHIFT_O1_FAST+8);
+            uint16_t F3 = S3>>(TF_SHIFT_O1_FAST+8);
+            uint16_t B2 = (S2>>8) & mask;
+            uint16_t B3 = (S3>>8) & mask;
+
+            R[2] = F2 * (R[2]>>TF_SHIFT_O1_FAST) + B2;
+            R[3] = F3 * (R[3]>>TF_SHIFT_O1_FAST) + B3;
+
+            if (ptr < ptr_end) {
+                RansDecRenorm(&R[0], &ptr);
+                RansDecRenorm(&R[1], &ptr);
+                RansDecRenorm(&R[2], &ptr);
+                RansDecRenorm(&R[3], &ptr);
+            } else {
+                RansDecRenormSafe(&R[0], &ptr, ptr_end+8);
+                RansDecRenormSafe(&R[1], &ptr, ptr_end+8);
+                RansDecRenormSafe(&R[2], &ptr, ptr_end+8);
+                RansDecRenormSafe(&R[3], &ptr, ptr_end+8);
+            }
+        }
+
+        // Remainder
+        for (; i4[3] < out_sz; i4[3]++) {
+            uint32_t S = s3[l3][R[3] & ((1u<<TF_SHIFT_O1_FAST)-1)];
+            l3 = out[i4[3]] = S;
+            R[3] = (S>>(TF_SHIFT_O1_FAST+8)) * (R[3]>>TF_SHIFT_O1_FAST)
+                + ((S>>8) & ((1u<<TF_SHIFT_O1_FAST)-1));
+            RansDecRenormSafe(&R[3], &ptr, ptr_end + 8);
+        }
+    }
+    //fprintf(stderr, "    1 Decoded %d bytes\n", (int)(ptr-in)); //c-size
+
+    htscodecs_tls_free(fb);
+    htscodecs_tls_free(sfb_);
+    return out;
+
+ err:
+    htscodecs_tls_free(fb);
+    htscodecs_tls_free(sfb_);
+    free(out_free);
+    free(c_freq);
+
+    return NULL;
+}
+
+/*-----------------------------------------------------------------------------
+ * r32x16 implementation, included here for now for simplicity
+ */
+#include "rANS_static32x16pr.h"
+
+static int rans_cpu = 0xFFFF; // all
+
+#if defined(__x86_64__) && \
+    defined(HAVE_DECL___CPUID_COUNT)   && HAVE_DECL___CPUID_COUNT && \
+    defined(HAVE_DECL___GET_CPUID_MAX) && HAVE_DECL___GET_CPUID_MAX
+#include <cpuid.h>
+
+#if defined(__clang__) && defined(__has_attribute)
+#  if __has_attribute(unused)
+#    define UNUSED __attribute__((unused))
+#  else
+#    define UNUSED
+#  endif
+#elif defined(__GNUC__) && __GNUC__ >= 3
+#  define UNUSED __attribute__((unused))
+#else
+#  define UNUSED
+#endif
+
+// CPU detection is performed once.  NB this has an assumption that we're
+// not migrating between processes with different instruction stes, but
+// to date the only systems I know of that support this don't have different
+// capabilities (that we use) per core.
+#ifndef NO_THREADS
+static pthread_once_t rans_cpu_once = PTHREAD_ONCE_INIT;
+#endif
+
+static int have_ssse3   UNUSED = 0;
+static int have_sse4_1  UNUSED = 0;
+static int have_popcnt  UNUSED = 0;
+static int have_avx2    UNUSED = 0;
+static int have_avx512f UNUSED = 0;
+static int is_amd       UNUSED = 0;
+
+#define HAVE_HTSCODECS_TLS_CPU_INIT
+static void htscodecs_tls_cpu_init(void) {
+    unsigned int eax = 0, ebx = 0, ecx = 0, edx = 0;
+    // These may be unused, depending on HAVE_* config.h macros
+
+    int level = __get_cpuid_max(0, NULL);
+    __cpuid_count(0, 0, eax, ebx, ecx, edx);
+    is_amd = (ecx == 0x444d4163);
+    if (level >= 1) {
+        __cpuid_count(1, 0, eax, ebx, ecx, edx);
+#if defined(bit_SSSE3)
+        have_ssse3 = ecx & bit_SSSE3;
+#endif
+#if defined(bit_POPCNT)
+        have_popcnt = ecx & bit_POPCNT;
+#endif
+#if defined(bit_SSE4_1)
+        have_sse4_1 = ecx & bit_SSE4_1;
+#endif
+    }
+    if (level >= 7) {
+        __cpuid_count(7, 0, eax, ebx, ecx, edx);
+#if defined(bit_AVX2)
+        have_avx2 = ebx & bit_AVX2;
+#endif
+#if defined(bit_AVX512F)
+        have_avx512f = ebx & bit_AVX512F;
+#endif
+    }
+
+    if (!have_popcnt) have_avx512f = have_avx2 = have_sse4_1 = 0;
+    if (!have_ssse3)  have_sse4_1 = 0;
+}
+
+static inline
+unsigned char *(*rans_enc_func(int do_simd, int order))
+    (unsigned char *in,
+     unsigned int in_size,
+     unsigned char *out,
+     unsigned int *out_size) {
+
+    int have_e_sse4_1  = have_sse4_1;
+    int have_e_avx2    = have_avx2;
+    int have_e_avx512f = have_avx512f;
+
+    if (!(rans_cpu & RANS_CPU_ENC_AVX512)) have_e_avx512f = 0;
+    if (!(rans_cpu & RANS_CPU_ENC_AVX2))   have_e_avx2    = 0;
+    if (!(rans_cpu & RANS_CPU_ENC_SSE4))   have_e_sse4_1  = 0;
+
+    if (!do_simd) { // SIMD disabled
+        return order & 1
+            ? rans_compress_O1_4x16
+            : rans_compress_O0_4x16;
+    }
+
+#ifdef NO_THREADS
+    htscodecs_tls_cpu_init();
+#else
+    int err = pthread_once(&rans_cpu_once, htscodecs_tls_cpu_init);
+    if (err != 0) {
+        fprintf(stderr, "Initialising TLS data failed: pthread_once: %s\n",
+                strerror(err));
+        fprintf(stderr, "Using scalar code only\n");
+    }
+#endif
+
+    if (order & 1) {
+        // With simulated gathers, the AVX512 is now slower than AVX2, so
+        // we avoid using it unless asking for the real avx512 gather.
+        // Note for testing we do -c 0x0404 to enable AVX512 and disable AVX2.
+        // We then need to call the avx512 func regardless.
+        int use_gather;
+#ifdef USE_GATHER
+        use_gather = 1;
+#else
+        use_gather = !have_e_avx2;
+#endif
+
+#if defined(HAVE_AVX512)
+        if (have_e_avx512f && (!is_amd || !have_e_avx2) && use_gather)
+            return rans_compress_O1_32x16_avx512;
+#endif
+#if defined(HAVE_AVX2)
+        if (have_e_avx2)
+            return rans_compress_O1_32x16_avx2;
+#endif
+#if defined(HAVE_SSE4_1) && defined(HAVE_SSSE3) && defined(HAVE_POPCNT)
+        if (have_e_sse4_1)
+            return rans_compress_O1_32x16;
+#endif
+        return rans_compress_O1_32x16;
+    } else {
+#if defined(HAVE_AVX512)
+        if (have_e_avx512f && (!is_amd || !have_e_avx2))
+            return rans_compress_O0_32x16_avx512;
+#endif
+#if defined(HAVE_AVX2)
+        if (have_e_avx2)
+            return rans_compress_O0_32x16_avx2;
+#endif
+#if defined(HAVE_SSE4_1) && defined(HAVE_SSSE3) && defined(HAVE_POPCNT)
+        if (have_e_sse4_1)
+            return rans_compress_O0_32x16;
+#endif
+        return rans_compress_O0_32x16;
+    }
+}
+
+static inline
+unsigned char *(*rans_dec_func(int do_simd, int order))
+    (unsigned char *in,
+     unsigned int in_size,
+     unsigned char *out,
+     unsigned int out_size) {
+
+    int have_d_sse4_1  = have_sse4_1;
+    int have_d_avx2    = have_avx2;
+    int have_d_avx512f = have_avx512f;
+
+    if (!(rans_cpu & RANS_CPU_DEC_AVX512)) have_d_avx512f = 0;
+    if (!(rans_cpu & RANS_CPU_DEC_AVX2))   have_d_avx2    = 0;
+    if (!(rans_cpu & RANS_CPU_DEC_SSE4))   have_d_sse4_1  = 0;
+
+    if (!do_simd) { // SIMD disabled
+        return order & 1
+            ? rans_uncompress_O1_4x16
+            : rans_uncompress_O0_4x16;
+    }
+
+#ifdef NO_THREADS
+    htscodecs_tls_cpu_init();
+#else
+    int err = pthread_once(&rans_cpu_once, htscodecs_tls_cpu_init);
+    if (err != 0) {
+        fprintf(stderr, "Initialising TLS data failed: pthread_once: %s\n",
+                strerror(err));
+        fprintf(stderr, "Using scalar code only\n");
+    }
+#endif
+
+    if (order & 1) {
+#if defined(HAVE_AVX512)
+        if (have_d_avx512f)
+            return rans_uncompress_O1_32x16_avx512;
+#endif
+#if defined(HAVE_AVX2)
+        if (have_d_avx2)
+            return rans_uncompress_O1_32x16_avx2;
+#endif
+#if defined(HAVE_SSE4_1) && defined(HAVE_SSSE3) && defined(HAVE_POPCNT)
+        if (have_d_sse4_1)
+            return rans_uncompress_O1_32x16_sse4;
+#endif
+        return rans_uncompress_O1_32x16;
+    } else {
+#if defined(HAVE_AVX512)
+        if (have_d_avx512f)
+            return rans_uncompress_O0_32x16_avx512;
+#endif
+#if defined(HAVE_AVX2)
+        if (have_d_avx2)
+            return rans_uncompress_O0_32x16_avx2;
+#endif
+#if defined(HAVE_SSE4_1) && defined(HAVE_SSSE3) && defined(HAVE_POPCNT)
+        if (have_d_sse4_1)
+            return rans_uncompress_O0_32x16_sse4;
+#endif
+        return rans_uncompress_O0_32x16;
+    }
+}
+
+#elif defined(__ARM_NEON) && defined(__aarch64__)
+
+#if defined(__linux__) || defined(__FreeBSD__)
+#include <sys/auxv.h>
+#elif defined(_WIN32)
+#include <processthreadsapi.h>
+#endif
+
+static inline int have_neon(void) {
+#if defined(__linux__) && defined(__arm__)
+    return (getauxval(AT_HWCAP) & HWCAP_NEON) != 0;
+#elif defined(__linux__) && defined(__aarch64__) && defined(HWCAP_ASIMD)
+    return (getauxval(AT_HWCAP) & HWCAP_ASIMD) != 0;
+#elif defined(__APPLE__)
+    return 1;
+#elif defined(__FreeBSD__) && defined(__arm__)
+    unsigned long cap;
+    if (elf_aux_info(AT_HWCAP, &cap, sizeof cap) != 0) return 0;
+    return (cap & HWCAP_NEON) != 0;
+#elif defined(__FreeBSD__) && defined(__aarch64__) && defined(HWCAP_ASIMD)
+    unsigned long cap;
+    if (elf_aux_info(AT_HWCAP, &cap, sizeof cap) != 0) return 0;
+    return (cap & HWCAP_ASIMD) != 0;
+#elif defined(_WIN32)
+    return IsProcessorFeaturePresent(PF_ARM_V8_INSTRUCTIONS_AVAILABLE) != 0;
+#else
+    return 0;
+#endif
+}
+
+static inline
+unsigned char *(*rans_enc_func(int do_simd, int order))
+    (unsigned char *in,
+     unsigned int in_size,
+     unsigned char *out,
+     unsigned int *out_size) {
+
+    if (do_simd) {
+        if ((rans_cpu & RANS_CPU_ENC_NEON) && have_neon())
+            return order & 1
+                ? rans_compress_O1_32x16_neon
+                : rans_compress_O0_32x16_neon;
+        else
+            return order & 1
+                ? rans_compress_O1_32x16
+                : rans_compress_O0_32x16;
+    } else {
+        return order & 1
+            ? rans_compress_O1_4x16
+            : rans_compress_O0_4x16;
+    }
+}
+
+static inline
+unsigned char *(*rans_dec_func(int do_simd, int order))
+    (unsigned char *in,
+     unsigned int in_size,
+     unsigned char *out,
+     unsigned int out_size) {
+
+    if (do_simd) {
+        if ((rans_cpu & RANS_CPU_DEC_NEON) && have_neon())
+            return order & 1
+                ? rans_uncompress_O1_32x16_neon
+                : rans_uncompress_O0_32x16_neon;
+        else
+            return order & 1
+                ? rans_uncompress_O1_32x16
+                : rans_uncompress_O0_32x16;
+    } else {
+        return order & 1
+            ? rans_uncompress_O1_4x16
+            : rans_uncompress_O0_4x16;
+    }
+}
+
+#else // !(defined(__GNUC__) && defined(__x86_64__)) && !defined(__ARM_NEON)
+
+static inline
+unsigned char *(*rans_enc_func(int do_simd, int order))
+    (unsigned char *in,
+     unsigned int in_size,
+     unsigned char *out,
+     unsigned int *out_size) {
+
+    if (do_simd) {
+        return order & 1
+            ? rans_compress_O1_32x16
+            : rans_compress_O0_32x16;
+    } else {
+        return order & 1
+            ? rans_compress_O1_4x16
+            : rans_compress_O0_4x16;
+    }
+}
+
+static inline
+unsigned char *(*rans_dec_func(int do_simd, int order))
+    (unsigned char *in,
+     unsigned int in_size,
+     unsigned char *out,
+     unsigned int out_size) {
+
+    if (do_simd) {
+        return order & 1
+            ? rans_uncompress_O1_32x16
+            : rans_uncompress_O0_32x16;
+    } else {
+        return order & 1
+            ? rans_uncompress_O1_4x16
+            : rans_uncompress_O0_4x16;
+    }
+}
+
+#endif
+
+// Test interface for restricting the auto-detection methods so we
+// can forcibly compare different implementations on the same machine.
+// See RANS_CPU_ defines in rANS_static4x16.h
+void rans_set_cpu(int opts) {
+    rans_cpu = opts;
+#ifdef HAVE_HTSCODECS_TLS_CPU_INIT
+    htscodecs_tls_cpu_init();
+#endif
+}
+
+/*-----------------------------------------------------------------------------
+ * Simple interface to the order-0 vs order-1 encoders and decoders.
+ *
+ * Smallest is method, <in_size> <input>, so worst case 2 bytes longer.
+ */
+unsigned char *rans_compress_to_4x16(unsigned char *in, unsigned int in_size,
+                                     unsigned char *out,unsigned int *out_size,
+                                     int order) {
+    if (in_size > INT_MAX) {
+        *out_size = 0;
+        return NULL;
+    }
+
+    unsigned int c_meta_len;
+    uint8_t *meta = NULL, *rle = NULL, *packed = NULL;
+    uint8_t *out_free = NULL;
+
+    if (!out) {
+        *out_size = rans_compress_bound_4x16(in_size, order);
+        if (*out_size == 0)
+            return NULL;
+        if (!(out_free = out = malloc(*out_size)))
+            return NULL;
+    }
+
+    unsigned char *out_end = out + *out_size;
+
+    // Permit 32-way unrolling for large blocks, paving the way for
+    // AVX2 and AVX512 SIMD variants.
+    if ((order & RANS_ORDER_SIMD_AUTO) && in_size >= 50000
+        && !(order & RANS_ORDER_STRIPE))
+        order |= X_32;
+
+    if (in_size <= 20)
+        order &= ~RANS_ORDER_STRIPE;
+#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    if (in_size <= 1000)
+        order &= ~RANS_ORDER_X32;
+#endif
+    if (order & RANS_ORDER_STRIPE) {
+        int N = (order>>8) & 0xff;
+        if (N == 0) N = 4; // default for compatibility with old tests
+
+        unsigned char *transposed = malloc(in_size);
+        unsigned int part_len[256];
+        unsigned int idx[256];
+        if (!transposed) {
+            free(out_free);
+            return NULL;
+        }
+        int i, j, x;
+
+        for (i = 0; i < N; i++) {
+            part_len[i] = in_size / N + ((in_size % N) > i);
+            idx[i] = i ? idx[i-1] + part_len[i-1] : 0; // cumulative index
+        }
+
+#define KN 8
+        i = x = 0;
+        if (in_size >= N*KN) {
+            for (; i < in_size-N*KN;) {
+                int k;
+                unsigned char *ink = in+i;
+                for (j = 0; j < N; j++)
+                    for (k = 0; k < KN; k++)
+                        transposed[idx[j]+x+k] = ink[j+N*k];
+                x += KN; i+=N*KN;
+            }
+        }
+#undef KN
+        for (; i < in_size-N; i += N, x++) {
+            for (j = 0; j < N; j++)
+                transposed[idx[j]+x] = in[i+j];
+        }
+
+        for (; i < in_size; i += N, x++) {
+            for (j = 0; i+j < in_size; j++)
+                transposed[idx[j]+x] = in[i+j];
+        }
+
+        unsigned int olen2;
+        unsigned char *out2, *out2_start;
+        c_meta_len = 1;
+        *out = order & ~RANS_ORDER_NOSZ;
+        c_meta_len += var_put_u32(out+c_meta_len, out_end, in_size);
+        out[c_meta_len++] = N;
+        
+        unsigned char *out_best = NULL;
+        unsigned int out_best_len = 0;
+
+        out2_start = out2 = out+7+5*N; // shares a buffer with c_meta
+        for (i = 0; i < N; i++) {
+            // Brute force try all methods.
+            int j, m[] = {1,64,128,0}, best_j = 0, best_sz = in_size+10;
+            for (j = 0; j < sizeof(m)/sizeof(*m); j++) {
+                if ((order & m[j]) != m[j])
+                    continue;
+
+                // order-1 *only*; bit check above cannot elide order-0
+                if ((order & RANS_ORDER_STRIPE_NO0) && (m[j]&1) == 0)
+                    continue;
+                olen2 = *out_size - (out2 - out);
+                rans_compress_to_4x16(transposed+idx[i], part_len[i],
+                                      out2, &olen2,
+                                      m[j] | RANS_ORDER_NOSZ
+                                      | (order&RANS_ORDER_X32));
+                if (best_sz > olen2) {
+                    best_sz = olen2;
+                    best_j = j;
+                    if (j < sizeof(m)/sizeof(*m) && olen2 > out_best_len) {
+                        unsigned char *tmp = realloc(out_best, olen2);
+                        if (!tmp) {
+                            free(out_free);
+                            return NULL;
+                        }
+                        out_best = tmp;
+                        out_best_len = olen2;
+                    }
+
+                    // Cache a copy of the best so far
+                    memcpy(out_best, out2, olen2);
+                }
+            }
+            if (best_j < sizeof(m)/sizeof(*m)) {
+                // Copy the best compression to output buffer if not current
+                memcpy(out2, out_best, best_sz);
+                olen2 = best_sz;
+            }
+
+            out2 += olen2;
+            c_meta_len += var_put_u32(out+c_meta_len, out_end, olen2);
+        }
+        if (out_best)
+            free(out_best);
+
+        memmove(out+c_meta_len, out2_start, out2-out2_start);
+        free(transposed);
+        *out_size = c_meta_len + out2-out2_start;
+        return out;
+    }
+
+    if (order & RANS_ORDER_CAT) {
+        out[0] = RANS_ORDER_CAT;
+        c_meta_len = 1;
+        c_meta_len += var_put_u32(&out[1], out_end, in_size);
+        if (in_size)
+            memcpy(out+c_meta_len, in, in_size);
+        *out_size = c_meta_len + in_size;
+        return out;
+    }
+
+    int do_pack = order & RANS_ORDER_PACK;
+    int do_rle  = order & RANS_ORDER_RLE;
+    int no_size = order & RANS_ORDER_NOSZ;
+    int do_simd = order & RANS_ORDER_X32;
+
+    out[0] = order;
+    c_meta_len = 1;
+
+    if (!no_size)
+        c_meta_len += var_put_u32(&out[1], out_end, in_size);
+
+    order &= 3;
+
+    // Format is compressed meta-data, compressed data.
+    // Meta-data can be empty, pack, rle lengths, or pack + rle lengths.
+    // Data is either the original data, bit-packed packed, rle literals or
+    // packed + rle literals.
+
+    if (do_pack && in_size) {
+        // PACK 2, 4 or 8 symbols into one byte.
+        int pmeta_len;
+        uint64_t packed_len;
+        packed = hts_pack(in, in_size, out+c_meta_len, &pmeta_len, &packed_len);
+        if (!packed) {
+            out[0] &= ~RANS_ORDER_PACK;
+            do_pack = 0;
+            free(packed);
+            packed = NULL;
+        } else {
+            in = packed;
+            in_size = packed_len;
+            c_meta_len += pmeta_len;
+
+            // Could derive this rather than storing verbatim.
+            // Orig size * 8/nbits (+1 if not multiple of 8/n)
+            int sz = var_put_u32(out+c_meta_len, out_end, in_size);
+            c_meta_len += sz;
+            *out_size -= sz;
+        }
+    } else if (do_pack) {
+        out[0] &= ~RANS_ORDER_PACK;
+    }
+
+    if (do_rle && in_size) {
+        // RLE 'in' -> rle_length + rle_literals arrays
+        unsigned int rmeta_len, c_rmeta_len;
+        uint64_t rle_len;
+        c_rmeta_len = in_size+257;
+        if (!(meta = malloc(c_rmeta_len))) {
+            free(out_free);
+            return NULL;
+        }
+
+        uint8_t rle_syms[256];
+        int rle_nsyms = 0;
+        uint64_t rmeta_len64;
+        rle = hts_rle_encode(in, in_size, meta, &rmeta_len64,
+                             rle_syms, &rle_nsyms, NULL, &rle_len);
+        memmove(meta+1+rle_nsyms, meta, rmeta_len64);
+        meta[0] = rle_nsyms;
+        memcpy(meta+1, rle_syms, rle_nsyms);
+        rmeta_len = rmeta_len64 + rle_nsyms+1;
+
+        if (!rle || rle_len + rmeta_len >= .99*in_size) {
+            // Not worth the speed hit.
+            out[0] &= ~RANS_ORDER_RLE;
+            do_rle = 0;
+            free(rle);
+            rle = NULL;
+        } else {
+            // Compress lengths with O0 and literals with O0/O1 ("order" param)
+            int sz = var_put_u32(out+c_meta_len, out_end, rmeta_len*2), sz2;
+            sz += var_put_u32(out+c_meta_len+sz, out_end, rle_len);
+            c_rmeta_len = *out_size - (c_meta_len+sz+5);
+            rans_enc_func(do_simd, 0)(meta, rmeta_len, out+c_meta_len+sz+5, &c_rmeta_len);
+            if (c_rmeta_len < rmeta_len) {
+                sz2 = var_put_u32(out+c_meta_len+sz, out_end, c_rmeta_len);
+                memmove(out+c_meta_len+sz+sz2, out+c_meta_len+sz+5, c_rmeta_len);
+            } else {
+                // Uncompressed RLE meta-data as too small
+                sz = var_put_u32(out+c_meta_len, out_end, rmeta_len*2+1);
+                sz2 = var_put_u32(out+c_meta_len+sz, out_end, rle_len);
+                memcpy(out+c_meta_len+sz+sz2, meta, rmeta_len);
+                c_rmeta_len = rmeta_len;
+            }
+
+            c_meta_len += sz + sz2 + c_rmeta_len;
+
+            in = rle;
+            in_size = rle_len;
+        }
+
+        free(meta);
+    } else if (do_rle) {
+        out[0] &= ~RANS_ORDER_RLE;
+    }
+
+    *out_size -= c_meta_len;
+    if (order && in_size < 8) {
+        out[0] &= ~1;
+        order  &= ~1;
+    }
+
+    rans_enc_func(do_simd, order)(in, in_size, out+c_meta_len, out_size);
+
+    if (*out_size >= in_size) {
+        out[0] &= ~3;
+        out[0] |= RANS_ORDER_CAT | no_size;
+        if (in_size)
+            memcpy(out+c_meta_len, in, in_size);
+        *out_size = in_size;
+    }
+
+    free(rle);
+    free(packed);
+
+    *out_size += c_meta_len;
+
+    return out;
+}
+
+unsigned char *rans_compress_4x16(unsigned char *in, unsigned int in_size,
+                                  unsigned int *out_size, int order) {
+    return rans_compress_to_4x16(in, in_size, NULL, out_size, order);
+}
+
+unsigned char *rans_uncompress_to_4x16(unsigned char *in,  unsigned int in_size,
+                                       unsigned char *out, unsigned int *out_size) {
+    unsigned char *in_end = in + in_size;
+    unsigned char *out_free = NULL, *tmp_free = NULL, *meta_free = NULL;
+
+    if (in_size == 0)
+        return NULL;
+
+    if (*in & RANS_ORDER_STRIPE) {
+        unsigned int ulen, olen, c_meta_len = 1;
+        int i;
+        uint64_t clen_tot = 0;
+
+        // Decode lengths
+        c_meta_len += var_get_u32(in+c_meta_len, in_end, &ulen);
+        if (c_meta_len >= in_size)
+            return NULL;
+        unsigned int N = in[c_meta_len++];
+        if (N < 1)  // Must be at least one stripe
+            return NULL;
+        unsigned int clenN[256], ulenN[256], idxN[256];
+        if (!out) {
+            if (ulen >= INT_MAX)
+                return NULL;
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+            if (ulen > 100000)
+                return NULL;
+#endif
+            if (!(out_free = out = malloc(ulen))) {
+                return NULL;
+            }
+            *out_size = ulen;
+        }
+        if (ulen != *out_size) {
+            free(out_free);
+            return NULL;
+        }
+
+        for (i = 0; i < N; i++) {
+            ulenN[i] = ulen / N + ((ulen % N) > i);
+            idxN[i] = i ? idxN[i-1] + ulenN[i-1] : 0;
+            c_meta_len += var_get_u32(in+c_meta_len, in_end, &clenN[i]);
+            clen_tot += clenN[i];
+            if (c_meta_len > in_size || clenN[i] > in_size || clenN[i] < 1) {
+                free(out_free);
+                return NULL;
+            }
+        }
+
+        // We can call this with a larger buffer, but once we've determined
+        // how much we really use we limit it so the recursion becomes easier
+        // to limit.
+        if (c_meta_len + clen_tot > in_size) {
+            free(out_free);
+            return NULL;
+        }
+        in_size = c_meta_len + clen_tot;
+
+        //fprintf(stderr, "    stripe meta %d\n", c_meta_len); //c-size
+
+        // Uncompress the N streams
+        unsigned char *outN = malloc(ulen);
+        if (!outN) {
+            free(out_free);
+            return NULL;
+        }
+        for (i = 0; i < N; i++) {
+            olen = ulenN[i];
+            if (in_size < c_meta_len) {
+                free(out_free);
+                free(outN);
+                return NULL;
+            }
+            if (!rans_uncompress_to_4x16(in+c_meta_len, in_size-c_meta_len, outN + idxN[i], &olen)
+                || olen != ulenN[i]) {
+                free(out_free);
+                free(outN);
+                return NULL;
+            }
+            c_meta_len += clenN[i];
+        }
+
+        unstripe(out, outN, ulen, N, idxN);
+
+        free(outN);
+        *out_size = ulen;
+        return out;
+    }
+
+    int order = *in++;  in_size--;
+    int do_pack = order & RANS_ORDER_PACK;
+    int do_rle  = order & RANS_ORDER_RLE;
+    int do_cat  = order & RANS_ORDER_CAT;
+    int no_size = order & RANS_ORDER_NOSZ;
+    int do_simd = order & RANS_ORDER_X32;
+    order &= 1;
+
+    int sz = 0;
+    unsigned int osz;
+    if (!no_size) {
+        sz = var_get_u32(in, in_end, &osz);
+    } else
+        sz = 0, osz = *out_size;
+    in += sz;
+    in_size -= sz;
+
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    if (osz > 100000)
+        return NULL;
+#endif
+
+    if (no_size && !out)
+        goto err; // Need one or the other
+
+    if (!out) {
+        *out_size = osz;
+        if (!(out = out_free = malloc(*out_size)))
+            return NULL;
+    } else {
+        if (*out_size < osz)
+        goto err;
+        *out_size = osz;
+    }
+
+//    if (do_pack || do_rle) {
+//      in += sz; // size field not needed when pure rANS
+//      in_size -= sz;
+//    }
+
+    uint32_t c_meta_size = 0;
+    unsigned int tmp1_size = *out_size;
+    unsigned int tmp2_size = *out_size;
+    unsigned int tmp3_size = *out_size;
+    unsigned char *tmp1 = NULL, *tmp2 = NULL, *tmp3 = NULL, *tmp = NULL;
+
+    // Need In, Out and Tmp buffers with temporary buffer of the same size
+    // as output.  All use rANS, but with optional transforms (none, RLE,
+    // Pack, or both).
+    //
+    //                    rans   unrle  unpack
+    // If none:     in -> out
+    // If RLE:      in -> tmp -> out
+    // If Pack:     in -> tmp        -> out
+    // If RLE+Pack: in -> out -> tmp -> out
+    //                    tmp1   tmp2   tmp3
+    //
+    // So rans is in   -> tmp1
+    // RLE     is tmp1 -> tmp2
+    // Unpack  is tmp2 -> tmp3
+
+    // Format is meta data (Pack and RLE in that order if present),
+    // followed by rANS compressed data.
+
+    if (do_pack || do_rle) {
+        if (!(tmp = tmp_free = malloc(*out_size)))
+            goto err;
+        if (do_pack && do_rle) {
+            tmp1 = out;
+            tmp2 = tmp;
+            tmp3 = out;
+        } else if (do_pack) {
+            tmp1 = tmp;
+            tmp2 = tmp1;
+            tmp3 = out;
+        } else if (do_rle) {
+            tmp1 = tmp;
+            tmp2 = out;
+            tmp3 = out;
+        }
+    } else {
+        // neither
+        tmp  = NULL;
+        tmp1 = out;
+        tmp2 = out;
+        tmp3 = out;
+    }
+
+    // Decode the bit-packing map.
+    uint8_t map[16] = {0};
+    int npacked_sym = 0;
+    uint64_t unpacked_sz = 0; // FIXME: rename to packed_per_byte
+    if (do_pack) {
+        c_meta_size = hts_unpack_meta(in, in_size, *out_size, map, &npacked_sym);
+        if (c_meta_size == 0)
+            goto err;
+
+        unpacked_sz = osz;
+        in      += c_meta_size;
+        in_size -= c_meta_size;
+
+        // New unpacked size.  We could derive this bit from *out_size
+        // and npacked_sym.
+        unsigned int osz;
+        sz = var_get_u32(in, in_end, &osz);
+        in += sz;
+        in_size -= sz;
+        if (osz > tmp1_size)
+            goto err;
+        tmp1_size = osz;
+    }
+
+    uint8_t *meta = NULL;
+    uint32_t u_meta_size = 0;
+    if (do_rle) {
+        // Uncompress meta data
+        uint32_t c_meta_size, rle_len, sz;
+        sz  = var_get_u32(in,    in_end, &u_meta_size);
+        sz += var_get_u32(in+sz, in_end, &rle_len);
+        if (rle_len > tmp1_size) // should never grow
+            goto err;
+        if (u_meta_size & 1) {
+            meta = in + sz;
+            u_meta_size = u_meta_size/2 > (in_end-meta) ? (in_end-meta) : u_meta_size/2;
+            c_meta_size = u_meta_size;
+        } else {
+            sz += var_get_u32(in+sz, in_end, &c_meta_size);
+            u_meta_size /= 2;
+
+            meta_free = meta = rans_dec_func(do_simd, 0)(in+sz, in_size-sz, NULL, u_meta_size);
+            if (!meta)
+                goto err;
+        }
+        if (c_meta_size+sz > in_size)
+            goto err;
+        in      += c_meta_size+sz;
+        in_size -= c_meta_size+sz;
+        tmp1_size = rle_len;
+    }
+    //fprintf(stderr, "    meta_size %d bytes\n", (int)(in - orig_in)); //c-size
+
+    // uncompress RLE data.  in -> tmp1
+    if (in_size) {
+        if (do_cat) {
+            //fprintf(stderr, "    CAT %d\n", tmp1_size); //c-size
+            if (tmp1_size > in_size)
+                goto err;
+            if (tmp1_size > *out_size)
+                goto err;
+            memcpy(tmp1, in, tmp1_size);
+        } else {
+            tmp1 = rans_dec_func(do_simd, order)(in, in_size, tmp1, tmp1_size);
+            if (!tmp1)
+                goto err;
+        }
+    } else {
+        tmp1_size = 0;
+    }
+    tmp2_size = tmp3_size = tmp1_size;
+
+    if (do_rle) {
+        // Unpack RLE.  tmp1 -> tmp2.
+        if (u_meta_size == 0)
+            goto err;
+        uint64_t unrle_size = *out_size;
+        int rle_nsyms = *meta ? *meta : 256;
+        if (u_meta_size < 1+rle_nsyms)
+            goto err;
+        if (!hts_rle_decode(tmp1, tmp1_size,
+                            meta+1+rle_nsyms, u_meta_size-(1+rle_nsyms),
+                            meta+1, rle_nsyms, tmp2, &unrle_size))
+            goto err;
+        tmp3_size = tmp2_size = unrle_size;
+        free(meta_free);
+        meta_free = NULL;
+    }
+    if (do_pack) {
+        // Unpack bits via pack-map.  tmp2 -> tmp3
+        if (npacked_sym == 1)
+            unpacked_sz = tmp2_size;
+        //uint8_t *porig = unpack(tmp2, tmp2_size, unpacked_sz, npacked_sym, map);
+        //memcpy(tmp3, porig, unpacked_sz);
+        if (!hts_unpack(tmp2, tmp2_size, tmp3, unpacked_sz, npacked_sym, map))
+            goto err;
+        tmp3_size = unpacked_sz;
+    }
+
+    if (tmp)
+        free(tmp);
+
+    *out_size = tmp3_size;
+    return tmp3;
+
+ err:
+    free(meta_free);
+    free(out_free);
+    free(tmp_free);
+    return NULL;
+}
+
+unsigned char *rans_uncompress_4x16(unsigned char *in, unsigned int in_size,
+                                    unsigned int *out_size) {
+    return rans_uncompress_to_4x16(in, in_size, NULL, out_size);
+}
diff --git a/ext/htslib/htscodecs/htscodecs/rANS_word.h b/ext/htslib/htscodecs/htscodecs/rANS_word.h
new file mode 100644
index 0000000..db60b04
--- /dev/null
+++ b/ext/htslib/htscodecs/htscodecs/rANS_word.h
@@ -0,0 +1,478 @@
+/* rans_byte.h originally from https://github.com/rygorous/ryg_rans
+ *
+ * This is a public-domain implementation of several rANS variants. rANS is an
+ * entropy coder from the ANS family, as described in Jarek Duda's paper
+ * "Asymmetric numeral systems" (http://arxiv.org/abs/1311.2540).
+ */
+
+/*-------------------------------------------------------------------------- */
+/* rans_byte.h from https://github.com/rygorous/ryg_rans */
+
+// Simple byte-aligned rANS encoder/decoder - public domain - Fabian 'ryg' Giesen 2014
+//
+// Not intended to be "industrial strength"; just meant to illustrate the general
+// idea.
+
+#ifndef RANS_WORD_HEADER
+#define RANS_WORD_HEADER
+
+#include <stdio.h>
+#include <stdint.h>
+#include <string.h>
+#include <assert.h>
+#include "htscodecs_endian.h"
+
+#ifdef assert
+#define RansAssert assert
+#else
+#define RansAssert(x)
+#endif
+
+// READ ME FIRST:
+//
+// This is designed like a typical arithmetic coder API, but there's three
+// twists you absolutely should be aware of before you start hacking:
+//
+// 1. You need to encode data in *reverse* - last symbol first. rANS works
+//    like a stack: last in, first out.
+// 2. Likewise, the encoder outputs bytes *in reverse* - that is, you give
+//    it a pointer to the *end* of your buffer (exclusive), and it will
+//    slowly move towards the beginning as more bytes are emitted.
+// 3. Unlike basically any other entropy coder implementation you might
+//    have used, you can interleave data from multiple independent rANS
+//    encoders into the same bytestream without any extra signaling;
+//    you can also just write some bytes by yourself in the middle if
+//    you want to. This is in addition to the usual arithmetic encoder
+//    property of being able to switch models on the fly. Writing raw
+//    bytes can be useful when you have some data that you know is
+//    incompressible, and is cheaper than going through the rANS encode
+//    function. Using multiple rANS coders on the same byte stream wastes
+//    a few bytes compared to using just one, but execution of two
+//    independent encoders can happen in parallel on superscalar and
+//    Out-of-Order CPUs, so this can be *much* faster in tight decoding
+//    loops.
+//
+//    This is why all the rANS functions take the write pointer as an
+//    argument instead of just storing it in some context struct.
+
+// --------------------------------------------------------------------------
+
+// L ('l' in the paper) is the lower bound of our normalization interval.
+// Between this and our byte-aligned emission, we use 31 (not 32!) bits.
+// This is done intentionally because exact reciprocals for 31-bit uints
+// fit in 32-bit uints: this permits some optimizations during encoding.
+#define RANS_BYTE_L (1u << 15)  // lower bound of our normalization interval
+
+// State for a rANS encoder. Yep, that's all there is to it.
+typedef uint32_t RansState;
+
+// Initialize a rANS encoder.
+static inline void RansEncInit(RansState* r)
+{
+    *r = RANS_BYTE_L;
+}
+
+// Renormalize the encoder. Internal function.
+static inline RansState RansEncRenorm(RansState x, uint8_t** pptr, uint32_t freq, uint32_t scale_bits)
+{
+    uint32_t x_max = ((RANS_BYTE_L >> scale_bits) << 16) * freq-1; // this turns into a shift.
+    if (x > x_max) {
+        uint16_t* ptr = (uint16_t *)*pptr;
+        *--ptr = (uint16_t) (x & 0xffff);
+        x >>= 16;
+        *pptr = (uint8_t *)ptr;
+    }
+    return x;
+}
+
+// Encodes a single symbol with range start "start" and frequency "freq".
+// All frequencies are assumed to sum to "1 << scale_bits", and the
+// resulting bytes get written to ptr (which is updated).
+//
+// NOTE: With rANS, you need to encode symbols in *reverse order*, i.e. from
+// beginning to end! Likewise, the output bytestream is written *backwards*:
+// ptr starts pointing at the end of the output buffer and keeps decrementing.
+static inline void RansEncPut(RansState* r, uint8_t** pptr, uint32_t start, uint32_t freq, uint32_t scale_bits)
+{
+    // renormalize
+    RansState x = RansEncRenorm(*r, pptr, freq, scale_bits);
+
+    // x = C(s,x)
+    *r = ((x / freq) << scale_bits) + (x % freq) + start;
+}
+
+// Flushes the rANS encoder.
+static inline void RansEncFlush(RansState* r, uint8_t** pptr)
+{
+    uint32_t x = *r;
+    uint8_t* ptr = *pptr;
+
+    ptr -= 4;
+    ptr[0] = (uint8_t) (x >> 0);
+    ptr[1] = (uint8_t) (x >> 8);
+    ptr[2] = (uint8_t) (x >> 16);
+    ptr[3] = (uint8_t) (x >> 24);
+
+    *pptr = ptr;
+}
+
+// Initializes a rANS decoder.
+// Unlike the encoder, the decoder works forwards as you'd expect.
+static inline void RansDecInit(RansState* r, uint8_t** pptr)
+{
+    uint32_t x;
+    uint8_t* ptr = *pptr;
+
+    x  = ptr[0] << 0;
+    x |= ptr[1] << 8;
+    x |= ptr[2] << 16;
+    x |= ((uint32_t)ptr[3]) << 24;
+    ptr += 4;
+
+    *pptr = ptr;
+    *r = x;
+}
+
+// Returns the current cumulative frequency (map it to a symbol yourself!)
+static inline uint32_t RansDecGet(RansState* r, uint32_t scale_bits)
+{
+    return *r & ((1u << scale_bits) - 1);
+}
+
+// Advances in the bit stream by "popping" a single symbol with range start
+// "start" and frequency "freq". All frequencies are assumed to sum to "1 << scale_bits",
+// and the resulting bytes get written to ptr (which is updated).
+static inline void RansDecAdvance(RansState* r, uint8_t** pptr, uint32_t start, uint32_t freq, uint32_t scale_bits)
+{
+    uint32_t mask = (1u << scale_bits) - 1;
+
+    // s, x = D(x)
+    uint32_t x = *r;
+    x = freq * (x >> scale_bits) + (x & mask) - start;
+
+    // renormalize
+    if (x < RANS_BYTE_L) {
+        uint8_t* ptr = *pptr;
+        do x = (x << 8) | *ptr++; while (x < RANS_BYTE_L);
+        *pptr = ptr;
+    }
+
+    *r = x;
+}
+
+// --------------------------------------------------------------------------
+
+// That's all you need for a full encoder; below here are some utility
+// functions with extra convenience or optimizations.
+
+// Encoder symbol description
+// This (admittedly odd) selection of parameters was chosen to make
+// RansEncPutSymbol as cheap as possible.
+typedef struct {
+    uint32_t x_max;     // (Exclusive) upper bound of pre-normalization interval
+    uint32_t rcp_freq;  // Fixed-point reciprocal frequency
+    uint32_t bias;      // Bias
+
+    // NB: This pair are read as a 32-bit value by the SIMD o1 encoder.
+    uint16_t cmpl_freq; // Complement of frequency: (1 << scale_bits) - freq
+    uint16_t rcp_shift; // Reciprocal shift
+} RansEncSymbol;
+
+// As above, but with cmpl_freq and rcp_shift combined into
+// a single value.  This could be done with a cast, but it avoids
+// a type punning error.  We could use a union, but anonymous unions
+// are C11 only (still that's 10 year old!).  For now we just cheat
+// instead.
+typedef struct {
+    uint32_t x_max;     // (Exclusive) upper bound of pre-normalization interval
+    uint32_t rcp_freq;  // Fixed-point reciprocal frequency
+    uint32_t bias;      // Bias
+
+    uint32_t cmpl_freq; // cmpl_freq+rcp_shift
+} RansEncSymbol_simd;
+
+// Decoder symbols are straightforward.
+typedef struct {
+    uint16_t start;     // Start of range.
+    uint16_t freq;      // Symbol frequency.
+} RansDecSymbol;
+
+// Initializes an encoder symbol to start "start" and frequency "freq"
+static inline void RansEncSymbolInit(RansEncSymbol* s, uint32_t start, uint32_t freq, uint32_t scale_bits)
+{
+    RansAssert(scale_bits <= 16);
+    RansAssert(start <= (1u << scale_bits));
+    RansAssert(freq <= (1u << scale_bits) - start);
+
+    // Say M := 1 << scale_bits.
+    //
+    // The original encoder does:
+    //   x_new = (x/freq)*M + start + (x%freq)
+    //
+    // The fast encoder does (schematically):
+    //   q     = mul_hi(x, rcp_freq) >> rcp_shift   (division)
+    //   r     = x - q*freq                         (remainder)
+    //   x_new = q*M + bias + r                     (new x)
+    // plugging in r into x_new yields:
+    //   x_new = bias + x + q*(M - freq)
+    //        =: bias + x + q*cmpl_freq             (*)
+    //
+    // and we can just precompute cmpl_freq. Now we just need to
+    // set up our parameters such that the original encoder and
+    // the fast encoder agree.
+    
+    s->x_max = ((RANS_BYTE_L >> scale_bits) << 16) * freq -1;
+    s->cmpl_freq = (uint16_t) ((1 << scale_bits) - freq);
+    if (freq < 2) {
+        // freq=0 symbols are never valid to encode, so it doesn't matter what
+        // we set our values to.
+        //
+        // freq=1 is tricky, since the reciprocal of 1 is 1; unfortunately,
+        // our fixed-point reciprocal approximation can only multiply by values
+        // smaller than 1.
+        //
+        // So we use the "next best thing": rcp_freq=0xffffffff, rcp_shift=0.
+        // This gives:
+        //   q = mul_hi(x, rcp_freq) >> rcp_shift
+        //     = mul_hi(x, (1<<32) - 1)) >> 0
+        //     = floor(x - x/(2^32))
+        //     = x - 1 if 1 <= x < 2^32
+        // and we know that x>0 (x=0 is never in a valid normalization interval).
+        //
+        // So we now need to choose the other parameters such that
+        //   x_new = x*M + start
+        // plug it in:
+        //     x*M + start                   (desired result)
+        //   = bias + x + q*cmpl_freq        (*)
+        //   = bias + x + (x - 1)*(M - 1)    (plug in q=x-1, cmpl_freq)
+        //   = bias + 1 + (x - 1)*M
+        //   = x*M + (bias + 1 - M)
+        //
+        // so we have start = bias + 1 - M, or equivalently
+        //   bias = start + M - 1.
+        s->rcp_freq = ~0u;
+        s->rcp_shift = 0;
+        s->bias = start + (1 << scale_bits) - 1;
+    } else {
+        // Alverson, "Integer Division using reciprocals"
+        // shift=ceil(log2(freq))
+        uint32_t shift = 0;
+        while (freq > (1u << shift))
+            shift++;
+
+        s->rcp_freq = (uint32_t) (((1ull << (shift + 31)) + freq-1) / freq);
+        s->rcp_shift = shift - 1;
+
+        // With these values, 'q' is the correct quotient, so we
+        // have bias=start.
+        s->bias = start;
+    }
+
+    s->rcp_shift += 32; // Avoid the extra >>32 in RansEncPutSymbol
+}
+
+// Initialize a decoder symbol to start "start" and frequency "freq"
+static inline void RansDecSymbolInit(RansDecSymbol* s, uint32_t start, uint32_t freq)
+{
+    RansAssert(start <= (1 << 16));
+    RansAssert(freq <= (1 << 16) - start);
+    s->start = (uint16_t) start;
+    s->freq = (uint16_t) freq;
+}
+
+// Encodes a given symbol. This is faster than straight RansEnc since we can do
+// multiplications instead of a divide.
+//
+// See RansEncSymbolInit for a description of how this works.
+static inline void RansEncPutSymbol(RansState* r, uint8_t** pptr, RansEncSymbol const* sym)
+{
+    //RansAssert(sym->x_max != 0); // can't encode symbol with freq=0
+
+    // renormalize
+    uint32_t x = *r;
+    uint32_t x_max = sym->x_max;
+
+#ifdef HTSCODECS_LITTLE_ENDIAN
+    // Branchless renorm.
+    //
+    // This works best on high entropy data where branch prediction
+    // is poor.
+    //
+    // Note the bit-packing and RLE modes are more likely to be used on
+    // low entropy data, making this assertion generally true.  See
+    // RansEncPutSymbol_branched for a low-entropy optimised function.
+
+    // NB: "(x > x_max)*2" turns back into branched code with gcc.
+    int c = (x > x_max); c*=2;
+    memcpy(*pptr-2, &x, 2);
+    x >>= c*8;
+    *pptr = *pptr - c;
+#else
+    if (x > x_max) {
+        uint8_t* ptr = *pptr;
+        ptr -= 2;
+        ptr[0] = x & 0xff;
+        ptr[1] = (x >> 8) & 0xff;
+        x >>= 16;
+        *pptr = ptr;
+    }
+#endif
+
+    // x = C(s,x)
+    // NOTE: written this way so we get a 32-bit "multiply high" when
+    // available. If you're on a 64-bit platform with cheap multiplies
+    // (e.g. x64), just bake the +32 into rcp_shift.
+    //uint32_t q = (uint32_t) (((uint64_t)x * sym->rcp_freq) >> 32) >> sym->rcp_shift;
+
+    // Slow method, but robust
+//    *r = ((x / sym->freq) << sym->scale_bits) + (x % sym->freq) + sym->start;
+//    return;
+
+    // The extra >>32 has already been added to RansEncSymbolInit
+    uint32_t q = (uint32_t) (((uint64_t)x * sym->rcp_freq) >> sym->rcp_shift);
+    *r = x + sym->bias + q * sym->cmpl_freq;
+
+//    assert(((x / sym->freq) << sym->scale_bits) + (x % sym->freq) + sym->start == *r);
+}
+
+static inline void RansEncPutSymbol_branched(RansState* r, uint8_t** pptr, RansEncSymbol const* sym)
+{
+    //RansAssert(sym->x_max != 0); // can't encode symbol with freq=0
+
+    // renormalize
+    uint32_t x = *r;
+    uint32_t x_max = sym->x_max;
+
+#ifdef HTSCODECS_LITTLE_ENDIAN
+    // The old non-branchless method
+    if (x > x_max) {
+        (*pptr) -= 2;
+        memcpy(*pptr, &x, 2);
+        x >>= 16;
+    }
+#else
+    if (x > x_max) {
+        uint8_t* ptr = *pptr;
+        ptr -= 2;
+        ptr[0] = x & 0xff;
+        ptr[1] = (x >> 8) & 0xff;
+        x >>= 16;
+        *pptr = ptr;
+    }
+#endif
+
+    // x = C(s,x)
+    // NOTE: written this way so we get a 32-bit "multiply high" when
+    // available. If you're on a 64-bit platform with cheap multiplies
+    // (e.g. x64), just bake the +32 into rcp_shift.
+    //uint32_t q = (uint32_t) (((uint64_t)x * sym->rcp_freq) >> 32) >> sym->rcp_shift;
+
+    // Slow method, but robust
+//    *r = ((x / sym->freq) << sym->scale_bits) + (x % sym->freq) + sym->start;
+//    return;
+
+    // The extra >>32 has already been added to RansEncSymbolInit
+    uint32_t q = (uint32_t) (((uint64_t)x * sym->rcp_freq) >> sym->rcp_shift);
+    *r = x + sym->bias + q * sym->cmpl_freq;
+
+//    assert(((x / sym->freq) << sym->scale_bits) + (x % sym->freq) + sym->start == *r);
+}
+
+// Equivalent to RansDecAdvance that takes a symbol.
+static inline void RansDecAdvanceSymbol(RansState* r, uint8_t** pptr, RansDecSymbol const* sym, uint32_t scale_bits)
+{
+    RansDecAdvance(r, pptr, sym->start, sym->freq, scale_bits);
+}
+
+// Advances in the bit stream by "popping" a single symbol with range start
+// "start" and frequency "freq". All frequencies are assumed to sum to "1 << scale_bits".
+// No renormalization or output happens.
+static inline void RansDecAdvanceStep(RansState* r, uint32_t start, uint32_t freq, uint32_t scale_bits)
+{
+    uint32_t mask = (1u << scale_bits) - 1;
+
+    // s, x = D(x)
+    uint32_t x = *r;
+    *r = freq * (x >> scale_bits) + (x & mask) - start;
+}
+
+// Equivalent to RansDecAdvanceStep that takes a symbol.
+static inline void RansDecAdvanceSymbolStep(RansState* r, RansDecSymbol const* sym, uint32_t scale_bits)
+{
+    RansDecAdvanceStep(r, sym->start, sym->freq, scale_bits);
+}
+
+// Renormalize.
+
+#if defined(__x86_64) && !defined(__ILP32__)
+
+/*
+ * Assembly variants of the RansDecRenorm code.
+ * These are based on joint ideas from Rob Davies and from looking at
+ * the clang assembly output.
+ */
+static inline void RansDecRenorm(RansState* r, uint8_t** pptr) {
+    //       q4        q40
+    // clang 730/608   717/467
+    // gcc8  733/588   737/458
+    uint32_t  x   = *r;
+    uint8_t  *ptr = *pptr;
+    __asm__ ("movzwl (%0),  %%eax\n\t"
+             "mov    %1,    %%edx\n\t"
+             "shl    $0x10, %%edx\n\t"
+             "or     %%eax, %%edx\n\t"
+             "xor    %%eax, %%eax\n\t"
+             "cmp    $0x8000,%1\n\t"
+             "cmovb  %%edx, %1\n\t"
+             "lea    2(%0), %%rax\n\t"
+             "cmovb  %%rax, %0\n\t"
+             : "=r" (ptr), "=r" (x)
+             : "0"  (ptr), "1"  (x)
+             : "eax", "edx"
+             );
+    *pptr = (uint8_t *)ptr;
+    *r = x;
+}
+
+#else /* __x86_64 */
+
+static inline void RansDecRenorm(RansState* r, uint8_t** pptr)
+{
+    // renormalize, branchless
+    uint32_t x = *r;
+    int cmp = (x < RANS_BYTE_L)*2;
+    uint32_t y = (*pptr)[0] + ((*pptr)[1]<<8);
+    uint32_t x2 = (x << 16) | y;
+    x = cmp ? x2 : x;
+    (*pptr) += cmp;
+    *r = x;
+
+//    // renormalize, branched.  Faster on low-complexity data, but generally
+//    // that is best compressed with PACK and/or RLE which turns it back
+//    // into high complexity data.
+//    uint32_t x = *r;
+//    uint32_t y = (*pptr)[0] | ((*pptr)[1]<<8);
+//
+//    if (x < RANS_BYTE_L)
+//      (*pptr)+=2;
+//    if (x < RANS_BYTE_L)
+//      x = (x << 16) | y;
+//
+//    *r = x;
+}
+#endif /* __x86_64 */
+
+// Note the data may not be word aligned here.
+// This function is only used sparingly, for the last few bytes in the buffer,
+// so speed isn't critical.
+static inline void RansDecRenormSafe(RansState* r, uint8_t** pptr, uint8_t *ptr_end)
+{
+    uint32_t x = *r;
+    if (x >= RANS_BYTE_L || *pptr+1 >= ptr_end) return;
+    uint16_t y = (*pptr)[0] + ((*pptr)[1]<<8);
+    x = (x << 16) | y;
+    (*pptr) += 2;
+    *r = x;
+}
+
+#endif // RANS_WORD_HEADER
diff --git a/ext/htslib/htscodecs/htscodecs/rle.c b/ext/htslib/htscodecs/htscodecs/rle.c
new file mode 100644
index 0000000..863dde2
--- /dev/null
+++ b/ext/htslib/htscodecs/htscodecs/rle.c
@@ -0,0 +1,207 @@
+/*
+ * Copyright (c) 2019-2021 Genome Research Ltd.
+ * Author(s): James Bonfield
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above
+ *       copyright notice, this list of conditions and the following
+ *       disclaimer in the documentation and/or other materials provided
+ *       with the distribution.
+ *
+ *    3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+ *       Institute nor the names of its contributors may be used to endorse
+ *       or promote products derived from this software without specific
+ *       prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH
+ * LTD OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+#include "varint.h"
+#include "rle.h"
+
+#define MAGIC 8
+
+//-----------------------------------------------------------------------------
+// Auto compute rle_syms / rle_nsyms
+static void rle_find_syms(uint8_t *data, uint64_t data_len,
+                          int64_t *saved, // dim >= 256 
+                          uint8_t *rle_syms, int *rle_nsyms) {
+    int last = -1, n;
+    uint64_t i;
+
+    if (data_len > 256) {
+        // 186/450
+        // Interleaved buffers to avoid cache collisions
+        int64_t saved2[256+MAGIC] = {0};
+        int64_t saved3[256+MAGIC] = {0};
+        int64_t saved4[256+MAGIC] = {0};
+        int64_t len4 = data_len&~3;
+        for (i = 0; i < len4; i+=4) {
+            int d1 = (data[i+0] == last)     <<1;
+            int d2 = (data[i+1] == data[i+0])<<1;
+            int d3 = (data[i+2] == data[i+1])<<1;
+            int d4 = (data[i+3] == data[i+2])<<1;
+            last = data[i+3];
+            saved [data[i+0]] += d1-1;
+            saved2[data[i+1]] += d2-1;
+            saved3[data[i+2]] += d3-1;
+            saved4[data[i+3]] += d4-1;
+        }
+        while (i < data_len) {
+            int d = (data[i] == last)<<1;
+            saved[data[i]] += d - 1;
+            last = data[i];
+            i++;
+        }
+        for (i = 0; i < 256; i++)
+            saved[i] += saved2[i] + saved3[i] + saved4[i];
+    } else {
+        // 163/391
+        for (i = 0; i < data_len; i++) {
+            if (data[i] == last) {
+                saved[data[i]]++;
+            } else {
+                saved[data[i]]--;
+                last = data[i];
+            }
+        }
+    }
+
+    // Map back to a list
+    for (i = n = 0; i < 256; i++) {
+        if (saved[i] > 0)
+            rle_syms[n++] = i;
+    }
+    *rle_nsyms = n;
+}
+
+uint8_t *hts_rle_encode(uint8_t *data, uint64_t data_len,
+                        uint8_t *run,  uint64_t *run_len,
+                        uint8_t *rle_syms, int *rle_nsyms,
+                        uint8_t *out, uint64_t *out_len) {
+    uint64_t i, j, k;
+    if (!out)
+        if (!(out = malloc(data_len*2)))
+            return NULL;
+
+    // Two pass:  Firstly compute which symbols are worth using RLE on.
+    int64_t saved[256+MAGIC] = {0};
+
+    if (*rle_nsyms) {
+        for (i = 0; i < *rle_nsyms; i++)
+            saved[rle_syms[i]] = 1;
+    } else {
+        // Writes back to rle_syms and rle_nsyms
+        rle_find_syms(data, data_len, saved, rle_syms, rle_nsyms);
+    }
+
+    // 2nd pass: perform RLE itself to out[] and run[] arrays.
+    for (i = j = k = 0; i < data_len; i++) {
+        out[k++] = data[i];
+        if (saved[data[i]] > 0) {
+            int rlen = i;
+            int last = data[i];
+            while (i < data_len && data[i] == last)
+                i++;
+            i--;
+            rlen = i-rlen;
+
+            j += var_put_u32(&run[j], NULL, rlen);
+        }
+    }
+    
+    *run_len = j;
+    *out_len = k;
+    return out;
+}
+
+// On input *out_len holds the allocated size of out[].
+// On output it holds the used size of out[].
+uint8_t *hts_rle_decode(uint8_t *lit, uint64_t lit_len,
+                        uint8_t *run, uint64_t run_len,
+                        uint8_t *rle_syms, int rle_nsyms,
+                        uint8_t *out, uint64_t *out_len) {
+    uint64_t j;
+    uint8_t *run_end = run + run_len;
+
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    if (*out_len > 100000)
+        return NULL;
+#endif
+
+    int saved[256] = {0};
+    for (j = 0; j < rle_nsyms; j++)
+        saved[rle_syms[j]] = 1;
+
+    uint8_t *lit_end = lit + lit_len;
+    uint8_t *out_end = out + *out_len;
+    uint8_t *outp = out;
+
+    while (lit < lit_end) {
+        if (outp >= out_end)
+            goto err;
+
+        uint8_t b = *lit;
+        if (saved[b]) {
+            uint32_t rlen;
+            run += var_get_u32(run, run_end, &rlen);
+            if (rlen) {
+                if (outp + rlen >= out_end)
+                    goto err;
+                memset(outp, b, rlen+1);
+                outp += rlen+1;
+            } else {
+                *outp++ = b;
+            }
+        } else {
+            *outp++ = b;
+        }
+        lit++;
+    }
+
+    *out_len = outp-out;
+    return out;
+
+ err:
+    return NULL;
+}
+
+// Deprecated interface; to remove when we next to an ABI breakage
+uint8_t *rle_encode(uint8_t *data, uint64_t data_len,
+                    uint8_t *run,  uint64_t *run_len,
+                    uint8_t *rle_syms, int *rle_nsyms,
+                    uint8_t *out, uint64_t *out_len) {
+    return hts_rle_encode(data, data_len, run, run_len,
+                          rle_syms, rle_nsyms, out, out_len);
+}
+
+// Deprecated interface; to remove when we next to an ABI breakage
+uint8_t *rle_decode(uint8_t *lit, uint64_t lit_len,
+                    uint8_t *run, uint64_t run_len,
+                    uint8_t *rle_syms, int rle_nsyms,
+                    uint8_t *out, uint64_t *out_len) {
+    return hts_rle_decode(lit, lit_len, run, run_len,
+                          rle_syms, rle_nsyms, out, out_len);
+}
diff --git a/ext/htslib/htscodecs/htscodecs/rle.h b/ext/htslib/htscodecs/htscodecs/rle.h
new file mode 100644
index 0000000..b2f0671
--- /dev/null
+++ b/ext/htslib/htscodecs/htscodecs/rle.h
@@ -0,0 +1,95 @@
+/*
+ * Copyright (c) 2019 Genome Research Ltd.
+ * Author(s): James Bonfield
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above
+ *       copyright notice, this list of conditions and the following
+ *       disclaimer in the documentation and/or other materials provided
+ *       with the distribution.
+ *
+ *    3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+ *       Institute nor the names of its contributors may be used to endorse
+ *       or promote products derived from this software without specific
+ *       prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH
+ * LTD OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef HTS_RLE_H
+#define HTS_RLE_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Performs run length encoding of a byte stream, turning it into a
+ * list of lengths and a list of literals.
+ *
+ * The method used is a bit different to traditional run length
+ * encoding.  It always outputs run-lengths for symbols in the
+ * 'rle_syms' list (even if that length is +0 more), and never outputs
+ * lengths for symbols not in that list.
+ *
+ * "run" should be preallocated to be large enough;
+ * e.g at least data_len bytes long as a worse case.
+ * "rle_syms" should be allocated to be at least 256 bytes.
+ *
+ * If *rle_nsyms is zero this function will survey the input data
+ * first to choose symbols automatically, writing back to rle_syms and
+ * rle_nsyms.
+ *
+ * The "out" buffer may be passed in as NULL in which case it is
+ * allocated and returned (and is up to the caller to free).
+ * Otherwise if specified as non-NULL it will be written to, but
+ * it is up to the caller to ensure the buffer size is large enough.
+ * A worst case scenario is 2*data_len.
+ *
+ * Returns the literal buffer on success with new length in out_len,
+ *         also fills out run buffer and run_len,  and potentially
+ *         updates rle_syms / rle_nsyms too.
+ * Returns NULL of failure
+ */
+uint8_t *hts_rle_encode(uint8_t *data, uint64_t data_len,
+                        uint8_t *run,  uint64_t *run_len,
+                        uint8_t *rle_syms, int *rle_nsyms,
+                        uint8_t *out, uint64_t *out_len);
+
+/*
+ * Expands a run lengthed data steam from a pair of literal and
+ * run-length buffers.
+ *
+ * On input *out_len holds the length of the supplied out
+ * buffer.  On exit, it holds the used portion of this buffer.
+ *
+ * Returns uncompressed data (out) on success,
+ *         NULL on failure.
+ */
+uint8_t *hts_rle_decode(uint8_t *lit, uint64_t lit_len,
+                        uint8_t *run, uint64_t run_len,
+                        uint8_t *rle_syms, int rle_nsyms,
+                        uint8_t *out, uint64_t *out_len);
+
+// TODO: Add rle scanning func to compute rle_syms.
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* HTS_RLE_H */
diff --git a/ext/htslib/htscodecs/htscodecs/tokenise_name3.c b/ext/htslib/htscodecs/htscodecs/tokenise_name3.c
new file mode 100644
index 0000000..7493579
--- /dev/null
+++ b/ext/htslib/htscodecs/htscodecs/tokenise_name3.c
@@ -0,0 +1,1819 @@
+/*
+ * Copyright (c) 2016-2022 Genome Research Ltd.
+ * Author(s): James Bonfield
+ * 
+ * Redistribution and use in source and binary forms, with or without 
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ *    1. Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimer.
+ * 
+ *    2. Redistributions in binary form must reproduce the above
+ *       copyright notice, this list of conditions and the following
+ *       disclaimer in the documentation and/or other materials provided
+ *       with the distribution.
+ * 
+ *    3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+ *    Institute nor the names of its contributors may be used to endorse
+ *    or promote products derived from this software without specific
+ *    prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH
+ * LTD OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+// cc -O3 -g -DTEST_TOKENISER tokenise_name3.c arith_dynamic.c rANS_static4x16pr.c pooled_alloc.c -I.. -I. -lbz2 -pthread
+
+// Name tokeniser.
+// It generates a series of byte streams (per token) and compresses these
+// either using static rANS or dynamic arithmetic coding.  Arith coding is
+// typically 1-5% smaller, but around 50-100% slower.  We only envisage it
+// being used at the higher compression levels.
+
+// TODO
+//
+// - Is it better when encoding 1, 2, 3, 3, 4, 5, 5, 6, 7, 9, 9, 10 to encode
+//   this as a mixture of MATCH and DELTA ops, or as entirely as DELTA ops
+//   with some delta values being zero?  I suspect the latter, but it is
+//   not implemented here.  See "last_token_delta" comments in code.
+//
+// - Consider variable size string implementations.
+//   Pascal style strings (length + str),
+//   C style strings (nul terminated),
+//   Or split blocks: length block and string contents block.
+//
+// - Is this one token-block or many serialised token-blocks?
+//   A) Lots of different models but feeding one bit-buffer emitted to
+//      by the entropy encoder => one block (fqzcomp).
+//   B) Lots of different models each feeding their own bit-buffers
+//      => many blocks.
+//
+// - multiple integer types depending on size; 1, 2, 4 byte long.
+//
+// - Consider token choice for isalnum instead of isalpha.  Sometimes better.
+//
+// - Consider token synchronisation (eg on matching chr symbols?) incase of
+//   variable number.  Eg consider foo:0999, foo:1000, foo:1001 (the leading
+//   zero adds an extra token).
+//
+// - Optimisation of tokens.  Eg:
+//     HS25_09827:2:2102:11274:80442#49
+//     HS25_09827:2:2109:12941:31311#49
+//
+//   We'll have tokens for HS 25 _ 09827 : 2 : that are entirely <MATCH>
+//   after the initial token.  These 7 tokens could be one ALPHA instead
+//   of 7 distinct tokens, with 1 MATCH instead of 7.  This is both a speed
+//   improvement for decoding as well as a space saving (fewer token-blocks
+//   and associated overhead).
+//
+// - XOR.  Like ALPHA, but used when previous symbol is ALPHA or XOR
+//   and string lengths match.  Useful when names are similar, eg:
+//   the sequence in 07.names:
+//
+//   @VP2-06:112:H7LNDMCVY:1:1105:26919:1172 1:N:0:ATTCAGAA+AGGAGAAG
+//   @VP2-06:112:H7LNDMCVY:1:1105:27100:1172 1:N:0:ATTCAGAA+AGGCGAAG
+//   @VP2-06:112:H7LNDMCVY:1:1105:27172:1172 1:N:0:ATTCAGAA+AGGCTAAG
+
+#include "config.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <limits.h>
+#include <ctype.h>
+#include <assert.h>
+#include <inttypes.h>
+#include <math.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <time.h>
+
+#include "pooled_alloc.h"
+#include "arith_dynamic.h"
+#include "rANS_static4x16.h"
+#include "tokenise_name3.h"
+#include "varint.h"
+#include "utils.h"
+
+// 128 is insufficient for SAM names (max 256 bytes) as
+// we may alternate a0a0a0a0a0 etc.  However if we fail,
+// we just give up and switch to another codec, so this
+// isn't a serious limit.  Maybe up to 256 to permit all
+// SAM names?
+#define MAX_TOKENS 128
+#define MAX_TBLOCKS (MAX_TOKENS<<4)
+
+// Number of names per block
+#define MAX_NAMES 1000000
+
+enum name_type {N_ERR = -1, N_TYPE = 0, N_ALPHA, N_CHAR, N_DIGITS0, N_DZLEN, N_DUP, N_DIFF, 
+                N_DIGITS, N_DDELTA, N_DDELTA0, N_MATCH, N_NOP, N_END, N_ALL};
+
+typedef struct trie {
+    struct trie *next, *sibling;
+    int count;
+    uint32_t c:8;
+    uint32_t n:24; // Nth line
+} trie_t;
+
+typedef struct {
+    enum name_type token_type;
+    int token_int;
+    int token_str;
+} last_context_tok;
+
+typedef struct {
+    char *last_name;
+    int last_ntok;
+    last_context_tok *last; // [last_ntok]
+} last_context;
+
+typedef struct {
+    uint8_t *buf;
+    size_t buf_a, buf_l; // alloc and used length.
+    int tnum, ttype;
+    int dup_from;
+} descriptor;
+
+typedef struct {
+    last_context *lc;
+
+    // For finding entire line dups
+    int counter;
+
+    // Trie used in encoder only
+    trie_t *t_head;
+    pool_alloc_t *pool;
+
+    // token blocks
+    descriptor desc[MAX_TBLOCKS];
+
+    // summary stats per token
+    int token_dcount[MAX_TOKENS];
+    int token_icount[MAX_TOKENS];
+    //int token_zcount[MAX_TOKENS];
+
+    int max_tok; // tracks which desc/[id]count elements have been initialised
+    int max_names;
+} name_context;
+
+static name_context *create_context(int max_names) {
+    if (max_names <= 0)
+        return NULL;
+
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    if (max_names > 100000)
+        return NULL;
+#endif
+
+    // An arbitrary limit to prevent malformed data from consuming excessive
+    // amounts of memory.  Consider upping this if we have genuine use cases
+    // for larger blocks.
+    if (max_names > 1e7) {
+        fprintf(stderr, "Name codec currently has a max of 10 million rec.\n");
+        return NULL;
+    }
+
+    name_context *ctx = htscodecs_tls_alloc(sizeof(*ctx) +
+                                            ++max_names*sizeof(*ctx->lc));
+    if (!ctx) return NULL;
+    ctx->max_names = max_names;
+
+    ctx->counter = 0;
+    ctx->t_head = NULL;
+
+    ctx->lc = (last_context *)(((char *)ctx) + sizeof(*ctx));
+    ctx->pool = NULL;
+
+     memset(&ctx->desc[0], 0, 2*16 * sizeof(ctx->desc[0]));
+     memset(&ctx->token_dcount[0], 0, sizeof(int));
+     memset(&ctx->token_icount[0], 0, sizeof(int));
+     memset(&ctx->lc[0], 0, max_names*sizeof(ctx->lc[0]));
+     ctx->max_tok = 1;
+
+     ctx->lc[0].last_ntok = 0;
+
+    return ctx;
+}
+
+static void free_context(name_context *ctx) {
+    if (!ctx)
+        return;
+
+    if (ctx->t_head)
+        free(ctx->t_head);
+    if (ctx->pool)
+        pool_destroy(ctx->pool);
+
+    int i;
+    for (i = 0; i < ctx->max_tok*16; i++)
+        free(ctx->desc[i].buf);
+
+    for (i = 0; i < ctx->max_names; i++)
+        free(ctx->lc[i].last);
+
+    htscodecs_tls_free(ctx);
+}
+
+//-----------------------------------------------------------------------------
+// Fast unsigned integer printing code.
+// Returns number of bytes written.
+static int append_uint32_fixed(char *cp, uint32_t i, uint8_t l) {
+    switch (l) {
+    case 9:*cp++ = i / 100000000 + '0', i %= 100000000; // fall-through
+    case 8:*cp++ = i / 10000000  + '0', i %= 10000000;  // fall-through
+    case 7:*cp++ = i / 1000000   + '0', i %= 1000000;   // fall-through
+    case 6:*cp++ = i / 100000    + '0', i %= 100000;    // fall-through
+    case 5:*cp++ = i / 10000     + '0', i %= 10000;     // fall-through
+    case 4:*cp++ = i / 1000      + '0', i %= 1000;      // fall-through
+    case 3:*cp++ = i / 100       + '0', i %= 100;       // fall-through
+    case 2:*cp++ = i / 10        + '0', i %= 10;        // fall-through
+    case 1:*cp++ = i             + '0';                 // fall-throuhg
+    case 0:break;
+    }
+    return l;
+}
+
+static int append_uint32_var(char *cp, uint32_t i) {
+    char *op = cp;
+    uint32_t j;
+
+    //if (i < 10)         goto b0;
+    if (i < 100)        goto b1;
+    //if (i < 1000)       goto b2;
+    if (i < 10000)      goto b3;
+    //if (i < 100000)     goto b4;
+    if (i < 1000000)    goto b5;
+    //if (i < 10000000)   goto b6;
+    if (i < 100000000)  goto b7;
+
+    if ((j = i / 1000000000)) {*cp++ = j + '0'; i -= j*1000000000; goto x8;}
+    if ((j = i / 100000000))  {*cp++ = j + '0'; i -= j*100000000;  goto x7;}
+ b7:if ((j = i / 10000000))   {*cp++ = j + '0'; i -= j*10000000;   goto x6;}
+    if ((j = i / 1000000))    {*cp++ = j + '0', i -= j*1000000;    goto x5;}
+ b5:if ((j = i / 100000))     {*cp++ = j + '0', i -= j*100000;     goto x4;}
+    if ((j = i / 10000))      {*cp++ = j + '0', i -= j*10000;      goto x3;}
+ b3:if ((j = i / 1000))       {*cp++ = j + '0', i -= j*1000;       goto x2;}
+    if ((j = i / 100))        {*cp++ = j + '0', i -= j*100;        goto x1;}
+ b1:if ((j = i / 10))         {*cp++ = j + '0', i -= j*10;         goto x0;}
+    if (i)                     *cp++ = i + '0';
+    return cp-op;
+
+ x8:*cp++ = i / 100000000 + '0', i %= 100000000;
+ x7:*cp++ = i / 10000000  + '0', i %= 10000000;
+ x6:*cp++ = i / 1000000   + '0', i %= 1000000;
+ x5:*cp++ = i / 100000    + '0', i %= 100000;
+ x4:*cp++ = i / 10000     + '0', i %= 10000;
+ x3:*cp++ = i / 1000      + '0', i %= 1000;
+ x2:*cp++ = i / 100       + '0', i %= 100;
+ x1:*cp++ = i / 10        + '0', i %= 10;
+ x0:*cp++ = i             + '0';
+
+    return cp-op;
+}
+
+//-----------------------------------------------------------------------------
+// Example descriptor encoding and IO.
+//
+// Here we just append to a buffer so we can dump out the results.
+// These could then be passed through a static entropy encoder that
+// encodes the entire buffer.
+//
+// Alternatively an adaptive entropy encoder could be place inline
+// here to encode as it goes using additional knowledge from the
+// supplied context.
+
+// Ensure room for sz more bytes.
+static int descriptor_grow(descriptor *fd, uint32_t sz) {
+    while (fd->buf_l + sz > fd->buf_a) {
+        size_t buf_a = fd->buf_a ? fd->buf_a*2 : 65536;
+        unsigned char *buf = realloc(fd->buf, buf_a);
+        if (!buf)
+            return -1;
+        fd->buf = buf;
+        fd->buf_a = buf_a;
+    }
+
+    return 0;
+}
+
+static int encode_token_type(name_context *ctx, int ntok,
+                             enum name_type type) {
+    int id = ntok<<4;
+
+    if (descriptor_grow(&ctx->desc[id], 1) < 0) return -1;
+
+    ctx->desc[id].buf[ctx->desc[id].buf_l++] = type;
+
+    return 0;
+}
+
+static int encode_token_match(name_context *ctx, int ntok) {
+    return encode_token_type(ctx, ntok, N_MATCH);
+}
+
+static int encode_token_end(name_context *ctx, int ntok) {
+    return encode_token_type(ctx, ntok, N_END);
+}
+
+static enum name_type decode_token_type(name_context *ctx, int ntok) {
+    int id = ntok<<4;
+    if (ctx->desc[id].buf_l >= ctx->desc[id].buf_a) return -1;
+    return ctx->desc[id].buf[ctx->desc[id].buf_l++];
+}
+
+// int stored as 32-bit quantities
+static int encode_token_int(name_context *ctx, int ntok,
+                            enum name_type type, uint32_t val) {
+    int id = (ntok<<4) | type;
+
+    if (encode_token_type(ctx, ntok, type) < 0) return -1;
+    if (descriptor_grow(&ctx->desc[id], 4) < 0) return -1;
+
+    uint8_t *cp = &ctx->desc[id].buf[ctx->desc[id].buf_l];
+    cp[0] = (val >>  0) & 0xff;
+    cp[1] = (val >>  8) & 0xff;
+    cp[2] = (val >> 16) & 0xff;
+    cp[3] = (val >> 24) & 0xff;
+    ctx->desc[id].buf_l += 4;
+
+    return 0;
+}
+
+// Return 0 on success, -1 on failure;
+static int decode_token_int(name_context *ctx, int ntok,
+                            enum name_type type, uint32_t *val) {
+    int id = (ntok<<4) | type;
+
+    if (ctx->desc[id].buf_l + 4 > ctx->desc[id].buf_a)
+        return -1;
+
+    uint8_t *cp = ctx->desc[id].buf + ctx->desc[id].buf_l;
+    *val = (cp[0]) + (cp[1]<<8) + (cp[2]<<16) + ((uint32_t)cp[3]<<24);
+    ctx->desc[id].buf_l += 4;
+
+    return 0;
+}
+
+// 8 bit integer quantity
+static int encode_token_int1(name_context *ctx, int ntok,
+                             enum name_type type, uint32_t val) {
+    int id = (ntok<<4) | type;
+
+    if (encode_token_type(ctx, ntok, type) < 0) return -1;
+    if (descriptor_grow(&ctx->desc[id], 1) < 0) return -1;
+
+    ctx->desc[id].buf[ctx->desc[id].buf_l++] = val;
+
+    return 0;
+}
+
+static int encode_token_int1_(name_context *ctx, int ntok,
+                              enum name_type type, uint32_t val) {
+    int id = (ntok<<4) | type;
+
+    if (descriptor_grow(&ctx->desc[id], 1) < 0) return -1;
+
+    ctx->desc[id].buf[ctx->desc[id].buf_l++] = val;
+
+    return 0;
+}
+
+// Return 0 on success, -1 on failure;
+static int decode_token_int1(name_context *ctx, int ntok,
+                             enum name_type type, uint32_t *val) {
+    int id = (ntok<<4) | type;
+
+    if (ctx->desc[id].buf_l  >= ctx->desc[id].buf_a)
+        return -1;
+    *val = ctx->desc[id].buf[ctx->desc[id].buf_l++];
+
+    return 0;
+}
+
+
+// Basic C-string style for now.
+//
+// Maybe XOR with previous string as context?
+// This permits partial match to be encoded efficiently.
+static int encode_token_alpha(name_context *ctx, int ntok,
+                              char *str, int len) {
+    int id = (ntok<<4) | N_ALPHA;
+
+    if (encode_token_type(ctx, ntok, N_ALPHA) < 0)  return -1;
+    if (descriptor_grow(&ctx->desc[id], len+1) < 0) return -1;
+    memcpy(&ctx->desc[id].buf[ctx->desc[id].buf_l], str, len);
+    ctx->desc[id].buf[ctx->desc[id].buf_l+len] = 0;
+    ctx->desc[id].buf_l += len+1;
+
+    return 0;
+}
+
+// FIXME: need limit on string length for security.
+// Return length on success, -1 on failure;
+static int decode_token_alpha(name_context *ctx, int ntok, char *str, int max_len) {
+    int id = (ntok<<4) | N_ALPHA;
+    char c;
+    int len = 0;
+    if (ctx->desc[id].buf_l  >= ctx->desc[id].buf_a)
+        return -1;
+    do {
+        c = ctx->desc[id].buf[ctx->desc[id].buf_l++];
+        str[len++] = c;
+    } while(c && len < max_len && ctx->desc[id].buf_l < ctx->desc[id].buf_a);
+
+    return len-1;
+}
+
+static int encode_token_char(name_context *ctx, int ntok, char c) {
+    int id = (ntok<<4) | N_CHAR;
+
+    if (encode_token_type(ctx, ntok, N_CHAR) < 0) return -1;
+    if (descriptor_grow(&ctx->desc[id], 1) < 0)    return -1;
+    ctx->desc[id].buf[ctx->desc[id].buf_l++] = c;
+
+    return 0;
+}
+
+// FIXME: need limit on string length for security
+// Return length on success, -1 on failure;
+static int decode_token_char(name_context *ctx, int ntok, char *str) {
+    int id = (ntok<<4) | N_CHAR;
+
+    if (ctx->desc[id].buf_l  >= ctx->desc[id].buf_a)
+        return -1;
+    *str = ctx->desc[id].buf[ctx->desc[id].buf_l++];
+
+    return 1;
+}
+
+
+// A duplicated name
+static int encode_token_dup(name_context *ctx, uint32_t val) {
+    return encode_token_int(ctx, 0, N_DUP, val);
+}
+
+// Which read to delta against
+static int encode_token_diff(name_context *ctx, uint32_t val) {
+    return encode_token_int(ctx, 0, N_DIFF, val);
+}
+
+
+//-----------------------------------------------------------------------------
+// Trie implementation for tracking common name prefixes.
+static
+int build_trie(name_context *ctx, char *data, size_t len, int n) {
+    int nlines = 0;
+    size_t i;
+    trie_t *t;
+
+    if (!ctx->t_head) {
+        ctx->t_head = calloc(1, sizeof(*ctx->t_head));
+        if (!ctx->t_head)
+            return -1;
+    }
+
+    // Build our trie, also counting input lines
+    for (nlines = i = 0; i < len; i++, nlines++) {
+        t = ctx->t_head;
+        t->count++;
+        while (i < len && (unsigned char)data[i] > '\n') {
+            unsigned char c = data[i++];
+            if (c & 0x80)
+                //fprintf(stderr, "8-bit ASCII is unsupported\n");
+                return -1;
+            c &= 127;
+
+
+            trie_t *x = t->next, *l = NULL;
+            while (x && x->c != c) {
+                l = x; x = x->sibling;
+            }
+            if (!x) {
+                if (!ctx->pool)
+                    ctx->pool = pool_create(sizeof(trie_t));
+                if (!(x = (trie_t *)pool_alloc(ctx->pool)))
+                    return -1;
+                memset(x, 0, sizeof(*x));
+                if (!l)
+                    x = t->next    = x;
+                else
+                    x = l->sibling = x;
+                x->n = n;
+                x->c = c;
+            }
+            t = x;
+            t->c = c;
+            t->count++;
+        }
+    }
+
+    return 0;
+}
+
+#if 0
+void dump_trie(trie_t *t, int depth) {
+    if (depth == 0) {
+        printf("graph x_%p {\n    splines = ortho\n    ranksep=2\n", t);
+        printf("    p_%p [label=\"\"];\n", t);
+        dump_trie(t, 1);
+        printf("}\n");
+    } else {
+        int j, k, count;//, cj;
+        char label[100], *cp;
+        trie_t *tp = t;
+
+//    patricia:
+//      for (count = j = 0; j < 128; j++)
+//          if (t->next[j])
+//              count++, cj=j;
+//
+//      if (count == 1) {
+//          t = t->next[cj];
+//          *cp++ = cj;
+//          goto patricia;
+//      }
+
+        trie_t *x;
+        for (x = t->next; x; x = x->sibling) {
+            printf("    p_%p [label=\"%c\"];\n", x, x->c);
+            printf("    p_%p -- p_%p [label=\"%d\", penwidth=\"%f\"];\n", tp, x, x->count, MAX((log(x->count)-3)*2,1));
+            //if (depth <= 11)
+                dump_trie(x, depth+1);
+        }
+
+#if 0       
+        for (j = 0; j < 128; j++) {
+            trie_t *tn;
+
+            if (!t->next[j])
+                continue;
+
+            cp = label;
+            tn = t->next[j];
+            *cp++ = j;
+//      patricia:
+
+            for (count = k = 0; k < 128; k++)
+                if (tn->next[k])
+                    count++;//, cj=k;
+
+//          if (count == 1) {
+//              tn = tn->next[cj];
+//              *cp++ = cj;
+//              goto patricia;
+//          }
+            *cp++ = 0;
+
+            printf("    p_%p [label=\"%s\"];\n", tn, label);
+            printf("    p_%p -- p_%p [label=\"%d\", penwidth=\"%f\"];\n", tp, tn, tn->count, MAX((log(tn->count)-3)*2,1));
+            if (depth <= 11)
+                dump_trie(tn, depth+1);
+        }
+#endif
+    }
+}
+#endif
+
+static
+int search_trie(name_context *ctx, char *data, size_t len, int n, int *exact, int *is_fixed, int *fixed_len) {
+    int nlines = 0;
+    size_t i;
+    trie_t *t;
+    int from = -1, p3 = -1;
+    *exact = 0;
+    *fixed_len = 0;
+    *is_fixed = 0;
+
+    // Horrid hack for the encoder only.
+    // We optimise per known name format here.
+    int prefix_len;
+    char *d = *data == '@' ? data+1 : data;
+    int l   = *data == '@' ? len-1  : len;
+    int f = (*data == '>') ? 1 : 0;
+    if (l > 70 && d[f+0] == 'm' && d[7] == '_' && d[f+14] == '_' && d[f+61] == '/') {
+        prefix_len = 60; // PacBio
+        *is_fixed = 0;
+    } else if (l == 17 && d[f+5] == ':' && d[f+11] == ':') {
+        prefix_len = 6;  // IonTorrent
+        *fixed_len = 6;
+        *is_fixed = 1;
+    } else if (l > 37 && d[f+8] == '-' && d[f+13] == '-' && d[f+18] == '-' && d[f+23] == '-' &&
+               ((d[f+0] >= '0' && d[f+0] <='9') || (d[f+0] >= 'a' && d[f+0] <= 'f')) &&
+               ((d[f+35] >= '0' && d[f+35] <='9') || (d[f+35] >= 'a' && d[f+35] <= 'f'))) {
+        // ONT: f33d30d5-6eb8-4115-8f46-154c2620a5da_Basecall_1D_template...
+        prefix_len = 37;
+        *fixed_len = 37;
+        *is_fixed = 1;
+    } else {
+        // Check Illumina and trim back to lane:tile:x:y.
+        int colons = 0;
+        for (i = 0; i < len && data[i] > ' '; i++)
+            ;
+        while (i > 0 && colons < 4)
+            if (data[--i] == ':')
+                colons++;
+
+        if (colons == 4) {
+            // Constant illumina prefix
+            *fixed_len = i+1;
+            prefix_len = i+1;
+            *is_fixed = 1;
+        } else {
+            // Unknown, don't use a fixed len, but still search
+            // for any exact matches.
+            prefix_len = INT_MAX;
+            *is_fixed = 0;
+        }
+    }
+    //prefix_len = INT_MAX;
+
+    if (!ctx->t_head) {
+        ctx->t_head = calloc(1, sizeof(*ctx->t_head));
+        if (!ctx->t_head)
+            return -1;
+    }
+
+    // Find an item in the trie
+    for (nlines = i = 0; i < len; i++, nlines++) {
+        t = ctx->t_head;
+        while (i < len && data[i] > '\n') {
+            unsigned char c = data[i++];
+            if (c & 0x80)
+                //fprintf(stderr, "8-bit ASCII is unsupported\n");
+                return -1;
+            c &= 127;
+
+            trie_t *x = t->next;
+            while (x && x->c != c)
+                x = x->sibling;
+            t = x;
+
+//          t = t->next[c];
+
+//          if (!t)
+//              return -1;
+
+            from = t->n;
+            if (i == prefix_len) p3 = t->n;
+            //if (t->count >= .0035*ctx->t_head->count && t->n != n) p3 = t->n; // pacbio
+            //if (i == 60) p3 = t->n; // pacbio
+            //if (i == 7) p3 = t->n; // iontorrent
+            t->n = n;
+        }
+    }
+
+    //printf("Looked for %d, found %d, prefix %d\n", n, from, p3);
+
+    *exact = (n != from) && len;
+    return *exact ? from : p3;
+}
+
+
+//-----------------------------------------------------------------------------
+// Name encoder
+
+/*
+ * Tokenises a read name using ctx as context as the previous
+ * tokenisation.
+ *
+ * Parsed elements are then emitted for encoding by calling the
+ * encode_token() function with the context, token number (Nth token
+ * in line), token type and token value.
+ *
+ * Returns 0 on success;
+ *        -1 on failure.
+ */
+static int encode_name(name_context *ctx, char *name, int len, int mode) {
+    int i, is_fixed, fixed_len;
+
+    int exact;
+    int cnum = ctx->counter++;
+    int pnum = search_trie(ctx, name, len, cnum, &exact, &is_fixed, &fixed_len);
+    if (pnum < 0) pnum = cnum ? cnum-1 : 0;
+    //pnum = pnum & (MAX_NAMES-1);
+    //cnum = cnum & (MAX_NAMES-1);
+    //if (pnum == cnum) {pnum = cnum ? cnum-1 : 0;}
+#ifdef ENC_DEBUG
+    fprintf(stderr, "%d: pnum=%d (%d), exact=%d\n%s\n%s\n",
+            ctx->counter, pnum, cnum-pnum, exact, ctx->lc[pnum].last_name, name);
+#endif
+
+    // Return DUP or DIFF switch, plus the distance.
+    if (exact && len == strlen(ctx->lc[pnum].last_name)) {
+        encode_token_dup(ctx, cnum-pnum);
+        ctx->lc[cnum].last_name = name;
+        ctx->lc[cnum].last_ntok = ctx->lc[pnum].last_ntok;
+        int nc = ctx->lc[cnum].last_ntok ? ctx->lc[cnum].last_ntok : MAX_TOKENS;
+        ctx->lc[cnum].last = malloc(nc * sizeof(*ctx->lc[cnum].last));
+        if (!ctx->lc[cnum].last)
+            return -1;
+        memcpy(ctx->lc[cnum].last, ctx->lc[pnum].last,
+               ctx->lc[cnum].last_ntok * sizeof(*ctx->lc[cnum].last));
+        return 0;
+    }
+
+    ctx->lc[cnum].last = malloc(MAX_TOKENS * sizeof(*ctx->lc[cnum].last));
+    if (!ctx->lc[cnum].last)
+        return -1;
+    encode_token_diff(ctx, cnum-pnum);
+
+    int ntok = 1;
+    i = 0;
+    if (is_fixed) {
+        if (ntok >= ctx->max_tok) {
+            memset(&ctx->desc[ctx->max_tok << 4], 0, 16*sizeof(ctx->desc[0]));
+            memset(&ctx->token_dcount[ctx->max_tok], 0, sizeof(int));
+            memset(&ctx->token_icount[ctx->max_tok], 0, sizeof(int));
+            ctx->max_tok = ntok+1;
+        }
+        if (pnum < cnum && ntok < ctx->lc[pnum].last_ntok && ctx->lc[pnum].last[ntok].token_type == N_ALPHA) {
+            if (ctx->lc[pnum].last[ntok].token_int == fixed_len && memcmp(name, ctx->lc[pnum].last_name, fixed_len) == 0) {
+                encode_token_match(ctx, ntok);
+            } else {
+                encode_token_alpha(ctx, ntok, name, fixed_len);
+            }
+        } else {
+            encode_token_alpha(ctx, ntok, name, fixed_len);
+        }
+        ctx->lc[cnum].last[ntok].token_int = fixed_len;
+        ctx->lc[cnum].last[ntok].token_str = 0;
+        ctx->lc[cnum].last[ntok++].token_type = N_ALPHA;
+        i = fixed_len;
+    }
+
+    for (; i < len; i++) {
+        if (ntok >= ctx->max_tok) {
+            if (ctx->max_tok >= MAX_TOKENS)
+                return -1;
+            memset(&ctx->desc[ctx->max_tok << 4], 0, 16*sizeof(ctx->desc[0]));
+            memset(&ctx->token_dcount[ctx->max_tok], 0, sizeof(int));
+            memset(&ctx->token_icount[ctx->max_tok], 0, sizeof(int));
+            ctx->max_tok = ntok+1;
+        }
+
+        /* Determine data type of this segment */
+        if (isalpha((uint8_t)name[i])) {
+            int s = i+1;
+//          int S = i+1;
+
+//          // FIXME: try which of these is best.  alnum is good sometimes.
+//          while (s < len && isalpha((uint8_t)name[s]))
+            while (s < len && (isalpha((uint8_t)name[s]) ||
+                               ispunct((uint8_t)name[s])))
+//          while (s < len && name[s] != ':')
+//          while (s < len && !isdigit((uint8_t)name[s]) && name[s] != ':')
+                s++;
+
+//          if (!is_fixed) {
+//              while (S < len && isalnum((uint8_t)name[S]))
+//                  S++;
+//              if (s < S)
+//                  s = S;
+//          }
+
+            // Single byte strings are better encoded as chars.
+            if (s-i == 1) goto n_char;
+
+            if (pnum < cnum && ntok < ctx->lc[pnum].last_ntok && ctx->lc[pnum].last[ntok].token_type == N_ALPHA) {
+                if (s-i == ctx->lc[pnum].last[ntok].token_int &&
+                    memcmp(&name[i], 
+                           &ctx->lc[pnum].last_name[ctx->lc[pnum].last[ntok].token_str],
+                           s-i) == 0) {
+#ifdef ENC_DEBUG
+                    fprintf(stderr, "Tok %d (alpha-mat, %.*s)\n", N_MATCH, s-i, &name[i]);
+#endif
+                    if (encode_token_match(ctx, ntok) < 0) return -1;
+                } else {
+#ifdef ENC_DEBUG
+                    fprintf(stderr, "Tok %d (alpha, %.*s / %.*s)\n", N_ALPHA,
+                            s-i, &ctx->lc[pnum].last_name[ctx->lc[pnum].last[ntok].token_str], s-i, &name[i]);
+#endif
+                    // same token/length, but mismatches
+                    if (encode_token_alpha(ctx, ntok, &name[i], s-i) < 0) return -1;
+                }
+            } else {
+#ifdef ENC_DEBUG
+                fprintf(stderr, "Tok %d (new alpha, %.*s)\n", N_ALPHA, s-i, &name[i]);
+#endif
+                if (encode_token_alpha(ctx, ntok, &name[i], s-i) < 0) return -1;
+            }
+
+            ctx->lc[cnum].last[ntok].token_int = s-i;
+            ctx->lc[cnum].last[ntok].token_str = i;
+            ctx->lc[cnum].last[ntok].token_type = N_ALPHA;
+
+            i = s-1;
+        } else if (name[i] == '0') digits0: {
+            // Digits starting with zero; encode length + value
+            uint32_t s = i;
+            uint32_t v = 0;
+            int d = 0;
+
+            while (s < len && isdigit((uint8_t)name[s]) && s-i < 9) {
+                v = v*10 + name[s] - '0';
+                //putchar(name[s]);
+                s++;
+            }
+
+            // TODO: optimise choice over whether to switch from DIGITS to DELTA
+            // regularly vs all DIGITS, also MATCH vs DELTA 0.
+            if (pnum < cnum && ntok < ctx->lc[pnum].last_ntok && ctx->lc[pnum].last[ntok].token_type == N_DIGITS0) {
+                d = v - ctx->lc[pnum].last[ntok].token_int;
+                if (d == 0 && ctx->lc[pnum].last[ntok].token_str == s-i) {
+#ifdef ENC_DEBUG
+                    fprintf(stderr, "Tok %d (dig-mat, %d)\n", N_MATCH, v);
+#endif
+                    if (encode_token_match(ctx, ntok) < 0) return -1;
+                    //ctx->lc[pnum].last[ntok].token_delta=0;
+                } else if (mode == 1 && d < 256 && d >= 0 && ctx->lc[pnum].last[ntok].token_str == s-i) {
+#ifdef ENC_DEBUG
+                    fprintf(stderr, "Tok %d (dig0-delta, %d / %d)\n", N_DDELTA0, ctx->lc[pnum].last[ntok].token_int, v);
+#endif
+                    //if (encode_token_int1_(ctx, ntok, N_DZLEN, s-i) < 0) return -1;
+                    if (encode_token_int1(ctx, ntok, N_DDELTA0, d) < 0) return -1;
+                    //ctx->lc[pnum].last[ntok].token_delta=1;
+                } else {
+#ifdef ENC_DEBUG
+                    fprintf(stderr, "Tok %d (dig0, %d / %d len %d)\n", N_DIGITS0, ctx->lc[pnum].last[ntok].token_int, v, s-i);
+#endif
+                    if (encode_token_int1_(ctx, ntok, N_DZLEN, s-i) < 0) return -1;
+                    if (encode_token_int(ctx, ntok, N_DIGITS0, v) < 0) return -1;
+                    //ctx->lc[pnum].last[ntok].token_delta=0;
+                }
+            } else {
+#ifdef ENC_DEBUG
+                fprintf(stderr, "Tok %d (new dig0, %d len %d)\n", N_DIGITS0, v, s-i);
+#endif
+                if (encode_token_int1_(ctx, ntok, N_DZLEN, s-i) < 0) return -1;
+                if (encode_token_int(ctx, ntok, N_DIGITS0, v) < 0) return -1;
+                //ctx->lc[pnum].last[ntok].token_delta=0;
+            }
+
+            ctx->lc[cnum].last[ntok].token_str = s-i; // length
+            ctx->lc[cnum].last[ntok].token_int = v;
+            ctx->lc[cnum].last[ntok].token_type = N_DIGITS0;
+
+            i = s-1;
+        } else if (isdigit((uint8_t)name[i])) {
+            // digits starting 1-9; encode value
+            uint32_t s = i;
+            uint32_t v = 0;
+            int d = 0;
+
+            while (s < len && isdigit((uint8_t)name[s]) && s-i < 9) {
+                v = v*10 + name[s] - '0';
+                //putchar(name[s]);
+                s++;
+            }
+
+            // dataset/10/K562_cytosol_LID8465_TopHat_v2.names
+            // col 4 is Illumina lane - we don't want match & delta in there
+            // as it has multiple lanes (so not ALL match) and delta is just
+            // random chance, increasing entropy instead.
+//          if (ntok == 4  || ntok == 8 || ntok == 10) {
+//              encode_token_int(ctx, ntok, N_DIGITS, v);
+//          } else {
+
+            // If the last token was DIGITS0 and we are the same length, then encode
+            // using that method instead as it seems likely the entire column is fixed
+            // width, sometimes with leading zeros.
+            if (pnum < cnum && ntok < ctx->lc[pnum].last_ntok &&
+                ctx->lc[pnum].last[ntok].token_type == N_DIGITS0 &&
+                ctx->lc[pnum].last[ntok].token_str == s-i)
+                goto digits0;
+            
+            // TODO: optimise choice over whether to switch from DIGITS to DELTA
+            // regularly vs all DIGITS, also MATCH vs DELTA 0.
+            if (pnum < cnum && ntok < ctx->lc[pnum].last_ntok && ctx->lc[pnum].last[ntok].token_type == N_DIGITS) {
+                d = v - ctx->lc[pnum].last[ntok].token_int;
+                if (d == 0) {
+#ifdef ENC_DEBUG
+                    fprintf(stderr, "Tok %d (dig-mat, %d)\n", N_MATCH, v);
+#endif
+                    if (encode_token_match(ctx, ntok) < 0) return -1;
+                    //ctx->lc[pnum].last[ntok].token_delta=0;
+                    //ctx->token_zcount[ntok]++;
+                } else if (mode == 1 && d < 256 && d >= 0
+                           //&& (10+ctx->token_dcount[ntok]) > (ctx->token_icount[ntok]+ctx->token_zcount[ntok])
+                           && (5+ctx->token_dcount[ntok]) > ctx->token_icount[ntok]
+                           ) {
+#ifdef ENC_DEBUG
+                    fprintf(stderr, "Tok %d (dig-delta, %d / %d)\n", N_DDELTA, ctx->lc[pnum].last[ntok].token_int, v);
+#endif
+                    if (encode_token_int1(ctx, ntok, N_DDELTA, d) < 0) return -1;
+                    //ctx->lc[pnum].last[ntok].token_delta=1;
+                    ctx->token_dcount[ntok]++;
+                } else {
+#ifdef ENC_DEBUG
+                    fprintf(stderr, "Tok %d (dig, %d / %d)\n", N_DIGITS, ctx->lc[pnum].last[ntok].token_int, v);
+#endif
+                    if (encode_token_int(ctx, ntok, N_DIGITS, v) < 0) return -1;
+                    //ctx->lc[pnum].last[ntok].token_delta=0;
+                    ctx->token_icount[ntok]++;
+                }
+            } else {
+#ifdef ENC_DEBUG
+                fprintf(stderr, "Tok %d (new dig, %d)\n", N_DIGITS, v);
+#endif
+                if (encode_token_int(ctx, ntok, N_DIGITS, v) < 0) return -1;
+                //ctx->lc[pnum].last[ntok].token_delta=0;
+            }
+//          }
+
+            ctx->lc[cnum].last[ntok].token_int = v;
+            ctx->lc[cnum].last[ntok].token_type = N_DIGITS;
+
+            i = s-1;
+        } else {
+        n_char:
+            //if (!isalpha((uint8_t)name[i])) putchar(name[i]);
+            if (pnum < cnum && ntok < ctx->lc[pnum].last_ntok && ctx->lc[pnum].last[ntok].token_type == N_CHAR) {
+                if (name[i] == ctx->lc[pnum].last[ntok].token_int) {
+#ifdef ENC_DEBUG
+                    fprintf(stderr, "Tok %d (chr-mat, %c)\n", N_MATCH, name[i]);
+#endif
+                    if (encode_token_match(ctx, ntok) < 0) return -1;
+                } else {
+#ifdef ENC_DEBUG
+                    fprintf(stderr, "Tok %d (chr, %c / %c)\n", N_CHAR, ctx->lc[pnum].last[ntok].token_int, name[i]);
+#endif
+                    if (encode_token_char(ctx, ntok, name[i]) < 0) return -1;
+                }
+            } else {
+#ifdef ENC_DEBUG
+                fprintf(stderr, "Tok %d (new chr, %c)\n", N_CHAR, name[i]);
+#endif
+                if (encode_token_char(ctx, ntok, name[i]) < 0) return -1;
+            }
+
+            ctx->lc[cnum].last[ntok].token_int = name[i];
+            ctx->lc[cnum].last[ntok].token_type = N_CHAR;
+        }
+
+        ntok++;
+        //putchar(' ');
+    }
+
+#ifdef ENC_DEBUG
+    fprintf(stderr, "Tok %d (end)\n", N_END);
+#endif
+    if (ntok >= ctx->max_tok) {
+        if (ctx->max_tok >= MAX_TOKENS)
+            return -1;
+        memset(&ctx->desc[ctx->max_tok << 4], 0, 16*sizeof(ctx->desc[0]));
+        memset(&ctx->token_dcount[ctx->max_tok], 0, sizeof(int));
+        memset(&ctx->token_icount[ctx->max_tok], 0, sizeof(int));
+        ctx->max_tok = ntok+1;
+    }
+    if (encode_token_end(ctx, ntok) < 0) return -1;
+#ifdef ENC_DEBUG
+    fprintf(stderr, "ntok=%d max_tok=%d\n", ntok, ctx->max_tok);
+#endif
+
+    //printf("Encoded %.*s with %d tokens\n", len, name, ntok);
+    
+    ctx->lc[cnum].last_name = name;
+    ctx->lc[cnum].last_ntok = ntok;
+    last_context_tok *shrunk = realloc(ctx->lc[cnum].last,
+                                       (ntok+1) * sizeof(*ctx->lc[cnum].last));
+    if (shrunk)
+        ctx->lc[cnum].last = shrunk;
+
+    if (!ctx->lc[cnum].last)
+        return -1;
+
+    return 0;
+}
+
+//-----------------------------------------------------------------------------
+// Name decoder
+
+static int decode_name(name_context *ctx, char *name, int name_len) {
+    int t0 = decode_token_type(ctx, 0);
+    uint32_t dist;
+    int pnum, cnum = ctx->counter++;
+
+    if (cnum >= ctx->max_names)
+        return -1;
+
+    if (t0 < 0 || t0 >= ctx->max_tok*16)
+        return 0;
+
+    if (decode_token_int(ctx, 0, t0, &dist) < 0 || dist > cnum)
+        return -1;
+    if ((pnum = cnum - dist) < 0) pnum = 0;
+
+    //fprintf(stderr, "t0=%d, dist=%d, pnum=%d, cnum=%d\n", t0, dist, pnum, cnum);
+
+    if (t0 == N_DUP) {
+        if (pnum == cnum)
+            return -1;
+
+        if (strlen(ctx->lc[pnum].last_name) +1 >= name_len) return -1;
+        strcpy(name, ctx->lc[pnum].last_name);
+        // FIXME: optimise this
+        ctx->lc[cnum].last_name = name;
+        ctx->lc[cnum].last_ntok = ctx->lc[pnum].last_ntok;
+
+        int nc = ctx->lc[cnum].last_ntok ? ctx->lc[cnum].last_ntok : MAX_TOKENS;
+        ctx->lc[cnum].last = malloc(nc * sizeof(*ctx->lc[cnum].last));
+        if (!ctx->lc[cnum].last)
+            return -1;
+        memcpy(ctx->lc[cnum].last, ctx->lc[pnum].last,
+               ctx->lc[cnum].last_ntok * sizeof(*ctx->lc[cnum].last));
+
+        return strlen(name)+1;
+    }
+
+    *name = 0;
+    int ntok, len = 0, len2;
+    ctx->lc[cnum].last = malloc(MAX_TOKENS * sizeof(*ctx->lc[cnum].last));
+    if (!ctx->lc[cnum].last)
+        return -1;
+
+    for (ntok = 1; ntok < MAX_TOKENS && ntok < ctx->max_tok; ntok++) {
+        uint32_t v, vl;
+        enum name_type tok;
+        tok = decode_token_type(ctx, ntok);
+        //fprintf(stderr, "Tok %d = %d\n", ntok, tok);
+
+        ctx->lc[cnum].last_ntok = 0;
+
+        switch (tok) {
+        case N_CHAR:
+            if (len+1 >= name_len) return -1;
+            if (decode_token_char(ctx, ntok, &name[len]) < 0) return -1;
+            //fprintf(stderr, "Tok %d CHAR %c\n", ntok, name[len]);
+            ctx->lc[cnum].last[ntok].token_type = N_CHAR;
+            ctx->lc[cnum].last[ntok].token_int  = name[len++];
+            break;
+
+        case N_ALPHA:
+            if ((len2 = decode_token_alpha(ctx, ntok, &name[len], name_len - len)) < 0)
+                return -1;
+            //fprintf(stderr, "Tok %d ALPHA %.*s\n", ntok, len2, &name[len]);
+            ctx->lc[cnum].last[ntok].token_type = N_ALPHA;
+            ctx->lc[cnum].last[ntok].token_str  = len;
+            ctx->lc[cnum].last[ntok].token_int  = len2;
+            len += len2;
+            break;
+
+        case N_DIGITS0: // [0-9]*
+            if (decode_token_int1(ctx, ntok, N_DZLEN, &vl) < 0) return -1;
+            if (decode_token_int(ctx, ntok, N_DIGITS0, &v) < 0) return -1;
+            if (len+20+vl >= name_len) return -1;
+            len += append_uint32_fixed(&name[len], v, vl);
+            //fprintf(stderr, "Tok %d DIGITS0 %0*d\n", ntok, vl, v);
+            ctx->lc[cnum].last[ntok].token_type = N_DIGITS0;
+            ctx->lc[cnum].last[ntok].token_int = v;
+            ctx->lc[cnum].last[ntok].token_str = vl;
+            break;
+
+        case N_DDELTA0:
+            if (ntok >= ctx->lc[pnum].last_ntok) return -1;
+            if (decode_token_int1(ctx, ntok, N_DDELTA0, &v) < 0) return -1;
+            v += ctx->lc[pnum].last[ntok].token_int;
+            if (len+ctx->lc[pnum].last[ntok].token_str+1 >= name_len) return -1;
+            len += append_uint32_fixed(&name[len], v, ctx->lc[pnum].last[ntok].token_str);
+            //fprintf(stderr, "Tok %d DELTA0 %0*d\n", ntok, ctx->lc[pnum].last[ntok].token_str, v);
+            ctx->lc[cnum].last[ntok].token_type = N_DIGITS0;
+            ctx->lc[cnum].last[ntok].token_int = v;
+            ctx->lc[cnum].last[ntok].token_str = ctx->lc[pnum].last[ntok].token_str;
+            break;
+
+        case N_DIGITS: // [1-9][0-9]*
+            if (decode_token_int(ctx, ntok, N_DIGITS, &v) < 0) return -1;
+            if (len+20 >= name_len) return -1;
+            len += append_uint32_var(&name[len], v);
+            //fprintf(stderr, "Tok %d DIGITS %d\n", ntok, v);
+            ctx->lc[cnum].last[ntok].token_type = N_DIGITS;
+            ctx->lc[cnum].last[ntok].token_int = v;
+            break;
+
+        case N_DDELTA:
+            if (ntok >= ctx->lc[pnum].last_ntok) return -1;
+            if (decode_token_int1(ctx, ntok, N_DDELTA, &v) < 0) return -1;
+            v += ctx->lc[pnum].last[ntok].token_int;
+            if (len+20 >= name_len) return -1;
+            len += append_uint32_var(&name[len], v);
+            //fprintf(stderr, "Tok %d DELTA %d\n", ntok, v);
+            ctx->lc[cnum].last[ntok].token_type = N_DIGITS;
+            ctx->lc[cnum].last[ntok].token_int = v;
+            break;
+
+        case N_NOP:
+            ctx->lc[cnum].last[ntok].token_type = N_NOP;
+            break;
+
+        case N_MATCH:
+            if (ntok >= ctx->lc[pnum].last_ntok) return -1;
+            switch (ctx->lc[pnum].last[ntok].token_type) {
+            case N_CHAR:
+                if (len+1 >= name_len) return -1;
+                name[len++] = ctx->lc[pnum].last[ntok].token_int;
+                //fprintf(stderr, "Tok %d MATCH CHAR %c\n", ntok, ctx->lc[pnum].last[ntok].token_int);
+                ctx->lc[cnum].last[ntok].token_type = N_CHAR;
+                ctx->lc[cnum].last[ntok].token_int = ctx->lc[pnum].last[ntok].token_int;
+                break;
+
+            case N_ALPHA:
+                if (ctx->lc[pnum].last[ntok].token_int < 0 ||
+                    len+ctx->lc[pnum].last[ntok].token_int >= name_len) return -1;
+                memcpy(&name[len],
+                       &ctx->lc[pnum].last_name[ctx->lc[pnum].last[ntok].token_str],
+                       ctx->lc[pnum].last[ntok].token_int);
+                //fprintf(stderr, "Tok %d MATCH ALPHA %.*s\n", ntok, ctx->lc[pnum].last[ntok].token_int, &name[len]);
+                ctx->lc[cnum].last[ntok].token_type = N_ALPHA;
+                ctx->lc[cnum].last[ntok].token_str = len;
+                ctx->lc[cnum].last[ntok].token_int = ctx->lc[pnum].last[ntok].token_int;
+                len += ctx->lc[pnum].last[ntok].token_int;
+                break;
+
+            case N_DIGITS:
+                if (len+20 >= name_len) return -1;
+                len += append_uint32_var(&name[len], ctx->lc[pnum].last[ntok].token_int);
+                //fprintf(stderr, "Tok %d MATCH DIGITS %d\n", ntok, ctx->lc[pnum].last[ntok].token_int);
+                ctx->lc[cnum].last[ntok].token_type = N_DIGITS;
+                ctx->lc[cnum].last[ntok].token_int = ctx->lc[pnum].last[ntok].token_int;
+                break;
+
+            case N_DIGITS0:
+                if (len+ctx->lc[pnum].last[ntok].token_str >= name_len) return -1;
+                len += append_uint32_fixed(&name[len], ctx->lc[pnum].last[ntok].token_int, ctx->lc[pnum].last[ntok].token_str);
+                //fprintf(stderr, "Tok %d MATCH DIGITS %0*d\n", ntok, ctx->lc[pnum].last[ntok].token_str, ctx->lc[pnum].last[ntok].token_int);
+                ctx->lc[cnum].last[ntok].token_type = N_DIGITS0;
+                ctx->lc[cnum].last[ntok].token_int = ctx->lc[pnum].last[ntok].token_int;
+                ctx->lc[cnum].last[ntok].token_str = ctx->lc[pnum].last[ntok].token_str;
+                break;
+
+            default:
+                return -1;
+            }
+            break;
+
+        default: // an elided N_END
+        case N_END:
+            if (len+1 >= name_len) return -1;
+            name[len++] = 0;
+            ctx->lc[cnum].last[ntok].token_type = N_END;
+
+            ctx->lc[cnum].last_name = name;
+            ctx->lc[cnum].last_ntok = ntok;
+
+            last_context_tok *shrunk
+                = realloc(ctx->lc[cnum].last,
+                          (ntok+1) * sizeof(*ctx->lc[cnum].last));
+            if (shrunk)
+                ctx->lc[cnum].last = shrunk;
+
+            if (!ctx->lc[cnum].last)
+                return -1;
+
+            return len;
+        }
+    }
+
+
+    return -1;
+}
+
+//-----------------------------------------------------------------------------
+// arith adaptive codec or static rANS 4x16pr codec
+static int arith_encode(uint8_t *in, uint64_t in_len, uint8_t *out, uint64_t *out_len, int method) {
+    unsigned int olen = *out_len-6, nb;
+    if (arith_compress_to(in, in_len, out+6, &olen, method) == NULL)
+        return -1;
+
+    nb = var_put_u32(out, out + *out_len, olen);
+    memmove(out+nb, out+6, olen);
+    *out_len = olen+nb;
+
+    return 0;
+}
+
+// Returns number of bytes read from 'in' on success,
+//        -1 on failure.
+static int64_t arith_decode(uint8_t *in, uint64_t in_len, uint8_t *out, uint64_t *out_len) {
+    unsigned int olen = *out_len;
+
+    uint32_t clen;
+    int nb = var_get_u32(in, in+in_len, &clen);
+    //fprintf(stderr, "Arith decode %x\n", in[nb]);
+    if (arith_uncompress_to(in+nb, in_len-nb, out, &olen) == NULL)
+        return -1;
+    //fprintf(stderr, "    Stored clen=%d\n", (int)clen);
+    *out_len = olen;
+    return clen+nb;
+}
+
+static int rans_encode(uint8_t *in, uint64_t in_len, uint8_t *out, uint64_t *out_len, int method) {
+    unsigned int olen = *out_len-6, nb;
+    if (rans_compress_to_4x16(in, in_len, out+6, &olen, method) == NULL)
+        return -1;
+
+    nb = var_put_u32(out, out + *out_len, olen);
+    memmove(out+nb, out+6, olen);
+    *out_len = olen+nb;
+
+    return 0;
+}
+
+// Returns number of bytes read from 'in' on success,
+//        -1 on failure.
+static int64_t rans_decode(uint8_t *in, uint64_t in_len, uint8_t *out, uint64_t *out_len) {
+    unsigned int olen = *out_len;
+
+    uint32_t clen;
+    int nb = var_get_u32(in, in+in_len, &clen);
+    //fprintf(stderr, "Arith decode %x\n", in[nb]);
+    if (rans_uncompress_to_4x16(in+nb, in_len-nb, out, &olen) == NULL)
+        return -1;
+    //fprintf(stderr, "    Stored clen=%d\n", (int)clen);
+    *out_len = olen;
+    return clen+nb;
+}
+
+static int compress(uint8_t *in, uint64_t in_len, enum name_type type,
+                    int level, int use_arith,
+                    uint8_t *out, uint64_t *out_len) {
+    uint64_t best_sz = UINT64_MAX;
+    uint64_t olen = *out_len;
+    int ret = -1;
+
+    // Map levels 1-9 to 0-4, for parameter lookup in R[] below
+    level = (level-1)/2;
+    if (level<0) level=0;
+    if (level>4) level=4;
+
+    // rANS4x16pr and arith_dynamic parameters to explore.
+    // We brute force these, so fast levels test 1 setting and slow test more
+    int R[5][N_ALL][7] = {
+        {   // -1
+            /* TYPE     */ {1, 128},
+            /* ALPHA    */ {1, 129},
+            /* CHAR     */ {1, 0},
+            /* DIGITS0  */ {1, 8},
+            /* DZLEN    */ {1, 0},
+            /* DUP      */ {1, 8},
+            /* DIFF     */ {1, 8},
+            /* DIGITS   */ {1, 8},
+            /* DDELTA   */ {1, 0},
+            /* DDELTA0  */ {1, 128},
+            /* MATCH    */ {1, 0},
+            /* NOP      */ {1, 0},
+            /* END      */ {1, 0}
+        },
+
+        {   // -3
+            /* TYPE     */ {2, 192,0},
+            /* ALPHA    */ {2, 129,1},
+            /* CHAR     */ {1, 0},
+            /* DIGITS0  */ {2, 128+8,0}, // size%4==0
+            /* DZLEN    */ {1, 0},
+            /* DUP      */ {1, 192+8},   // size%4==0
+            /* DIFF     */ {1, 128+8},   // size%4==0
+            /* DIGITS   */ {1, 192+8},   // size%4==0
+            /* DDELTA   */ {1, 0},
+            /* DDELTA0  */ {1, 128},
+            /* MATCH    */ {1, 0},
+            /* NOP      */ {1, 0},
+            /* END      */ {1, 0}
+        },
+
+        {   // -5
+            /* TYPE     */ {2, 192,0},
+            /* ALPHA    */ {4, 1,128,0,129},
+            /* CHAR     */ {1, 0},
+            /* DIGITS0  */ {2, 200,0},
+            /* DZLEN    */ {1, 0},
+            /* DUP      */ {1, 200},
+            /* DIFF     */ {2, 192,200},
+            /* DIGITS   */ {2, 132,201},
+            /* DDELTA   */ {1, 0},
+            /* DDELTA0  */ {1, 128},
+            /* MATCH    */ {1, 0},
+            /* NOP      */ {1, 0},
+            /* END      */ {1, 0}
+        },
+
+        {   // -7
+            /* TYPE     */ {3, 193,0,1},
+            /* ALPHA    */ {5, 128, 1,128,0,129},
+            /* CHAR     */ {2, 1,0},
+            /* DIGITS0  */ {2, 200,0},    // or 201,0
+            /* DZLEN    */ {1, 0},
+            /* DUP      */ {1, 201},
+            /* DIFF     */ {2, 192,200},  // or 192,201
+            /* DIGITS   */ {2, 132, 201}, // +bz2 here and -9
+            /* DDELTA   */ {1, 0},
+            /* DDELTA0  */ {1, 128},
+            /* MATCH    */ {1, 0},
+            /* NOP      */ {1, 0},
+            /* END      */ {1, 0}
+        },
+
+        {   // -9
+            /* TYPE     */ {6, 192,0,1,  65,  193,132},
+            /* ALPHA    */ {4, 132, 1, 0,129},
+            /* CHAR     */ {3, 1,0,192},
+            /* DIGITS0  */ {4, 201,0, 192,64},
+            /* DZLEN    */ {3, 0,128,1},
+            /* DUP      */ {1, 201},
+            /* DIFF     */ {3, 192,  201,65},
+            /* DIGITS   */ {6, 132, 201,1,  192,129,  193},
+            /* DDELTA   */ {3, 1,0,  192},
+            /* DDELTA0  */ {3, 192,1,  0},
+            /* MATCH    */ {1, 0},
+            /* NOP      */ {1, 0},
+            /* END      */ {1, 0}
+        },
+    };
+    // Minor tweak to level 3 DIGITS if arithmetic, to use O(201) instead.
+    if (use_arith) R[1][N_DIGITS][1]=201;
+
+    int *meth = R[level][type];
+
+    int last = 0, m;
+    uint8_t best_static[8192];
+    uint8_t *best_dat = best_static;
+    for (m = 1; m <= meth[0]; m++) {
+        *out_len = olen;
+
+        if (!use_arith && (meth[m] & 4))
+            meth[m] &= ~4;
+
+        if (in_len % 4 != 0 && (meth[m] & 8))
+            continue;
+
+        last = 0;
+        if (use_arith) {
+            if (arith_encode(in, in_len, out, out_len, meth[m]) <0)
+                goto err;
+        } else {
+            if (rans_encode(in, in_len, out, out_len, meth[m]) < 0)
+                goto err;
+        }
+
+        if (best_sz > *out_len) {
+            best_sz = *out_len;
+            last = 1;
+
+            if (m+1 > meth[0])
+                // no need to memcpy if we're not going to overwrite out
+                break;
+
+            if (best_sz > 8192 && best_dat == best_static) {
+                // No need to realloc as best_sz only ever decreases
+                best_dat = malloc(best_sz);
+                if (!best_dat)
+                    return -1;
+            }
+            memcpy(best_dat, out, best_sz);
+        }
+    }
+
+    if (!last)
+        memcpy(out, best_dat, best_sz);
+    *out_len = best_sz;
+    ret = 0;
+
+ err:
+    if (best_dat != best_static)
+        free(best_dat);
+
+    return ret;
+}
+
+static uint64_t uncompressed_size(uint8_t *in, uint64_t in_len) {
+    uint32_t clen, ulen;
+
+    // in[0] in part of buffer written by us
+    int nb = var_get_u32(in, in+in_len, &clen);
+
+    // in[nb] is part of buffer written to by arith_dynamic.
+    var_get_u32(in+nb+1, in+in_len, &ulen);
+
+    return ulen;
+}
+
+static int uncompress(int use_arith, uint8_t *in, uint64_t in_len,
+                      uint8_t *out, uint64_t *out_len) {
+    uint32_t clen;
+    var_get_u32(in, in+in_len, &clen);
+    return use_arith
+        ? arith_decode(in, in_len, out, out_len)
+        : rans_decode(in, in_len, out, out_len);
+}
+
+//-----------------------------------------------------------------------------
+
+/*
+ * Converts a line or \0 separated block of reading names to a compressed buffer.
+ * The code can only encode whole lines and will not attempt a partial line.
+ * Use the "last_start_p" return value to identify the partial line start
+ * offset, for continuation purposes.
+ *
+ * Returns a malloced buffer holding compressed data of size *out_len,
+ *         or NULL on failure
+ */
+uint8_t *tok3_encode_names(char *blk, int len, int level, int use_arith,
+                           int *out_len, int *last_start_p) {
+    int last_start = 0, i, j, nreads;
+
+    if (len < 0) {
+        *out_len = 0;
+        return NULL;
+    }
+
+    // Count lines
+    for (nreads = i = 0; i < len; i++)
+        if (blk[i] <= '\n') // \n or \0 separated entries
+            nreads++;
+
+    name_context *ctx = create_context(nreads);
+    if (!ctx)
+        return NULL;
+
+    // Construct trie
+    int ctr = 0;
+    for (i = j = 0; i < len; j=++i) {
+        while (i < len && blk[i] > '\n')
+            i++;
+        if (i >= len)
+            break;
+
+        //blk[i] = '\0';
+        last_start = i+1;
+        if (build_trie(ctx, &blk[j], i-j, ctr++) < 0) {
+            free_context(ctx);
+            return NULL;
+        }
+    }
+    if (last_start_p)
+        *last_start_p = last_start;
+
+    //fprintf(stderr, "Processed %d of %d in block, line %d\n", last_start, len, ctr);
+
+    // Encode name
+    for (i = j = 0; i < len; j=++i) {
+        while (i < len && (signed char)blk[i] >= ' ') // non-ASCII check
+            i++;
+        if (i >= len)
+            break;
+
+        if (blk[i] != '\0' && blk[i] != '\n') {
+            // Names must be 7-bit ASCII printable
+            free_context(ctx);
+            return NULL;
+        }
+
+        blk[i] = '\0';
+        // try both 0 and 1 and pick best?
+        if (encode_name(ctx, &blk[j], i-j, 1) < 0) {
+            free_context(ctx);
+            return NULL;
+        }
+    }
+
+#if 0
+    for (i = 0; i < ctx->max_tok*16; i++) {
+        char fn[1024];
+        if (!ctx->desc[i].buf_l) continue;
+        sprintf(fn, "_tok.%02d_%02d.%d", i>>4,i&15,i);
+        FILE *fp = fopen(fn, "w");
+        fwrite(ctx->desc[i].buf, 1, ctx->desc[i].buf_l, fp);
+        fclose(fp);
+    }
+#endif
+
+    //dump_trie(t_head, 0);
+
+    // FIXME: merge descriptors
+    //
+    // If we see foo7:1 foo7:12 foo7:7 etc then foo: is constant,
+    // but it's encoded as alpha<foo>+dig<7>+char<:> instead of alpha<foo7:>.
+    // Any time token type 0 is all match beyond the first location we have
+    // a candidate for merging in string form.
+    //
+    // This saves around .1 to 1.3 percent on varying data sets.
+    // Cruder hack is dedicated prefix/suffix matching to short-cut this.
+
+
+    // Drop N_TYPE blocks if they all contain matches bar the first item,
+    // as we can regenerate these from the subsequent blocks types during
+    // decode.
+    for (i = 0; i < ctx->max_tok*16; i+=16) {
+        if (!ctx->desc[i].buf_l) continue;
+
+        int z;
+        for (z=1; z<ctx->desc[i].buf_l; z++) {
+            if (ctx->desc[i].buf[z] != N_MATCH)
+                break;
+        }
+        if (z == ctx->desc[i].buf_l) {
+            int k;
+            for (k=1; k<16; k++)
+                if (ctx->desc[i+k].buf_l)
+                    break;
+
+            if (k < 16) {
+                ctx->desc[i].buf_l = 0;
+                free(ctx->desc[i].buf);
+                ctx->desc[i].buf = NULL;
+            }
+        }
+    }
+
+    // Serialise descriptors
+    uint32_t tot_size = 9;
+    for (i = 0; i < ctx->max_tok*16; i++) {
+        if (!ctx->desc[i].buf_l) continue;
+
+        int tnum = i>>4;
+        int ttype = i&15;
+
+        uint64_t out_len = 1.5 * arith_compress_bound(ctx->desc[i].buf_l, 1); // guesswork
+        uint8_t *out = malloc(out_len);
+        if (!out) {
+            free_context(ctx);
+            return NULL;
+        }
+
+        if (compress(ctx->desc[i].buf, ctx->desc[i].buf_l, i&0xf, level,
+                     use_arith, out, &out_len) < 0) {
+            free_context(ctx);
+            return NULL;
+        }
+
+        free(ctx->desc[i].buf);
+        ctx->desc[i].buf = out;
+        ctx->desc[i].buf_l = out_len;
+        ctx->desc[i].tnum = tnum;
+        ctx->desc[i].ttype = ttype;
+
+        // Find dups
+        int j;
+        for (j = 0; j < i; j++) {
+            if (!ctx->desc[j].buf)
+                continue;
+            if (ctx->desc[i].buf_l != ctx->desc[j].buf_l || ctx->desc[i].buf_l <= 4)
+                continue;
+            if (memcmp(ctx->desc[i].buf, ctx->desc[j].buf, ctx->desc[i].buf_l) == 0)
+                break;
+        }
+        if (j < i) {
+            ctx->desc[i].dup_from = j;
+            tot_size += 3; // flag, dup_from, ttype
+        } else {
+            ctx->desc[i].dup_from = -1;
+            tot_size += out_len + 1; // ttype
+        }
+    }
+
+#if 0
+    for (i = 0; i < ctx->max_tok*16; i++) {
+        char fn[1024];
+        if (!ctx->desc[i].buf_l && ctx->desc[i].dup_from == -1) continue;
+        sprintf(fn, "_tok.%02d_%02d.%d.comp", i>>4,i&15,i);
+        FILE *fp = fopen(fn, "w");
+        fwrite(ctx->desc[i].buf, 1, ctx->desc[i].buf_l, fp);
+        fclose(fp);
+    }
+#endif
+
+    // Write
+    uint8_t *out = malloc(tot_size+13);
+    if (!out) {
+        free_context(ctx);
+        return NULL;
+    }
+
+    uint8_t *cp = out;
+
+    *out_len = tot_size;
+//    *(uint32_t *)cp = last_start; cp += 4;
+//    *(uint32_t *)cp = nreads;     cp += 4;
+    *cp++ = (last_start >>  0) & 0xff;
+    *cp++ = (last_start >>  8) & 0xff;
+    *cp++ = (last_start >> 16) & 0xff;
+    *cp++ = (last_start >> 24) & 0xff;
+    *cp++ = (nreads     >>  0) & 0xff;
+    *cp++ = (nreads     >>  8) & 0xff;
+    *cp++ = (nreads     >> 16) & 0xff;
+    *cp++ = (nreads     >> 24) & 0xff;
+    *cp++ = use_arith;
+    //write(1, &nreads, 4);
+    int last_tnum = -1;
+    for (i = 0; i < ctx->max_tok*16; i++) {
+        if (!ctx->desc[i].buf_l) continue;
+        uint8_t ttype8 = ctx->desc[i].ttype;
+        if (ctx->desc[i].tnum != last_tnum) {
+            ttype8 |= 128;
+            last_tnum = ctx->desc[i].tnum;
+        }
+        if (ctx->desc[i].dup_from >= 0) {
+            //fprintf(stderr, "Dup %d from %d, sz %d\n", i, ctx->desc[i].dup_from, ctx->desc[i].buf_l);
+            *cp++ = ttype8 | 64;
+            *cp++ = ctx->desc[i].dup_from >> 4;
+            *cp++ = ctx->desc[i].dup_from & 15;
+        } else {
+            *cp++ = ttype8;
+            memcpy(cp, ctx->desc[i].buf, ctx->desc[i].buf_l);
+            cp += ctx->desc[i].buf_l;
+        }
+    }
+
+    //assert(cp-out == tot_size);
+
+    free_context(ctx);
+
+    return out;
+}
+
+// Deprecated interface; to remove when we next to an ABI breakage
+uint8_t *encode_names(char *blk, int len, int level, int use_arith,
+                      int *out_len, int *last_start_p) {
+    return tok3_encode_names(blk, len, level, use_arith, out_len,
+                             last_start_p);
+}
+
+/*
+ * Decodes a compressed block of read names into \0 separated names.
+ * The size of the data returned (malloced) is in *out_len.
+ *
+ * Returns NULL on failure.
+ */
+uint8_t *tok3_decode_names(uint8_t *in, uint32_t sz, uint32_t *out_len) {
+    if (sz < 9)
+        return NULL;
+
+    int i, o = 9;
+    //int ulen   = *(uint32_t *)in;
+    int ulen   = (in[0]<<0) | (in[1]<<8) | (in[2]<<16) |
+        (((uint32_t)in[3])<<24);
+
+    if (ulen < 0 || ulen >= INT_MAX-1024)
+        return NULL;
+
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    // Speed up fuzzing by blocking excessive sizes
+    if (ulen > 100000)
+        return NULL;
+#endif
+
+    //int nreads = *(uint32_t *)(in+4);
+    int nreads = (in[4]<<0) | (in[5]<<8) | (in[6]<<16) | (((uint32_t)in[7])<<24);
+    int use_arith = in[8];
+    name_context *ctx = create_context(nreads);
+    if (!ctx)
+        return NULL;
+
+    // Unpack descriptors
+    int tnum = -1;
+    while (o < sz) {
+        uint8_t ttype = in[o++];
+        if (ttype & 64) {
+            if (o+2 > sz) goto err;
+            int j = in[o++]<<4;
+            j += in[o++];
+            if (ttype & 128) {
+                tnum++;
+                if (tnum >= MAX_TOKENS)
+                    goto err;
+                ctx->max_tok = tnum+1;
+                memset(&ctx->desc[tnum<<4], 0, 16*sizeof(ctx->desc[tnum]));
+            }
+
+            if ((ttype & 15) != 0 && (ttype & 128)) {
+                if (tnum < 0) goto err;
+                ctx->desc[tnum<<4].buf = malloc(nreads);
+                if (!ctx->desc[tnum<<4].buf)
+                    goto err;
+
+                ctx->desc[tnum<<4].buf_l = 0;
+                ctx->desc[tnum<<4].buf_a = nreads;
+                ctx->desc[tnum<<4].buf[0] = ttype&15;
+                memset(&ctx->desc[tnum<<4].buf[1], N_MATCH, nreads-1);
+            }
+
+            if (tnum < 0) goto err;
+            i = (tnum<<4) | (ttype&15);
+            if (j >= i)
+                goto err;
+            if (!ctx->desc[j].buf)
+                goto err; // Attempt to copy a non-existent stream
+
+            ctx->desc[i].buf_l = 0;
+            ctx->desc[i].buf_a = ctx->desc[j].buf_a;
+            if (ctx->desc[i].buf) free(ctx->desc[i].buf);
+            ctx->desc[i].buf = malloc(ctx->desc[i].buf_a);
+            if (!ctx->desc[i].buf)
+                goto err;
+
+            memcpy(ctx->desc[i].buf, ctx->desc[j].buf, ctx->desc[i].buf_a);
+            //fprintf(stderr, "Copy ttype %d, i=%d,j=%d, size %d\n", ttype, i, j, (int)ctx->desc[i].buf_a);
+            continue;
+        }
+
+        //if (ttype == 0)
+        if (ttype & 128) {
+            tnum++;
+            if (tnum >= MAX_TOKENS)
+                goto err;
+            ctx->max_tok = tnum+1;
+            memset(&ctx->desc[tnum<<4], 0, 16*sizeof(ctx->desc[tnum]));
+        }
+
+        if ((ttype & 15) != 0 && (ttype & 128)) {
+            if (tnum < 0) goto err;
+            if (ctx->desc[tnum<<4].buf) free(ctx->desc[tnum<<4].buf);
+            ctx->desc[tnum<<4].buf = malloc(nreads);
+            if (!ctx->desc[tnum<<4].buf)
+                goto err;
+            ctx->desc[tnum<<4].buf_l = 0;
+            ctx->desc[tnum<<4].buf_a = nreads;
+            ctx->desc[tnum<<4].buf[0] = ttype&15;
+            memset(&ctx->desc[tnum<<4].buf[1], N_MATCH, nreads-1);
+        }
+
+        //fprintf(stderr, "Read %02x\n", c);
+
+        // Load compressed block
+        int64_t clen, ulen = uncompressed_size(&in[o], sz-o);
+        if (ulen < 0 || ulen >= INT_MAX)
+            goto err;
+        if (tnum < 0) goto err;
+        i = (tnum<<4) | (ttype&15);
+
+        if (i >= MAX_TBLOCKS || i < 0)
+            goto err;
+
+        ctx->desc[i].buf_l = 0;
+        if (ctx->desc[i].buf) free(ctx->desc[i].buf);
+        ctx->desc[i].buf = malloc(ulen);
+        if (!ctx->desc[i].buf)
+            goto err;
+
+        ctx->desc[i].buf_a = ulen;
+        uint64_t usz = ctx->desc[i].buf_a; // convert from size_t for 32-bit sys
+        clen = uncompress(use_arith, &in[o], sz-o, ctx->desc[i].buf, &usz);
+        ctx->desc[i].buf_a = usz;
+        if (clen < 0 || ctx->desc[i].buf_a != ulen)
+            goto err;
+
+        // fprintf(stderr, "%d: Decode tnum %d type %d clen %d ulen %d via %d\n",
+        //      o, tnum, ttype, (int)clen, (int)ctx->desc[i].buf_a, ctx->desc[i].buf[0]);
+
+        o += clen;
+
+        // Encode tnum 0 type 0 ulen 100000 clen 12530 via 2
+        // Encode tnum 0 type 6 ulen 196800 clen 43928 via 3
+        // Encode tnum 0 type 7 ulen 203200 clen 17531 via 3
+        // Encode tnum 1 type 0 ulen 50800 clen 10 via 1
+        // Encode tnum 1 type 1 ulen 3 clen 5 via 0
+        // Encode tnum 2 type 0 ulen 50800 clen 10 via 1
+        //      
+    }
+
+    int ret;
+    ulen += 1024; // for easy coding in decode_name.
+    uint8_t *out = malloc(ulen);
+    if (!out)
+        goto err;
+
+    size_t out_sz = 0;
+    while ((ret = decode_name(ctx, (char *)out+out_sz, ulen)) > 0) {
+        out_sz += ret;
+        ulen -= ret;
+    }
+
+    if (ret < 0)
+        free(out);
+
+    free_context(ctx);
+
+    *out_len = out_sz;
+    return ret == 0 ? out : NULL;
+
+ err:
+    free_context(ctx);
+    return NULL;
+}
+
+// Deprecated interface; to remove when we next to an ABI breakage
+uint8_t *decode_names(uint8_t *in, uint32_t sz, uint32_t *out_len) {
+    return tok3_decode_names(in, sz, out_len);
+}
diff --git a/ext/htslib/htscodecs/htscodecs/tokenise_name3.h b/ext/htslib/htscodecs/htscodecs/tokenise_name3.h
new file mode 100644
index 0000000..ef341df
--- /dev/null
+++ b/ext/htslib/htscodecs/htscodecs/tokenise_name3.h
@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2017, 2019 Genome Research Ltd.
+ * Author(s): James Bonfield
+ * 
+ * Redistribution and use in source and binary forms, with or without 
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ *    1. Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimer.
+ * 
+ *    2. Redistributions in binary form must reproduce the above
+ *       copyright notice, this list of conditions and the following
+ *       disclaimer in the documentation and/or other materials provided
+ *       with the distribution.
+ * 
+ *    3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+ *    Institute nor the names of its contributors may be used to endorse
+ *    or promote products derived from this software without specific
+ *    prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH
+ * LTD OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _TOKENISE_NAME3_H_
+#define _TOKENISE_NAME3_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Converts a line or \0 separated block of reading names to a compressed buffer.
+ * The code can only encode whole lines and will not attempt a partial line.
+ * Use the "last_start_p" return value to identify the partial line start
+ * offset, for continuation purposes.
+ *
+ * Returns a malloced buffer holding compressed data of size *out_len,
+ *         or NULL on failure
+ */
+uint8_t *tok3_encode_names(char *blk, int len, int level, int use_arith,
+                           int *out_len, int *last_start_p);
+
+/*
+ * Decodes a compressed block of read names into \0 separated names.
+ * The size of the data returned (malloced) is in *out_len.
+ *
+ * Returns NULL on failure.
+ */
+uint8_t *tok3_decode_names(uint8_t *in, uint32_t sz, uint32_t *out_len);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _TOKENISE_NAME3_H_ */
diff --git a/ext/htslib/htscodecs/htscodecs/utils.c b/ext/htslib/htscodecs/htscodecs/utils.c
new file mode 100644
index 0000000..399b055
--- /dev/null
+++ b/ext/htslib/htscodecs/htscodecs/utils.c
@@ -0,0 +1,234 @@
+/*
+ * Copyright (c) 2022 Genome Research Ltd.
+ * Author(s): James Bonfield
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above
+ *       copyright notice, this list of conditions and the following
+ *       disclaimer in the documentation and/or other materials provided
+ *       with the distribution.
+ *
+ *    3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+ *       Institute nor the names of its contributors may be used to endorse
+ *       or promote products derived from this software without specific
+ *       prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH
+ * LTD OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <inttypes.h>
+
+#include "utils.h"
+
+#ifndef NO_THREADS
+#include <pthread.h>
+#endif
+
+//#define TLS_DEBUG
+
+#ifndef NO_THREADS
+/*
+ * Thread local storage per thread in the pool.
+ *
+ * We have some large memory blocks for rANS which we cannot store on the
+ * stack due to various system limitations.  Allocaitng them can be
+ * expensive as some OSes use mmap and will pass the pages back to the OS
+ * on each free.  This unfortunately then means zeroing the pages out again
+ * on each new malloc, plus additional switching into the kernel.
+ *
+ * Instead where available, we use pthread_once to allocate a small arena
+ * of memory buffers and we continually reuse these same buffers.  We don't
+ * need to memset it (calloc equivalent) either as we're sure that any
+ * leakage of data is simply an earlier set of precomputed frequency
+ * lookups, and not something more sinister such as an encryption key.
+ *
+ * If we don't have pthreads, then we have to fall back to the slow
+ * traditional calloc instead.
+ */
+
+#define MAX_TLS_BUFS 10
+typedef struct {
+    void   *bufs[MAX_TLS_BUFS];
+    size_t sizes[MAX_TLS_BUFS];
+    int     used[MAX_TLS_BUFS];
+} tls_pool;
+
+static pthread_once_t rans_once = PTHREAD_ONCE_INIT;
+static pthread_key_t rans_key;
+
+/*
+ * Frees all local storage for this thread.
+ * Note: this isn't a function to free a specific allocated item.
+ */
+static void htscodecs_tls_free_all(void *ptr) {
+    tls_pool *tls = (tls_pool *)ptr;
+    if (!tls)
+        return;
+
+    int i;
+    for (i = 0; i < MAX_TLS_BUFS; i++) {
+#ifdef TLS_DEBUG
+        if (tls->bufs[i])
+            fprintf(stderr, "Free %ld = %p\n", tls->sizes[i], tls->bufs[i]);
+#endif
+        if (tls->used[i]) {
+            fprintf(stderr, "Closing thread while TLS data is in use\n");
+        }
+        free(tls->bufs[i]);
+    }
+
+    free(tls);
+}
+
+static void htscodecs_tls_init(void) {
+    pthread_key_create(&rans_key, htscodecs_tls_free_all);
+}
+
+/*
+ * Allocates size bytes from the global Thread Local Storage pool.
+ * This is shared by all subsequent calls within this thread.
+ *
+ * An simpler alternative could be possible where we have a fixed number
+ * of types of alloc, say 5, and specify the correct label when allocating.
+ * Eg histogram, name_context, fqzcomp, rans.  We can have multiple types
+ * in use in different stack frames (such name_context + hist + rans), but
+ * the number is very limited.  That then paves the way to simply check and
+ * realloc without needing to keep track of use status or overflowing
+ * the maximum number permitted.
+ */
+void *htscodecs_tls_alloc(size_t size) {
+    int i;
+
+    int err = pthread_once(&rans_once, htscodecs_tls_init);
+    if (err != 0) {
+        fprintf(stderr, "Initialising TLS data failed: pthread_once: %s\n",
+                strerror(err));
+        return NULL;
+    }
+
+    // Initialise tls_pool on first usage
+    tls_pool *tls = pthread_getspecific(rans_key);
+    if (!tls) {
+        if (!(tls = calloc(1, sizeof(*tls))))
+            return NULL;
+        pthread_setspecific(rans_key, tls);
+    }
+
+    // Query pool for size
+    int avail = -1;
+    for (i = 0; i < MAX_TLS_BUFS; i++) {
+        if (!tls->used[i]) {
+            if (size <= tls->sizes[i]) {
+                tls->used[i] = 1;
+#ifdef TLS_DEBUG
+                fprintf(stderr, "Reuse %d: %ld/%ld = %p\n",
+                        i, size, tls->sizes[i], tls->bufs[i]);
+#endif
+                return tls->bufs[i];
+            } else if (avail == -1) {
+                avail = i;
+            }
+        }
+    }
+
+    if (i == MAX_TLS_BUFS && avail == -1) {
+        // Shouldn't happen given our very limited use of this function
+        fprintf(stderr, "Error: out of rans_tls_alloc slots\n");
+        return NULL;
+    }
+
+    if (tls->bufs[avail])
+        free(tls->bufs[avail]);
+    if (!(tls->bufs[avail] = calloc(1, size)))
+        return NULL;
+#ifdef TLS_DEBUG
+    fprintf(stderr, "Alloc %d: %ld = %p\n", avail, size, tls->bufs[avail]);
+#endif
+    tls->sizes[avail] = size;
+    tls->used[avail] = 1;
+
+    return tls->bufs[avail];
+}
+
+void *htscodecs_tls_calloc(size_t nmemb, size_t size) {
+#ifdef TLS_DEBUG
+    fprintf(stderr, "htscodecs_tls_calloc(%ld)\n", nmemb*size);
+#endif
+    void *ptr = htscodecs_tls_alloc(nmemb * size);
+    if (ptr)
+        memset(ptr, 0, nmemb * size);
+    return ptr;
+}
+
+void htscodecs_tls_free(void *ptr) {
+    if (!ptr)
+        return;
+
+    tls_pool *tls = pthread_getspecific(rans_key);
+
+    int i;
+    for (i = 0; i < MAX_TLS_BUFS; i++) {
+        if (tls->bufs[i] == ptr)
+            break;
+    }
+#ifdef TLS_DEBUG
+    fprintf(stderr, "Fake free %d size %ld ptr %p\n",
+            i, tls->sizes[i], tls->bufs[i]);
+#endif
+    if (i == MAX_TLS_BUFS) {
+        fprintf(stderr, "Attempt to htscodecs_tls_free a buffer not allocated"
+                " with htscodecs_tls_alloc\n");
+        return;
+    }
+    if (!tls->used[i]) {
+        fprintf(stderr, "Attempt to htscodecs_tls_free a buffer twice\n");
+        return;
+    }
+    tls->used[i] = 0;
+}
+
+#else
+/*
+ * Calloc/free equivalents instead.
+ *
+ * We use calloc instead of malloc as a sufficiently malformed set of input
+ * frequencies may not sum to the expected total frequency size, leaving
+ * some elements uninitialised.  It's unlikely, but potentially a crafty
+ * attacker could somehow exploit this to pull out parts of this allocated
+ * buffer and leak them into the decompressed data stream, potentially
+ * compromising previous buffers such as encryption keys.  (Although
+ * frankly any well-written crypto library should be zeroing such memory
+ * before freeing it to ensure it's never visible to a subsequent malloc.)
+ */
+void *htscodecs_tls_alloc(size_t size) {
+    return calloc(1, size);
+}
+
+void *htscodecs_tls_calloc(size_t nmemb, size_t size) {
+    return calloc(nmemb, size);
+}
+
+void htscodecs_tls_free(void *ptr) {
+    free(ptr);
+}
+#endif
diff --git a/ext/htslib/htscodecs/htscodecs/utils.h b/ext/htslib/htscodecs/htscodecs/utils.h
new file mode 100644
index 0000000..ae3a3dd
--- /dev/null
+++ b/ext/htslib/htscodecs/htscodecs/utils.h
@@ -0,0 +1,412 @@
+/*
+ * Copyright (c) 2019-2022 Genome Research Ltd.
+ * Author(s): James Bonfield
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above
+ *       copyright notice, this list of conditions and the following
+ *       disclaimer in the documentation and/or other materials provided
+ *       with the distribution.
+ *
+ *    3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+ *       Institute nor the names of its contributors may be used to endorse
+ *       or promote products derived from this software without specific
+ *       prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH
+ * LTD OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef RANS_UTILS_H
+#define RANS_UTILS_H
+
+#include <string.h>
+#include <stdlib.h>
+#include <math.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+#  if !defined(__clang__) && __GNUC__ >= 100
+     // better still on gcc10 for O1 decode of old rans 4x8
+     // gcc 10=246/205 11=243/205 12=230/197
+#    define likely(x)      __builtin_expect_with_probability((x), 1, 0.99)
+#  else
+     // gcc 10=193/168 11=195/161 12=199/176
+#    define likely(x)      __builtin_expect((x), 1)
+#  endif
+#  define unlikely(x)     __builtin_expect((x), 0)
+#else
+#  define likely(x)   (x)
+#  define unlikely(x) (x)
+#endif
+
+/*
+ * Allocates size bytes from the global Thread Local Storage pool.
+ * This is shared by all subsequent calls within this thread.
+ *
+ * Note this is NOT a general purpose allocator and usage outside of this
+ * library is not advised due to assumptions and limitations in the design.
+ */
+void *htscodecs_tls_alloc(size_t size);
+void *htscodecs_tls_calloc(size_t nmemb, size_t size);
+void  htscodecs_tls_free(void *ptr);
+
+
+/* Fast approximate log base 2 */
+static inline double fast_log(double a) {
+  union { double d; long long x; } u = { a };
+  return (u.x - 4606921278410026770) * 1.539095918623324e-16; /* 1 / 6497320848556798.0; */
+}
+
+/*
+ * Data transpose by N.  Common to rANS4x16 and arith_dynamic decoders.
+ *
+ * Tuned for specific common cases of N.
+ */
+static inline void unstripe(unsigned char *out, unsigned char *outN,
+                            unsigned int ulen, unsigned int N,
+                            unsigned int idxN[256]) {
+    int j = 0, k;
+
+    if (ulen >= N) {
+        switch (N) {
+        case 4:
+#define LLN 16
+            if (ulen >= 4*LLN) {
+                while (j < ulen-4*LLN) {
+                    int l;
+                    for (l = 0; l < LLN; l++) {
+                        for (k = 0; k < 4; k++)
+                            out[j+k+l*4] = outN[idxN[k]+l];
+                    }
+                    for (k = 0; k < 4; k++)
+                        idxN[k] += LLN;
+                    j += 4*LLN;
+                }
+            }
+            while (j < ulen-4) {
+                for (k = 0; k < 4; k++)
+                    out[j++] = outN[idxN[k]++];
+            }
+#undef LLN
+            break;
+
+        case 2:
+#define LLN 4
+            if (ulen >= 2*LLN) {
+                while (j < ulen-2*LLN) {
+                    int l;
+                    for (l = 0; l < LLN; l++) {
+                        for (k = 0; k < 2; k++)
+                            out[j++] = outN[idxN[k]+l];
+                    }
+                    for (k = 0; k < 2; k++)
+                        idxN[k] += l;
+                }
+            }
+            while (j < ulen-2) {
+                for (k = 0; k < 2; k++)
+                    out[j++] = outN[idxN[k]++];
+            }
+#undef LLN
+            break;
+
+        default:
+            // General case, around 25% slower overall decode
+            while (j < ulen-N) {
+                for (k = 0; k < N; k++)
+                    out[j++] = outN[idxN[k]++];
+            }
+            break;
+        }
+    }
+    for (k = 0; j < ulen; k++)
+        out[j++] = outN[idxN[k]++];
+}
+
+#define MAGIC 8
+
+/*
+ * Order 0 histogram construction.  8-way unrolled to avoid cache collisions.
+ */
+static inline
+int hist8(unsigned char *in, unsigned int in_size, uint32_t F0[256]) {
+    if (in_size > 500000) {
+        uint32_t *f0 = htscodecs_tls_calloc((65536+37)*3, sizeof(*f0));
+        if (f0 == NULL)
+            return -1;
+        uint32_t *f1 = f0 + 65536+37;
+        uint32_t *f2 = f1 + 65536+37;
+
+        uint32_t i, i8 = in_size & ~15;
+
+        for (i = 0; i < i8; i+=16) {
+            uint16_t i16a[4], i16b[4];
+            memcpy(i16a, in+i, 8);
+            f0[i16a[0]]++;
+            f1[i16a[1]]++;
+            f2[i16a[2]]++;
+            f0[i16a[3]]++;
+
+            memcpy(i16b, in+i+8, 8);
+            f1[i16b[0]]++;
+            f0[i16b[1]]++;
+            f1[i16b[2]]++;
+            f2[i16b[3]]++;
+        }
+
+        while (i < in_size)
+            F0[in[i++]]++;
+
+        for (i = 0; i < 65536; i++) {
+            F0[i & 0xff] += f0[i] + f1[i] + f2[i];
+            F0[i >> 8  ] += f0[i] + f1[i] + f2[i];
+        }
+        htscodecs_tls_free(f0);
+    } else {
+        uint32_t F1[256+MAGIC] = {0}, F2[256+MAGIC] = {0}, F3[256+MAGIC] = {0};
+        uint32_t i, i8 = in_size & ~7;
+
+        for (i = 0; i < i8; i+=8) {
+            F0[in[i+0]]++;
+            F1[in[i+1]]++;
+            F2[in[i+2]]++;
+            F3[in[i+3]]++;
+            F0[in[i+4]]++;
+            F1[in[i+5]]++;
+            F2[in[i+6]]++;
+            F3[in[i+7]]++;
+        }
+
+        while (i < in_size)
+            F0[in[i++]]++;
+
+        for (i = 0; i < 256; i++)
+            F0[i] += F1[i] + F2[i] + F3[i];
+    }
+
+    return 0;
+}
+
+// Hist8 with a crude entropy (bits / byte) estimator.
+static inline
+double hist8e(unsigned char *in, unsigned int in_size, uint32_t F0[256]) {
+    uint32_t F1[256+MAGIC] = {0}, F2[256+MAGIC] = {0}, F3[256+MAGIC] = {0};
+    uint32_t F4[256+MAGIC] = {0}, F5[256+MAGIC] = {0}, F6[256+MAGIC] = {0};
+    uint32_t F7[256+MAGIC] = {0};
+
+#ifdef __GNUC__
+    double e = 0, in_size_r2 = log(1.0/in_size)/log(2);
+#else
+    double e = 0, in_size_r2 = log(1.0/in_size);
+#endif
+
+    unsigned int i, i8 = in_size & ~7;
+    for (i = 0; i < i8; i+=8) {
+        F0[in[i+0]]++;
+        F1[in[i+1]]++;
+        F2[in[i+2]]++;
+        F3[in[i+3]]++;
+        F4[in[i+4]]++;
+        F5[in[i+5]]++;
+        F6[in[i+6]]++;
+        F7[in[i+7]]++;
+    }
+    while (i < in_size)
+        F0[in[i++]]++;
+
+    for (i = 0; i < 256; i++) {
+        F0[i] += F1[i] + F2[i] + F3[i] + F4[i] + F5[i] + F6[i] + F7[i];
+#ifdef __GNUC__
+        e -= F0[i] * (32 - __builtin_clz(F0[i]|1) + in_size_r2);
+#else
+        e -= F0[i] * (fast_log(F0[i]) + in_size_r2);
+#endif
+    }
+
+#ifndef __GNUC__
+    e /= log(2);
+#endif
+    return e/in_size;
+}
+
+/*
+ * A variant of hist8 that simply marks the presence of a symbol rather
+ * than its frequency.
+ */
+static inline
+void present8(unsigned char *in, unsigned int in_size,
+              uint32_t F0[256]) {
+    uint32_t F1[256+MAGIC] = {0}, F2[256+MAGIC] = {0}, F3[256+MAGIC] = {0};
+    uint32_t F4[256+MAGIC] = {0}, F5[256+MAGIC] = {0}, F6[256+MAGIC] = {0};
+    uint32_t F7[256+MAGIC] = {0};
+
+    unsigned int i, i8 = in_size & ~7;
+    for (i = 0; i < i8; i+=8) {
+        F0[in[i+0]]=1;
+        F1[in[i+1]]=1;
+        F2[in[i+2]]=1;
+        F3[in[i+3]]=1;
+        F4[in[i+4]]=1;
+        F5[in[i+5]]=1;
+        F6[in[i+6]]=1;
+        F7[in[i+7]]=1;
+    }
+    while (i < in_size)
+        F0[in[i++]]=1;
+
+    for (i = 0; i < 256; i++)
+        F0[i] += F1[i] + F2[i] + F3[i] + F4[i] + F5[i] + F6[i] + F7[i];
+}
+
+/*
+ * Order 1 histogram construction.  4-way unrolled to avoid cache collisions.
+ */
+#if 1
+static inline
+int hist1_4(unsigned char *in, unsigned int in_size,
+            uint32_t F0[256][256], uint32_t *T0) {
+    unsigned char l = 0, c;
+    unsigned char *in_end = in + in_size;
+
+    unsigned char cc[5] = {0};
+    if (in_size > 500000) {
+        uint32_t (*F1)[259] = htscodecs_tls_calloc(256, sizeof(*F1));
+        if (F1 == NULL)
+            return -1;
+        while (in < in_end-8) {
+            memcpy(cc, in, 4); in += 4;
+            F0[cc[4]][cc[0]]++;
+            F1[cc[0]][cc[1]]++;
+            F0[cc[1]][cc[2]]++;
+            F1[cc[2]][cc[3]]++;
+            cc[4] = cc[3];
+
+            memcpy(cc, in, 4); in += 4;
+            F0[cc[4]][cc[0]]++;
+            F1[cc[0]][cc[1]]++;
+            F0[cc[1]][cc[2]]++;
+            F1[cc[2]][cc[3]]++;
+            cc[4] = cc[3];
+        }
+        l = cc[3];
+
+        while (in < in_end) {
+            F0[l][c = *in++]++;
+            l = c;
+        }
+        T0[l]++;
+
+        int i, j;
+        for (i = 0; i < 256; i++) {
+            int tt = 0;
+            for (j = 0; j < 256; j++) {
+                F0[i][j] += F1[i][j];
+                tt += F0[i][j];
+            }
+            T0[i]+=tt;
+        }
+        htscodecs_tls_free(F1);
+    } else {
+        while (in < in_end-8) {
+            memcpy(cc, in, 4); in += 4;
+            F0[cc[4]][cc[0]]++;
+            F0[cc[0]][cc[1]]++;
+            F0[cc[1]][cc[2]]++;
+            F0[cc[2]][cc[3]]++;
+            cc[4] = cc[3];
+
+            memcpy(cc, in, 4); in += 4;
+            F0[cc[4]][cc[0]]++;
+            F0[cc[0]][cc[1]]++;
+            F0[cc[1]][cc[2]]++;
+            F0[cc[2]][cc[3]]++;
+            cc[4] = cc[3];
+        }
+        l = cc[3];
+
+        while (in < in_end) {
+            F0[l][c = *in++]++;
+            l = c;
+        }
+        T0[l]++;
+
+        int i, j;
+        for (i = 0; i < 256; i++) {
+            int tt = 0;
+            for (j = 0; j < 256; j++)
+                tt += F0[i][j];
+            T0[i]+=tt;
+        }
+    }
+
+    return 0;
+}
+
+#else
+// 16 bit mode, similar to O0 freq.
+// This is better on some low entropy data, but generally we prefer to do
+// bit-packing and/or RLE to turn it into higher-entropy data first.
+//
+// Kept here for posterity incase we need it again, as it's quick tricky.
+static inline
+int hist1_4(unsigned char *in, unsigned int in_size,
+            uint32_t F0[256][256], uint32_t *T0) {
+    uint32_t f0[65536+MAGIC] = {0};
+    uint32_t f1[65536+MAGIC] = {0};
+
+    uint32_t i, i8 = (in_size-1) & ~15;
+
+    T0[0]++; f0[in[0]<<8]++;
+    for (i = 0; i < i8; i+=16) {
+        uint16_t i16a[16];
+        memcpy(i16a,   in+i,   16); // faster in 2 as gcc recognises this
+        memcpy(i16a+8, in+i+1, 16); // faster in 2 as gcc recognises this
+
+        f0[i16a[0]]++;
+        f1[i16a[1]]++;
+        f0[i16a[2]]++;
+        f1[i16a[3]]++;
+        f0[i16a[4]]++;
+        f1[i16a[5]]++;
+        f0[i16a[6]]++;
+        f1[i16a[7]]++;
+        f0[i16a[8]]++;
+        f1[i16a[9]]++;
+        f0[i16a[10]]++;
+        f1[i16a[11]]++;
+        f0[i16a[12]]++;
+        f1[i16a[13]]++;
+        f0[i16a[14]]++;
+        f1[i16a[15]]++;
+    }
+
+    while (i < in_size-1) {
+        F0[in[i]][in[i+1]]++;
+        T0[in[i+1]]++;
+        i++;
+    }
+
+    for (i = 0; i < 65536; i++) {
+        F0[i&0xff][i>>8] += f0[i] + f1[i];
+        T0[i>>8]         += f0[i] + f1[i];
+    }
+
+    return 0;
+}
+#endif
+
+#endif /* RANS_UTILS_H */
diff --git a/ext/htslib/htscodecs/htscodecs/varint.h b/ext/htslib/htscodecs/htscodecs/varint.h
new file mode 100644
index 0000000..c4a5168
--- /dev/null
+++ b/ext/htslib/htscodecs/htscodecs/varint.h
@@ -0,0 +1,446 @@
+// FIXME: make get functions const uint8_t *
+
+/*
+ * Copyright (c) 2019-2021 Genome Research Ltd.
+ * Author(s): James Bonfield
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above
+ *       copyright notice, this list of conditions and the following
+ *       disclaimer in the documentation and/or other materials provided
+ *       with the distribution.
+ *
+ *    3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+ *       Institute nor the names of its contributors may be used to endorse
+ *       or promote products derived from this software without specific
+ *       prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH
+ * LTD OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef VARINT_H
+#define VARINT_H
+
+#include <stdint.h>
+
+#ifdef VARINT2
+#include "varint2.h"
+#else
+
+// General API scheme is var_{get,put}_{s,u}{32,64}
+// s/u for signed/unsigned;  32/64 for integer size.
+
+// FIXME: consider returning the value and having nbytes passed in by
+// reference instead of vice-versa.
+//
+// ie uint64_t var_get_u64(uint8_t *cp, int *nbytes)
+// vs int      var_get_u64(uint8_t *cp, uint64_t *val)
+//
+// The return value can then be assigned to 32-bit or 64-bit type
+// without need of a new function name.  The cost is we can't then
+// do "cp += var_get_u32(cp, endp, &u_freq_sz);".  Maybe we can't do
+// overflow detection with former? (Want 32-bit but got, say, 40 bit)
+
+
+// Big endian.
+// Harder for encoding, but a simpler and faster decoder.
+#define BIG_END
+#ifdef BIG_END
+
+static inline
+int var_put_u64_safe(uint8_t *cp, const uint8_t *endp, uint64_t i) {
+    uint8_t *op = cp;
+    int s = 0;
+    uint64_t X = i;
+
+    // safe method when we're near end of buffer
+    do {
+        s += 7;
+        X >>= 7;
+    } while (X);
+
+    if (endp && (endp-cp)*7 < s)
+        return 0;
+
+    int n;
+    for (n = 0; n < 10; n++) {
+        s -= 7;
+        *cp++ = ((i>>s) & 0x7f) + (s?128:0);
+        if (!s)
+            break;
+    }
+
+    return cp-op;
+}
+
+// This can be optimised further with __builtin_clzl(i) and goto various
+// bits of the if/else-if structure, but it's not a vast improvement and
+// we are dominated by small values.  Simplicity wins for now
+static inline
+int var_put_u64(uint8_t *cp, const uint8_t *endp, uint64_t i) {
+    if (endp && (endp-cp) < 10)
+        return var_put_u64_safe(cp, endp, i);
+
+    // maximum of 10 bytes written
+    if (i < (1<<7)) {
+        *cp = i;
+        return 1;
+    } else if (i < (1<<14)) {
+        *cp++ = ((i>> 7) & 0x7f) | 128;
+        *cp++ =   i      & 0x7f;
+        return 2;
+    } else if (i < (1<<21)) {
+        *cp++ = ((i>>14) & 0x7f) | 128;
+        *cp++ = ((i>> 7) & 0x7f) | 128;
+        *cp++ =   i      & 0x7f;
+        return 3;
+    } else if (i < (1<<28)) {
+        *cp++ = ((i>>21) & 0x7f) | 128;
+        *cp++ = ((i>>14) & 0x7f) | 128;
+        *cp++ = ((i>> 7) & 0x7f) | 128;
+        *cp++ =   i      & 0x7f;
+        return 4;
+    } else if (i < (1ULL<<35)) {
+        *cp++ = ((i>>28) & 0x7f) | 128;
+        *cp++ = ((i>>21) & 0x7f) | 128;
+        *cp++ = ((i>>14) & 0x7f) | 128;
+        *cp++ = ((i>> 7) & 0x7f) | 128;
+        *cp++ =   i      & 0x7f;
+        return 5;
+    } else if (i < (1ULL<<42)) {
+        *cp++ = ((i>>35) & 0x7f) | 128;
+        *cp++ = ((i>>28) & 0x7f) | 128;
+        *cp++ = ((i>>21) & 0x7f) | 128;
+        *cp++ = ((i>>14) & 0x7f) | 128;
+        *cp++ = ((i>> 7) & 0x7f) | 128;
+        *cp++ =   i      & 0x7f;
+        return 6;
+    } else if (i < (1ULL<<49)) {
+        *cp++ = ((i>>42) & 0x7f) | 128;
+        *cp++ = ((i>>35) & 0x7f) | 128;
+        *cp++ = ((i>>28) & 0x7f) | 128;
+        *cp++ = ((i>>21) & 0x7f) | 128;
+        *cp++ = ((i>>14) & 0x7f) | 128;
+        *cp++ = ((i>> 7) & 0x7f) | 128;
+        *cp++ =   i      & 0x7f;
+        return 7;
+    } else if (i < (1ULL<<56)) {
+        *cp++ = ((i>>49) & 0x7f) | 128;
+        *cp++ = ((i>>42) & 0x7f) | 128;
+        *cp++ = ((i>>35) & 0x7f) | 128;
+        *cp++ = ((i>>28) & 0x7f) | 128;
+        *cp++ = ((i>>21) & 0x7f) | 128;
+        *cp++ = ((i>>14) & 0x7f) | 128;
+        *cp++ = ((i>> 7) & 0x7f) | 128;
+        *cp++ =   i      & 0x7f;
+        return 8;
+    } else if (i < (1ULL<<63)) {
+        *cp++ = ((i>>56) & 0x7f) | 128;
+        *cp++ = ((i>>49) & 0x7f) | 128;
+        *cp++ = ((i>>42) & 0x7f) | 128;
+        *cp++ = ((i>>35) & 0x7f) | 128;
+        *cp++ = ((i>>28) & 0x7f) | 128;
+        *cp++ = ((i>>21) & 0x7f) | 128;
+        *cp++ = ((i>>14) & 0x7f) | 128;
+        *cp++ = ((i>> 7) & 0x7f) | 128;
+        *cp++ =   i      & 0x7f;
+        return 9;
+    } else {
+        *cp++ = ((i>>63) & 0x7f) | 128;
+        *cp++ = ((i>>56) & 0x7f) | 128;
+        *cp++ = ((i>>49) & 0x7f) | 128;
+        *cp++ = ((i>>42) & 0x7f) | 128;
+        *cp++ = ((i>>35) & 0x7f) | 128;
+        *cp++ = ((i>>28) & 0x7f) | 128;
+        *cp++ = ((i>>21) & 0x7f) | 128;
+        *cp++ = ((i>>14) & 0x7f) | 128;
+        *cp++ = ((i>> 7) & 0x7f) | 128;
+        *cp++ =   i      & 0x7f;
+    }
+
+    return 10;
+}
+
+static inline
+int var_put_u32_safe(uint8_t *cp, const uint8_t *endp, uint32_t i) {
+    uint8_t *op = cp;
+    int s = 0;
+    uint32_t X = i;
+
+    // safe method when we're near end of buffer
+    do {
+        s += 7;
+        X >>= 7;
+    } while (X);
+
+    if (endp && (endp-cp)*7 < s)
+        return 0;
+
+    int n;
+    for (n = 0; n < 5; n++) {
+        s -= 7;
+        *cp++ = ((i>>s) & 0x7f) + (s?128:0);
+        if (!s)
+            break;
+    }
+
+    return cp-op;
+}
+
+static inline
+int var_put_u32(uint8_t *cp, const uint8_t *endp, uint32_t i) {
+    if (endp && (endp-cp) < 5)
+        return var_put_u32_safe(cp, endp, i);
+
+    if (i < (1<<7)) {
+        *cp = i;
+        return 1;
+    } else if (i < (1<<14)) {
+        *cp++ = ((i>> 7) & 0x7f) | 128;
+        *cp++ =   i      & 0x7f;
+        return 2;
+    } else if (i < (1<<21)) {
+        *cp++ = ((i>>14) & 0x7f) | 128;
+        *cp++ = ((i>> 7) & 0x7f) | 128;
+        *cp++ =   i      & 0x7f;
+        return 3;
+    } else if (i < (1<<28)) {
+        *cp++ = ((i>>21) & 0x7f) | 128;
+        *cp++ = ((i>>14) & 0x7f) | 128;
+        *cp++ = ((i>> 7) & 0x7f) | 128;
+        *cp++ =   i      & 0x7f;
+        return 4;
+    } else {
+        *cp++ = ((i>>28) & 0x7f) | 128;
+        *cp++ = ((i>>21) & 0x7f) | 128;
+        *cp++ = ((i>>14) & 0x7f) | 128;
+        *cp++ = ((i>> 7) & 0x7f) | 128;
+        *cp++ =   i      & 0x7f;
+    }
+
+    return 5;
+}
+
+static inline
+int var_get_u64(uint8_t *cp, const uint8_t *endp, uint64_t *i) {
+    uint8_t *op = cp, c;
+    uint64_t j = 0;
+
+    if (!endp || endp - cp >= 11) {
+        int n = 10;
+        do {
+            c = *cp++;
+            j = (j<<7) | (c & 0x7f);
+        } while ((c & 0x80) && n-- > 0);
+    } else {
+        if (cp >= endp) {
+            *i = 0;
+            return 0;
+        }
+
+        do {
+            c = *cp++;
+            j = (j<<7) | (c & 0x7f);
+        } while ((c & 0x80) && cp < endp);
+    }
+
+    *i = j;
+    return cp-op;
+}
+
+static inline
+int var_get_u32(uint8_t *cp, const uint8_t *endp, uint32_t *i) {
+    uint8_t *op = cp, c;
+    uint32_t j = 0;
+
+    if (!endp || endp - cp >= 6) {
+        // Known maximum loop count helps optimiser.
+        // NB: this helps considerably at -O3 level, but may harm -O2.
+        // (However we optimise for those that want optimal code.)
+        int n = 5;
+        do {
+            c = *cp++;
+            j = (j<<7) | (c & 0x7f);
+        } while ((c & 0x80) && n-- > 0);
+    } else {
+        if (cp >= endp) {
+            *i = 0;
+            return 0;
+        }
+
+        if (*cp < 128) {
+            *i = *cp;
+            return 1;
+        }
+
+        do {
+            c = *cp++;
+            j = (j<<7) | (c & 0x7f);
+        } while ((c & 0x80) && cp < endp);
+    }
+
+    *i = j;
+    return cp-op;
+}
+
+//-----------------------------------------------------------------------------
+#else // BIG_END
+
+// Little endian 7-bit variable sized integer encoding.
+// The unsigned value is equivalent to LEB128 encoding.
+// For signed, see below.
+// This is also the Google Protocol Buffer and WebAssembly format.
+static inline int var_put_u64(uint8_t *cp, const uint8_t *endp, uint64_t i) {
+    uint8_t *op = cp;
+
+    if (!endp || (endp-cp)*7 >= 10) {
+        // Unsafe or big-enough anyway
+        do {
+            *cp++ = (i&0x7f) + ((i>=0x80)<<7);
+            i >>= 7;
+        } while (i);
+    } else if (cp < endp) {
+        // End checked variant
+        do {
+            *cp++ = (i&0x7f) + ((i>=0x80)<<7);
+            i >>= 7;
+        } while (i && cp < endp);
+    }
+
+    return cp-op;
+}
+
+static inline int var_put_u32(uint8_t *cp, const uint8_t *endp, uint32_t i) {
+    uint8_t *op = cp;
+
+    if (!endp || (endp-cp)*7 >= 5) {
+        // Unsafe or big-enough anyway
+        do {
+            *cp++ = (i&0x7f) + ((i>=0x80)<<7);
+            i >>= 7;
+        } while (i);
+    } else if (cp < endp) {
+        // End checked variant
+        do {
+            *cp++ = (i&0x7f) + ((i>=0x80)<<7);
+            i >>= 7;
+        } while (i && cp < endp);
+    }
+
+    return cp-op;
+}
+
+static inline int var_get_u64(uint8_t *cp, const uint8_t *endp, uint64_t *i) {
+    uint8_t *op = cp, c;
+    uint64_t j = 0, s = 0;
+
+    if (endp) {
+        // Safe variant
+        if (cp >= endp) {
+            *i = 0;
+            return 0;
+        }
+
+        do {
+            c = *cp++;
+            j |= (c & 0x7f) << s;
+            s += 7;
+        } while ((c & 0x80) && cp < endp);
+    } else {
+        // Unsafe variant
+        do {
+            c = *cp++;
+            j |= (c & 0x7f) << s;
+            s += 7;
+        } while ((c & 0x80));
+    }
+
+    *i = j;
+    return cp-op;
+}
+
+static inline int var_get_u32(uint8_t *cp, const uint8_t *endp, uint32_t *i) {
+    uint8_t *op = cp, c;
+    uint32_t j = 0, s = 0;
+
+    if (endp) {
+        // Safe variant
+        if (cp >= endp) {
+            *i = 0;
+            return 0;
+        }
+
+        do {
+            c = *cp++;
+            j |= (c & 0x7f) << s;
+            s += 7;
+        } while ((c & 0x80) && cp < endp);
+    } else {
+        // Unsafe variant
+        do {
+            c = *cp++;
+            j |= (c & 0x7f) << s;
+            s += 7;
+        } while ((c & 0x80));
+    }
+
+    *i = j;
+    return cp-op;
+}
+#endif // BIG_END
+
+//-----------------------------------------------------------------------------
+// Signed versions of the above using zig-zag integer encoding.
+// This folds the sign bit into the bottom bit so we iterate
+// 0, -1, +1, -2, +2, etc.
+static inline int var_put_s32(uint8_t *cp, const uint8_t *endp, int32_t i) {
+    return var_put_u32(cp, endp, ((uint32_t)i << 1) ^ (i >> 31));
+}
+static inline int var_put_s64(uint8_t *cp, const uint8_t *endp, int64_t i) {
+    return var_put_u64(cp, endp, ((uint64_t)i << 1) ^ (i >> 63));
+}
+
+static inline int var_get_s32(uint8_t *cp, const uint8_t *endp, int32_t *i) {
+    int b = var_get_u32(cp, endp, (uint32_t *)i);
+    *i = ((uint32_t)*i >> 1) ^ -(int32_t)(*i & 1);
+    return b;
+}
+static inline int var_get_s64(uint8_t *cp, const uint8_t *endp, int64_t *i) {
+    int b = var_get_u64(cp, endp, (uint64_t *)i);
+    *i = ((uint64_t)*i >> 1) ^ -(int64_t)(*i & 1);
+    return b;
+}
+
+static inline int var_size_u64(uint64_t v) {
+    int i = 0;
+    do {
+        i++;
+        v >>= 7;
+    } while (v);
+    return i;
+}
+#define var_size_u32 var_size_u64
+
+static inline int var_size_s64(int64_t v) {
+    return var_size_u64(((uint64_t)v << 1) ^ (v >> 63));
+}
+#define var_size_s32 var_size_s64
+
+#endif /* VARINT2 */
+
+#endif /* VARINT_H */
diff --git a/ext/htslib/htscodecs/htscodecs/varint2.h b/ext/htslib/htscodecs/htscodecs/varint2.h
new file mode 100644
index 0000000..4d6cc5f
--- /dev/null
+++ b/ext/htslib/htscodecs/htscodecs/varint2.h
@@ -0,0 +1,318 @@
+//#include <stdio.h>
+
+// FIXME: make get functions const uint8_t *
+
+/*
+ * Copyright (c) 2019 Genome Research Ltd.
+ * Author(s): James Bonfield
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above
+ *       copyright notice, this list of conditions and the following
+ *       disclaimer in the documentation and/or other materials provided
+ *       with the distribution.
+ *
+ *    3. Neither the names Genome Research Ltd and Wellcome Trust Sanger
+ *       Institute nor the names of its contributors may be used to endorse
+ *       or promote products derived from this software without specific
+ *       prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY GENOME RESEARCH LTD AND CONTRIBUTORS "AS
+ * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENOME RESEARCH
+ * LTD OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef VARINT2_H
+#define VARINT2_H
+
+#include <stdint.h>
+
+// General API scheme is var_{get,put}_{s,u}{32,64}
+// s/u for signed/unsigned;  32/64 for integer size.
+
+// The ideas here are taken from the vbenc code in TurboPFor
+// (https://github.com/powturbo/TurboPFor) with analysis at
+// https://github.com/stoklund/varint.
+
+// Unlike the ITF8 and standard 7-bit at a time encodings, this
+// tries to ensure a larger portion of small numbers still fit in 1 byte.
+// This trades more space for long integers with less space for short ones,
+// which seems like a good tradeoff given the typical distribution curves.
+//
+// Like ITF8 and LTF8, the first byte also indicates the total number of
+// bytes we need to decode, but unlike those it uses the same format for
+// both meaning changing data type doesn't change encoding.
+//
+// Size comparison examples.
+//
+//              Max value
+// Bytes        ITF8/7bit               This
+// 1                  127                176
+// 2               16,383             16,560
+// 3            2,097,151            540,848
+// 4          268,435,455         16,777,215
+// 5       34,359,738,368      4,294,967,296
+// 6    4,398,046,511,104  1,099,511,627,776
+// ...
+//
+// The format is as follows:
+// 0-176                     1 byte:  0 + 8 bit
+// 177-16560 (14 bit range)  2 bytes: 177 + 6bit, 0 + 8bit, for x-177
+// 16561-540848 (19 bits)    3 bytes: 241 + 3bit, 0+8, 0+8, for x-16561
+// 540849-16777215 (~24 bit) 4 bytes: 249, 0+8, 0+8, 0+8, for x
+// 2^24 - 2^32-1             5 bytes: 250, 0+8 x4
+// 2^32 - 2^40-1             6 bytes: 251, 0+8 x5
+// 2^40 - 2^48-1             7 bytes: 252, 0+8 x6
+// 2^48 - 2^56-1             8 bytes: 253, 0+8 x7
+// 2^56 - 2^64-1             9 bytes: 254, 0+8 x8
+//
+// Hence first byte value 255 is not possible and permits future
+// escape code.
+
+
+// FIXME: consider returning the value and having nbytes passed in by
+// reference instead of vice-versa.
+//
+// ie uint64_t var_get_u64(uint8_t *cp, int *nbytes)
+// vs int      var_get_u64(uint8_t *cp, uint64_t *val)
+//
+// The return value can then be assigned to 32-bit or 64-bit type
+// without need of a new function name.  The cost is we can't then
+// do "cp += var_get_u32(cp, endp, &u_freq_sz);".  Maybe we can't do
+// overflow detection with former? (Want 32-bit but got, say, 40 bit)
+
+
+// static inline char *var_dump(const uint8_t *cp, int n) {
+//     static char buf[1000];
+//     int i, o = 0;
+//     for (i = 0; i < n; i++)
+//      o += sprintf(&buf[o], " %d", cp[i]);
+//     return buf;
+// }
+
+static inline int var_put_u64(uint8_t *cp, const uint8_t *endp, uint64_t x) {
+    uint8_t *op = cp;
+
+    if (x < 177) {
+        if (endp && endp - cp < 1) return 0;
+        // 0 to 176 in single byte as-is
+        *cp++ = x;
+    } else if (x < 16561) {
+        if (endp && endp - cp < 2) return 0;
+        *cp++ = ((x-177)>>8)+177;
+        *cp++ = x-177;
+    } else if (x < 540849) {
+        if (endp && endp - cp < 3) return 0;
+        *cp++ = ((x-16561)>>16)+241;
+        *cp++ = (x-16561)>>8;
+        *cp++ = x-16561;
+    } else if (x < (1<<24)) {
+        if (endp && endp - cp < 4) return 0;
+        *cp++ = 249;
+        *cp++ = x>>16;
+        *cp++ = x>>8;
+        *cp++ = x;
+    } else if (x < (1LL<<32)) {
+        if (endp && endp - cp < 5) return 0;
+        *cp++ = 250;
+        *cp++ = x>>24;
+        *cp++ = x>>16;
+        *cp++ = x>>8;
+        *cp++ = x;
+    } else if (x < (1LL<<40)) {
+        if (endp && endp - cp < 6) return 0;
+        *cp++ = 251;
+        *cp++ = x>>32;
+        *cp++ = x>>24;
+        *cp++ = x>>16;
+        *cp++ = x>>8;
+        *cp++ = x;
+    } else if (x < (1LL<<48)) {
+        if (endp && endp - cp < 7) return 0;
+        *cp++ = 252;
+        *cp++ = x>>40;
+        *cp++ = x>>32;
+        *cp++ = x>>24;
+        *cp++ = x>>16;
+        *cp++ = x>>8;
+        *cp++ = x;
+    } else if (x < (1LL<<56)) {
+        if (endp && endp - cp < 8) return 0;
+        *cp++ = 253;
+        *cp++ = x>>48;
+        *cp++ = x>>40;
+        *cp++ = x>>32;
+        *cp++ = x>>24;
+        *cp++ = x>>16;
+        *cp++ = x>>8;
+        *cp++ = x;
+    } else {
+        if (endp && endp - cp < 9) return 0;
+        *cp++ = 254;
+        *cp++ = x>>56;
+        *cp++ = x>>48;
+        *cp++ = x>>40;
+        *cp++ = x>>32;
+        *cp++ = x>>24;
+        *cp++ = x>>16;
+        *cp++ = x>>8;
+        *cp++ = x;
+    }
+
+//    fprintf(stderr, "Put64 %d (%s)\n", x, var_dump(op, cp-op));
+
+    return cp-op;
+}
+
+static inline int var_put_u32(uint8_t *cp, const uint8_t *endp, uint32_t x) {
+    uint8_t *op = cp;
+
+    if (x < 177) {
+        if (endp && endp - cp < 1) abort();//return 0;
+        // 0 to 176 in single byte as-is
+        *cp++ = x;
+    } else if (x < 16561) {
+        if (endp && endp - cp < 2) abort();//return 0;
+        *cp++ = ((x-177)>>8)+177;
+        *cp++ = x-177;
+    } else if (x < 540849) {
+        if (endp && endp - cp < 3) abort();//return 0;
+        *cp++ = ((x-16561)>>16)+241;
+        *cp++ = (x-16561)>>8;
+        *cp++ = x-16561;
+    } else if (x < (1<<24)) {
+        if (endp && endp - cp < 4) abort();//return 0;
+        *cp++ = 249;
+        *cp++ = x>>16;
+        *cp++ = x>>8;
+        *cp++ = x;
+    } else {
+        if (endp && endp - cp < 5) abort();//return 0;
+        *cp++ = 250;
+        *cp++ = x>>24;
+        *cp++ = x>>16;
+        *cp++ = x>>8;
+        *cp++ = x;
+    }
+
+//    fprintf(stderr, "Put32 %d (%s)\n", x, var_dump(op, cp-op));
+
+    return cp-op;
+}
+
+static inline int var_get_u64(uint8_t *cp, const uint8_t *endp, uint64_t *i) {
+    uint8_t *op = cp;
+    uint64_t j = 0;
+
+    if (endp && cp >= endp) {
+        *i = 0;
+        return 0;
+    }
+    if (*cp < 177) {
+        j = *cp++;
+    } else if (*cp < 241) {
+        j = ((cp[0] - 177)<<8) + cp[1] + 177;
+        cp += 2;
+    } else if (*cp < 249) {
+        j = ((cp[0] - 241)<<16) + (cp[1]<<8) + cp[2] + 16561;
+        cp += 3;
+    } else {
+        int n = *cp++ - 249 + 3;
+        while (n--)
+            j = (j<<8) + *cp++;
+    }
+
+//    fprintf(stderr, "Get64 %ld (%s)\n", j, var_dump(op, cp-op));
+
+    *i = j;
+    return cp-op;
+}
+
+static inline int var_get_u32(uint8_t *cp, const uint8_t *endp, uint32_t *i) {
+    uint8_t *op = cp;
+    uint32_t j = 0;
+
+    if (endp && cp >= endp) {
+        *i = 0;
+        return 0;
+    }
+    if (*cp < 177) {
+        j = *cp++;
+    } else if (*cp < 241) {
+        j = ((cp[0] - 177)<<8) + cp[1] + 177;
+        cp += 2;
+    } else if (*cp < 249) {
+        j = ((cp[0] - 241)<<16) + (cp[1]<<8) + cp[2] + 16561;
+        cp += 3;
+    } else {
+        int n = *cp++ - 249 + 3;
+        while (n--)
+            j = (j<<8) + *cp++;
+    }
+
+//    fprintf(stderr, "Get32 %d (%s)\n", j, var_dump(op, cp-op));
+
+    *i = j;
+    return cp-op;
+}
+
+// Signed versions of the above using zig-zag integer encoding.
+// This folds the sign bit into the bottom bit so we iterate
+// 0, -1, +1, -2, +2, etc.
+static inline int var_put_s32(uint8_t *cp, const uint8_t *endp, int32_t i) {
+    return var_put_u32(cp, endp, (i << 1) ^ (i >> 31));
+}
+static inline int var_put_s64(uint8_t *cp, const uint8_t *endp, int64_t i) {
+    return var_put_u64(cp, endp, (i << 1) ^ (i >> 63));
+}
+
+static inline int var_get_s32(uint8_t *cp, const uint8_t *endp, int32_t *i) {
+    int b = var_get_u32(cp, endp, (uint32_t *)i);
+    *i = (*i >> 1) ^ -(*i & 1);
+    return b;
+}
+static inline int var_get_s64(uint8_t *cp, const uint8_t *endp, int64_t *i) {
+    int b = var_get_u64(cp, endp, (uint64_t *)i);
+    *i = (*i >> 1) ^ -(*i & 1);
+    return b;
+}
+
+static inline int var_size_u64(uint64_t v) {
+    if (v < 177)
+        return 1;
+    else if (v < 16561)
+        return 2;
+    else if (v < 540849)
+        return 3;
+
+    int i = 0;
+    do {
+        v >>= 8;
+        i++;
+    } while (v);
+
+//    fprintf(stderr, "Size %ld (%d)\n", v, i+1);
+
+    return i+1;
+}
+#define var_size_u32 var_size_u64
+
+static inline int var_size_s64(int64_t v) {
+    return var_size_u64((v >> 63) ^ (v << 1));
+}
+#define var_size_s32 var_size_s64
+
+#endif /* VARINT2_H */
diff --git a/ext/htslib/htscodecs/htscodecs/version.h b/ext/htslib/htscodecs/htscodecs/version.h
new file mode 100644
index 0000000..048dcab
--- /dev/null
+++ b/ext/htslib/htscodecs/htscodecs/version.h
@@ -0,0 +1 @@
+#define HTSCODECS_VERSION_TEXT "1.6.1"
diff --git a/ext/htslib/htscodecs_bundled.mk b/ext/htslib/htscodecs_bundled.mk
new file mode 100644
index 0000000..6274350
--- /dev/null
+++ b/ext/htslib/htscodecs_bundled.mk
@@ -0,0 +1,72 @@
+# Makefile fragment to add settings needed when bundling htscodecs functions
+#
+#    Copyright (C) 2021-2022 Genome Research Ltd.
+#
+#    Author: Rob Davies <rmd@sanger.ac.uk>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+
+HTSCODECS_SOURCES = $(HTSPREFIX)htscodecs/htscodecs/arith_dynamic.c \
+        $(HTSPREFIX)htscodecs/htscodecs/fqzcomp_qual.c \
+        $(HTSPREFIX)htscodecs/htscodecs/htscodecs.c \
+        $(HTSPREFIX)htscodecs/htscodecs/pack.c \
+        $(HTSPREFIX)htscodecs/htscodecs/rANS_static4x16pr.c \
+	$(HTSPREFIX)htscodecs/htscodecs/rANS_static32x16pr_avx2.c \
+	$(HTSPREFIX)htscodecs/htscodecs/rANS_static32x16pr_avx512.c \
+	$(HTSPREFIX)htscodecs/htscodecs/rANS_static32x16pr_sse4.c \
+	$(HTSPREFIX)htscodecs/htscodecs/rANS_static32x16pr_neon.c \
+	$(HTSPREFIX)htscodecs/htscodecs/rANS_static32x16pr.c \
+        $(HTSPREFIX)htscodecs/htscodecs/rANS_static.c \
+        $(HTSPREFIX)htscodecs/htscodecs/rle.c \
+        $(HTSPREFIX)htscodecs/htscodecs/tokenise_name3.c \
+	$(HTSPREFIX)htscodecs/htscodecs/utils.c
+
+
+HTSCODECS_OBJS = $(HTSCODECS_SOURCES:.c=.o)
+
+# htscodecs public headers
+htscodecs_arith_dynamic_h = htscodecs/htscodecs/arith_dynamic.h
+htscodecs_fqzcomp_qual_h = htscodecs/htscodecs/fqzcomp_qual.h
+htscodecs_htscodecs_h = htscodecs/htscodecs/htscodecs.h $(htscodecs_version_h)
+htscodecs_pack_h = htscodecs/htscodecs/pack.h
+htscodecs_rANS_static_h = htscodecs/htscodecs/rANS_static.h
+htscodecs_rANS_static4x16_h = htscodecs/htscodecs/rANS_static4x16.h
+htscodecs_rle_h = htscodecs/htscodecs/rle.h
+htscodecs_tokenise_name3_h = htscodecs/htscodecs/tokenise_name3.h
+htscodecs_varint_h = htscodecs/htscodecs/varint.h
+
+# htscodecs internal headers
+htscodecs_htscodecs_endian_h = htscodecs/htscodecs/htscodecs_endian.h
+htscodecs_c_range_coder_h = htscodecs/htscodecs/c_range_coder.h
+htscodecs_c_simple_model_h = htscodecs/htscodecs/c_simple_model.h $(htscodecs_c_range_coder_h)
+htscodecs_permute_h = htscodecs/htscodecs/permute.h
+htscodecs_pooled_alloc_h = htscodecs/htscodecs/pooled_alloc.h
+htscodecs_rANS_byte_h = htscodecs/htscodecs/rANS_byte.h
+htscodecs_rANS_static16_int_h = htscodecs/htscodecs/rANS_static16_int.h $(htscodecs_varint_h) $(htscodecs_utils_h)
+htscodecs_rANS_static32x16pr_h = htscodecs/htscodecs/rANS_static32x16pr.h
+htscodecs_rANS_word_h = htscodecs/htscodecs/rANS_word.h $(htscodecs_htscodecs_endian_h)
+htscodecs_utils_h = htscodecs/htscodecs/utils.h
+htscodecs_version_h = htscodecs/htscodecs/version.h
+
+# Add htscodecs tests into the HTSlib test framework
+
+HTSCODECS_TEST_TARGETS = test_htscodecs_rans4x8 \
+    test_htscodecs_rans4x16 test_htscodecs_arith test_htscodecs_tok3 \
+    test_htscodecs_fqzcomp test_htscodecs_varint
diff --git a/ext/htslib/htscodecs_external.mk b/ext/htslib/htscodecs_external.mk
new file mode 100644
index 0000000..3f86811
--- /dev/null
+++ b/ext/htslib/htscodecs_external.mk
@@ -0,0 +1,46 @@
+# Makefile fragment for use when linking to an external libhtscodecs
+#
+#    Copyright (C) 2021 Genome Research Ltd.
+#
+#    Author: Rob Davies <rmd@sanger.ac.uk>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+HTSCODECS_SOURCES =
+HTSCODECS_OBJS =
+HTSCODECS_TEST_TARGETS =
+
+htscodecs_arith_dynamic_h =
+htscodecs_fqzcomp_qual_h =
+htscodecs_htscodecs_h =
+htscodecs_pack_h =
+htscodecs_rANS_static_h =
+htscodecs_rANS_static4x16_h =
+htscodecs_rle_h =
+htscodecs_tokenise_name3_h =
+htscodecs_varint_h =
+
+htscodecs_htscodecs_endian_h =
+htscodecs_c_range_coder_h =
+htscodecs_c_simple_model_h =
+htscodecs_pooled_alloc_h =
+htscodecs_rANS_byte_h =
+htscodecs_rANS_word_h =
+htscodecs_utils_h =
+htscodecs_version_h =
diff --git a/ext/htslib/htsfile.1 b/ext/htslib/htsfile.1
new file mode 100644
index 0000000..e22fdbc
--- /dev/null
+++ b/ext/htslib/htsfile.1
@@ -0,0 +1,94 @@
+.TH htsfile 1 "12 September 2024" "htslib-1.21" "Bioinformatics tools"
+.SH NAME
+htsfile \- identify high-throughput sequencing data files
+.\"
+.\" Copyright (C) 2015, 2017-2018 Genome Research Ltd.
+.\"
+.\" Author: John Marshall <jm18@sanger.ac.uk>
+.\"
+.\" Permission is hereby granted, free of charge, to any person obtaining a
+.\" copy of this software and associated documentation files (the "Software"),
+.\" to deal in the Software without restriction, including without limitation
+.\" the rights to use, copy, modify, merge, publish, distribute, sublicense,
+.\" and/or sell copies of the Software, and to permit persons to whom the
+.\" Software is furnished to do so, subject to the following conditions:
+.\"
+.\" The above copyright notice and this permission notice shall be included in
+.\" all copies or substantial portions of the Software.
+.\"
+.\" THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+.\" IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+.\" FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+.\" THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+.\" LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+.\" FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+.\" DEALINGS IN THE SOFTWARE.
+.\"
+.SH SYNOPSIS
+.B htsfile
+.RB [ -chHv ]
+.IR FILE ...
+.br
+.B htsfile --copy
+.RB [ -v ]
+.I FILE DESTFILE
+.SH DESCRIPTION
+The \fBhtsfile\fR utility attempts to identify what kind of high-throughput
+sequencing data files the specified files are, and provides minimal viewing
+capabilities for some kinds of data file.
+.P
+It can identify sequencing data files such as SAM, BAM, and CRAM;
+variant calling data files such as VCF and BCF;
+index files used to index these data files;
+and compressed versions of many of them.
+.P
+For each \fIFILE\fR given, \fBhtsfile\fP prints a description of the file
+format determined, using similar keyword conventions to \fBfile\fP(1):
+"text" indicates a textual file that can probably be viewed on a terminal;
+"data" indicates binary data;
+"sequence", "variant calling", and "index" indicate different categories of
+data file.
+When it can be identified, the name of the particular file format (such as
+"BAM" or "VCF") is printed at the start of the description.
+.P
+When used to view file contents as text, \fBhtsfile\fP can optionally show
+only headers or only data records, but has no other filtering capabilities.
+Use \fBsamtools\fR or \fBbcftools\fR if you need more extensive viewing or
+filtering capabilities.
+.P
+Alternatively, when \fB--copy\fR is used, \fBhtsfile\fR takes exactly two
+arguments and performs a byte-for-byte copy from \fIFILE\fR to \fIDESTFILE\fR.
+This is similar to \fBcp\fR(1), but HTSlib's remote file access facilities
+are available for both source and destination.
+.P
+The following options are accepted:
+.TP 4n
+.BR -c ", " --view
+Instead of identifying the specified files, display a textual representation
+of their contents on standard output.
+.IP
+By default, \fB--view\fR refuses to display files in unknown formats.
+When \fB--verbose\fR is also given, the raw contents of such files are
+displayed, with non-printable characters shown via C-style "\\x" hexadecimal
+escape sequences.
+.TP
+.BR -C ", " --copy
+Instead of identifying or displaying the specified files, copy the source
+\fIFILE\fR to the destination \fIDESTFILE\fR.
+Only \fB--verbose\fR may be used in conjunction with \fB--copy\fR.
+.TP
+.BR -h ", " --header-only
+Display data file headers only.
+Implies \fB--view\fR.
+.TP
+.BR -H ", " --no-header
+When viewing files, display data records only.
+.TP
+.BR -v ", " --verbose
+Display additional warnings and diagnostic messages.
+Using \fB--verbose\fR repeatedly further raises the verbosity.
+.PP
+.SH SEE ALSO
+.IR bcftools (1),
+.IR file (1),
+.IR samtools (1)
diff --git a/ext/htslib/htsfile.c b/ext/htslib/htsfile.c
new file mode 100644
index 0000000..25af3f5
--- /dev/null
+++ b/ext/htslib/htsfile.c
@@ -0,0 +1,324 @@
+/*  htsfile.c -- file identifier and minimal viewer.
+
+    Copyright (C) 2014-2019 Genome Research Ltd.
+
+    Author: John Marshall <jm18@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#include <config.h>
+
+#include <ctype.h>
+#include <errno.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <getopt.h>
+
+#include "htslib/hfile.h"
+#include "htslib/hts.h"
+#include "htslib/sam.h"
+#include "htslib/vcf.h"
+
+#ifndef EFTYPE
+#define EFTYPE ENOEXEC
+#endif
+
+enum { identify, view_headers, view_all, copy } mode = identify;
+int show_headers = 1;
+int verbose = 0;
+int status = EXIT_SUCCESS;  /* Exit status from main */
+
+void HTS_FORMAT(HTS_PRINTF_FMT, 1, 2) error(const char *format, ...)
+{
+    int err = errno;
+    va_list args;
+    va_start(args, format);
+    fflush(stdout);
+    fprintf(stderr, "htsfile: ");
+    vfprintf(stderr, format, args);
+    if (err) fprintf(stderr, ": %s\n", strerror(err));
+    else fprintf(stderr, "\n");
+    fflush(stderr);
+    va_end(args);
+    status = EXIT_FAILURE;
+}
+
+static void view_sam(samFile *in, const char *filename)
+{
+    bam1_t *b = NULL;
+    sam_hdr_t *hdr = NULL;
+    samFile *out = NULL;
+
+    hdr = sam_hdr_read(in);
+    if (hdr == NULL) {
+        errno = 0; error("reading headers from \"%s\" failed", filename);
+        goto clean;
+    }
+
+    out = hts_open("-", "w");
+    if (out == NULL) { error("reopening standard output failed"); goto clean; }
+
+    if (show_headers) {
+        if (sam_hdr_write(out, hdr) != 0) {
+            error("writing headers to standard output failed");
+            goto clean;
+        }
+    }
+
+    if (mode == view_all) {
+        int ret;
+
+        b = bam_init1();
+        if (b == NULL) { error("can't create record"); goto clean; }
+
+        while ((ret = sam_read1(in, hdr, b)) >= 0) {
+            if (sam_write1(out, hdr, b) < 0) {
+                error("writing to standard output failed");
+                goto clean;
+            }
+        }
+
+        if (ret < -1) { error("reading \"%s\" failed", filename); goto clean; }
+    }
+
+ clean:
+    sam_hdr_destroy(hdr);
+    bam_destroy1(b);
+    if (out) hts_close(out);
+}
+
+static void view_vcf(vcfFile *in, const char *filename)
+{
+    bcf1_t *rec = NULL;
+    bcf_hdr_t *hdr = NULL;
+    vcfFile *out = NULL;
+
+    hdr = bcf_hdr_read(in);
+    if (hdr == NULL) {
+        errno = 0; error("reading headers from \"%s\" failed", filename);
+        goto clean;
+    }
+
+    out = hts_open("-", "w");
+    if (out == NULL) { error("reopening standard output failed"); goto clean; }
+
+    if (show_headers) {
+        if (bcf_hdr_write(out, hdr) != 0) {
+            error("writing headers to standard output failed");
+            goto clean;
+        }
+    }
+
+    if (mode == view_all) {
+        int ret;
+
+        rec = bcf_init();
+        if (rec == NULL) { error("can't create record"); goto clean; }
+
+        while ((ret = bcf_read(in, hdr, rec)) >= 0) {
+            if (bcf_write(out, hdr, rec) < 0) {
+                error("writing to standard output failed");
+                goto clean;
+            }
+        }
+
+        if (ret < -1) { error("reading \"%s\" failed", filename); goto clean; }
+    }
+
+ clean:
+    if (hdr) bcf_hdr_destroy(hdr);
+    if (rec) bcf_destroy(rec);
+    if (out) hts_close(out);
+}
+
+static void view_raw(hFILE *fp, const char *filename)
+{
+    int c, prev;
+    for (prev = '\n'; (c = hgetc(fp)) != EOF; prev = c)
+        if (isprint(c) || c == '\n' || c == '\t') putchar(c);
+        else if (c == '\r') fputs("\\r", stdout);
+        else if (c == '\0') fputs("\\0", stdout);
+        else printf("\\x%02x", c);
+
+    if (prev != '\n') putchar('\n');
+
+    if (herrno(fp)) {
+        errno = herrno(fp);
+        error("reading \"%s\" failed", filename);
+    }
+}
+
+static void copy_raw(const char *srcfilename, const char *destfilename)
+{
+    hFILE *src = hopen(srcfilename, "r");
+    if (src == NULL) {
+        error("can't open \"%s\"", srcfilename);
+        return;
+    }
+
+    size_t bufsize = 1048576;
+    char *buffer = malloc(bufsize);
+    if (buffer == NULL) {
+        error("can't allocate copy buffer");
+        hclose_abruptly(src);
+        return;
+    }
+
+    hFILE *dest = hopen(destfilename, "w");
+    if (dest == NULL) {
+        error("can't create \"%s\"", destfilename);
+        hclose_abruptly(src);
+        free(buffer);
+        return;
+    }
+
+    ssize_t n;
+    while ((n = hread(src, buffer, bufsize)) > 0)
+        if (hwrite(dest, buffer, n) != n) {
+            error("writing to \"%s\" failed", destfilename);
+            hclose_abruptly(dest);
+            dest = NULL;
+            break;
+        }
+
+    if (n < 0) {
+        error("reading from \"%s\" failed", srcfilename);
+        hclose_abruptly(src);
+        src = NULL;
+    }
+
+    if (dest && hclose(dest) < 0) error("closing \"%s\" failed", destfilename);
+    if (src && hclose(src) < 0)   error("closing \"%s\" failed", srcfilename);
+    free(buffer);
+}
+
+static void usage(FILE *fp, int status)
+{
+    fprintf(fp,
+"Usage: htsfile [-chHv] FILE...\n"
+"       htsfile --copy [-v] FILE DESTFILE\n"
+"Options:\n"
+"  -c, --view         Write textual form of FILEs to standard output\n"
+"  -C, --copy         Copy the exact contents of FILE to DESTFILE\n"
+"  -h, --header-only  Display only headers in view mode, not records\n"
+"  -H, --no-header    Suppress header display in view mode\n"
+"  -v, --verbose      Increase verbosity of warnings and diagnostics\n");
+    exit(status);
+}
+
+int main(int argc, char **argv)
+{
+    static const struct option options[] = {
+        { "copy", no_argument, NULL, 'C' },
+        { "header-only", no_argument, NULL, 'h' },
+        { "no-header", no_argument, NULL, 'H' },
+        { "view", no_argument, NULL, 'c' },
+        { "verbose", no_argument, NULL, 'v' },
+        { "help", no_argument, NULL, 2 },
+        { "version", no_argument, NULL, 1 },
+        { NULL, 0, NULL, 0 }
+    };
+
+    int c, i;
+
+    status = EXIT_SUCCESS;
+    while ((c = getopt_long(argc, argv, "cChHv", options, NULL)) >= 0)
+        switch (c) {
+        case 'c': mode = view_all; break;
+        case 'C': mode = copy; break;
+        case 'h': mode = view_headers; show_headers = 1; break;
+        case 'H': show_headers = 0; break;
+        case 'v': hts_verbose++; verbose++; break;
+        case 1:
+            printf(
+"htsfile (htslib) %s\n"
+"Copyright (C) 2024 Genome Research Ltd.\n",
+                   hts_version());
+            exit(EXIT_SUCCESS);
+            break;
+        case 2:   usage(stdout, EXIT_SUCCESS); break;
+        default:  usage(stderr, EXIT_FAILURE); break;
+        }
+
+    if (optind == argc) usage(stderr, EXIT_FAILURE);
+
+    if (mode == copy) {
+        if (optind + 2 != argc) usage(stderr, EXIT_FAILURE);
+        copy_raw(argv[optind], argv[optind + 1]);
+        return status;
+    }
+
+    for (i = optind; i < argc; i++) {
+        hFILE *fp = hopen(argv[i], "r");
+        if (fp == NULL) {
+            error("can't open \"%s\"", argv[i]);
+            continue;
+        }
+
+        if (mode == identify) {
+            htsFormat fmt;
+            if (hts_detect_format2(fp, argv[i], &fmt) < 0) {
+                error("detecting \"%s\" format failed", argv[i]);
+                hclose_abruptly(fp);
+                continue;
+            }
+
+            char *description = hts_format_description(&fmt);
+            printf("%s:\t%s\n", argv[i], description);
+            free(description);
+        }
+        else {
+            htsFile *hts = hts_hopen(fp, argv[i], "r");
+            if (hts) {
+                switch (hts_get_format(hts)->category) {
+                case sequence_data:
+                    view_sam(hts, argv[i]);
+                    break;
+                case variant_data:
+                    view_vcf(hts, argv[i]);
+                    break;
+                default:
+                    if (verbose)
+                        view_raw(fp, argv[i]);
+                    else {
+                        errno = 0;
+                        error("can't view \"%s\": unknown format", argv[i]);
+                    }
+                    break;
+                }
+
+                if (hts_close(hts) < 0) error("closing \"%s\" failed", argv[i]);
+                fp = NULL;
+            }
+            else if ((errno == EFTYPE || errno == ENOEXEC) && verbose)
+                view_raw(fp, argv[i]);
+            else
+                error("can't view \"%s\"", argv[i]);
+        }
+
+        if (fp && hclose(fp) < 0) error("closing \"%s\" failed", argv[i]);
+    }
+
+    if (fclose(stdout) != 0 && errno != EBADF)
+        error("closing standard output failed");
+
+    return status;
+}
diff --git a/ext/htslib/htslib-s3-plugin.7 b/ext/htslib/htslib-s3-plugin.7
new file mode 100644
index 0000000..44de657
--- /dev/null
+++ b/ext/htslib/htslib-s3-plugin.7
@@ -0,0 +1,215 @@
+.TH htslib-s3-plugin 7 "12 September 2024" "htslib-1.21" "Bioinformatics tools"
+.SH NAME
+htslib-s3-plugin \- htslib AWS S3 plugin
+.\"
+.\" Copyright (C) 2021-2022 Genome Research Ltd.
+.\"
+.\" Author: Andrew Whitwham <aw7@sanger.ac.uk>
+.\"
+.\" Permission is hereby granted, free of charge, to any person obtaining a
+.\" copy of this software and associated documentation files (the "Software"),
+.\" to deal in the Software without restriction, including without limitation
+.\" the rights to use, copy, modify, merge, publish, distribute, sublicense,
+.\" and/or sell copies of the Software, and to permit persons to whom the
+.\" Software is furnished to do so, subject to the following conditions:
+.\"
+.\" The above copyright notice and this permission notice shall be included in
+.\" all copies or substantial portions of the Software.
+.\"
+.\" THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+.\" IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+.\" FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+.\" THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+.\" LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+.\" FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+.\" DEALINGS IN THE SOFTWARE.
+.\"
+.
+.\" For code blocks and examples (cf groff's Ultrix-specific man macros)
+.de EX
+
+.  in +\\$1
+.  nf
+.  ft CR
+..
+.de EE
+.  ft
+.  fi
+.  in
+
+..
+
+.SH DESCRIPTION
+The S3 plugin allows htslib file functions to communicate with servers that use
+the AWS S3 protocol.  Files are identified by their bucket and object key in a
+URL format e.g.
+
+.B s3://mybucket/path/to/file
+
+With \fIpath/to/file\fR being the object key.
+
+Necessary security information can be provided in as part of the URL, in
+environment variables or from configuration files.
+
+The full URL format is:
+
+.B s3[+SCHEME]://[ID[:SECRET[:TOKEN]]@]BUCKET/PATH
+
+The elements are:
+.TP
+.I SCHEME
+The protocol used.  Defaults to \fIhttps\fR.
+.TP
+.I ID
+The user AWS access key.
+.TP
+.I SECRET
+The secret key for use with the access key.
+.TP
+.I TOKEN
+Token used for temporary security credentials.
+.TP
+.I BUCKET
+AWS S3 bucket.
+.TP
+.I PATH
+Path to the object under the bucket.
+.LP
+
+The environment variables below will be used if the user ID is not set.
+.TP
+.B AWS_ACCESS_KEY_ID
+The user AWS access key.
+.TP
+.B AWS_SECRET_ACCESS_KEY
+The secret key for use with the access key.
+.TP
+.B AWS_DEFAULT_REGION
+The region to use. Defaults to
+.IR us-east-1 .
+.TP
+.B AWS_SESSION_TOKEN
+Token used for temporary security credentials.
+.TP
+.B AWS_DEFAULT_PROFILE
+The profile to use in \fIcredentials\fR, \fIconfig\fR or \fIs3cfg\fR files.
+Defaults to
+.IR default .
+.TP
+.B AWS_PROFILE
+Same as above.
+.TP
+.B AWS_SHARED_CREDENTIALS_FILE
+Location of the credentials file.  Defaults to
+.IR ~/.aws/credentials .
+.TP
+.B HTS_S3_S3CFG
+Location of the s3cfg file.  Defaults to
+.IR ~/.s3cfg .
+.TP
+.B HTS_S3_HOST
+Sets the host.  Defaults to
+.IR s3.amazonaws.com .
+.TP
+.B HTS_S3_V2
+If set use signature v2 rather the default v4.  This will limit the plugin to
+reading only.
+.TP
+.B HTS_S3_PART_SIZE
+Sets the upload part size in Mb, the minimum being 5Mb.
+By default the part size starts at 5Mb and expands at regular intervals to
+accommodate bigger files (up to 2.5 Tbytes with the current rate).
+Using this setting disables the automatic part size expansion.
+.TP
+.B HTS_S3_ADDRESS_STYLE
+Sets the URL style.  Options are auto (default), virtual or path.  
+.LP
+In the absence of an ID from the previous two methods the credential/config
+files will be used.  The default file locations are either
+\fI~/.aws/credentials\fR or \fI~/.s3cfg\fR (in that order).
+
+Entries used in aws style credentials file are aws_access_key_id, 
+aws_secret_access_key, aws_session_token, region, addressing_style and
+expiry_time (unofficial, see SHORT-LIVED CREDENTIALS below).
+Only the first two are usually needed.
+
+Entries used in s3cmd style config files are access_key, secret_key,
+access_token, host_base, bucket_location and host_bucket. Again only the first
+two are usually needed. The host_bucket option is only used to set a path-style
+URL, see below.
+
+.SH SHORT-LIVED CREDENTIALS
+
+Some cloud identity and access management (IAM) systems can make short-lived
+credentials that allow access to resources.
+These credentials will expire after a time and need to be renewed to
+give continued access.
+To enable this, the S3 plugin allows an \fIexpiry_time\fR entry to be set in the
+\fI.aws/credentials\fR file.
+The value for this entry should be the time when the token expires,
+following the format in RFC3339 section 5.6, which takes the form:
+
+   2012-04-29T05:20:48Z
+
+That is, year - month - day, the letter "T", hour : minute : second.
+The time can be followed by the letter "Z", indicating the UTC timezone,
+or an offset from UTC which is a "+" or "-" sign followed by two digits for
+the hours offset, ":", and two digits for the minutes.
+
+The S3 plugin will attempt to re-read the credentials file up to 1 minute
+before the given expiry time, which means the file needs to be updated with
+new credentials before then.
+As the exact way of doing this can vary between services and IAM providers,
+the S3 plugin expects this to be done by an external user-supplied process.
+This may be achieved by running a program that replaces the file as new
+credentials become available.
+The following script shows how it might be done for AWS instance credentials:
+.EX 2
+#!/bin/sh
+instance='http://169.254.169.254'
+tok_url="$instance/latest/api/token"
+ttl_hdr='X-aws-ec2-metadata-token-ttl-seconds: 10'
+creds_url="$instance/latest/meta-data/iam/security-credentials"
+key1='aws_access_key_id = \(rs(.AccessKeyId)\(rsn'
+key2='aws_secret_access_key = \(rs(.SecretAccessKey)\(rsn'
+key3='aws_session_token = \(rs(.Token)\(rsn'
+key4='expiry_time = \(rs(.Expiration)\(rsn'
+while true; do
+    token=`curl -X PUT -H "$ttl_hdr" "$tok_url"`
+    tok_hdr="X-aws-ec2-metadata-token: $token"
+    role=`curl -H "$tok_hdr" "$creds_url/"`
+    expires='now'
+    ( curl -H "$tok_hdr" "$creds_url/$role" \(rs
+      | jq -r "\(rs"${key1}${key2}${key3}${key4}\(rs"" > credentials.new ) \(rs
+      && mv -f credentials.new credentials \(rs
+      && expires=`grep expiry_time credentials | cut -d ' ' -f 3-`
+    if test $? -ne 0 ; then break ; fi
+    expiry=`date -d "$expires - 3 minutes" '+%s'`
+    now=`date '+%s'`
+    test "$expiry" -gt "$now" && sleep $((($expiry - $now) / 2))
+    sleep 30
+done
+.EE
+
+Note that the \fIexpiry_time\fR key is currently only supported for the
+\fI.aws/credentials\fR file (or the file referred to in the
+.B AWS_SHARED_CREDENTIALS_FILE
+environment variable).
+
+.SH NOTES
+In most cases this plugin transforms the given URL into a virtual host-style
+format e.g. \fIhttps://bucket.host/path/to/file\fR.  A path-style format is used
+where the URL is not DNS compliant or the bucket name contains a dot e.g.
+\fIhttps://host/bu.cket/path/to/file\fR.
+
+Path-style can be forced by setting one either HTS_S3_ADDRESS_STYLE,
+addressing_style or host_bucket.  The first two can be set to \fBpath\fR while
+host_bucket must \fBnot\fR include the \fB%(bucket).s\fR string.
+
+.SH "SEE ALSO"
+.IR htsfile (1)
+.IR samtools (1)
+.PP
+RFC 3339: <https://www.rfc-editor.org/rfc/rfc3339#section-5.6>
+.PP
+htslib website: <http://www.htslib.org/>
diff --git a/ext/htslib/htslib.map b/ext/htslib/htslib.map
new file mode 100644
index 0000000..52ad738
--- /dev/null
+++ b/ext/htslib/htslib.map
@@ -0,0 +1,652 @@
+HTSLIB_1.0 {
+    bam_aux2A;
+    bam_aux2Z;
+    bam_aux2f;
+    bam_aux2i;
+    bam_aux_append;
+    bam_aux_del;
+    bam_aux_get;
+    bam_cigar2qlen;
+    bam_cigar2rlen;
+    bam_copy1;
+    bam_destroy1;
+    bam_dup1;
+    bam_endpos;
+    bam_flag2str;
+    bam_hdr_read;
+    bam_hdr_write;
+    bam_init1;
+    bam_mplp_auto;
+    bam_mplp_destroy;
+    bam_mplp_init;
+    bam_mplp_init_overlaps;
+    bam_mplp_set_maxcnt;
+    bam_plp_auto;
+    bam_plp_destroy;
+    bam_plp_init;
+    bam_plp_next;
+    bam_plp_push;
+    bam_plp_reset;
+    bam_plp_set_maxcnt;
+    bam_read1;
+    bam_str2flag;
+    bam_write1;
+    bcf_add_filter;
+    bcf_calc_ac;
+    bcf_clear;
+    bcf_destroy;
+    bcf_dup;
+    bcf_enc_vchar;
+    bcf_enc_vfloat;
+    bcf_enc_vint;
+    bcf_float_missing;
+    bcf_float_vector_end;
+    bcf_fmt_array;
+    bcf_fmt_sized_array;
+    bcf_get_fmt;
+    bcf_get_format_string;
+    bcf_get_format_values;
+    bcf_get_info;
+    bcf_get_info_values;
+    bcf_get_variant_type;
+    bcf_get_variant_types;
+    bcf_gt_type;
+    bcf_has_filter;
+    bcf_hdr_add_hrec;
+    bcf_hdr_add_sample;
+    bcf_hdr_append;
+    bcf_hdr_combine;
+    bcf_hdr_destroy;
+    bcf_hdr_dup;
+    bcf_hdr_fmt_text;
+    bcf_hdr_get_hrec;
+    bcf_hdr_get_version;
+    bcf_hdr_id2int;
+    bcf_hdr_init;
+    bcf_hdr_parse;
+    bcf_hdr_parse_line;
+    bcf_hdr_printf;
+    bcf_hdr_read;
+    bcf_hdr_remove;
+    bcf_hdr_seqnames;
+    bcf_hdr_set;
+    bcf_hdr_set_samples;
+    bcf_hdr_set_version;
+    bcf_hdr_subset;
+    bcf_hdr_sync;
+    bcf_hdr_write;
+    bcf_hrec_add_key;
+    bcf_hrec_destroy;
+    bcf_hrec_dup;
+    bcf_hrec_find_key;
+    bcf_hrec_format;
+    bcf_hrec_set_val;
+    bcf_index_build;
+    bcf_init;
+    bcf_is_snp;
+    bcf_read;
+    bcf_readrec;
+    bcf_remove_alleles;
+    bcf_remove_filter;
+    bcf_sr_add_reader;
+    bcf_sr_destroy;
+    bcf_sr_init;
+    bcf_sr_next_line;
+    bcf_sr_regions_destroy;
+    bcf_sr_regions_flush;
+    bcf_sr_regions_init;
+    bcf_sr_regions_next;
+    bcf_sr_regions_overlap;
+    bcf_sr_regions_seek;
+    bcf_sr_remove_reader;
+    bcf_sr_seek;
+    bcf_sr_set_regions;
+    bcf_sr_set_samples;
+    bcf_sr_set_targets;
+    bcf_subset;
+    bcf_subset_format;
+    bcf_sweep_bwd;
+    bcf_sweep_destroy;
+    bcf_sweep_fwd;
+    bcf_sweep_hdr;
+    bcf_sweep_init;
+    bcf_translate;
+    bcf_trim_alleles;
+    bcf_type_shift;
+    bcf_unpack;
+    bcf_update_alleles;
+    bcf_update_alleles_str;
+    bcf_update_filter;
+    bcf_update_format;
+    bcf_update_format_string;
+    bcf_update_id;
+    bcf_update_info;
+    bcf_write;
+    bgzf_check_EOF;
+    bgzf_close;
+    bgzf_dopen;
+    bgzf_flush;
+    bgzf_flush_try;
+    bgzf_getc;
+    bgzf_getline;
+    bgzf_hopen;
+    bgzf_index_build_init;
+    bgzf_index_dump;
+    bgzf_index_load;
+    bgzf_is_bgzf;
+    bgzf_mt;
+    bgzf_open;
+    bgzf_raw_read;
+    bgzf_raw_write;
+    bgzf_read;
+    bgzf_read_block;
+    bgzf_seek;
+    bgzf_set_cache_size;
+    bgzf_useek;
+    bgzf_utell;
+    bgzf_write;
+    cram_close;
+    cram_compress_block;
+    cram_dopen;
+    cram_eof;
+    cram_flush;
+    cram_free_block;
+    cram_free_container;
+    cram_new_block;
+    cram_new_container;
+    cram_open;
+    cram_read_block;
+    cram_read_container;
+    cram_seek;
+    cram_set_header;
+    cram_set_option;
+    cram_set_voption;
+    cram_uncompress_block;
+    cram_write_block;
+    cram_write_container;
+    fai_build;
+    fai_destroy;
+    fai_fetch;
+    fai_load;
+    faidx_fetch_nseq;
+    faidx_fetch_seq;
+    faidx_has_seq;
+    hclose;
+    hclose_abruptly;
+    hdopen;
+    hfile_destroy;
+    hfile_init;
+    hfile_oflags;
+    hflush;
+    hgetc2;
+    hopen;
+    hpeek;
+    hputc2;
+    hputs2;
+    hread2;
+    hrec_add_idx;
+    hseek;
+    hts_close;
+    hts_file_type;
+    hts_get_bgzfp;
+    hts_getline;
+    hts_idx_destroy;
+    hts_idx_finish;
+    hts_idx_get_meta;
+    hts_idx_get_n_no_coor;
+    hts_idx_get_stat;
+    hts_idx_init;
+    hts_idx_load;
+    hts_idx_push;
+    hts_idx_save;
+    hts_idx_seqnames;
+    hts_idx_set_meta;
+    hts_itr_destroy;
+    hts_itr_next;
+    hts_itr_query;
+    hts_itr_querys;
+    hts_open;
+    hts_parse_reg;
+    hts_readlines;
+    hts_readlist;
+    hts_set_fai_filename;
+    hts_set_threads;
+    hts_verbose;
+    hts_version;
+    hwrite2;
+    kf_betai;
+    kf_erfc;
+    kf_gammap;
+    kf_gammaq;
+    kf_lgamma;
+    kmemmem;
+    knet_close;
+    knet_dopen;
+    knet_open;
+    knet_read;
+    knet_seek;
+    ksplit_core;
+    ksprintf;
+    kstrnstr;
+    kstrstr;
+    kstrtok;
+    kt_fisher_exact;
+    kvsprintf;
+    sam_format1;
+    sam_hdr_add_lines;
+    sam_hdr_dup;
+    sam_hdr_incr_ref;
+    sam_hdr_length;
+    sam_hdr_parse;
+    sam_hdr_read;
+    sam_hdr_str;
+    sam_hdr_write;
+    sam_index_load;
+    sam_itr_queryi;
+    sam_itr_querys;
+    sam_open_mode;
+    sam_parse1;
+    sam_read1;
+    sam_write1;
+    seq_nt16_str;
+    seq_nt16_table;
+    stringify_argv;
+    tbx_conf_bed;
+    tbx_conf_gff;
+    tbx_conf_psltbl;
+    tbx_conf_sam;
+    tbx_conf_vcf;
+    tbx_destroy;
+    tbx_index;
+    tbx_index_build;
+    tbx_index_load;
+    tbx_name2id;
+    tbx_readrec;
+    tbx_seqnames;
+    vcf_format;
+    vcf_hdr_read;
+    vcf_hdr_write;
+    vcf_parse;
+    vcf_read;
+    vcf_write;
+    vcf_write_line;
+};
+
+HTSLIB_1.1 {
+    bcf_get_fmt_id;
+    bcf_get_info_id;
+    faidx_iseq;
+    faidx_nseq;
+    faidx_seq_len;
+} HTSLIB_1.0;
+
+
+HTSLIB_1.2.1 {
+    bcf_copy;
+    bcf_sr_strerror;
+    hisremote;
+    hts_detect_format;
+    hts_format_description;
+    hts_get_format;
+    hts_hopen;
+    hts_set_opt;
+    regidx_destroy;
+    regidx_init;
+    regidx_insert;
+    regidx_nregs;
+    regidx_overlap;
+    regidx_parse_bed;
+    regidx_parse_tab;
+    regidx_seq_names;
+    regidx_seq_nregs;
+    seq_nt16_int;
+} HTSLIB_1.1;
+
+HTSLIB_1.3 {
+    bcf_add_id;
+    bcf_empty;
+    bcf_hdr_merge;
+    bcf_index_build2;
+    bcf_index_load2;
+    bcf_remove_allele_set;
+    bgzf_compress;
+    cram_block_append;
+    cram_block_get_comp_size;
+    cram_block_get_content_id;
+    cram_block_get_content_type;
+    cram_block_get_crc32;
+    cram_block_get_data;
+    cram_block_get_offset;
+    cram_block_get_uncomp_size;
+    cram_block_set_comp_size;
+    cram_block_set_content_id;
+    cram_block_set_crc32;
+    cram_block_set_data;
+    cram_block_set_offset;
+    cram_block_set_uncomp_size;
+    cram_block_size;
+    cram_block_update_size;
+    cram_container_get_landmarks;
+    cram_container_get_length;
+    cram_container_get_num_blocks;
+    cram_container_is_empty;
+    cram_container_set_landmarks;
+    cram_container_set_length;
+    cram_container_set_num_blocks;
+    cram_container_size;
+    cram_copy_slice;
+    cram_fd_get_fp;
+    cram_fd_get_header;
+    cram_fd_get_version;
+    cram_fd_set_fp;
+    cram_fd_set_header;
+    cram_fd_set_version;
+    cram_major_vers;
+    cram_minor_vers;
+    cram_store_container;
+    cram_transcode_rg;
+    hfile_add_scheme_handler;
+    hfile_always_local;
+    hfile_always_remote;
+    hts_format_file_extension;
+    hts_idx_load2;
+    hts_idx_save_as;
+    hts_md5_destroy;
+    hts_md5_final;
+    hts_md5_hex;
+    hts_md5_init;
+    hts_md5_reset;
+    hts_md5_update;
+    hts_open_format;
+    hts_opt_add;
+    hts_opt_apply;
+    hts_opt_free;
+    hts_parse_decimal;
+    hts_parse_format;
+    hts_parse_opt_list;
+    int32_put_blk;
+    kgetline;
+    sam_index_build;
+    sam_index_build2;
+    sam_index_load2;
+    sam_open_mode_opts;
+    tbx_index_build2;
+    tbx_index_load2;
+} HTSLIB_1.2.1;
+
+HTSLIB_1.4 {
+    bam_auxB2f;
+    bam_auxB2i;
+    bam_auxB_len;
+    bam_aux_update_str;
+    bam_mplp_constructor;
+    bam_mplp_destructor;
+    bam_mplp_reset;
+    bam_plp_constructor;
+    bam_plp_destructor;
+    bcf_hdr_format;
+    bcf_index_build3;
+    bcf_sr_destroy_threads;
+    bcf_sr_set_opt;
+    bcf_sr_set_threads;
+    bgzf_block_write;
+    bgzf_compression;
+    bgzf_index_dump_hfile;
+    bgzf_index_load_hfile;
+    bgzf_thread_pool;
+    cram_check_EOF;
+    cram_get_refs;
+    errmod_cal;
+    errmod_destroy;
+    errmod_init;
+    fai_build3;
+    fai_load3;
+    hgetdelim;
+    hgets;
+    hts_check_EOF;
+    hts_json_fnext;
+    hts_json_fskip_value;
+    hts_json_snext;
+    hts_json_sskip_value;
+    hts_realloc_or_die;
+    hts_set_cache_size;
+    hts_set_thread_pool;
+    hts_tpool_delete_result;
+    hts_tpool_destroy;
+    hts_tpool_dispatch;
+    hts_tpool_dispatch2;
+    hts_tpool_init;
+    hts_tpool_kill;
+    hts_tpool_next_result;
+    hts_tpool_next_result_wait;
+    hts_tpool_process_attach;
+    hts_tpool_process_destroy;
+    hts_tpool_process_detach;
+    hts_tpool_process_empty;
+    hts_tpool_process_flush;
+    hts_tpool_process_init;
+    hts_tpool_process_len;
+    hts_tpool_process_qsize;
+    hts_tpool_process_ref_decr;
+    hts_tpool_process_ref_incr;
+    hts_tpool_process_reset;
+    hts_tpool_process_shutdown;
+    hts_tpool_process_sz;
+    hts_tpool_result_data;
+    hts_tpool_size;
+    hts_tpool_wake_dispatch;
+    kputd;
+    probaln_glocal;
+    sam_cap_mapq;
+    sam_index_build3;
+    sam_prob_realn;
+    tbx_index_build3;
+} HTSLIB_1.3;
+
+HTSLIB_1.5 {
+    hfile_set_blksize;
+    hts_get_log_level;
+    hts_log;
+    hts_set_log_level;
+} HTSLIB_1.4;
+
+HTSLIB_1.6 {
+    hts_drand48;
+    hts_erand48;
+    hts_lrand48;
+    hts_srand48;
+} HTSLIB_1.5;
+
+HTSLIB_1.7 {
+    hfile_mem_get_buffer;
+    hfile_mem_steal_buffer;
+    hts_itr_multi_bam;
+    hts_itr_multi_cram;
+    hts_itr_multi_next;
+    hts_itr_regions;
+    hts_json_alloc_token;
+    hts_json_free_token;
+    hts_json_token_str;
+    hts_json_token_type;
+    hts_reglist_free;
+    sam_hdr_change_HD;
+    sam_itr_regions;
+} HTSLIB_1.6;
+
+HTSLIB_1.9 {
+    bam_aux_update_array;
+    bam_aux_update_float;
+    bam_aux_update_int;
+    fai_fetchqual;
+    fai_load3_format;
+    fai_load_format;
+    faidx_fetch_qual;
+} HTSLIB_1.7;
+
+HTSLIB_1.10 {
+    bam_cigar_table;
+    bam_mplp64_auto;
+    bam_plp64_auto;
+    bam_plp64_next;
+    bam_plp_insertion;
+    bam_set_qname;
+    bcf_idx_init;
+    bcf_idx_save;
+    bcf_index_load3;
+    bgzf_peek;
+    fai_fetch64;
+    fai_fetchqual64;
+    fai_parse_region;
+    fai_set_cache_size;
+    faidx_fetch_qual64;
+    faidx_fetch_seq64;
+    haddextension;
+    hts_free;
+    hts_idx_fmt;
+    hts_idx_load3;
+    hts_idx_tbi_name;
+    hts_parse_reg64;
+    hts_parse_region;
+    hts_reglist_create;
+    hts_resize_array_;
+    hts_tpool_dispatch3;
+    kgetline2;
+    regidx_init_string;
+    regidx_insert_list;
+    regidx_parse_reg;
+    regidx_parse_vcf;
+    regidx_push;
+    regitr_copy;
+    regitr_destroy;
+    regitr_init;
+    regitr_loop;
+    regitr_overlap;
+    regitr_reset;
+    sam_hdr_add_line;
+    sam_hdr_add_pg;
+    sam_hdr_count_lines;
+    sam_hdr_destroy;
+    sam_hdr_find_line_id;
+    sam_hdr_find_line_pos;
+    sam_hdr_find_tag_id;
+    sam_hdr_find_tag_pos;
+    sam_hdr_init;
+    sam_hdr_line_index;
+    sam_hdr_line_name;
+    sam_hdr_name2tid;
+    sam_hdr_nref;
+    sam_hdr_pg_id;
+    sam_hdr_remove_except;
+    sam_hdr_remove_line_id;
+    sam_hdr_remove_line_pos;
+    sam_hdr_remove_lines;
+    sam_hdr_remove_tag_id;
+    sam_hdr_tid2len;
+    sam_hdr_tid2name;
+    sam_hdr_update_line;
+    sam_idx_init;
+    sam_idx_save;
+    sam_index_load3;
+    sam_itr_regarray;
+    sam_parse_region;
+    tbx_index_load3;
+} HTSLIB_1.9;
+
+HTSLIB_1.11 {
+    fai_path;
+    hts_lib_shutdown;
+    hts_tpool_process_is_shutdown;
+    vcf_open_mode;
+} HTSLIB_1.10;
+
+HTSLIB_1.12 {
+    bam_parse_cigar;
+    bam_set1;
+    hfile_has_plugin;
+    hfile_list_plugins;
+    hfile_list_schemes;
+    hts_feature_string;
+    hts_features;
+    hts_filter_eval;
+    hts_filter_free;
+    hts_filter_init;
+    hts_set_filter_expression;
+    hts_test_feature;
+    sam_parse_cigar;
+    sam_passes_filter;
+} HTSLIB_1.11;
+
+HTSLIB_1.13 {
+    hts_idx_nseq;
+} HTSLIB_1.12;
+
+HTSLIB_1.14 {
+    bam_mods_at_next_pos;
+    bam_mods_at_qpos;
+    bam_next_basemod;
+    bam_parse_basemod;
+    bam_plp_insertion_mod;
+    hts_base_mod_state_alloc;
+    hts_base_mod_state_free;
+    hts_flush;
+} HTSLIB_1.13;
+
+HTSLIB_1.15 {
+    hts_detect_format2;
+} HTSLIB_1.14;
+
+HTSLIB_1.16 {
+    bam_mods_query_type;
+    bam_mods_recorded;
+    bcf_has_variant_type;
+    bcf_has_variant_types;
+    bcf_variant_length;
+    cram_decode_slice_header;
+    cram_free_slice_header;
+    cram_slice_hdr_get_coords;
+    cram_slice_hdr_get_embed_ref_id;
+    cram_slice_hdr_get_num_blocks;
+    hts_filter_eval2;
+} HTSLIB_1.15;
+
+HTSLIB_1.17 {
+    bam_aux_first;
+    bam_aux_next;
+    bam_aux_remove;
+    bcf_strerror;
+    cram_block_get_method;
+    cram_cid2ds_free;
+    cram_cid2ds_query;
+    cram_codec_describe;
+    cram_codec_get_content_ids;
+    cram_container_get_num_bases;
+    cram_container_get_num_records;
+    cram_decode_compression_header;
+    cram_describe_encodings;
+    cram_expand_method;
+    cram_free_compression_header;
+    cram_update_cid2ds_map;
+    fai_adjust_region;
+    fai_line_length;
+    faidx_seq_len64;
+} HTSLIB_1.16;
+
+HTSLIB_1.18 {
+    bam_mods_queryi;
+    bam_parse_basemod2;
+    fai_thread_pool;
+} HTSLIB_1.17;
+
+HTSLIB_1.20 {
+    tbx_conf_gaf;
+} HTSLIB_1.18;
+
+HTSLIB_1.21 {
+    cram_container_get_coords;
+    cram_container_num2offset;
+    cram_container_offset2num;
+    cram_filter_container;
+    cram_index_extents;
+    cram_num_containers;
+    cram_num_containers_between;
+} HTSLIB_1.20;
diff --git a/ext/htslib/htslib.mk b/ext/htslib/htslib.mk
new file mode 100644
index 0000000..57dffae
--- /dev/null
+++ b/ext/htslib/htslib.mk
@@ -0,0 +1,197 @@
+# Makefile rules useful for third-party code using htslib's public API.
+#
+#    Copyright (C) 2013-2017, 2019, 2021 Genome Research Ltd.
+#
+#    Author: John Marshall <jm18@sanger.ac.uk>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+# The makefile fragment included below provides variables that can be used
+# to express dependencies on headers supplied by an in-development htslib.
+# If your source file foo.c #includes <htslib/hts.h> and <htslib/kstring.h>,
+# you can write the correct prerequisites for foo.o as:
+#
+#	HTSDIR = <path to htslib top-level (build) directory>
+#	include $(HTSDIR)/htslib.mk
+#
+#	foo.o: foo.c $(htslib_hts_h) $(htslib_kstring_h)
+
+HTSSRCDIR = $(HTSDIR)
+HTSPREFIX = $(HTSSRCDIR)/
+include $(HTSDIR)/htslib_vars.mk
+
+# This file provides the HTSCODECS_SOURCES variable.  It may not be present
+# in a freshly checked-out htslib, so is only included if available.  The
+# absence is unlikely to cause a problem as there will be plenty of other
+# missing files that will trigger a build in htslib, and when that happens
+# htslib's makefile will create it.
+-include $(HTSDIR)/htscodecs.mk
+
+# Rules for rebuilding an in-development htslib's static and shared libraries.
+# If your program foo links with libhts, adding the appropriate prerequisite
+# will cause the library to be rebuilt as necessary:
+#
+#	foo: foo.o $(HTSDIR)/libhts.a
+#
+# or similarly if your target requires any of the tools supplied:
+#
+#	bar.bed.bgz.tbi: bar.bed.bgz $(HTSDIR)/tabix
+#		$(HTSDIR)/tabix -p bed bar.bed.bgz
+
+HTSLIB_PUBLIC_HEADERS = \
+	$(HTSSRCDIR)/htslib/bgzf.h \
+	$(HTSSRCDIR)/htslib/cram.h \
+	$(HTSSRCDIR)/htslib/faidx.h \
+	$(HTSSRCDIR)/htslib/hfile.h \
+	$(HTSSRCDIR)/htslib/hts.h \
+	$(HTSSRCDIR)/htslib/hts_defs.h \
+	$(HTSSRCDIR)/htslib/hts_endian.h \
+	$(HTSSRCDIR)/htslib/hts_expr.h \
+	$(HTSSRCDIR)/htslib/hts_log.h \
+	$(HTSSRCDIR)/htslib/hts_os.h \
+	$(HTSSRCDIR)/htslib/kbitset.h \
+	$(HTSSRCDIR)/htslib/kfunc.h \
+	$(HTSSRCDIR)/htslib/khash.h \
+	$(HTSSRCDIR)/htslib/khash_str2int.h \
+	$(HTSSRCDIR)/htslib/klist.h \
+	$(HTSSRCDIR)/htslib/kseq.h \
+	$(HTSSRCDIR)/htslib/ksort.h \
+	$(HTSSRCDIR)/htslib/kstring.h \
+	$(HTSSRCDIR)/htslib/regidx.h \
+	$(HTSSRCDIR)/htslib/sam.h \
+	$(HTSSRCDIR)/htslib/synced_bcf_reader.h \
+	$(HTSSRCDIR)/htslib/tbx.h \
+	$(HTSSRCDIR)/htslib/thread_pool.h \
+	$(HTSSRCDIR)/htslib/vcf.h \
+	$(HTSSRCDIR)/htslib/vcf_sweep.h \
+	$(HTSSRCDIR)/htslib/vcfutils.h
+
+HTSLIB_ALL = \
+	$(HTSLIB_PUBLIC_HEADERS) \
+	$(HTSSRCDIR)/bcf_sr_sort.c \
+	$(HTSSRCDIR)/bcf_sr_sort.h \
+	$(HTSSRCDIR)/bgzf.c \
+	$(HTSDIR)/config.h \
+	$(HTSSRCDIR)/errmod.c \
+	$(HTSSRCDIR)/faidx.c \
+	$(HTSSRCDIR)/header.c \
+	$(HTSSRCDIR)/header.h \
+	$(HTSSRCDIR)/hfile_internal.h \
+	$(HTSSRCDIR)/hfile.c \
+	$(HTSSRCDIR)/hfile_gcs.c \
+	$(HTSSRCDIR)/hfile_libcurl.c \
+	$(HTSSRCDIR)/hfile_s3.c \
+	$(HTSSRCDIR)/hfile_s3_write.c \
+	$(HTSSRCDIR)/hts.c \
+	$(HTSSRCDIR)/hts_expr.c \
+	$(HTSSRCDIR)/hts_internal.h \
+	$(HTSSRCDIR)/hts_os.c \
+	$(HTSSRCDIR)/kfunc.c \
+	$(HTSSRCDIR)/kstring.c \
+	$(HTSSRCDIR)/md5.c \
+	$(HTSSRCDIR)/multipart.c \
+	$(HTSSRCDIR)/plugin.c \
+	$(HTSSRCDIR)/probaln.c \
+	$(HTSSRCDIR)/realn.c \
+	$(HTSSRCDIR)/regidx.c \
+	$(HTSSRCDIR)/region.c \
+	$(HTSSRCDIR)/sam.c \
+	$(HTSSRCDIR)/sam_internal.h \
+	$(HTSSRCDIR)/synced_bcf_reader.c \
+	$(HTSSRCDIR)/tbx.c \
+	$(HTSSRCDIR)/textutils.c \
+	$(HTSSRCDIR)/textutils_internal.h \
+	$(HTSSRCDIR)/thread_pool.c \
+	$(HTSSRCDIR)/thread_pool_internal.h \
+	$(HTSSRCDIR)/vcf.c \
+	$(HTSSRCDIR)/vcf_sweep.c \
+	$(HTSSRCDIR)/vcfutils.c \
+	$(HTSSRCDIR)/cram/cram.h \
+	$(HTSSRCDIR)/cram/cram_codecs.c \
+	$(HTSSRCDIR)/cram/cram_codecs.h \
+	$(HTSSRCDIR)/cram/cram_decode.c \
+	$(HTSSRCDIR)/cram/cram_decode.h \
+	$(HTSSRCDIR)/cram/cram_encode.c \
+	$(HTSSRCDIR)/cram/cram_encode.h \
+	$(HTSSRCDIR)/cram/cram_external.c \
+	$(HTSSRCDIR)/cram/cram_index.c \
+	$(HTSSRCDIR)/cram/cram_index.h \
+	$(HTSSRCDIR)/cram/cram_io.c \
+	$(HTSSRCDIR)/cram/cram_io.h \
+	$(HTSSRCDIR)/cram/cram_samtools.h \
+	$(HTSSRCDIR)/cram/cram_stats.c \
+	$(HTSSRCDIR)/cram/cram_stats.h \
+	$(HTSSRCDIR)/cram/cram_structs.h \
+	$(HTSSRCDIR)/cram/mFILE.c \
+	$(HTSSRCDIR)/cram/mFILE.h \
+	$(HTSSRCDIR)/cram/misc.h \
+	$(HTSSRCDIR)/cram/open_trace_file.c \
+	$(HTSSRCDIR)/cram/open_trace_file.h \
+	$(HTSSRCDIR)/cram/os.h \
+	$(HTSSRCDIR)/cram/pooled_alloc.c \
+	$(HTSSRCDIR)/cram/pooled_alloc.h \
+	$(HTSSRCDIR)/cram/string_alloc.c \
+	$(HTSSRCDIR)/cram/string_alloc.h \
+	$(HTSSRCDIR)/os/lzma_stub.h \
+	$(HTSSRCDIR)/os/rand.c \
+	$(HTSCODECS_SOURCES)
+
+$(HTSDIR)/config.h:
+	+cd $(HTSDIR) && $(MAKE) config.h
+
+$(HTSDIR)/hts-object-files : $(HTSLIB_ALL)
+	+cd $(HTSDIR) && $(MAKE) hts-object-files
+
+$(HTSDIR)/libhts.a: $(HTSDIR)/hts-object-files
+	+cd $(HTSDIR) && $(MAKE) lib-static
+
+$(HTSDIR)/libhts.so: $(HTSLIB_ALL)
+	+cd $(HTSDIR) && $(MAKE) lib-shared
+
+$(HTSDIR)/libhts.dylib $(HTSDIR)/libhts.dll.a $(HTSDIR)/hts.dll.a: $(HTSDIR)/hts-object-files
+	+cd $(HTSDIR) && $(MAKE) lib-shared
+
+$(HTSDIR)/bgzip: $(HTSSRCDIR)/bgzip.c $(HTSLIB_PUBLIC_HEADERS) $(HTSDIR)/libhts.a
+	+cd $(HTSDIR) && $(MAKE) bgzip
+
+$(HTSDIR)/htsfile: $(HTSSRCDIR)/htsfile.c $(HTSLIB_PUBLIC_HEADERS) $(HTSDIR)/libhts.a
+	+cd $(HTSDIR) && $(MAKE) htsfile
+
+$(HTSDIR)/tabix: $(HTSSRCDIR)/tabix.c $(HTSLIB_PUBLIC_HEADERS) $(HTSDIR)/libhts.a
+	+cd $(HTSDIR) && $(MAKE) tabix
+
+$(HTSDIR)/annot-tsv: $(HTSSRCDIR)/annot-tsv.c $(HTSLIB_PUBLIC_HEADERS) $(HTSDIR)/libhts.a
+	+cd $(HTSDIR) && $(MAKE) annot-tsv
+
+$(HTSDIR)/htslib_static.mk: $(HTSDIR)/htslib.pc.tmp
+	+cd $(HTSDIR) && $(MAKE) htslib_static.mk
+
+$(HTSDIR)/htslib.pc.tmp:
+	+cd $(HTSDIR) && $(MAKE) htslib.pc.tmp
+
+# Rules for phony targets.  You may wish to have your corresponding phony
+# targets invoke these in addition to their own recipes:
+#
+#	clean: clean-htslib
+
+all-htslib check-htslib clean-htslib distclean-htslib install-htslib mostlyclean-htslib plugins-htslib test-htslib testclean-htslib:
+	+cd $(HTSDIR) && $(MAKE) $(@:-htslib=)
+
+.PHONY: all-htslib check-htslib clean-htslib distclean-htslib install-htslib
+.PHONY: mostlyclean-htslib plugins-htslib test-htslib testclean-htslib
diff --git a/ext/htslib/htslib.pc.in b/ext/htslib/htslib.pc.in
new file mode 100644
index 0000000..d969d6b
--- /dev/null
+++ b/ext/htslib/htslib.pc.in
@@ -0,0 +1,15 @@
+includedir=@-includedir@
+libdir=@-libdir@
+
+# Flags and libraries needed when linking against a static libhts.a
+# (used by manual and semi-manual pkg-config(1)-style enquiries).
+static_ldflags=@static_LDFLAGS@
+static_libs=@static_LIBS@
+
+Name: htslib
+Description: C library for high-throughput sequencing data formats
+Version: @-PACKAGE_VERSION@
+Cflags: -I${includedir}
+Libs: -L${libdir} -lhts
+Libs.private: -L${libdir} @private_LIBS@ -lhts -lm -lpthread
+Requires.private: zlib @pc_requires@
diff --git a/ext/htslib/htslib/bgzf.h b/ext/htslib/htslib/bgzf.h
new file mode 100644
index 0000000..87d4c6a
--- /dev/null
+++ b/ext/htslib/htslib/bgzf.h
@@ -0,0 +1,506 @@
+/// @file htslib/bgzf.h
+/// Low-level routines for direct BGZF operations.
+/*
+   Copyright (c) 2008 Broad Institute / Massachusetts Institute of Technology
+                 2011, 2012 Attractive Chaos <attractor@live.co.uk>
+   Copyright (C) 2009, 2013, 2014, 2017, 2018-2019, 2022-2024 Genome Research Ltd
+
+   Permission is hereby granted, free of charge, to any person obtaining a copy
+   of this software and associated documentation files (the "Software"), to deal
+   in the Software without restriction, including without limitation the rights
+   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+   copies of the Software, and to permit persons to whom the Software is
+   furnished to do so, subject to the following conditions:
+
+   The above copyright notice and this permission notice shall be included in
+   all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+   THE SOFTWARE.
+*/
+
+/* The BGZF library was originally written by Bob Handsaker from the Broad
+ * Institute. It was later improved by the SAMtools developers. */
+
+#ifndef HTSLIB_BGZF_H
+#define HTSLIB_BGZF_H
+
+#include <stdint.h>
+#include <string.h>
+#include <sys/types.h>
+
+#include "hts_defs.h"
+
+// Ensure ssize_t exists within this header. All #includes must precede this,
+// and ssize_t must be undefined again at the end of this header.
+#if defined _MSC_VER && defined _INTPTR_T_DEFINED && !defined _SSIZE_T_DEFINED && !defined ssize_t
+#define HTSLIB_SSIZE_T
+#define ssize_t intptr_t
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define BGZF_BLOCK_SIZE     0xff00 // make sure compressBound(BGZF_BLOCK_SIZE) < BGZF_MAX_BLOCK_SIZE
+#define BGZF_MAX_BLOCK_SIZE 0x10000
+
+#define BGZF_ERR_ZLIB   1
+#define BGZF_ERR_HEADER 2
+#define BGZF_ERR_IO     4
+#define BGZF_ERR_MISUSE 8
+#define BGZF_ERR_MT     16 // stream cannot be multi-threaded
+#define BGZF_ERR_CRC    32
+
+struct hFILE;
+struct hts_tpool;
+struct kstring_t;
+struct bgzf_mtaux_t;
+typedef struct bgzidx_t bgzidx_t;
+typedef struct bgzf_cache_t bgzf_cache_t;
+struct z_stream_s;
+
+struct BGZF {
+    // Reserved bits should be written as 0; read as "don't care"
+    unsigned errcode:16, reserved:1, is_write:1, no_eof_block:1, is_be:1;
+    signed compress_level:9;
+    unsigned last_block_eof:1, is_compressed:1, is_gzip:1;
+    int cache_size;
+    int block_length, block_clength, block_offset;
+    int64_t block_address, uncompressed_address;
+    void *uncompressed_block, *compressed_block;
+    bgzf_cache_t *cache;
+    struct hFILE *fp; // actual file handle
+    struct bgzf_mtaux_t *mt; // only used for multi-threading
+    bgzidx_t *idx;      // BGZF index
+    int idx_build_otf;  // build index on the fly, set by bgzf_index_build_init()
+    struct z_stream_s *gz_stream; // for gzip-compressed files
+    int64_t seeked;     // virtual offset of last seek
+};
+#ifndef HTS_BGZF_TYPEDEF
+typedef struct BGZF BGZF;
+#define HTS_BGZF_TYPEDEF
+#endif
+
+    /******************
+     * Basic routines *
+     ******************/
+
+    /**
+     * Open an existing file descriptor for reading or writing.
+     *
+     * @param fd    file descriptor
+     *              Note that the file must be opened in binary mode, or else
+     *              there will be problems on platforms that make a difference
+     *              between text and binary mode.
+     * @param mode  mode matching /[rwag][u0-9]+/: 'r' for reading, 'w' for
+     *              writing, 'a' for appending, 'g' for gzip rather than BGZF
+     *              compression (with 'w' only), and digit specifies the zlib
+     *              compression level.
+     *              Note that there is a distinction between 'u' and '0': the
+     *              first yields plain uncompressed output whereas the latter
+     *              outputs uncompressed data wrapped in the zlib format.
+     * @return      BGZF file handler; 0 on error
+     */
+    HTSLIB_EXPORT
+    BGZF* bgzf_dopen(int fd, const char *mode);
+
+    #define bgzf_fdopen(fd, mode) bgzf_dopen((fd), (mode)) // for backward compatibility
+
+    /**
+     * Open the specified file for reading or writing.
+     */
+    HTSLIB_EXPORT
+    BGZF* bgzf_open(const char* path, const char *mode);
+
+    /**
+     * Open an existing hFILE stream for reading or writing.
+     */
+    HTSLIB_EXPORT
+    BGZF* bgzf_hopen(struct hFILE *fp, const char *mode);
+
+    /**
+     * Close the BGZF and free all associated resources.
+     *
+     * @param fp    BGZF file handler
+     * @return      0 on success and -1 on error
+     */
+    HTSLIB_EXPORT
+    int bgzf_close(BGZF *fp);
+
+    /**
+     * Read up to _length_ bytes from the file storing into _data_.
+     *
+     * @param fp     BGZF file handler
+     * @param data   data array to read into
+     * @param length size of data to read
+     * @return       number of bytes actually read; 0 on end-of-file and -1 on error
+     */
+    HTSLIB_EXPORT
+    ssize_t bgzf_read(BGZF *fp, void *data, size_t length) HTS_RESULT_USED;
+
+/**
+ * bgzf_read optimised for small quantities, as a static inline
+ * See bgzf_read() normal function for return values.
+ */
+static inline ssize_t bgzf_read_small(BGZF *fp, void *data, size_t length) {
+    // A block length of 0 implies current block isn't loaded (see
+    // bgzf_seek_common).  That gives negative available so careful on types
+    if ((ssize_t)length < fp->block_length - fp->block_offset) {
+        // Short cut the common and easy mode
+        memcpy((uint8_t *)data,
+               (uint8_t *)fp->uncompressed_block + fp->block_offset,
+               length);
+        fp->block_offset += length;
+        fp->uncompressed_address += length;
+        return length;
+    } else {
+        return bgzf_read(fp, data, length);
+    }
+}
+
+    /**
+     * Write _length_ bytes from _data_ to the file.  If no I/O errors occur,
+     * the complete _length_ bytes will be written (or queued for writing).
+     *
+     * @param fp     BGZF file handler
+     * @param data   data array to write
+     * @param length size of data to write
+     * @return       number of bytes written (i.e., _length_); negative on error
+     */
+    HTSLIB_EXPORT
+    ssize_t bgzf_write(BGZF *fp, const void *data, size_t length) HTS_RESULT_USED;
+
+/**
+ * bgzf_write optimised for small quantities, as a static inline
+ * See bgzf_write() normal function for return values.
+ */
+static inline
+ssize_t bgzf_write_small(BGZF *fp, const void *data, size_t length) {
+    if (fp->is_compressed
+        && (size_t) (BGZF_BLOCK_SIZE - fp->block_offset) > length) {
+        // Short cut the common and easy mode
+        memcpy((uint8_t *)fp->uncompressed_block + fp->block_offset,
+               data, length);
+        fp->block_offset += length;
+        return length;
+    } else {
+        return bgzf_write(fp, data, length);
+    }
+}
+
+    /**
+     * Write _length_ bytes from _data_ to the file, the index will be used to
+     * decide the amount of uncompressed data to be written to each bgzip block.
+     * If no I/O errors occur, the complete _length_ bytes will be written (or
+     * queued for writing).
+     * @param fp     BGZF file handler
+     * @param data   data array to write
+     * @param length size of data to write
+     * @return       number of bytes written (i.e., _length_); negative on error
+     */
+    HTSLIB_EXPORT
+    ssize_t bgzf_block_write(BGZF *fp, const void *data, size_t length);
+
+    /**
+     * Returns the next byte in the file without consuming it.
+     * @param fp     BGZF file handler
+     * @return       -1 on EOF,
+     *               -2 on error,
+     *               otherwise the unsigned byte value.
+     */
+    HTSLIB_EXPORT
+    int bgzf_peek(BGZF *fp);
+
+    /**
+     * Read up to _length_ bytes directly from the underlying stream without
+     * decompressing.  Bypasses BGZF blocking, so must be used with care in
+     * specialised circumstances only.
+     *
+     * @param fp     BGZF file handler
+     * @param data   data array to read into
+     * @param length number of raw bytes to read
+     * @return       number of bytes actually read; 0 on end-of-file and -1 on error
+     */
+    HTSLIB_EXPORT
+    ssize_t bgzf_raw_read(BGZF *fp, void *data, size_t length) HTS_RESULT_USED;
+
+    /**
+     * Write _length_ bytes directly to the underlying stream without
+     * compressing.  Bypasses BGZF blocking, so must be used with care
+     * in specialised circumstances only.
+     *
+     * @param fp     BGZF file handler
+     * @param data   data array to write
+     * @param length number of raw bytes to write
+     * @return       number of bytes actually written; -1 on error
+     */
+    HTSLIB_EXPORT
+    ssize_t bgzf_raw_write(BGZF *fp, const void *data, size_t length) HTS_RESULT_USED;
+
+    /**
+     * Write the data in the buffer to the file.
+     *
+     * @param fp     BGZF file handle
+     * @return       0 on success and -1 on error
+     */
+    HTSLIB_EXPORT
+    int bgzf_flush(BGZF *fp) HTS_RESULT_USED;
+
+    /**
+     * Return a virtual file pointer to the current location in the file.
+     * No interpretation of the value should be made, other than a subsequent
+     * call to bgzf_seek can be used to position the file at the same point.
+     * Return value is non-negative on success.
+     */
+    #define bgzf_tell(fp) (((fp)->block_address << 16) | ((fp)->block_offset & 0xFFFF))
+
+    /**
+     * Set the file to read from the location specified by _pos_.
+     *
+     * @param fp     BGZF file handler
+     * @param pos    virtual file offset returned by bgzf_tell()
+     * @param whence must be SEEK_SET
+     * @return       0 on success and -1 on error
+     *
+     * @note It is not permitted to seek on files open for writing,
+     * or files compressed with gzip (as opposed to bgzip).
+     */
+    HTSLIB_EXPORT
+    int64_t bgzf_seek(BGZF *fp, int64_t pos, int whence) HTS_RESULT_USED;
+
+    /**
+     * Check if the BGZF end-of-file (EOF) marker is present
+     *
+     * @param fp    BGZF file handler opened for reading
+     * @return      1 if the EOF marker is present and correct;
+     *              2 if it can't be checked, e.g., because fp isn't seekable;
+     *              0 if the EOF marker is absent;
+     *              -1 (with errno set) on error
+     */
+    HTSLIB_EXPORT
+    int bgzf_check_EOF(BGZF *fp);
+
+    /** Return the file's compression format
+     *
+     * @param fp  BGZF file handle
+     * @return    A small integer matching the corresponding
+     *            `enum htsCompression` value:
+     *   - 0 / `no_compression` if the file is uncompressed
+     *   - 1 / `gzip` if the file is plain GZIP-compressed
+     *   - 2 / `bgzf` if the file is BGZF-compressed
+     * @since 1.4
+     */
+    HTSLIB_EXPORT
+    int bgzf_compression(BGZF *fp);
+
+    /**
+     * Check if a file is in the BGZF format
+     *
+     * @param fn    file name
+     * @return      1 if _fn_ is BGZF; 0 if not or on I/O error
+     */
+    HTSLIB_EXPORT
+    int bgzf_is_bgzf(const char *fn) HTS_DEPRECATED("Use bgzf_compression() or hts_detect_format() instead");
+
+    /*********************
+     * Advanced routines *
+     *********************/
+
+    /**
+     * Set the cache size. Only effective when compiled with -DBGZF_CACHE.
+     *
+     * @param fp    BGZF file handler
+     * @param size  size of cache in bytes; 0 to disable caching (default)
+     */
+    HTSLIB_EXPORT
+    void bgzf_set_cache_size(BGZF *fp, int size);
+
+    /**
+     * Flush the file if the remaining buffer size is smaller than _size_
+     * @return      0 if flushing succeeded or was not needed; negative on error
+     */
+    HTSLIB_EXPORT
+    int bgzf_flush_try(BGZF *fp, ssize_t size) HTS_RESULT_USED;
+
+    /**
+     * Read one byte from a BGZF file. It is faster than bgzf_read()
+     * @param fp     BGZF file handler
+     * @return       byte read; -1 on end-of-file or error
+     */
+    HTSLIB_EXPORT
+    int bgzf_getc(BGZF *fp);
+
+    /**
+     * Read one line from a BGZF file. It is faster than bgzf_getc()
+     *
+     * @param fp     BGZF file handler
+     * @param delim  delimiter
+     * @param str    string to write to; must be initialized
+     * @return       length of the string (capped at INT_MAX);
+     *               -1 on end-of-file; <= -2 on error
+     */
+    HTSLIB_EXPORT
+    int bgzf_getline(BGZF *fp, int delim, struct kstring_t *str);
+
+    /**
+     * Read the next BGZF block.
+     */
+    HTSLIB_EXPORT
+    int bgzf_read_block(BGZF *fp) HTS_RESULT_USED;
+
+    /**
+     * Enable multi-threading via a shared thread pool.  This means
+     * both encoder and decoder can balance usage across a single pool
+     * of worker jobs.
+     *
+     * @param fp          BGZF file handler
+     * @param pool        The thread pool (see hts_create_threads)
+     * @param qsize       The size of the job queue.  If 0 this is twice the
+     *                    number of threads in the pool.
+     */
+    HTSLIB_EXPORT
+    int bgzf_thread_pool(BGZF *fp, struct hts_tpool *pool, int qsize);
+
+    /**
+     * Enable multi-threading
+     *
+     * @param fp          BGZF file handler
+     * @param n_threads   #threads used for reading / writing
+     * @param n_sub_blks  Unused (was #blocks processed by each thread)
+     */
+    HTSLIB_EXPORT
+    int bgzf_mt(BGZF *fp, int n_threads, int n_sub_blks);
+
+    /**
+     * Compress a single BGZF block.
+     *
+     * @param dst    output buffer (must have size >= BGZF_MAX_BLOCK_SIZE)
+     * @param dlen   size of output buffer; updated on return to the number
+     *               of bytes actually written to dst
+     * @param src    buffer to be compressed
+     * @param slen   size of data to compress (must be <= BGZF_BLOCK_SIZE)
+     * @param level  compression level
+     * @return       0 on success and negative on error
+     */
+    HTSLIB_EXPORT
+    int bgzf_compress(void *dst, size_t *dlen, const void *src, size_t slen, int level);
+
+    /*******************
+     * bgzidx routines *
+     *******************/
+
+    /**
+     *  Position BGZF at the uncompressed offset
+     *
+     *  @param fp           BGZF file handler; must be opened for reading
+     *  @param uoffset      file offset in the uncompressed data
+     *  @param where        must be SEEK_SET
+     *
+     *  Returns 0 on success and -1 on error.
+     *
+     *  @note It is not permitted to seek on files open for writing,
+     *  or files compressed with gzip (as opposed to bgzip).
+     */
+    HTSLIB_EXPORT
+    int bgzf_useek(BGZF *fp, off_t uoffset, int where) HTS_RESULT_USED;
+
+    /**
+     *  Position in uncompressed BGZF
+     *
+     *  @param fp           BGZF file handler; must be opened for reading
+     *
+     *  Returns the current offset on success and -1 on error.
+     */
+    HTSLIB_EXPORT
+    off_t bgzf_utell(BGZF *fp);
+
+    /**
+     * Tell BGZF to build index while compressing.
+     *
+     * @param fp          BGZF file handler; can be opened for reading or writing.
+     *
+     * Returns 0 on success and -1 on error.
+     *
+     * @note This function must be called before any data has been read or
+     * written, and in particular before calling bgzf_mt() on the same
+     * file handle (as threads may start reading data before the index
+     * has been set up).
+     */
+    HTSLIB_EXPORT
+    int bgzf_index_build_init(BGZF *fp);
+
+    /// Load BGZF index
+    /**
+     * @param fp          BGZF file handler
+     * @param bname       base name
+     * @param suffix      suffix to add to bname (can be NULL)
+     * @return 0 on success and -1 on error.
+     */
+    HTSLIB_EXPORT
+    int bgzf_index_load(BGZF *fp,
+                        const char *bname, const char *suffix) HTS_RESULT_USED;
+
+    /// Load BGZF index from an hFILE
+    /**
+     * @param fp   BGZF file handle
+     * @param idx  hFILE to read from
+     * @param name file name (for error reporting only; can be NULL)
+     * @return 0 on success and -1 on error.
+     *
+     * Populates @p fp with index data read from the hFILE handle @p idx.
+     * The file pointer to @idx should point to the start of the index
+     * data when this function is called.
+     *
+     * The file name can optionally be passed in the @p name parameter.  This
+     * is only used for printing error messages; if NULL the word "index" is
+     * used instead.
+     */
+    HTSLIB_EXPORT
+    int bgzf_index_load_hfile(BGZF *fp, struct hFILE *idx,
+                              const char *name) HTS_RESULT_USED;
+
+    /// Save BGZF index
+    /**
+     * @param fp          BGZF file handler
+     * @param bname       base name
+     * @param suffix      suffix to add to bname (can be NULL)
+     * @return 0 on success and -1 on error.
+     */
+    HTSLIB_EXPORT
+    int bgzf_index_dump(BGZF *fp,
+                        const char *bname, const char *suffix) HTS_RESULT_USED;
+
+    /// Write a BGZF index to an hFILE
+    /**
+     * @param fp     BGZF file handle
+     * @param idx    hFILE to write to
+     * @param name   file name (for error reporting only, can be NULL)
+     * @return 0 on success and -1 on error.
+     *
+     * Write index data from @p fp to the file @p idx.
+     *
+     * The file name can optionally be passed in the @p name parameter.  This
+     * is only used for printing error messages; if NULL the word "index" is
+     * used instead.
+     */
+
+    HTSLIB_EXPORT
+    int bgzf_index_dump_hfile(BGZF *fp, struct hFILE *idx,
+                              const char *name) HTS_RESULT_USED;
+
+#ifdef __cplusplus
+}
+#endif
+
+#ifdef HTSLIB_SSIZE_T
+#undef HTSLIB_SSIZE_T
+#undef ssize_t
+#endif
+
+#endif
diff --git a/ext/htslib/htslib/cram.h b/ext/htslib/htslib/cram.h
new file mode 100644
index 0000000..ddc44bb
--- /dev/null
+++ b/ext/htslib/htslib/cram.h
@@ -0,0 +1,826 @@
+/// @file htslib/cram.h
+/// CRAM format-specific API functions.
+/*
+    Copyright (C) 2015, 2016, 2018-2020, 2022-2024 Genome Research Ltd.
+
+    Author: James Bonfield <jkb@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+/** @file
+ * Consider using the higher level hts_*() API for programs that wish to
+ * be file format agnostic (see htslib/hts.h).
+ *
+ * This API should be used for CRAM specific code. The specifics of the
+ * public API are implemented in cram_io.h, cram_encode.h and cram_decode.h
+ * although these should not be included directly (use this file instead).
+ */
+
+#ifndef HTSLIB_CRAM_H
+#define HTSLIB_CRAM_H
+
+#include <stdarg.h>
+#include <stdint.h>
+#include <sys/types.h>
+
+#include "hts_defs.h"
+#include "hts.h"
+#include "sam.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// see cram/cram_structs.h for an internal more complete copy of this enum
+
+// Htslib 1.11 had these listed without any hts prefix, and included
+// some internal values such as RANS1 and GZIP_RLE (which shouldn't have ever
+// been public).
+//
+// We can't find evidence of these being used and the data type occurs
+// nowhere in functions or structures meaning using it would be pointless.
+// However for safety, if you absolute need the API to not change then
+// define HTS_COMPAT to 101100 (XYYYZZ for X.Y[.Z], meaning 1.11).
+#if defined(HTS_COMPAT) && HTS_COMPAT <= 101100
+enum cram_block_method {
+    // Public methods as defined in the CRAM spec.
+    BM_ERROR = -1,
+
+    // CRAM 2.x and 3.0
+    RAW      = 0,
+    GZIP     = 1,
+    BZIP2    = 2,
+    LZMA     = 3,
+    RANS     = 4,
+
+    // NB: the subsequent numbers may change.  They're simply here for
+    // compatibility with the old API, but may have no bearing on the
+    // internal way htslib works.  DO NOT USE
+    RANS0    = 4,
+    RANS1    = 10,
+    GZIP_RLE = 11,
+};
+#else
+
+// Values as defined in the CRAM specifications.
+// See cram/cram_structs.h cram_block_method_int for an expanded version of
+// this with local specialisations assigned to codes.
+enum cram_block_method {
+    CRAM_COMP_UNKNOWN = -1,
+
+    // CRAM 2.x and 3.0
+    CRAM_COMP_RAW      = 0,
+    CRAM_COMP_GZIP     = 1,
+    CRAM_COMP_BZIP2    = 2,
+
+    // CRAM 3.0
+    CRAM_COMP_LZMA     = 3,
+    CRAM_COMP_RANS4x8  = 4, // 4-way interleaving, 8-bit renormalisation
+
+    // CRAM 3.1
+    CRAM_COMP_RANSNx16 = 5, // both 4x16 and 32x16 variants, plus transforms
+    CRAM_COMP_ARITH    = 6, // aka Range coding
+    CRAM_COMP_FQZ      = 7, // FQZComp
+    CRAM_COMP_TOK3     = 8, // Name tokeniser
+};
+#endif
+
+/* NOTE this structure may be expanded in future releases by appending
+ * additional fields.
+ *
+ * Do not assume the size is fixed and avoid using arrays of this struct.
+ */
+typedef struct {
+    enum cram_block_method method;
+
+    // Generic compression level if known (0 if not).
+    // 1 or 9 for gzip min/max flag (else 5).  1-9 for bzip2
+    // 1 or 11 for for tok3 (rans/arith encoder).
+    int level;
+
+    // For rans* and arith codecs
+    int order;
+
+    // ransNx16/arith specific
+    int rle;
+    int pack;
+    int stripe;
+    int cat;
+    int nosz;
+    int Nway;
+
+    // Arithmetic coder only
+    int ext; // external: use gz, xz or bzip2
+} cram_method_details;
+
+enum cram_content_type {
+    CT_ERROR           = -1,
+    FILE_HEADER        = 0,
+    COMPRESSION_HEADER = 1,
+    MAPPED_SLICE       = 2,
+    UNMAPPED_SLICE     = 3, // CRAM V1.0 only
+    EXTERNAL           = 4,
+    CORE               = 5,
+};
+
+// Opaque data types, see cram_structs for the fully fledged versions.
+typedef struct cram_file_def cram_file_def;
+typedef struct cram_fd cram_fd;
+typedef struct cram_container cram_container;
+typedef struct cram_block cram_block;
+typedef struct cram_slice cram_slice;
+typedef struct cram_metrics cram_metrics;
+typedef struct cram_block_slice_hdr cram_block_slice_hdr;
+typedef struct cram_block_compression_hdr cram_block_compression_hdr;
+typedef struct cram_codec cram_codec;
+typedef struct refs_t refs_t;
+
+struct hFILE;
+
+// Accessor functions
+
+/*
+ *-----------------------------------------------------------------------------
+ * cram_fd
+ */
+HTSLIB_EXPORT
+sam_hdr_t *cram_fd_get_header(cram_fd *fd);
+
+HTSLIB_EXPORT
+void cram_fd_set_header(cram_fd *fd, sam_hdr_t *hdr);
+
+HTSLIB_EXPORT
+int cram_fd_get_version(cram_fd *fd);
+
+HTSLIB_EXPORT
+void cram_fd_set_version(cram_fd *fd, int vers);
+
+HTSLIB_EXPORT
+int cram_major_vers(cram_fd *fd);
+HTSLIB_EXPORT
+int cram_minor_vers(cram_fd *fd);
+
+HTSLIB_EXPORT
+struct hFILE *cram_fd_get_fp(cram_fd *fd);
+HTSLIB_EXPORT
+void cram_fd_set_fp(cram_fd *fd, struct hFILE *fp);
+
+
+/*
+ *-----------------------------------------------------------------------------
+ * cram_container
+ */
+HTSLIB_EXPORT
+int32_t cram_container_get_length(cram_container *c);
+HTSLIB_EXPORT
+void cram_container_set_length(cram_container *c, int32_t length);
+HTSLIB_EXPORT
+int32_t cram_container_get_num_blocks(cram_container *c);
+HTSLIB_EXPORT
+void cram_container_set_num_blocks(cram_container *c, int32_t num_blocks);
+HTSLIB_EXPORT
+int32_t *cram_container_get_landmarks(cram_container *c, int32_t *num_landmarks);
+HTSLIB_EXPORT
+void cram_container_set_landmarks(cram_container *c, int32_t num_landmarks,
+                                  int32_t *landmarks);
+HTSLIB_EXPORT
+int32_t cram_container_get_num_records(cram_container *c);
+HTSLIB_EXPORT
+int64_t cram_container_get_num_bases(cram_container *c);
+
+/* Returns true if the container is empty (EOF marker) */
+HTSLIB_EXPORT
+int cram_container_is_empty(cram_fd *fd);
+
+
+/* Returns chromosome and start/span from container struct */
+HTSLIB_EXPORT
+void cram_container_get_coords(cram_container *c,
+                               int *refid, hts_pos_t *start, hts_pos_t *span);
+
+/*
+ *-----------------------------------------------------------------------------
+ * cram_block
+ */
+HTSLIB_EXPORT
+int32_t cram_block_get_content_id(cram_block *b);
+HTSLIB_EXPORT
+int32_t cram_block_get_comp_size(cram_block *b);
+HTSLIB_EXPORT
+int32_t cram_block_get_uncomp_size(cram_block *b);
+HTSLIB_EXPORT
+int32_t cram_block_get_crc32(cram_block *b);
+HTSLIB_EXPORT
+void *  cram_block_get_data(cram_block *b);
+HTSLIB_EXPORT
+enum cram_content_type cram_block_get_content_type(cram_block *b);
+HTSLIB_EXPORT
+enum cram_block_method cram_block_get_method(cram_block *b);
+
+HTSLIB_EXPORT
+cram_method_details *cram_expand_method(uint8_t *data, int32_t size,
+                                        enum cram_block_method comp);
+
+HTSLIB_EXPORT
+void cram_block_set_content_id(cram_block *b, int32_t id);
+HTSLIB_EXPORT
+void cram_block_set_comp_size(cram_block *b, int32_t size);
+HTSLIB_EXPORT
+void cram_block_set_uncomp_size(cram_block *b, int32_t size);
+HTSLIB_EXPORT
+void cram_block_set_crc32(cram_block *b, int32_t crc);
+HTSLIB_EXPORT
+void cram_block_set_data(cram_block *b, void *data);
+
+HTSLIB_EXPORT
+int cram_block_append(cram_block *b, const void *data, int size);
+HTSLIB_EXPORT
+void cram_block_update_size(cram_block *b);
+
+// Offset is known as "size" internally, but it can be confusing.
+HTSLIB_EXPORT
+size_t cram_block_get_offset(cram_block *b);
+HTSLIB_EXPORT
+void cram_block_set_offset(cram_block *b, size_t offset);
+
+/*
+ * Computes the size of a cram block, including the block
+ * header itself.
+ */
+HTSLIB_EXPORT
+uint32_t cram_block_size(cram_block *b);
+
+/*
+ * Returns the Block Content ID values referred to by a cram_codec in
+ * ids[2].
+ *
+ * -2 is unused.
+ * -1 is CORE
+ * >= 0 is the block with that Content ID
+ */
+HTSLIB_EXPORT
+void cram_codec_get_content_ids(cram_codec *c, int ids[2]);
+
+/*
+ * Produces a human readable description of the codec parameters.
+ * This is appended to an existing kstring 'ks'.
+ *
+ * Returns 0 on succes,
+ *        <0 on failure
+ */
+HTSLIB_EXPORT
+int cram_codec_describe(cram_codec *c, kstring_t *ks);
+
+/*
+ * Renumbers RG numbers in a cram compression header.
+ *
+ * CRAM stores RG as the Nth number in the header, rather than a
+ * string holding the ID: tag.  This is smaller in space, but means
+ * "samtools cat" to join files together that contain single but
+ * different RG lines needs a way of renumbering them.
+ *
+ * The file descriptor is expected to be immediately after the
+ * cram_container structure (ie before the cram compression header).
+ * Due to the nature of the CRAM format, this needs to read and write
+ * the blocks itself.  Note that there may be multiple slices within
+ * the container, meaning multiple compression headers to manipulate.
+ * Changing RG may change the size of the compression header and
+ * therefore the length field in the container.  Hence we rewrite all
+ * blocks just in case and also emit the adjusted container.
+ *
+ * The current implementation can only cope with renumbering a single
+ * RG (and only then if it is using HUFFMAN or BETA codecs).  In
+ * theory it *may* be possible to renumber multiple RGs if they use
+ * HUFFMAN to the CORE block or use an external block unshared by any
+ * other data series.  So we have an API that can be upgraded to
+ * support this, but do not implement it for now.  An example
+ * implementation of RG as an EXTERNAL block would be to find that
+ * block and rewrite it, returning the number of blocks consumed.
+ *
+ * Returns 0 on success;
+ *        -1 if unable to edit;
+ *        -2 on other errors (eg I/O).
+ */
+HTSLIB_EXPORT
+int cram_transcode_rg(cram_fd *in, cram_fd *out,
+                      cram_container *c,
+                      int nrg, int *in_rg, int *out_rg);
+
+/*
+ * Copies the blocks representing the next num_slice slices from a
+ * container from 'in' to 'out'.  It is expected that the file pointer
+ * is just after the read of the cram_container and cram compression
+ * header.
+ *
+ * Returns 0 on success
+ *        -1 on failure
+ */
+HTSLIB_EXPORT
+int cram_copy_slice(cram_fd *in, cram_fd *out, int32_t num_slice);
+
+/*
+ * Copies a container, but filtering it down to a specific region (as
+ * already specified in 'in'
+ *
+ * Returns 0 on success
+ *        -1 on EOF
+ *        -2 on error
+ */
+HTSLIB_EXPORT
+int cram_filter_container(cram_fd *in, cram_fd *out, cram_container *c,
+                          int *ref_id);
+
+/*
+ * Decodes a CRAM block compression header.
+ * Returns header ptr on success
+ *         NULL on failure
+ */
+HTSLIB_EXPORT
+cram_block_compression_hdr *cram_decode_compression_header(cram_fd *fd,
+                                                           cram_block *b);
+/*
+ * Frees a cram_block_compression_hdr structure.
+ */
+HTSLIB_EXPORT
+void cram_free_compression_header(cram_block_compression_hdr *hdr);
+
+typedef struct cram_cid2ds_t cram_cid2ds_t;
+
+/*
+ * Map cram block numbers to data-series.  It's normally a 1:1 mapping,
+ * but in rare cases it can be 1:many (or even many:many).
+ * The key is the block number and the value is an index into the data-series
+ * array, which we iterate over until reaching a negative value.
+ *
+ * Provide cid2ds as NULL to allocate a new map or pass in an existing one
+ * to append to this map.  The new (or existing) map is returned.
+ *
+ * Returns the cid2ds (newly allocated or as provided) on success,
+ *         NULL on failure.
+ */
+HTSLIB_EXPORT
+cram_cid2ds_t *cram_update_cid2ds_map(cram_block_compression_hdr *hdr,
+                                      cram_cid2ds_t *cid2ds);
+
+/*
+ * Return a list of data series observed as belonging to a block with
+ * the specified content_id.  *n is the number of data series
+ * returned, or 0 if block is unused.
+ * Block content_id of -1 is used to indicate the CORE block.
+ *
+ * The pointer returned is owned by the cram_cid2ds state and should
+ * not be freed by the caller.
+ */
+HTSLIB_EXPORT
+int *cram_cid2ds_query(cram_cid2ds_t *c2d, int content_id, int *n);
+
+/*
+ * Frees a cram_cid2ds_t allocated by cram_update_cid2ds_map
+ */
+HTSLIB_EXPORT
+void cram_cid2ds_free(cram_cid2ds_t *cid2ds);
+
+/*
+ * Produces a description of the record and tag encodings held within
+ * a compression header and appends to 'ks'.
+ *
+ * Returns 0 on success,
+ *        <0 on failure.
+ */
+HTSLIB_EXPORT
+int cram_describe_encodings(cram_block_compression_hdr *hdr, kstring_t *ks);
+
+/*
+ *-----------------------------------------------------------------------------
+ * cram slice interrogation
+ */
+
+/*
+ * Returns the number of cram blocks within this slice.
+ */
+HTSLIB_EXPORT
+int32_t cram_slice_hdr_get_num_blocks(cram_block_slice_hdr *hdr);
+
+/*
+ * Returns the block content_id for the block containing an embedded reference
+ * sequence.  If none is present, -1 is returned.
+ */
+HTSLIB_EXPORT
+int cram_slice_hdr_get_embed_ref_id(cram_block_slice_hdr *h);
+
+/*
+ * Returns slice reference ID, start and span (length) coordinates.
+ * Return parameters may be NULL in which case they are ignored.
+ */
+HTSLIB_EXPORT
+void cram_slice_hdr_get_coords(cram_block_slice_hdr *h,
+                               int *refid, hts_pos_t *start, hts_pos_t *span);
+
+/*
+ * Decodes a slice header from a cram block.
+ * Returns the opaque cram_block_slice_hdr pointer on success,
+ *         NULL on failure.
+ */
+HTSLIB_EXPORT
+cram_block_slice_hdr *cram_decode_slice_header(cram_fd *fd, cram_block *b);
+
+/*
+ * Frees a cram_block_slice_hdr structure.
+ */
+HTSLIB_EXPORT
+void cram_free_slice_header(cram_block_slice_hdr *hdr);
+
+/*
+ *-----------------------------------------------------------------------------
+ * cram_io basics
+ */
+
+/**@{ ----------------------------------------------------------------------
+ * CRAM blocks - the dynamically growable data block. We have code to
+ * create, update, (un)compress and read/write.
+ *
+ * These are derived from the deflate_interlaced.c blocks, but with the
+ * CRAM extension of content types and IDs.
+ */
+
+/*! Allocates a new cram_block structure with a specified content_type and
+ * id.
+ *
+ * @return
+ * Returns block pointer on success;
+ *         NULL on failure
+ *
+ * The cram_block struct returned by a successful call should be freed
+ * via cram_free_block() when it is no longer needed.
+ */
+HTSLIB_EXPORT
+cram_block *cram_new_block(enum cram_content_type content_type,
+                           int content_id);
+
+/*! Reads a block from a cram file.
+ *
+ * @return
+ * Returns cram_block pointer on success;
+ *         NULL on failure
+ *
+ * The cram_block struct returned by a successful call should be freed
+ * via cram_free_block() when it is no longer needed.
+ */
+HTSLIB_EXPORT
+cram_block *cram_read_block(cram_fd *fd);
+
+/*! Writes a CRAM block.
+ *
+ * @return
+ * Returns 0 on success;
+ *        -1 on failure
+ */
+HTSLIB_EXPORT
+int cram_write_block(cram_fd *fd, cram_block *b);
+
+/*! Frees a CRAM block, deallocating internal data too.
+ */
+HTSLIB_EXPORT
+void cram_free_block(cram_block *b);
+
+/*! Uncompresses a CRAM block, if compressed.
+ *
+ * @return
+ * Returns 0 on success;
+ *        -1 on failure
+ */
+HTSLIB_EXPORT
+int cram_uncompress_block(cram_block *b);
+
+/*! Compresses a block.
+ *
+ * Compresses a block using one of two different zlib strategies. If we only
+ * want one choice set strat2 to be -1.
+ *
+ * The logic here is that sometimes Z_RLE does a better job than Z_FILTERED
+ * or Z_DEFAULT_STRATEGY on quality data. If so, we'd rather use it as it is
+ * significantly faster.
+ *
+ * @return
+ * Returns 0 on success;
+ *        -1 on failure
+ */
+HTSLIB_EXPORT
+int cram_compress_block(cram_fd *fd, cram_block *b, cram_metrics *metrics,
+                        int method, int level);
+int cram_compress_block2(cram_fd *fd, cram_slice *s,
+                         cram_block *b, cram_metrics *metrics,
+                         int method, int level);
+
+/**@}*/
+/**@{ ----------------------------------------------------------------------
+ * Containers
+ */
+
+/*! Creates a new container, specifying the maximum number of slices
+ * and records permitted.
+ *
+ * @return
+ * Returns cram_container ptr on success;
+ *         NULL on failure
+ *
+ * The cram_container struct returned by a successful call should be freed
+ * via cram_free_container() when it is no longer needed.
+ */
+HTSLIB_EXPORT
+cram_container *cram_new_container(int nrec, int nslice);
+HTSLIB_EXPORT
+void cram_free_container(cram_container *c);
+
+/*! Reads a container header.
+ *
+ * @return
+ * Returns cram_container on success;
+ *         NULL on failure or no container left (fd->err == 0).
+ *
+ * The cram_container struct returned by a successful call should be freed
+ * via cram_free_container() when it is no longer needed.
+ */
+HTSLIB_EXPORT
+cram_container *cram_read_container(cram_fd *fd);
+
+/*! Writes a container structure.
+ *
+ * @return
+ * Returns 0 on success;
+ *        -1 on failure
+ */
+HTSLIB_EXPORT
+int cram_write_container(cram_fd *fd, cram_container *h);
+
+/*
+ * Stores the container structure in dat and returns *size as the
+ * number of bytes written to dat[].  The input size of dat is also
+ * held in *size and should be initialised to cram_container_size(c).
+ *
+ * Returns 0 on success;
+ *        -1 on failure
+ */
+HTSLIB_EXPORT
+int cram_store_container(cram_fd *fd, cram_container *c, char *dat, int *size);
+
+HTSLIB_EXPORT
+int cram_container_size(cram_container *c);
+
+/**@}*/
+/**@{ ----------------------------------------------------------------------
+ * The top-level cram opening, closing and option handling
+ */
+
+/*! Opens a CRAM file for read (mode "rb") or write ("wb").
+ *
+ * The filename may be "-" to indicate stdin or stdout.
+ *
+ * @return
+ * Returns file handle on success;
+ *         NULL on failure.
+ */
+HTSLIB_EXPORT
+cram_fd *cram_open(const char *filename, const char *mode);
+
+/*! Opens an existing stream for reading or writing.
+ *
+ * @return
+ * Returns file handle on success;
+ *         NULL on failure.
+ */
+HTSLIB_EXPORT
+cram_fd *cram_dopen(struct hFILE *fp, const char *filename, const char *mode);
+
+/*! Closes a CRAM file.
+ *
+ * @return
+ * Returns 0 on success;
+ *        -1 on failure
+ */
+HTSLIB_EXPORT
+int cram_close(cram_fd *fd);
+
+/*
+ * Seek within a CRAM file.
+ *
+ * Returns 0 on success
+ *        -1 on failure
+ */
+HTSLIB_EXPORT
+int cram_seek(cram_fd *fd, off_t offset, int whence);
+
+/*
+ * Flushes a CRAM file.
+ * Useful for when writing to stdout without wishing to close the stream.
+ *
+ * Returns 0 on success
+ *        -1 on failure
+ */
+HTSLIB_EXPORT
+int cram_flush(cram_fd *fd);
+
+/*! Checks for end of file on a cram_fd stream.
+ *
+ * @return
+ * Returns 0 if not at end of file
+ *         1 if we hit an expected EOF (end of range or EOF block)
+ *         2 for other EOF (end of stream without EOF block)
+ */
+HTSLIB_EXPORT
+int cram_eof(cram_fd *fd);
+
+/*! Sets options on the cram_fd.
+ *
+ * See CRAM_OPT_* definitions in hts.h.
+ * Use this immediately after opening.
+ *
+ * @return
+ * Returns 0 on success;
+ *        -1 on failure
+ */
+HTSLIB_EXPORT
+int cram_set_option(cram_fd *fd, enum hts_fmt_option opt, ...);
+
+/*! Sets options on the cram_fd.
+ *
+ * See CRAM_OPT_* definitions in hts.h.
+ * Use this immediately after opening.
+ *
+ * @return
+ * Returns 0 on success;
+ *        -1 on failure
+ */
+HTSLIB_EXPORT
+int cram_set_voption(cram_fd *fd, enum hts_fmt_option opt, va_list args);
+
+/*!
+ * Attaches a header to a cram_fd.
+ *
+ * This should be used when creating a new cram_fd for writing where
+ * we have an SAM_hdr already constructed (eg from a file we've read
+ * in).
+ *
+ * @return
+ * Returns 0 on success;
+ *        -1 on failure
+ */
+HTSLIB_EXPORT
+int cram_set_header(cram_fd *fd, sam_hdr_t *hdr);
+
+/*! Check if this file has a proper EOF block
+ *
+ * @return
+ * Returns 3 if the file is a version of CRAM that does not contain EOF blocks
+ *         2 if the file is a stream and thus unseekable
+ *         1 if the file contains an EOF block
+ *         0 if the file does not contain an EOF block
+ *        -1 if an error occurred whilst reading the file or we could not seek back to where we were
+ *
+ */
+HTSLIB_EXPORT
+int cram_check_EOF(cram_fd *fd);
+
+/* As int32_decoded/encode, but from/to blocks instead of cram_fd */
+HTSLIB_EXPORT
+int int32_put_blk(cram_block *b, int32_t val);
+
+/**@}*/
+/**@{ -------------------------------------------------------------------
+ * Old typedef and function names for compatibility with existing code.
+ * Header functionality is now provided by sam.h's sam_hdr_t functions.
+ */
+
+typedef sam_hdr_t SAM_hdr;
+
+/*! Tokenises a SAM header into a hash table.
+ *
+ * Also extracts a few bits on specific data types, such as @RG lines.
+ *
+ * @return
+ * Returns a SAM_hdr struct on success (free with sam_hdr_free());
+ *         NULL on failure
+ */
+static inline SAM_hdr *sam_hdr_parse_(const char *hdr, size_t len) { return sam_hdr_parse(len, hdr); }
+
+/*! Deallocates all storage used by a SAM_hdr struct.
+ *
+ * This also decrements the header reference count. If after decrementing
+ * it is still non-zero then the header is assumed to be in use by another
+ * caller and the free is not done.
+ */
+static inline void sam_hdr_free(SAM_hdr *hdr) { sam_hdr_destroy(hdr); }
+
+/* sam_hdr_length() and sam_hdr_str() are now provided by sam.h. */
+
+/*! Add an @PG line.
+ *
+ * If we wish complete control over this use sam_hdr_add_line() directly. This
+ * function uses that, but attempts to do a lot of tedious house work for
+ * you too.
+ *
+ * - It will generate a suitable ID if the supplied one clashes.
+ * - It will generate multiple @PG records if we have multiple PG chains.
+ *
+ * Call it as per sam_hdr_add_line() with a series of key,value pairs ending
+ * in NULL.
+ *
+ * @return
+ * Returns 0 on success;
+ *        -1 on failure
+ */
+#define sam_hdr_add_PG sam_hdr_add_pg
+
+/**@{ -------------------------------------------------------------------*/
+
+/*!
+ * Returns the refs_t structure used by a cram file handle.
+ *
+ * This may be used in conjunction with option CRAM_OPT_SHARED_REF to
+ * share reference memory between multiple file handles.
+ *
+ * @return
+ * Returns NULL if none exists or the file handle is not a CRAM file.
+ */
+HTSLIB_EXPORT
+refs_t *cram_get_refs(htsFile *fd);
+
+/*!
+ * Returns the file offsets of CRAM slices covering a specific region
+ * query.  Note both offsets are the START of the slice.
+ *
+ * first will point to the start of the first overlapping slice
+ * last will point to the start of the last overlapping slice
+ *
+ * @return
+ * Returns 0 on success
+ *        <0 on failure
+ */
+HTSLIB_EXPORT
+int cram_index_extents(cram_fd *fd, int refid, hts_pos_t start, hts_pos_t end,
+                       off_t *first, off_t *last);
+
+/*! Returns the total number of containers in the CRAM index.
+ *
+ * Note the index is not required to have an entry for every container, but it
+ * will always have an index entry for the start of each chromosome.
+ * (Although in practice our indices do container one entry per container.)
+ *
+ * This is equivalent to cram_num_containers_between(fd, 0, 0, NULL, NULL)
+ */
+HTSLIB_EXPORT
+int64_t cram_num_containers(cram_fd *fd);
+
+/*! Returns the number of containers in the CRAM index within given offsets.
+ *
+ * The cstart and cend offsets are the locations of the start of containers
+ * as returned by index_container_offset.
+ *
+ * If non-NULL, first and last will hold the inclusive range of container
+ * numbers, counting from zero.
+ *
+ * @return
+ * Returns the number of containers, equivalent to *last-*first+1.
+ */
+HTSLIB_EXPORT
+int64_t cram_num_containers_between(cram_fd *fd,
+                                    off_t cstart, off_t cend,
+                                    int64_t *first, int64_t *last);
+
+/*! Returns the byte offset for the start of the n^th container.
+ *
+ * The index must have previously been loaded, otherwise <0 is returned.
+ */
+HTSLIB_EXPORT
+off_t cram_container_num2offset(cram_fd *fd, int64_t n);
+
+/*! Returns the container number for the first container at offset >= pos.
+ *
+ * The index must have previously been loaded, otherwise <0 is returned.
+ */
+HTSLIB_EXPORT
+int64_t cram_container_offset2num(cram_fd *fd, off_t pos);
+
+/**@}*/
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/ext/htslib/htslib/faidx.h b/ext/htslib/htslib/faidx.h
new file mode 100644
index 0000000..4351b3f
--- /dev/null
+++ b/ext/htslib/htslib/faidx.h
@@ -0,0 +1,391 @@
+/// @file htslib/faidx.h
+/// FASTA random access.
+/*
+   Copyright (C) 2008, 2009, 2013, 2014, 2016, 2017-2020, 2022-2023 Genome Research Ltd.
+
+   Author: Heng Li <lh3@sanger.ac.uk>
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be
+   included in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+*/
+
+#ifndef HTSLIB_FAIDX_H
+#define HTSLIB_FAIDX_H
+
+#include <stdint.h>
+#include "hts_defs.h"
+#include "hts.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** @file
+
+  Index FASTA or FASTQ files and extract subsequence.
+
+  The fai file index columns for FASTA are:
+    - chromosome name
+    - chromosome length: number of bases
+    - offset: number of bytes to skip to get to the first base
+        from the beginning of the file, including the length
+        of the sequence description string (`>chr ..\n`)
+    - line length: number of bases per line (excluding `\n`)
+    - binary line length: number of bytes, including `\n`
+
+   The index for FASTQ is similar to above:
+    - chromosome name
+    - chromosome length: number of bases
+    - sequence offset: number of bytes to skip to get to the first base
+        from the beginning of the file, including the length
+        of the sequence description string (`@chr ..\n`)
+    - line length: number of bases per line (excluding `\n`)
+    - binary line length: number of bytes, including `\n`
+    - quality offset: number of bytes to skip from the beginning of the file
+        to get to the first quality value in the indexed entry.
+
+    The FASTQ version of the index uses line length and binary line length
+    for both the sequence and the quality values, so they must be line
+    wrapped in the same way.
+ */
+
+struct faidx_t;
+/// Opaque structure representing FASTA index
+typedef struct faidx_t faidx_t;
+
+/// Opaque structure; sole item needed from htslib/thread_pool.h
+struct hts_tpool;
+
+/// File format to be dealing with.
+enum fai_format_options {
+    FAI_NONE,
+    FAI_FASTA,
+    FAI_FASTQ
+};
+
+/// Build index for a FASTA or FASTQ or bgzip-compressed FASTA or FASTQ file.
+/**  @param  fn  FASTA/FASTQ file name
+     @param  fnfai Name of .fai file to build.
+     @param  fngzi Name of .gzi file to build (if fn is bgzip-compressed).
+     @return     0 on success; or -1 on failure
+
+If fnfai is NULL, ".fai" will be appended to fn to make the FAI file name.
+If fngzi is NULL, ".gzi" will be appended to fn for the GZI file.  The GZI
+file will only be built if fn is bgzip-compressed.
+*/
+HTSLIB_EXPORT
+int fai_build3(const char *fn, const char *fnfai, const char *fngzi) HTS_RESULT_USED;
+
+/// Build index for a FASTA or FASTQ or bgzip-compressed FASTA or FASTQ file.
+/** @param  fn  FASTA/FASTQ file name
+    @return     0 on success; or -1 on failure
+
+File "fn.fai" will be generated.  This function is equivalent to
+fai_build3(fn, NULL, NULL);
+*/
+HTSLIB_EXPORT
+int fai_build(const char *fn) HTS_RESULT_USED;
+
+/// Destroy a faidx_t struct
+HTSLIB_EXPORT
+void fai_destroy(faidx_t *fai);
+
+enum fai_load_options {
+    FAI_CREATE = 0x01,
+};
+
+/// Load FASTA indexes.
+/** @param  fn  File name of the FASTA file (can be compressed with bgzip).
+    @param  fnfai File name of the FASTA index.
+    @param  fngzi File name of the bgzip index.
+    @param  flags Option flags to control index file caching and creation.
+    @return Pointer to a faidx_t struct on success, NULL on failure.
+
+If fnfai is NULL, ".fai" will be appended to fn to make the FAI file name.
+If fngzi is NULL, ".gzi" will be appended to fn for the bgzip index name.
+The bgzip index is only needed if fn is compressed.
+
+If (flags & FAI_CREATE) is true, the index files will be built using
+fai_build3() if they are not already present.
+
+The struct returned by a successful call should be freed via fai_destroy()
+when it is no longer needed.
+*/
+HTSLIB_EXPORT
+faidx_t *fai_load3(const char *fn, const char *fnfai, const char *fngzi,
+                   int flags);
+
+/// Load index from "fn.fai".
+/** @param  fn  File name of the FASTA file
+    @return Pointer to a faidx_t struct on success, NULL on failure.
+
+This function is equivalent to fai_load3(fn, NULL, NULL, FAI_CREATE|FAI_CACHE);
+*/
+HTSLIB_EXPORT
+faidx_t *fai_load(const char *fn);
+
+/// Load FASTA or FASTQ indexes.
+/** @param  fn  File name of the FASTA/FASTQ file (can be compressed with bgzip).
+    @param  fnfai File name of the FASTA/FASTQ index.
+    @param  fngzi File name of the bgzip index.
+    @param  flags Option flags to control index file caching and creation.
+    @param  format FASTA or FASTQ file format
+    @return Pointer to a faidx_t struct on success, NULL on failure.
+
+If fnfai is NULL, ".fai" will be appended to fn to make the FAI file name.
+If fngzi is NULL, ".gzi" will be appended to fn for the bgzip index name.
+The bgzip index is only needed if fn is compressed.
+
+If (flags & FAI_CREATE) is true, the index files will be built using
+fai_build3() if they are not already present.
+
+The struct returned by a successful call should be freed via fai_destroy()
+when it is no longer needed.
+*/
+HTSLIB_EXPORT
+faidx_t *fai_load3_format(const char *fn, const char *fnfai, const char *fngzi,
+                   int flags, enum fai_format_options format);
+
+/// Load index from "fn.fai".
+/** @param  fn  File name of the FASTA/FASTQ file
+    @param  format FASTA or FASTQ file format
+    @return Pointer to a faidx_t struct on success, NULL on failure.
+
+This function is equivalent to fai_load3_format(fn, NULL, NULL, FAI_CREATE|FAI_CACHE, format);
+*/
+HTSLIB_EXPORT
+faidx_t *fai_load_format(const char *fn, enum fai_format_options format);
+
+/// Fetch the sequence in a region
+/** @param  fai  Pointer to the faidx_t struct
+    @param  reg  Region in the format "chr2:20,000-30,000"
+    @param  len  Length of the region; -2 if seq not present, -1 general error
+    @return      Pointer to the sequence; `NULL` on failure
+
+The returned sequence is allocated by `malloc()` family and should be destroyed
+by end users by calling `free()` on it.
+
+To work around ambiguous parsing issues, eg both "chr1" and "chr1:100-200"
+are reference names, quote using curly braces.
+Thus "{chr1}:100-200" and "{chr1:100-200}" disambiguate the above example.
+*/
+HTSLIB_EXPORT
+char *fai_fetch(const faidx_t *fai, const char *reg, int *len);
+HTSLIB_EXPORT
+char *fai_fetch64(const faidx_t *fai, const char *reg, hts_pos_t *len);
+
+/// Query the line-wrap length for a chromosome specified as part of a region
+/** @param  fai  Pointer to the faidx_t struct
+    @param  reg  Region in the format "chr2:20,000-30,000"
+    @return      The line length (excluding newline),
+                 negative on error.
+*/
+HTSLIB_EXPORT
+hts_pos_t fai_line_length(const faidx_t *fai, const char *reg);
+
+/// Fetch the quality string for a region for FASTQ files
+/** @param  fai  Pointer to the faidx_t struct
+    @param  reg  Region in the format "chr2:20,000-30,000"
+    @param  len  Length of the region; -2 if seq not present, -1 general error
+    @return      Pointer to the quality string; null on failure
+
+The returned quality string is allocated by `malloc()` family and should be
+destroyed by end users by calling `free()` on it.
+
+Region names can be quoted with curly braces, as for fai_fetch().
+*/
+HTSLIB_EXPORT
+char *fai_fetchqual(const faidx_t *fai, const char *reg, int *len);
+HTSLIB_EXPORT
+char *fai_fetchqual64(const faidx_t *fai, const char *reg, hts_pos_t *len);
+
+/// Fetch the number of sequences
+/** @param  fai  Pointer to the faidx_t struct
+    @return      The number of sequences
+*/
+HTSLIB_EXPORT
+int faidx_fetch_nseq(const faidx_t *fai) HTS_DEPRECATED("Please use faidx_nseq instead");
+
+/// Fetch the sequence in a region
+/** @param  fai  Pointer to the faidx_t struct
+    @param  c_name Region name
+    @param  p_beg_i  Beginning position number (zero-based)
+    @param  p_end_i  End position number (zero-based)
+    @param  len  Length of the region; -2 if c_name not present, -1 general error
+    @return      Pointer to the sequence; null on failure
+
+The returned sequence is allocated by `malloc()` family and should be destroyed
+by end users by calling `free()` on it.
+*/
+HTSLIB_EXPORT
+char *faidx_fetch_seq(const faidx_t *fai, const char *c_name, int p_beg_i, int p_end_i, int *len);
+
+/// Fetch the sequence in a region
+/** @param  fai  Pointer to the faidx_t struct
+    @param  c_name Region name
+    @param  p_beg_i  Beginning position number (zero-based)
+    @param  p_end_i  End position number (zero-based)
+    @param  len  Length of the region; -2 if c_name not present, -1 general error
+    @return      Pointer to the sequence; null on failure
+
+The returned sequence is allocated by `malloc()` family and should be destroyed
+by end users by calling `free()` on it.
+*/
+HTSLIB_EXPORT
+char *faidx_fetch_seq64(const faidx_t *fai, const char *c_name, hts_pos_t p_beg_i, hts_pos_t p_end_i, hts_pos_t *len);
+
+/// Fetch the quality string in a region for FASTQ files
+/** @param  fai  Pointer to the faidx_t struct
+    @param  c_name Region name
+    @param  p_beg_i  Beginning position number (zero-based)
+    @param  p_end_i  End position number (zero-based)
+    @param  len  Length of the region; -2 if c_name not present, -1 general error
+    @return      Pointer to the sequence; null on failure
+
+The returned sequence is allocated by `malloc()` family and should be destroyed
+by end users by calling `free()` on it.
+*/
+HTSLIB_EXPORT
+char *faidx_fetch_qual(const faidx_t *fai, const char *c_name, int p_beg_i, int p_end_i, int *len);
+
+/// Fetch the quality string in a region for FASTQ files
+/** @param  fai  Pointer to the faidx_t struct
+    @param  c_name Region name
+    @param  p_beg_i  Beginning position number (zero-based)
+    @param  p_end_i  End position number (zero-based)
+    @param  len  Length of the region; -2 if c_name not present, -1 general error
+    @return      Pointer to the sequence; null on failure
+
+The returned sequence is allocated by `malloc()` family and should be destroyed
+by end users by calling `free()` on it.
+*/
+HTSLIB_EXPORT
+char *faidx_fetch_qual64(const faidx_t *fai, const char *c_name, hts_pos_t p_beg_i, hts_pos_t p_end_i, hts_pos_t *len);
+
+/// Query if sequence is present
+/**   @param  fai  Pointer to the faidx_t struct
+      @param  seq  Sequence name
+      @return      1 if present or 0 if absent
+*/
+HTSLIB_EXPORT
+int faidx_has_seq(const faidx_t *fai, const char *seq);
+
+/// Return number of sequences in fai index
+HTSLIB_EXPORT
+int faidx_nseq(const faidx_t *fai);
+
+/// Return name of i-th sequence
+HTSLIB_EXPORT
+const char *faidx_iseq(const faidx_t *fai, int i);
+
+/// Return sequence length
+/** @param  fai  Pointer to the faidx_t struct
+    @param  seq  Name of the sequence
+    @return Sequence length, or -1 if not present
+*/
+HTSLIB_EXPORT
+hts_pos_t faidx_seq_len64(const faidx_t *fai, const char *seq);
+
+/// Return sequence length
+/** @param  fai  Pointer to the faidx_t struct
+    @param  seq  Name of the sequence
+    @return Sequence length, or -1 if not present
+
+    @deprecated This funtion cannot handle very long sequences.
+                Use faidx_seq_len64() instead.
+*/
+HTSLIB_EXPORT
+int faidx_seq_len(const faidx_t *fai, const char *seq);
+
+/// Parses a region string.
+/** @param  fai   Pointer to the faidx_t struct
+    @param  s     Region string
+    @param  tid   Returns which i-th sequence is described in the region.
+    @param  beg   Returns the start of the region (0 based)
+    @param  end   Returns the one past last of the region (0 based)
+    @param  flags Parsing method, see HTS_PARSE_* in hts.h.
+    @return       Pointer to end of parsed s if successful, NULL if not.
+
+    To work around ambiguous parsing issues, eg both "chr1" and "chr1:100-200"
+    are reference names, quote using curly braces.
+    Thus "{chr1}:100-200" and "{chr1:100-200}" disambiguate the above example.
+*/
+HTSLIB_EXPORT
+const char *fai_parse_region(const faidx_t *fai, const char *s,
+                             int *tid, hts_pos_t *beg, hts_pos_t *end,
+                             int flags);
+
+/// Adjust region to the actual sequence length
+/** @param  fai   Pointer to the faidx_t struct
+    @param  tid   Sequence index, as returned by fai_parse_region()
+    @param  beg[in,out]   The start of the region (0 based)
+    @param  end[in,out]   One past end of the region (0 based)
+    @return 1, 2, or 3 if @p beg, @p end, or both are adjusted,
+            0 if @p beg and @p end are unchanged
+            -1 on error
+
+    Looks up the length of @p tid, and then adjusts the values of @p beg
+    and @p end if they fall outside the boundaries of the sequence.
+
+    If @p beg > @p end, it will be set to @p end.
+
+    The return value indicates which, if any, of the inputs have been
+    adjusted.  -1 will be returned if @p tid is not a valid sequence index.
+*/
+HTSLIB_EXPORT
+int fai_adjust_region(const faidx_t *fai, int tid,
+                      hts_pos_t *beg, hts_pos_t *end);
+
+/// Sets the cache size of the underlying BGZF compressed file
+/** @param  fai         Pointer to the faidx_t struct
+ *  @param  cache_size  Selected cache size in bytes
+ */
+HTSLIB_EXPORT
+void fai_set_cache_size(faidx_t *fai, int cache_size);
+
+/// Adds a thread pool to the underlying BGZF layer.
+/** @param fai         FAI file handler
+ *  @param pool        The thread pool (see hts_create_threads)
+ *  @param qsize       The size of the job queue.  If 0 this is twice the
+ *                     number of threads in the pool.
+ */
+HTSLIB_EXPORT
+int fai_thread_pool(faidx_t *fai, struct hts_tpool *pool, int qsize);
+
+/// Determines the path to the reference index file
+/** @param  fa    String with the path to the reference file
+ *  @return       String with the path to the reference index file, or NULL on failure
+
+    If the reference path has the format reference.fa##idx##index.fa.fai,
+    the index path is taken directly from it as index.fa.fai.
+    If the reference file is local and the index file cannot be found, it
+    will be created alongside the reference file.
+    If the reference file is remote and the index file cannot be found,
+    the method returns NULL.
+
+    The returned string has to be freed by the user at the end of its scope.
+ */
+HTSLIB_EXPORT
+char *fai_path(const char *fa);
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/ext/htslib/htslib/hfile.h b/ext/htslib/htslib/hfile.h
new file mode 100644
index 0000000..e851faf
--- /dev/null
+++ b/ext/htslib/htslib/hfile.h
@@ -0,0 +1,393 @@
+/// @file htslib/hfile.h
+/// Buffered low-level input/output streams.
+/*
+    Copyright (C) 2013-2022 Genome Research Ltd.
+
+    Author: John Marshall <jm18@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#ifndef HTSLIB_HFILE_H
+#define HTSLIB_HFILE_H
+
+#include <string.h>
+
+#include <sys/types.h>
+
+#include "hts_defs.h"
+
+// Ensure ssize_t exists within this header. All #includes must precede this,
+// and ssize_t must be undefined again at the end of this header.
+#if defined _MSC_VER && defined _INTPTR_T_DEFINED && !defined _SSIZE_T_DEFINED && !defined ssize_t
+#define HTSLIB_SSIZE_T
+#define ssize_t intptr_t
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct hFILE_backend;
+struct kstring_t;
+
+/// Low-level input/output stream handle
+/** The fields of this structure are declared here solely for the benefit
+of the hFILE-related inline functions.  They may change in future releases.
+User code should not use them directly; you should imagine that hFILE is an
+opaque incomplete type.
+*/
+typedef struct hFILE {
+    // @cond internal
+    char *buffer, *begin, *end, *limit;
+    const struct hFILE_backend *backend;
+    off_t offset;
+    unsigned at_eof:1, mobile:1, readonly:1, preserve:1;
+    int has_errno;
+    // @endcond
+} hFILE;
+
+/// Open the named file or URL as a stream
+/** @return An hFILE pointer, or `NULL` (with _errno_ set) if an error occurred.
+
+The usual `fopen(3)` _mode_ letters are supported: one of
+`r` (read), `w` (write), `a` (append), optionally followed by any of
+`+` (update), `e` (close on `exec(2)`), `x` (create exclusively),
+`:` (indicates scheme-specific variable arguments follow).
+*/
+HTSLIB_EXPORT
+hFILE *hopen(const char *filename, const char *mode, ...) HTS_RESULT_USED;
+
+/// Associate a stream with an existing open file descriptor
+/** @return An hFILE pointer, or `NULL` (with _errno_ set) if an error occurred.
+
+Note that the file must be opened in binary mode, or else
+there will be problems on platforms that make a difference
+between text and binary mode.
+
+By default, the returned hFILE "takes ownership" of the file descriptor
+and _fd_ will be closed by hclose(). When _mode_ contains `S` (shared fd),
+hclose() will destroy the hFILE but not close the underlying _fd_.
+
+For socket descriptors (on Windows), _mode_ should contain `s`.
+*/
+HTSLIB_EXPORT
+hFILE *hdopen(int fd, const char *mode) HTS_RESULT_USED;
+
+/// Report whether the file name or URL denotes remote storage
+/** @return  0 if local, 1 if remote.
+
+"Remote" means involving e.g. explicit network access, with the implication
+that callers may wish to cache such files' contents locally.
+*/
+HTSLIB_EXPORT
+int hisremote(const char *filename) HTS_RESULT_USED;
+
+/// Append an extension or replace an existing extension
+/** @param buffer     The kstring to be used to store the modified filename
+    @param filename   The filename to be (copied and) adjusted
+    @param replace    If non-zero, one extension (if any) is removed first
+    @param extension  The extension to be added (e.g. ".csi")
+    @return  The modified filename (i.e., `buffer->s`), or NULL on error.
+    @since   1.10
+
+If _filename_ is an URL, alters extensions at the end of the `hier-part`,
+leaving any trailing `?query` or `#fragment` unchanged.
+*/
+HTSLIB_EXPORT
+char *haddextension(struct kstring_t *buffer, const char *filename,
+                    int replace, const char *extension) HTS_RESULT_USED;
+
+/// Flush (for output streams) and close the stream
+/** @return  0 if successful, or `EOF` (with _errno_ set) if an error occurred.
+*/
+HTSLIB_EXPORT
+int hclose(hFILE *fp) HTS_RESULT_USED;
+
+/// Close the stream, without flushing or propagating errors
+/** For use while cleaning up after an error only.  Preserves _errno_.
+*/
+HTSLIB_EXPORT
+void hclose_abruptly(hFILE *fp);
+
+/// Return the stream's error indicator
+/** @return  Non-zero (in fact, an _errno_ value) if an error has occurred.
+
+This would be called `herror()` and return true/false to parallel `ferror(3)`,
+but a networking-related `herror(3)` function already exists.
+*/
+static inline int herrno(hFILE *fp)
+{
+    return fp->has_errno;
+}
+
+/// Clear the stream's error indicator
+static inline void hclearerr(hFILE *fp)
+{
+    fp->has_errno = 0;
+}
+
+/// Reposition the read/write stream offset
+/** @return  The resulting offset within the stream (as per `lseek(2)`),
+    or negative if an error occurred.
+*/
+HTSLIB_EXPORT
+off_t hseek(hFILE *fp, off_t offset, int whence) HTS_RESULT_USED;
+
+/// Report the current stream offset
+/** @return  The offset within the stream, starting from zero.
+*/
+static inline off_t htell(hFILE *fp)
+{
+    return fp->offset + (fp->begin - fp->buffer);
+}
+
+/// Read one character from the stream
+/** @return  The character read, or `EOF` on end-of-file or error.
+*/
+static inline int hgetc(hFILE *fp)
+{
+    HTSLIB_EXPORT
+    extern int hgetc2(hFILE *);
+    return (fp->end > fp->begin)? (unsigned char) *(fp->begin++) : hgetc2(fp);
+}
+
+/// Read from the stream until the delimiter, up to a maximum length
+/** @param buffer  The buffer into which bytes will be written
+    @param size    The size of the buffer
+    @param delim   The delimiter (interpreted as an `unsigned char`)
+    @param fp      The file stream
+    @return  The number of bytes read, or negative on error.
+    @since   1.4
+
+Bytes will be read into the buffer up to and including a delimiter, until
+EOF is reached, or _size-1_ bytes have been written, whichever comes first.
+The string will then be terminated with a NUL byte (`\0`).
+*/
+HTSLIB_EXPORT
+ssize_t hgetdelim(char *buffer, size_t size, int delim, hFILE *fp)
+    HTS_RESULT_USED;
+
+/// Read a line from the stream, up to a maximum length
+/** @param buffer  The buffer into which bytes will be written
+    @param size    The size of the buffer
+    @param fp      The file stream
+    @return  The number of bytes read, or negative on error.
+    @since   1.4
+
+Specialization of hgetdelim() for a `\n` delimiter.
+*/
+static inline ssize_t HTS_RESULT_USED
+hgetln(char *buffer, size_t size, hFILE *fp)
+{
+    return hgetdelim(buffer, size, '\n', fp);
+}
+
+/// Read a line from the stream, up to a maximum length
+/** @param buffer  The buffer into which bytes will be written
+    @param size    The size of the buffer (must be > 1 to be useful)
+    @param fp      The file stream
+    @return  _buffer_ on success, or `NULL` if an error occurred.
+    @since   1.4
+
+This function can be used as a replacement for `fgets(3)`, or together with
+kstring's `kgetline()` to read arbitrarily-long lines into a _kstring_t_.
+*/
+HTSLIB_EXPORT
+char *hgets(char *buffer, int size, hFILE *fp) HTS_RESULT_USED;
+
+/// Peek at characters to be read without removing them from buffers
+/** @param fp      The file stream
+    @param buffer  The buffer to which the peeked bytes will be written
+    @param nbytes  The number of bytes to peek at; limited by the size of the
+                   internal buffer, which could be as small as 4K.
+    @return  The number of bytes peeked, which may be less than _nbytes_
+             if EOF is encountered; or negative, if there was an I/O error.
+
+The characters peeked at remain in the stream's internal buffer, and will be
+returned by later hread() etc calls.
+*/
+HTSLIB_EXPORT
+ssize_t hpeek(hFILE *fp, void *buffer, size_t nbytes) HTS_RESULT_USED;
+
+/// Read a block of characters from the file
+/** @return  The number of bytes read, or negative if an error occurred.
+
+The full _nbytes_ requested will be returned, except as limited by EOF
+or I/O errors.
+*/
+static inline ssize_t HTS_RESULT_USED
+hread(hFILE *fp, void *buffer, size_t nbytes)
+{
+    HTSLIB_EXPORT
+    extern ssize_t hread2(hFILE *, void *, size_t, size_t);
+
+    size_t n = fp->end - fp->begin;
+    if (n > nbytes) n = nbytes;
+    memcpy(buffer, fp->begin, n);
+    fp->begin += n;
+    return (n == nbytes || !fp->mobile)? (ssize_t) n : hread2(fp, buffer, nbytes, n);
+}
+
+/// Write a character to the stream
+/** @return  The character written, or `EOF` if an error occurred.
+*/
+static inline int hputc(int c, hFILE *fp)
+{
+    HTSLIB_EXPORT
+    extern int hputc2(int, hFILE *);
+    if (fp->begin < fp->limit) *(fp->begin++) = c;
+    else c = hputc2(c, fp);
+    return c;
+}
+
+/// Write a string to the stream
+/** @return  0 if successful, or `EOF` if an error occurred.
+*/
+static inline int hputs(const char *text, hFILE *fp)
+{
+    HTSLIB_EXPORT
+    extern int hputs2(const char *, size_t, size_t, hFILE *);
+
+    size_t nbytes = strlen(text), n = fp->limit - fp->begin;
+    if (n > nbytes) n = nbytes;
+    memcpy(fp->begin, text, n);
+    fp->begin += n;
+    return (n == nbytes)? 0 : hputs2(text, nbytes, n, fp);
+}
+
+/// Write a block of characters to the file
+/** @return  Either _nbytes_, or negative if an error occurred.
+
+In the absence of I/O errors, the full _nbytes_ will be written.
+*/
+static inline ssize_t HTS_RESULT_USED
+hwrite(hFILE *fp, const void *buffer, size_t nbytes)
+{
+    HTSLIB_EXPORT
+    extern ssize_t hwrite2(hFILE *, const void *, size_t, size_t);
+    HTSLIB_EXPORT
+    extern int hfile_set_blksize(hFILE *fp, size_t bufsiz);
+
+    if (!fp->mobile) {
+        size_t n = fp->limit - fp->begin;
+        if (n < nbytes) {
+            hfile_set_blksize(fp, fp->limit - fp->buffer + nbytes);
+            fp->end = fp->limit;
+        }
+    }
+
+    size_t n = fp->limit - fp->begin;
+    if (nbytes >= n && fp->begin == fp->buffer) {
+        // Go straight to hwrite2 if the buffer is empty and the request
+        // won't fit.
+        return hwrite2(fp, buffer, nbytes, 0);
+    }
+
+    if (n > nbytes) n = nbytes;
+    memcpy(fp->begin, buffer, n);
+    fp->begin += n;
+    return (n==nbytes)? (ssize_t) n : hwrite2(fp, buffer, nbytes, n);
+}
+
+/// For writing streams, flush buffered output to the underlying stream
+/** @return  0 if successful, or `EOF` if an error occurred.
+
+This includes low-level flushing such as via `fdatasync(2)`.
+*/
+HTSLIB_EXPORT
+int hflush(hFILE *fp) HTS_RESULT_USED;
+
+/// For hfile_mem: get the internal buffer and it's size from a hfile
+/** @return  buffer if successful, or NULL if an error occurred
+
+The buffer returned should not be freed as this will happen when the
+hFILE is closed.
+*/
+HTSLIB_EXPORT
+char *hfile_mem_get_buffer(hFILE *file, size_t *length);
+
+/// For hfile_mem: get the internal buffer and it's size from a hfile.
+/** @return  buffer if successful, or NULL if an error occurred
+
+This is similar to hfile_mem_get_buffer except that ownership of the
+buffer is granted to the caller, who now has responsibility for freeing
+it.  From this point onwards, the hFILE should not be used for any
+purpose other than closing.
+*/
+HTSLIB_EXPORT
+char *hfile_mem_steal_buffer(hFILE *file, size_t *length);
+
+/// Fills out sc_list[] with the list of known URL schemes.
+/**
+ * @param plugin   [in]     Restricts schemes to only those from 'plugin.
+ * @param sc_list  [out]    Filled out with the scheme names
+ * @param nschemes [in/out] Size of sc_list (in) and number returned (out)
+ *
+ * Plugin may be passed in as NULL in which case all schemes are returned.
+ * Use plugin "built-in" to list the built in schemes.
+ * The size of sc_list is determined by the input value of *nschemes.
+ * This is updated to return the output size.  It is up to the caller to
+ * determine whether to call again with a larger number if this is too small.
+ *
+ * The return value represents the total number found matching plugin, which
+ * may be larger than *nschemes if too small a value was specified.
+ *
+ * @return the number of schemes found on success.
+ *         -1 on failure
+ */
+HTSLIB_EXPORT
+int hfile_list_schemes(const char *plugin, const char *sc_list[], int *nschemes);
+
+/// Fills out plist[] with the list of known hFILE plugins.
+/*
+ * @param plist    [out]    Filled out with the plugin names
+ * @param nplugins [in/out] Size of plist (in) and number returned (out)
+ *
+ * The size of plist is determined by the input value of *nplugins.
+ * This is updated to return the output size.  It is up to the caller to
+ * determine whether to call again with a larger number if this is too small.
+ *
+ * The return value represents the total number found, which may be
+ * larger than *nplugins if too small a value was specified.
+ *
+ * @return the number of plugins found on success.
+ *         -1 on failure
+ */
+HTSLIB_EXPORT
+int hfile_list_plugins(const char *plist[], int *nplugins);
+
+/// Tests for the presence of a specific hFILE plugin.
+/*
+ * @param name     The name of the plugin to query.
+ *
+ * @return 1 if found, 0 otherwise.
+ */
+HTSLIB_EXPORT
+int hfile_has_plugin(const char *name);
+
+#ifdef __cplusplus
+}
+#endif
+
+#ifdef HTSLIB_SSIZE_T
+#undef HTSLIB_SSIZE_T
+#undef ssize_t
+#endif
+
+#endif
diff --git a/ext/htslib/htslib/hts.h b/ext/htslib/htslib/hts.h
new file mode 100644
index 0000000..4f85424
--- /dev/null
+++ b/ext/htslib/htslib/hts.h
@@ -0,0 +1,1588 @@
+/// @file htslib/hts.h
+/// Format-neutral I/O, indexing, and iterator API functions.
+/*
+    Copyright (C) 2012-2022 Genome Research Ltd.
+    Copyright (C) 2010, 2012 Broad Institute.
+    Portions copyright (C) 2003-2006, 2008-2010 by Heng Li <lh3@live.co.uk>
+
+    Author: Heng Li <lh3@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#ifndef HTSLIB_HTS_H
+#define HTSLIB_HTS_H
+
+#include <stddef.h>
+#include <stdint.h>
+#include <inttypes.h>
+
+#include "hts_defs.h"
+#include "hts_log.h"
+#include "kstring.h"
+#include "kroundup.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Separator used to split HTS_PATH (for plugins); REF_PATH (cram references)
+#if defined(_WIN32) || defined(__MSYS__)
+#define HTS_PATH_SEPARATOR_CHAR ';'
+#define HTS_PATH_SEPARATOR_STR  ";"
+#else
+#define HTS_PATH_SEPARATOR_CHAR ':'
+#define HTS_PATH_SEPARATOR_STR  ":"
+#endif
+
+#ifndef HTS_BGZF_TYPEDEF
+typedef struct BGZF BGZF;
+#define HTS_BGZF_TYPEDEF
+#endif
+struct cram_fd;
+struct hFILE;
+struct hts_tpool;
+struct sam_hdr_t;
+
+/**
+ * @hideinitializer
+ * Deprecated macro to expand a dynamic array of a given type
+ *
+ * @param         type_t The type of the array elements
+ * @param[in]     n      Requested number of elements of type type_t
+ * @param[in,out] m      Size of memory allocated
+ * @param[in,out] ptr    Pointer to the array
+ *
+ * @discussion
+ * Do not use this macro.  Use hts_resize() instead as allows allocation
+ * failures to be handled more gracefully.
+ *
+ * The array *ptr will be expanded if necessary so that it can hold @p n
+ * or more elements.  If the array is expanded then the new size will be
+ * written to @p m and the value in @p ptr may change.
+ *
+ * It must be possible to take the address of @p ptr and @p m must be usable
+ * as an lvalue.
+ *
+ * @bug
+ * If the memory allocation fails, this will call exit(1).  This is
+ * not ideal behaviour in a library.
+ */
+#define hts_expand(type_t, n, m, ptr) do {                              \
+        if ((n) > (m)) {                                                \
+            size_t hts_realloc_or_die(size_t, size_t, size_t, size_t,   \
+                                      int, void **, const char *);      \
+            (m) = hts_realloc_or_die((n) >= 1 ? (n) : 1, (m), sizeof(m), \
+                                     sizeof(type_t),  0,                \
+                                     (void **)&(ptr), __func__);        \
+        }                                                               \
+    } while (0)
+
+/**
+ * @hideinitializer
+ * Macro to expand a dynamic array, zeroing any newly-allocated memory
+ *
+ * @param         type_t The type of the array elements
+ * @param[in]     n      Requested number of elements of type type_t
+ * @param[in,out] m      Size of memory allocated
+ * @param[in,out] ptr    Pointer to the array
+ *
+ * @discussion
+ * Do not use this macro.  Use hts_resize() instead as allows allocation
+ * failures to be handled more gracefully.
+ *
+ * As for hts_expand(), except the bytes that make up the array elements
+ * between the old and new values of @p m are set to zero using memset().
+ *
+ * @bug
+ * If the memory allocation fails, this will call exit(1).  This is
+ * not ideal behaviour in a library.
+ */
+
+
+#define hts_expand0(type_t, n, m, ptr) do {                             \
+        if ((n) > (m)) {                                                \
+            size_t hts_realloc_or_die(size_t, size_t, size_t, size_t,   \
+                                      int, void **, const char *);      \
+            (m) = hts_realloc_or_die((n) >= 1 ? (n) : 1, (m), sizeof(m), \
+                                     sizeof(type_t), 1,                 \
+                                     (void **)&(ptr), __func__);        \
+        }                                                               \
+    } while (0)
+
+// For internal use (by hts_resize()) only
+HTSLIB_EXPORT
+int hts_resize_array_(size_t, size_t, size_t, void *, void **, int,
+                      const char *);
+
+#define HTS_RESIZE_CLEAR 1
+
+/**
+ * @hideinitializer
+ * Macro to expand a dynamic array of a given type
+ *
+ * @param         type_t    The type of the array elements
+ * @param[in]     num       Requested number of elements of type type_t
+ * @param[in,out] size_ptr  Pointer to where the size (in elements) of the
+                            array is stored.
+ * @param[in,out] ptr       Location of the pointer to the array
+ * @param[in]     flags     Option flags
+ *
+ * @return        0 for success, or negative if an error occurred.
+ *
+ * @discussion
+ * The array *ptr will be expanded if necessary so that it can hold @p num
+ * or more elements.  If the array is expanded then the new size will be
+ * written to @p *size_ptr and the value in @p *ptr may change.
+ *
+ * If ( @p flags & HTS_RESIZE_CLEAR ) is set, any newly allocated memory will
+ * be cleared.
+ */
+
+#define hts_resize(type_t, num, size_ptr, ptr, flags)       \
+    ((num) > (*(size_ptr))                                  \
+     ? hts_resize_array_(sizeof(type_t), (num),             \
+                         sizeof(*(size_ptr)), (size_ptr),   \
+                         (void **)(ptr), (flags), __func__) \
+     : 0)
+
+/// Release resources when dlclosing a dynamically loaded HTSlib
+/** @discussion
+ *  Normally HTSlib cleans up automatically when your program exits,
+ *  whether that is via exit(3) or returning from main(). However if you
+ *  have dlopen(3)ed HTSlib and wish to close it before your main program
+ *  exits, you must call hts_lib_shutdown() before dlclose(3).
+*/
+HTSLIB_EXPORT
+void hts_lib_shutdown(void);
+
+/**
+ * Wrapper function for free(). Enables memory deallocation across DLL
+ * boundary. Should be used by all applications, which are compiled
+ * with a different standard library than htslib and call htslib
+ * methods that return dynamically allocated data.
+ */
+HTSLIB_EXPORT
+void hts_free(void *ptr);
+
+/************
+ * File I/O *
+ ************/
+
+// Add new entries only at the end (but before the *_maximum entry)
+// of these enums, as their numbering is part of the htslib ABI.
+
+enum htsFormatCategory {
+    unknown_category,
+    sequence_data,    // Sequence data -- SAM, BAM, CRAM, etc
+    variant_data,     // Variant calling data -- VCF, BCF, etc
+    index_file,       // Index file associated with some data file
+    region_list,      // Coordinate intervals or regions -- BED, etc
+    category_maximum = 32767
+};
+
+enum htsExactFormat {
+    unknown_format,
+    binary_format, text_format,
+    sam, bam, bai, cram, crai, vcf, bcf, csi, gzi, tbi, bed,
+    htsget,
+    json HTS_DEPRECATED_ENUM("Use htsExactFormat 'htsget' instead") = htsget,
+    empty_format,  // File is empty (or empty after decompression)
+    fasta_format, fastq_format, fai_format, fqi_format,
+    hts_crypt4gh_format,
+    d4_format,
+    format_maximum = 32767
+};
+
+enum htsCompression {
+    no_compression, gzip, bgzf, custom, bzip2_compression, razf_compression,
+    xz_compression, zstd_compression,
+    compression_maximum = 32767
+};
+
+typedef struct htsFormat {
+    enum htsFormatCategory category;
+    enum htsExactFormat format;
+    struct { short major, minor; } version;
+    enum htsCompression compression;
+    short compression_level;  // currently unused
+    void *specific;  // format specific options; see struct hts_opt.
+} htsFormat;
+
+struct hts_idx_t;
+typedef struct hts_idx_t hts_idx_t;
+struct hts_filter_t;
+
+/**
+ * @brief File handle returned by hts_open() etc.
+ * This structure should be considered opaque by end users. There should be
+ * no need to access most fields directly in user code, and in cases where
+ * it is desirable accessor functions such as hts_get_format() are provided.
+ */
+// Maintainers note htsFile cannot be an incomplete struct because some of its
+// fields are part of libhts.so's ABI (hence these fields must not be moved):
+//  - fp is used in the public sam_itr_next()/etc macros
+//  - is_bin is used directly in samtools <= 1.1 and bcftools <= 1.1
+//  - is_write and is_cram are used directly in samtools <= 1.1
+//  - fp is used directly in samtools (up to and including current develop)
+//  - line is used directly in bcftools (up to and including current develop)
+//  - is_bgzf and is_cram flags indicate which fp union member to use.
+//    Note is_bgzf being set does not indicate the flag is BGZF compressed,
+//    nor even whether it is compressed at all (eg on naked BAMs).
+typedef struct htsFile {
+    uint32_t is_bin:1, is_write:1, is_be:1, is_cram:1, is_bgzf:1, dummy:27;
+    int64_t lineno;
+    kstring_t line;
+    char *fn, *fn_aux;
+    union {
+        BGZF *bgzf;
+        struct cram_fd *cram;
+        struct hFILE *hfile;
+    } fp;
+    void *state;  // format specific state information
+    htsFormat format;
+    hts_idx_t *idx;
+    const char *fnidx;
+    struct sam_hdr_t *bam_header;
+    struct hts_filter_t *filter;
+} htsFile;
+
+// A combined thread pool and queue allocation size.
+// The pool should already be defined, but qsize may be zero to
+// indicate an appropriate queue size is taken from the pool.
+//
+// Reasons for explicitly setting it could be where many more file
+// descriptors are in use than threads, so keeping memory low is
+// important.
+typedef struct htsThreadPool {
+    struct hts_tpool *pool; // The shared thread pool itself
+    int qsize;    // Size of I/O queue to use for this fp
+} htsThreadPool;
+
+// REQUIRED_FIELDS
+enum sam_fields {
+    SAM_QNAME = 0x00000001,
+    SAM_FLAG  = 0x00000002,
+    SAM_RNAME = 0x00000004,
+    SAM_POS   = 0x00000008,
+    SAM_MAPQ  = 0x00000010,
+    SAM_CIGAR = 0x00000020,
+    SAM_RNEXT = 0x00000040,
+    SAM_PNEXT = 0x00000080,
+    SAM_TLEN  = 0x00000100,
+    SAM_SEQ   = 0x00000200,
+    SAM_QUAL  = 0x00000400,
+    SAM_AUX   = 0x00000800,
+    SAM_RGAUX = 0x00001000,
+};
+
+// Mostly CRAM only, but this could also include other format options
+enum hts_fmt_option {
+    // CRAM specific
+    CRAM_OPT_DECODE_MD,
+    CRAM_OPT_PREFIX,
+    CRAM_OPT_VERBOSITY,  // obsolete, use hts_set_log_level() instead
+    CRAM_OPT_SEQS_PER_SLICE,
+    CRAM_OPT_SLICES_PER_CONTAINER,
+    CRAM_OPT_RANGE,
+    CRAM_OPT_VERSION,    // rename to cram_version?
+    CRAM_OPT_EMBED_REF,
+    CRAM_OPT_IGNORE_MD5,
+    CRAM_OPT_REFERENCE,  // make general
+    CRAM_OPT_MULTI_SEQ_PER_SLICE,
+    CRAM_OPT_NO_REF,
+    CRAM_OPT_USE_BZIP2,
+    CRAM_OPT_SHARED_REF,
+    CRAM_OPT_NTHREADS,   // deprecated, use HTS_OPT_NTHREADS
+    CRAM_OPT_THREAD_POOL,// make general
+    CRAM_OPT_USE_LZMA,
+    CRAM_OPT_USE_RANS,
+    CRAM_OPT_REQUIRED_FIELDS,
+    CRAM_OPT_LOSSY_NAMES,
+    CRAM_OPT_BASES_PER_SLICE,
+    CRAM_OPT_STORE_MD,
+    CRAM_OPT_STORE_NM,
+    CRAM_OPT_RANGE_NOSEEK, // CRAM_OPT_RANGE minus the seek
+    CRAM_OPT_USE_TOK,
+    CRAM_OPT_USE_FQZ,
+    CRAM_OPT_USE_ARITH,
+    CRAM_OPT_POS_DELTA,  // force delta for AP, even on non-pos sorted data
+
+    // General purpose
+    HTS_OPT_COMPRESSION_LEVEL = 100,
+    HTS_OPT_NTHREADS,
+    HTS_OPT_THREAD_POOL,
+    HTS_OPT_CACHE_SIZE,
+    HTS_OPT_BLOCK_SIZE,
+    HTS_OPT_FILTER,
+    HTS_OPT_PROFILE,
+
+    // Fastq
+
+    // Boolean.
+    // Read / Write CASAVA 1.8 format.
+    // See https://emea.support.illumina.com/content/dam/illumina-support/documents/documentation/software_documentation/bcl2fastq/bcl2fastq_letterbooklet_15038058brpmi.pdf
+    //
+    // The CASAVA tag matches \d:[YN]:\d+:[ACGTN]+
+    // The first \d is read 1/2 (1 or 2), [YN] is QC-PASS/FAIL flag,
+    // \d+ is a control number, and the sequence at the end is
+    // for barcode sequence.  Barcodes are read into the aux tag defined
+    // by FASTQ_OPT_BARCODE ("BC" by default).
+    FASTQ_OPT_CASAVA = 1000,
+
+    // String.
+    // Whether to read / write extra SAM format aux tags from the fastq
+    // identifier line.  For reading this can simply be "1" to request
+    // decoding aux tags.  For writing it is a comma separated list of aux
+    // tag types to be written out.
+    FASTQ_OPT_AUX,
+
+    // Boolean.
+    // Whether to add /1 and /2 to read identifiers when writing FASTQ.
+    // These come from the BAM_FREAD1 or BAM_FREAD2 flags.
+    // (Detecting the /1 and /2 is automatic when reading fastq.)
+    FASTQ_OPT_RNUM,
+
+    // Two character string.
+    // Barcode aux tag for CASAVA; defaults to "BC".
+    FASTQ_OPT_BARCODE,
+
+    // Process SRA and ENA read names which pointlessly move the original
+    // name to the second field and insert a constructed <run>.<number>
+    // name in its place.
+    FASTQ_OPT_NAME2,
+};
+
+// Profile options for encoding; primarily used at present in CRAM
+// but also usable in BAM as a synonym for deflate compression levels.
+enum hts_profile_option {
+    HTS_PROFILE_FAST,
+    HTS_PROFILE_NORMAL,
+    HTS_PROFILE_SMALL,
+    HTS_PROFILE_ARCHIVE,
+};
+
+// For backwards compatibility
+#define cram_option hts_fmt_option
+
+typedef struct hts_opt {
+    char *arg;                // string form, strdup()ed
+    enum hts_fmt_option opt;  // tokenised key
+    union {                   // ... and value
+        int i;
+        char *s;
+    } val;
+    struct hts_opt *next;
+} hts_opt;
+
+#define HTS_FILE_OPTS_INIT {{0},0}
+
+/*
+ * Explicit index file name delimiter, see below
+ */
+#define HTS_IDX_DELIM "##idx##"
+
+
+/**********************
+ * Exported functions *
+ **********************/
+
+/*
+ * Parses arg and appends it to the option list.
+ *
+ * Returns 0 on success;
+ *        -1 on failure.
+ */
+HTSLIB_EXPORT
+int hts_opt_add(hts_opt **opts, const char *c_arg);
+
+/*
+ * Applies an hts_opt option list to a given htsFile.
+ *
+ * Returns 0 on success
+ *        -1 on failure
+ */
+HTSLIB_EXPORT
+int hts_opt_apply(htsFile *fp, hts_opt *opts);
+
+/*
+ * Frees an hts_opt list.
+ */
+HTSLIB_EXPORT
+void hts_opt_free(hts_opt *opts);
+
+/*
+ * Accepts a string file format (sam, bam, cram, vcf, bam) optionally
+ * followed by a comma separated list of key=value options and splits
+ * these up into the fields of htsFormat struct.
+ *
+ * Returns 0 on success
+ *        -1 on failure.
+ */
+HTSLIB_EXPORT
+int hts_parse_format(htsFormat *opt, const char *str);
+
+/*
+ * Tokenise options as (key(=value)?,)*(key(=value)?)?
+ * NB: No provision for ',' appearing in the value!
+ * Add backslashing rules?
+ *
+ * This could be used as part of a general command line option parser or
+ * as a string concatenated onto the file open mode.
+ *
+ * Returns 0 on success
+ *        -1 on failure.
+ */
+HTSLIB_EXPORT
+int hts_parse_opt_list(htsFormat *opt, const char *str);
+
+/*! @abstract Table for converting a nucleotide character to 4-bit encoding.
+The input character may be either an IUPAC ambiguity code, '=' for 0, or
+'0'/'1'/'2'/'3' for a result of 1/2/4/8.  The result is encoded as 1/2/4/8
+for A/C/G/T or combinations of these bits for ambiguous bases.
+*/
+HTSLIB_EXPORT
+extern const unsigned char seq_nt16_table[256];
+
+/*! @abstract Table for converting a 4-bit encoded nucleotide to an IUPAC
+ambiguity code letter (or '=' when given 0).
+*/
+HTSLIB_EXPORT
+extern const char seq_nt16_str[];
+
+/*! @abstract Table for converting a 4-bit encoded nucleotide to about 2 bits.
+Returns 0/1/2/3 for 1/2/4/8 (i.e., A/C/G/T), or 4 otherwise (0 or ambiguous).
+*/
+HTSLIB_EXPORT
+extern const int seq_nt16_int[];
+
+/*!
+  @abstract  Get the htslib version number
+  @return    For released versions, a string like "N.N[.N]"; or git describe
+  output if using a library built within a Git repository.
+*/
+HTSLIB_EXPORT
+const char *hts_version(void);
+
+/*!
+  @abstract  Compile-time HTSlib version number, for use in #if checks
+  @return    For released versions X.Y[.Z], an integer of the form XYYYZZ;
+  useful for preprocessor conditionals such as
+      #if HTS_VERSION >= 101000  // Check for v1.10 or later
+*/
+// Maintainers: Bump this in the final stage of preparing a new release.
+// Immediately after release, bump ZZ to 90 to distinguish in-development
+// Git repository builds from the release; you may wish to increment this
+// further when significant features are merged.
+#define HTS_VERSION 102100
+
+/*! @abstract Introspection on the features enabled in htslib
+ *
+ * @return a bitfield of HTS_FEATURE_* macros.
+ */
+HTSLIB_EXPORT
+unsigned int hts_features(void);
+
+HTSLIB_EXPORT
+const char *hts_test_feature(unsigned int id);
+
+/*! @abstract Introspection on the features enabled in htslib, string form
+ *
+ * @return a string describing htslib build features
+ */
+HTSLIB_EXPORT
+const char *hts_feature_string(void);
+
+// Whether ./configure was used or vanilla Makefile
+#define HTS_FEATURE_CONFIGURE    1
+
+// Whether --enable-plugins was used
+#define HTS_FEATURE_PLUGINS      2
+
+// Transport specific
+#define HTS_FEATURE_LIBCURL      (1u<<10)
+#define HTS_FEATURE_S3           (1u<<11)
+#define HTS_FEATURE_GCS          (1u<<12)
+
+// Compression options
+#define HTS_FEATURE_LIBDEFLATE   (1u<<20)
+#define HTS_FEATURE_LZMA         (1u<<21)
+#define HTS_FEATURE_BZIP2        (1u<<22)
+#define HTS_FEATURE_HTSCODECS    (1u<<23) // htscodecs library version
+
+// Build params
+#define HTS_FEATURE_CC           (1u<<27)
+#define HTS_FEATURE_CFLAGS       (1u<<28)
+#define HTS_FEATURE_CPPFLAGS     (1u<<29)
+#define HTS_FEATURE_LDFLAGS      (1u<<30)
+
+
+/*!
+  @abstract    Determine format by peeking at the start of a file
+  @param fp    File opened for reading, positioned at the beginning
+  @param fmt   Format structure that will be filled out on return
+  @return      0 for success, or negative if an error occurred.
+
+  Equivalent to hts_detect_format2(fp, NULL, fmt).
+*/
+HTSLIB_EXPORT
+int hts_detect_format(struct hFILE *fp, htsFormat *fmt);
+
+/*!
+  @abstract    Determine format primarily by peeking at the start of a file
+  @param fp    File opened for reading, positioned at the beginning
+  @param fname Name of the file, or NULL if not available
+  @param fmt   Format structure that will be filled out on return
+  @return      0 for success, or negative if an error occurred.
+  @since       1.15
+
+Some formats are only recognised if the filename is available and has the
+expected extension, as otherwise more generic files may be misrecognised.
+In particular:
+ - FASTA/Q indexes must have .fai/.fqi extensions; without this requirement,
+   some similar BED files would be misrecognised as indexes.
+*/
+HTSLIB_EXPORT
+int hts_detect_format2(struct hFILE *fp, const char *fname, htsFormat *fmt);
+
+/*!
+  @abstract    Get a human-readable description of the file format
+  @param fmt   Format structure holding type, version, compression, etc.
+  @return      Description string, to be freed by the caller after use.
+*/
+HTSLIB_EXPORT
+char *hts_format_description(const htsFormat *format);
+
+/*!
+  @abstract       Open a sequence data (SAM/BAM/CRAM) or variant data (VCF/BCF)
+                  or possibly-compressed textual line-orientated file
+  @param fn       The file name or "-" for stdin/stdout. For indexed files
+                  with a non-standard naming, the file name can include the
+                  name of the index file delimited with HTS_IDX_DELIM
+  @param mode     Mode matching / [rwa][bcefFguxz0-9]* /
+  @discussion
+      With 'r' opens for reading; any further format mode letters are ignored
+      as the format is detected by checking the first few bytes or BGZF blocks
+      of the file.  With 'w' or 'a' opens for writing or appending, with format
+      specifier letters:
+        b  binary format (BAM, BCF, etc) rather than text (SAM, VCF, etc)
+        c  CRAM format
+        f  FASTQ format
+        F  FASTA format
+        g  gzip compressed
+        u  uncompressed
+        z  bgzf compressed
+        [0-9]  zlib compression level
+      and with non-format option letters (for any of 'r'/'w'/'a'):
+        e  close the file on exec(2) (opens with O_CLOEXEC, where supported)
+        x  create the file exclusively (opens with O_EXCL, where supported)
+      Note that there is a distinction between 'u' and '0': the first yields
+      plain uncompressed output whereas the latter outputs uncompressed data
+      wrapped in the zlib format.
+  @example
+      [rw]b  .. compressed BCF, BAM, FAI
+      [rw]bu .. uncompressed BCF
+      [rw]z  .. compressed VCF
+      [rw]   .. uncompressed VCF
+*/
+HTSLIB_EXPORT
+htsFile *hts_open(const char *fn, const char *mode);
+
+/*!
+  @abstract       Open a SAM/BAM/CRAM/VCF/BCF/etc file
+  @param fn       The file name or "-" for stdin/stdout
+  @param mode     Open mode, as per hts_open()
+  @param fmt      Optional format specific parameters
+  @discussion
+      See hts_open() for description of fn and mode.
+      // TODO Update documentation for s/opts/fmt/
+      Opts contains a format string (sam, bam, cram, vcf, bcf) which will,
+      if defined, override mode.  Opts also contains a linked list of hts_opt
+      structures to apply to the open file handle.  These can contain things
+      like pointers to the reference or information on compression levels,
+      block sizes, etc.
+*/
+HTSLIB_EXPORT
+htsFile *hts_open_format(const char *fn, const char *mode, const htsFormat *fmt);
+
+/*!
+  @abstract       Open an existing stream as a SAM/BAM/CRAM/VCF/BCF/etc file
+  @param fn       The already-open file handle
+  @param mode     Open mode, as per hts_open()
+*/
+HTSLIB_EXPORT
+htsFile *hts_hopen(struct hFILE *fp, const char *fn, const char *mode);
+
+/*!
+  @abstract  For output streams, flush any buffered data
+  @param fp  The file handle to be flushed
+  @return    0 for success, or negative if an error occurred.
+  @since     1.14
+*/
+HTSLIB_EXPORT
+int hts_flush(htsFile *fp);
+
+/*!
+  @abstract  Close a file handle, flushing buffered data for output streams
+  @param fp  The file handle to be closed
+  @return    0 for success, or negative if an error occurred.
+*/
+HTSLIB_EXPORT
+int hts_close(htsFile *fp);
+
+/*!
+  @abstract  Returns the file's format information
+  @param fp  The file handle
+  @return    Read-only pointer to the file's htsFormat.
+*/
+HTSLIB_EXPORT
+const htsFormat *hts_get_format(htsFile *fp);
+
+/*!
+  @ abstract      Returns a string containing the file format extension.
+  @ param format  Format structure containing the file type.
+  @ return        A string ("sam", "bam", etc) or "?" for unknown formats.
+ */
+HTSLIB_EXPORT
+const char *hts_format_file_extension(const htsFormat *format);
+
+/*!
+  @abstract  Sets a specified CRAM option on the open file handle.
+  @param fp  The file handle open the open file.
+  @param opt The CRAM_OPT_* option.
+  @param ... Optional arguments, dependent on the option used.
+  @return    0 for success, or negative if an error occurred.
+*/
+HTSLIB_EXPORT
+int hts_set_opt(htsFile *fp, enum hts_fmt_option opt, ...);
+
+/*!
+  @abstract         Read a line (and its \n or \r\n terminator) from a file
+  @param fp         The file handle
+  @param delimiter  Unused, but must be '\n' (or KS_SEP_LINE)
+  @param str        The line (not including the terminator) is written here
+  @return           Length of the string read (capped at INT_MAX);
+                    -1 on end-of-file; <= -2 on error
+*/
+HTSLIB_EXPORT
+int hts_getline(htsFile *fp, int delimiter, kstring_t *str);
+
+HTSLIB_EXPORT
+char **hts_readlines(const char *fn, int *_n);
+/*!
+    @abstract       Parse comma-separated list or read list from a file
+    @param list     File name or comma-separated list
+    @param is_file
+    @param _n       Size of the output array (number of items read)
+    @return         NULL on failure or pointer to newly allocated array of
+                    strings
+*/
+HTSLIB_EXPORT
+char **hts_readlist(const char *fn, int is_file, int *_n);
+
+/*!
+  @abstract  Create extra threads to aid compress/decompression for this file
+  @param fp  The file handle
+  @param n   The number of worker threads to create
+  @return    0 for success, or negative if an error occurred.
+  @notes     This function creates non-shared threads for use solely by fp.
+             The hts_set_thread_pool function is the recommended alternative.
+*/
+HTSLIB_EXPORT
+int hts_set_threads(htsFile *fp, int n);
+
+/*!
+  @abstract  Create extra threads to aid compress/decompression for this file
+  @param fp  The file handle
+  @param p   A pool of worker threads, previously allocated by hts_create_threads().
+  @return    0 for success, or negative if an error occurred.
+*/
+HTSLIB_EXPORT
+int hts_set_thread_pool(htsFile *fp, htsThreadPool *p);
+
+/*!
+  @abstract  Adds a cache of decompressed blocks, potentially speeding up seeks.
+             This may not work for all file types (currently it is bgzf only).
+  @param fp  The file handle
+  @param n   The size of cache, in bytes
+*/
+HTSLIB_EXPORT
+void hts_set_cache_size(htsFile *fp, int n);
+
+/*!
+  @abstract  Set .fai filename for a file opened for reading
+  @return    0 for success, negative on failure
+  @discussion
+      Called before *_hdr_read(), this provides the name of a .fai file
+      used to provide a reference list if the htsFile contains no @SQ headers.
+*/
+HTSLIB_EXPORT
+int hts_set_fai_filename(htsFile *fp, const char *fn_aux);
+
+
+/*!
+  @abstract  Sets a filter expression
+  @return    0 for success, negative on failure
+  @discussion
+      To clear an existing filter, specifying expr as NULL.
+*/
+HTSLIB_EXPORT
+int hts_set_filter_expression(htsFile *fp, const char *expr);
+
+/*!
+  @abstract  Determine whether a given htsFile contains a valid EOF block
+  @return    3 for a non-EOF checkable filetype;
+             2 for an unseekable file type where EOF cannot be checked;
+             1 for a valid EOF block;
+             0 for if the EOF marker is absent when it should be present;
+            -1 (with errno set) on failure
+  @discussion
+      Check if the BGZF end-of-file (EOF) marker is present
+*/
+HTSLIB_EXPORT
+int hts_check_EOF(htsFile *fp);
+
+/************
+ * Indexing *
+ ************/
+
+/*!
+These HTS_IDX_* macros are used as special tid values for hts_itr_query()/etc,
+producing iterators operating as follows:
+ - HTS_IDX_NOCOOR iterates over unmapped reads sorted at the end of the file
+ - HTS_IDX_START  iterates over the entire file
+ - HTS_IDX_REST   iterates from the current position to the end of the file
+ - HTS_IDX_NONE   always returns "no more alignment records"
+When one of these special tid values is used, beg and end are ignored.
+When REST or NONE is used, idx is also ignored and may be NULL.
+*/
+#define HTS_IDX_NOCOOR (-2)
+#define HTS_IDX_START  (-3)
+#define HTS_IDX_REST   (-4)
+#define HTS_IDX_NONE   (-5)
+
+#define HTS_FMT_CSI 0
+#define HTS_FMT_BAI 1
+#define HTS_FMT_TBI 2
+#define HTS_FMT_CRAI 3
+#define HTS_FMT_FAI 4
+
+// Almost INT64_MAX, but when cast into a 32-bit int it's
+// also INT_MAX instead of -1.  This avoids bugs with old code
+// using the new hts_pos_t data type.
+#define HTS_POS_MAX ((((int64_t)INT_MAX)<<32)|INT_MAX)
+#define HTS_POS_MIN INT64_MIN
+#define PRIhts_pos PRId64
+typedef int64_t hts_pos_t;
+
+// For comparison with previous release:
+//
+// #define HTS_POS_MAX INT_MAX
+// #define HTS_POS_MIN INT_MIN
+// #define PRIhts_pos PRId32
+// typedef int32_t hts_pos_t;
+
+typedef struct hts_pair_pos_t {
+   hts_pos_t beg, end;
+} hts_pair_pos_t;
+
+typedef hts_pair_pos_t hts_pair32_t;  // For backwards compatibility
+
+typedef struct hts_pair64_t {
+    uint64_t u, v;
+} hts_pair64_t;
+
+typedef struct hts_pair64_max_t {
+    uint64_t u, v;
+    uint64_t max;
+} hts_pair64_max_t;
+
+typedef struct hts_reglist_t {
+    const char *reg;
+    hts_pair_pos_t *intervals;
+    int tid;
+    uint32_t count;
+    hts_pos_t min_beg, max_end;
+} hts_reglist_t;
+
+typedef int hts_readrec_func(BGZF *fp, void *data, void *r, int *tid, hts_pos_t *beg, hts_pos_t *end);
+typedef int hts_seek_func(void *fp, int64_t offset, int where);
+typedef int64_t hts_tell_func(void *fp);
+
+/**
+ * @brief File iterator that can handle multiple target regions.
+ * This structure should be considered opaque by end users.
+ * It does both the stepping inside the file and the filtering of alignments.
+ * It can operate in single or multi-region mode, and depending on this,
+ * it uses different fields.
+ *
+ * read_rest (1) - read everything from the current offset, without filtering
+ * finished  (1) - no more iterations
+ * is_cram   (1) - current file has CRAM format
+ * nocoor    (1) - read all unmapped reads
+ *
+ * multi     (1) - multi-region moode
+ * reg_list  - List of target regions
+ * n_reg     - Size of the above list
+ * curr_reg  - List index of the current region of search
+ * curr_intv - Interval index inside the current region; points to a (beg, end)
+ * end       - Used for CRAM files, to preserve the max end coordinate
+ *
+ * multi     (0) - single-region mode
+ * tid       - Reference id of the target region
+ * beg       - Start position of the target region
+ * end       - End position of the target region
+ *
+ * Common fields:
+ * off        - List of file offsets computed from the index
+ * n_off      - Size of the above list
+ * i          - List index of the current file offset
+ * curr_off   - File offset for the next file read
+ * curr_tid   - Reference id of the current alignment
+ * curr_beg   - Start position of the current alignment
+ * curr_end   - End position of the current alignment
+ * nocoor_off - File offset where the unmapped reads start
+ *
+ * readrec    - File specific function that reads an alignment
+ * seek       - File specific function for changing the file offset
+ * tell       - File specific function for indicating the file offset
+ */
+
+typedef struct hts_itr_t {
+    uint32_t read_rest:1, finished:1, is_cram:1, nocoor:1, multi:1, dummy:27;
+    int tid, n_off, i, n_reg;
+    hts_pos_t beg, end;
+    hts_reglist_t *reg_list;
+    int curr_tid, curr_reg, curr_intv;
+    hts_pos_t curr_beg, curr_end;
+    uint64_t curr_off, nocoor_off;
+    hts_pair64_max_t *off;
+    hts_readrec_func *readrec;
+    hts_seek_func *seek;
+    hts_tell_func *tell;
+    struct {
+        int n, m;
+        int *a;
+    } bins;
+} hts_itr_t;
+
+typedef hts_itr_t hts_itr_multi_t;
+
+/// Compute the first bin on a given level
+#define hts_bin_first(l) (((1<<(((l)<<1) + (l))) - 1) / 7)
+/// Compute the parent bin of a given bin
+#define hts_bin_parent(b) (((b) - 1) >> 3)
+
+///////////////////////////////////////////////////////////
+// Low-level API for building indexes.
+
+/// Create a BAI/CSI/TBI type index structure
+/** @param n          Initial number of targets
+    @param fmt        Format, one of HTS_FMT_CSI, HTS_FMT_BAI or HTS_FMT_TBI
+    @param offset0    Initial file offset
+    @param min_shift  Number of bits for the minimal interval
+    @param n_lvls     Number of levels in the binning index
+    @return An initialised hts_idx_t struct on success; NULL on failure
+
+The struct returned by a successful call should be freed via hts_idx_destroy()
+when it is no longer needed.
+*/
+HTSLIB_EXPORT
+hts_idx_t *hts_idx_init(int n, int fmt, uint64_t offset0, int min_shift, int n_lvls);
+
+/// Free a BAI/CSI/TBI type index
+/** @param idx   Index structure to free
+ */
+HTSLIB_EXPORT
+void hts_idx_destroy(hts_idx_t *idx);
+
+/// Push an index entry
+/** @param idx        Index
+    @param tid        Target id
+    @param beg        Range start (zero-based)
+    @param end        Range end (zero-based, half-open)
+    @param offset     File offset
+    @param is_mapped  Range corresponds to a mapped read
+    @return 0 on success; -1 on failure
+
+The @p is_mapped parameter is used to update the n_mapped / n_unmapped counts
+stored in the meta-data bin.
+ */
+HTSLIB_EXPORT
+int hts_idx_push(hts_idx_t *idx, int tid, hts_pos_t beg, hts_pos_t end, uint64_t offset, int is_mapped);
+
+/// Finish building an index
+/** @param idx          Index
+    @param final_offset Last file offset
+    @return 0 on success; non-zero on failure.
+*/
+HTSLIB_EXPORT
+int hts_idx_finish(hts_idx_t *idx, uint64_t final_offset);
+
+/// Returns index format
+/** @param idx   Index
+    @return One of HTS_FMT_CSI, HTS_FMT_BAI or HTS_FMT_TBI
+*/
+HTSLIB_EXPORT
+int hts_idx_fmt(hts_idx_t *idx);
+
+/// Add name to TBI index meta-data
+/** @param idx   Index
+    @param tid   Target identifier
+    @param name  Target name
+    @return Index number of name in names list on success; -1 on failure.
+*/
+HTSLIB_EXPORT
+int hts_idx_tbi_name(hts_idx_t *idx, int tid, const char *name);
+
+// Index loading and saving
+
+/// Save an index to a file
+/** @param idx  Index to be written
+    @param fn   Input BAM/BCF/etc filename, to which .bai/.csi/etc will be added
+    @param fmt  One of the HTS_FMT_* index formats
+    @return  0 if successful, or negative if an error occurred.
+*/
+HTSLIB_EXPORT
+int hts_idx_save(const hts_idx_t *idx, const char *fn, int fmt) HTS_RESULT_USED;
+
+/// Save an index to a specific file
+/** @param idx    Index to be written
+    @param fn     Input BAM/BCF/etc filename
+    @param fnidx  Output filename, or NULL to add .bai/.csi/etc to @a fn
+    @param fmt    One of the HTS_FMT_* index formats
+    @return  0 if successful, or negative if an error occurred.
+*/
+HTSLIB_EXPORT
+int hts_idx_save_as(const hts_idx_t *idx, const char *fn, const char *fnidx, int fmt) HTS_RESULT_USED;
+
+/// Load an index file
+/** @param fn   BAM/BCF/etc filename, to which .bai/.csi/etc will be added or
+                the extension substituted, to search for an existing index file.
+                In case of a non-standard naming, the file name can include the
+                name of the index file delimited with HTS_IDX_DELIM.
+    @param fmt  One of the HTS_FMT_* index formats
+    @return  The index, or NULL if an error occurred.
+
+If @p fn contains the string "##idx##" (HTS_IDX_DELIM), the part before
+the delimiter will be used as the name of the data file and the part after
+it will be used as the name of the index.
+
+Otherwise, this function tries to work out the index name as follows:
+
+  It will try appending ".csi" to @p fn
+  It will try substituting an existing suffix (e.g. .bam, .vcf) with ".csi"
+  Then, if @p fmt is HTS_FMT_BAI:
+    It will try appending ".bai" to @p fn
+    To will substituting the existing suffix (e.g. .bam) with ".bai"
+  else if @p fmt is HTS_FMT_TBI:
+    It will try appending ".tbi" to @p fn
+    To will substituting the existing suffix (e.g. .vcf) with ".tbi"
+
+If the index file is remote (served over a protocol like https), first a check
+is made to see is a locally cached copy is available.  This is done for all
+of the possible names listed above.  If a cached copy is not available then
+the index will be downloaded and stored in the current working directory,
+with the same name as the remote index.
+
+    Equivalent to hts_idx_load3(fn, NULL, fmt, HTS_IDX_SAVE_REMOTE);
+*/
+HTSLIB_EXPORT
+hts_idx_t *hts_idx_load(const char *fn, int fmt);
+
+/// Load a specific index file
+/** @param fn     Input BAM/BCF/etc filename
+    @param fnidx  The input index filename
+    @return  The index, or NULL if an error occurred.
+
+    Equivalent to hts_idx_load3(fn, fnidx, 0, 0);
+
+    This function will not attempt to save index files locally.
+*/
+HTSLIB_EXPORT
+hts_idx_t *hts_idx_load2(const char *fn, const char *fnidx);
+
+/// Load a specific index file
+/** @param fn     Input BAM/BCF/etc filename
+    @param fnidx  The input index filename
+    @param fmt    One of the HTS_FMT_* index formats
+    @param flags  Flags to alter behaviour (see description)
+    @return  The index, or NULL if an error occurred.
+
+    If @p fnidx is NULL, the index name will be derived from @p fn in the
+    same way as hts_idx_load().
+
+    If @p fnidx is not NULL, @p fmt is ignored.
+
+    The @p flags parameter can be set to a combination of the following
+    values:
+
+        HTS_IDX_SAVE_REMOTE   Save a local copy of any remote indexes
+        HTS_IDX_SILENT_FAIL   Fail silently if the index is not present
+
+    The index struct returned by a successful call should be freed
+    via hts_idx_destroy() when it is no longer needed.
+*/
+HTSLIB_EXPORT
+hts_idx_t *hts_idx_load3(const char *fn, const char *fnidx, int fmt, int flags);
+
+/// Flags for hts_idx_load3() ( and also sam_idx_load3(), tbx_idx_load3() )
+#define HTS_IDX_SAVE_REMOTE 1
+#define HTS_IDX_SILENT_FAIL 2
+
+///////////////////////////////////////////////////////////
+// Functions for accessing meta-data stored in indexes
+
+typedef const char *(*hts_id2name_f)(void*, int);
+
+/// Get extra index meta-data
+/** @param idx    The index
+    @param l_meta Pointer to where the length of the extra data is stored
+    @return Pointer to the extra data if present; NULL otherwise
+
+    Indexes (both .tbi and .csi) made by tabix include extra data about
+    the indexed file.  The returns a pointer to this data.  Note that the
+    data is stored exactly as it is in the index.  Callers need to interpret
+    the results themselves, including knowing what sort of data to expect;
+    byte swapping etc.
+*/
+HTSLIB_EXPORT
+uint8_t *hts_idx_get_meta(hts_idx_t *idx, uint32_t *l_meta);
+
+/// Set extra index meta-data
+/** @param idx     The index
+    @param l_meta  Length of data
+    @param meta    Pointer to the extra data
+    @param is_copy If not zero, a copy of the data is taken
+    @return 0 on success; -1 on failure (out of memory).
+
+    Sets the data that is returned by hts_idx_get_meta().
+
+    If is_copy != 0, a copy of the input data is taken.  If not, ownership of
+    the data pointed to by *meta passes to the index.
+*/
+HTSLIB_EXPORT
+int hts_idx_set_meta(hts_idx_t *idx, uint32_t l_meta, uint8_t *meta, int is_copy);
+
+/// Get number of mapped and unmapped reads from an index
+/** @param      idx      Index
+    @param      tid      Target ID
+    @param[out] mapped   Location to store number of mapped reads
+    @param[out] unmapped Location to store number of unmapped reads
+    @return 0 on success; -1 on failure (data not available)
+
+    BAI and CSI indexes store information on the number of reads for each
+    target that were mapped or unmapped (unmapped reads will generally have
+    a paired read that is mapped to the target).  This function returns this
+    information if it is available.
+
+    @note Cram CRAI indexes do not include this information.
+*/
+HTSLIB_EXPORT
+int hts_idx_get_stat(const hts_idx_t* idx, int tid, uint64_t* mapped, uint64_t* unmapped);
+
+/// Return the number of unplaced reads from an index
+/** @param idx    Index
+    @return Unplaced reads count
+
+    Unplaced reads are not linked to any reference (e.g. RNAME is '*' in SAM
+    files).
+*/
+HTSLIB_EXPORT
+uint64_t hts_idx_get_n_no_coor(const hts_idx_t* idx);
+
+/// Return a list of target names from an index
+/** @param      idx    Index
+    @param[out] n      Location to store the number of targets
+    @param      getid  Callback function to get the name for a target ID
+    @param      hdr    Header from indexed file
+    @return An array of pointers to the names on success; NULL on failure
+
+    @note The names are pointers into the header data structure.  When cleaning
+    up, only the array should be freed, not the names.
+ */
+HTSLIB_EXPORT
+const char **hts_idx_seqnames(const hts_idx_t *idx, int *n, hts_id2name_f getid, void *hdr); // free only the array, not the values
+
+/// Return the number of targets from an index
+/** @param      idx    Index
+    @return The number of targets
+ */
+HTSLIB_EXPORT
+int hts_idx_nseq(const hts_idx_t *idx);
+
+///////////////////////////////////////////////////////////
+// Region parsing
+
+#define HTS_PARSE_THOUSANDS_SEP 1  ///< Ignore ',' separators within numbers
+#define HTS_PARSE_ONE_COORD     2  ///< chr:pos means chr:pos-pos and not chr:pos-end
+#define HTS_PARSE_LIST          4  ///< Expect a comma separated list of regions. (Disables HTS_PARSE_THOUSANDS_SEP)
+
+/// Parse a numeric string
+/** The number may be expressed in scientific notation, and optionally may
+    contain commas in the integer part (before any decimal point or E notation).
+    @param str     String to be parsed
+    @param strend  If non-NULL, set on return to point to the first character
+                   in @a str after those forming the parsed number
+    @param flags   Or'ed-together combination of HTS_PARSE_* flags
+    @return  Integer value of the parsed number, or 0 if no valid number
+
+    The input string is parsed as: optional whitespace; an optional '+' or
+    '-' sign; decimal digits possibly including ',' characters (if @a flags
+    includes HTS_PARSE_THOUSANDS_SEP) and a '.' decimal point; and an optional
+    case-insensitive suffix, which may be either 'k', 'M', 'G', or scientific
+    notation consisting of 'e'/'E' followed by an optional '+' or '-' sign and
+    decimal digits. To be considered a valid numeric value, the main part (not
+    including any suffix or scientific notation) must contain at least one
+    digit (either before or after the decimal point).
+
+    When @a strend is NULL, @a str is expected to contain only (optional
+    whitespace followed by) the numeric value. A warning will be printed
+    (if hts_verbose is HTS_LOG_WARNING or more) if no valid parsable number
+    is found or if there are any unused characters after the number.
+
+    When @a strend is non-NULL, @a str starts with (optional whitespace
+    followed by) the numeric value. On return, @a strend is set to point
+    to the first unused character after the numeric value, or to @a str
+    if no valid parsable number is found.
+*/
+HTSLIB_EXPORT
+long long hts_parse_decimal(const char *str, char **strend, int flags);
+
+typedef int (*hts_name2id_f)(void*, const char*);
+
+/// Parse a "CHR:START-END"-style region string
+/** @param str  String to be parsed
+    @param beg  Set on return to the 0-based start of the region
+    @param end  Set on return to the 1-based end of the region
+    @return  Pointer to the colon or '\0' after the reference sequence name,
+             or NULL if @a str could not be parsed.
+
+    NOTE: For compatibility with hts_parse_reg only.
+    Please use hts_parse_region instead.
+*/
+HTSLIB_EXPORT
+const char *hts_parse_reg64(const char *str, hts_pos_t *beg, hts_pos_t *end);
+
+/// Parse a "CHR:START-END"-style region string
+/** @param str  String to be parsed
+    @param beg  Set on return to the 0-based start of the region
+    @param end  Set on return to the 1-based end of the region
+    @return  Pointer to the colon or '\0' after the reference sequence name,
+             or NULL if @a str could not be parsed.
+*/
+HTSLIB_EXPORT
+const char *hts_parse_reg(const char *str, int *beg, int *end);
+
+/// Parse a "CHR:START-END"-style region string
+/** @param str   String to be parsed
+    @param tid   Set on return (if not NULL) to be reference index (-1 if invalid)
+    @param beg   Set on return to the 0-based start of the region
+    @param end   Set on return to the 1-based end of the region
+    @param getid Function pointer.  Called if not NULL to set tid.
+    @param hdr   Caller data passed to getid.
+    @param flags Bitwise HTS_PARSE_* flags listed above.
+    @return      Pointer to the byte after the end of the entire region
+                 specifier (including any trailing comma) on success,
+                 or NULL if @a str could not be parsed.
+
+    A variant of hts_parse_reg which is reference-id aware.  It uses
+    the iterator name2id callbacks to validate the region tokenisation works.
+
+    This is necessary due to GRCh38 HLA additions which have reference names
+    like "HLA-DRB1*12:17".
+
+    To work around ambiguous parsing issues, eg both "chr1" and "chr1:100-200"
+    are reference names, quote using curly braces.
+    Thus "{chr1}:100-200" and "{chr1:100-200}" disambiguate the above example.
+
+    Flags are used to control how parsing works, and can be one of the below.
+
+    HTS_PARSE_THOUSANDS_SEP:
+        Ignore commas in numbers.  For example with this flag 1,234,567
+        is interpreted as 1234567.
+
+    HTS_PARSE_LIST:
+        If present, the region is assmed to be a comma separated list and
+        position parsing will not contain commas (this implicitly
+        clears HTS_PARSE_THOUSANDS_SEP in the call to hts_parse_decimal).
+        On success the return pointer will be the start of the next region, ie
+        the character after the comma.  (If *ret != '\0' then the caller can
+        assume another region is present in the list.)
+
+        If not set then positions may contain commas.  In this case the return
+        value should point to the end of the string, or NULL on failure.
+
+    HTS_PARSE_ONE_COORD:
+        If present, X:100 is treated as the single base pair region X:100-100.
+        In this case X:-100 is shorthand for X:1-100 and X:100- is X:100-<end>.
+        (This is the standard bcftools region convention.)
+
+        When not set X:100 is considered to be X:100-<end> where <end> is
+        the end of chromosome X (set to INT_MAX here).  X:100- and X:-100 are
+        invalid.
+        (This is the standard samtools region convention.)
+
+    Note the supplied string expects 1 based inclusive coordinates, but the
+    returned coordinates start from 0 and are half open, so pos0 is valid
+    for use in e.g. "for (pos0 = beg; pos0 < end; pos0++) {...}"
+
+    If NULL is returned, the value in tid mat give additional information
+    about the error:
+
+        -2   Failed to parse @p hdr; or out of memory
+        -1   The reference in @p str has mismatched braces, or does not
+             exist in @p hdr
+        >= 0 The specified range in @p str could not be parsed
+*/
+HTSLIB_EXPORT
+const char *hts_parse_region(const char *s, int *tid, hts_pos_t *beg,
+                             hts_pos_t *end, hts_name2id_f getid, void *hdr,
+                             int flags);
+
+
+///////////////////////////////////////////////////////////
+// Generic iterators
+//
+// These functions provide the low-level infrastructure for iterators.
+// Wrappers around these are used to make iterators for specific file types.
+// See:
+//     htslib/sam.h  for SAM/BAM/CRAM iterators
+//     htslib/vcf.h  for VCF/BCF iterators
+//     htslib/tbx.h  for files indexed by tabix
+
+/// Create a single-region iterator
+/** @param idx      Index
+    @param tid      Target ID
+    @param beg      Start of region
+    @param end      End of region
+    @param readrec  Callback to read a record from the input file
+    @return An iterator on success; NULL on failure
+
+    The iterator struct returned by a successful call should be freed
+    via hts_itr_destroy() when it is no longer needed.
+ */
+HTSLIB_EXPORT
+hts_itr_t *hts_itr_query(const hts_idx_t *idx, int tid, hts_pos_t beg, hts_pos_t end, hts_readrec_func *readrec);
+
+/// Free an iterator
+/** @param iter   Iterator to free
+ */
+HTSLIB_EXPORT
+void hts_itr_destroy(hts_itr_t *iter);
+
+typedef hts_itr_t *hts_itr_query_func(const hts_idx_t *idx, int tid, hts_pos_t beg, hts_pos_t end, hts_readrec_func *readrec);
+
+/// Create a single-region iterator from a text region specification
+/** @param idx       Index
+    @param reg       Region specifier
+    @param getid     Callback function to return the target ID for a name
+    @param hdr       Input file header
+    @param itr_query Callback function returning an iterator for a numeric tid,
+                     start and end position
+    @param readrec   Callback to read a record from the input file
+    @return An iterator on success; NULL on error
+
+    The iterator struct returned by a successful call should be freed
+    via hts_itr_destroy() when it is no longer needed.
+ */
+HTSLIB_EXPORT
+hts_itr_t *hts_itr_querys(const hts_idx_t *idx, const char *reg, hts_name2id_f getid, void *hdr, hts_itr_query_func *itr_query, hts_readrec_func *readrec);
+
+/// Return the next record from an iterator
+/** @param fp      Input file handle
+    @param iter    Iterator
+    @param r       Pointer to record placeholder
+    @param data    Data passed to the readrec callback
+    @return >= 0 on success, -1 when there is no more data, < -1 on error
+ */
+HTSLIB_EXPORT
+int hts_itr_next(BGZF *fp, hts_itr_t *iter, void *r, void *data) HTS_RESULT_USED;
+
+/**********************************
+ * Iterator with multiple regions *
+ **********************************/
+
+typedef int hts_itr_multi_query_func(const hts_idx_t *idx, hts_itr_t *itr);
+HTSLIB_EXPORT
+int hts_itr_multi_bam(const hts_idx_t *idx, hts_itr_t *iter);
+HTSLIB_EXPORT
+int hts_itr_multi_cram(const hts_idx_t *idx, hts_itr_t *iter);
+
+/// Create a multi-region iterator from a region list
+/** @param idx          Index
+    @param reglist      Region list
+    @param count        Number of items in region list
+    @param getid        Callback to convert names to target IDs
+    @param hdr          Indexed file header (passed to getid)
+    @param itr_specific Filetype-specific callback function
+    @param readrec      Callback to read an input file record
+    @param seek         Callback to seek in the input file
+    @param tell         Callback to return current input file location
+    @return An iterator on success; NULL on failure
+
+    The iterator struct returned by a successful call should be freed
+    via hts_itr_destroy() when it is no longer needed.
+ */
+HTSLIB_EXPORT
+hts_itr_t *hts_itr_regions(const hts_idx_t *idx, hts_reglist_t *reglist, int count, hts_name2id_f getid, void *hdr, hts_itr_multi_query_func *itr_specific, hts_readrec_func *readrec, hts_seek_func *seek, hts_tell_func *tell);
+
+/// Return the next record from an iterator
+/** @param fp      Input file handle
+    @param iter    Iterator
+    @param r       Pointer to record placeholder
+    @return >= 0 on success, -1 when there is no more data, < -1 on error
+ */
+HTSLIB_EXPORT
+int hts_itr_multi_next(htsFile *fd, hts_itr_t *iter, void *r);
+
+/// Create a region list from a char array
+/** @param argv      Char array of target:interval elements, e.g. chr1:2500-3600, chr1:5100, chr2
+    @param argc      Number of items in the array
+    @param r_count   Pointer to the number of items in the resulting region list
+    @param hdr       Header for the sam/bam/cram file
+    @param getid     Callback to convert target names to target ids.
+    @return  A region list on success, NULL on failure
+
+    The hts_reglist_t struct returned by a successful call should be freed
+    via hts_reglist_free() when it is no longer needed.
+ */
+HTSLIB_EXPORT
+hts_reglist_t *hts_reglist_create(char **argv, int argc, int *r_count, void *hdr,  hts_name2id_f getid);
+
+/// Free a region list
+/** @param reglist    Region list
+    @param count      Number of items in the list
+ */
+HTSLIB_EXPORT
+void hts_reglist_free(hts_reglist_t *reglist, int count);
+
+/// Free a multi-region iterator
+/** @param iter   Iterator to free
+ */
+#define hts_itr_multi_destroy(iter) hts_itr_destroy(iter)
+
+
+    /**
+     * hts_file_type() - Convenience function to determine file type
+     * DEPRECATED:  This function has been replaced by hts_detect_format().
+     * It and these FT_* macros will be removed in a future HTSlib release.
+     */
+    #define FT_UNKN   0
+    #define FT_GZ     1
+    #define FT_VCF    2
+    #define FT_VCF_GZ (FT_GZ|FT_VCF)
+    #define FT_BCF    (1<<2)
+    #define FT_BCF_GZ (FT_GZ|FT_BCF)
+    #define FT_STDIN  (1<<3)
+    HTSLIB_EXPORT
+    int hts_file_type(const char *fname);
+
+
+/***************************
+ * Revised MAQ error model *
+ ***************************/
+
+struct errmod_t;
+typedef struct errmod_t errmod_t;
+
+HTSLIB_EXPORT
+errmod_t *errmod_init(double depcorr);
+HTSLIB_EXPORT
+void errmod_destroy(errmod_t *em);
+
+/*
+    n: number of bases
+    m: maximum base
+    bases[i]: qual:6, strand:1, base:4
+    q[i*m+j]: phred-scaled likelihood of (i,j)
+ */
+HTSLIB_EXPORT
+int errmod_cal(const errmod_t *em, int n, int m, uint16_t *bases, float *q);
+
+
+/*****************************************************
+ * Probabilistic banded glocal alignment             *
+ * See https://doi.org/10.1093/bioinformatics/btr076 *
+ *****************************************************/
+
+typedef struct probaln_par_t {
+    float d, e;
+    int bw;
+} probaln_par_t;
+
+/// Perform probabilistic banded glocal alignment
+/** @param      ref     Reference sequence
+    @param      l_ref   Length of reference
+    @param      query   Query sequence
+    @param      l_query Length of query sequence
+    @param      iqual   Query base qualities
+    @param      c       Alignment parameters
+    @param[out] state   Output alignment
+    @param[out] q    Phred scaled posterior probability of state[i] being wrong
+    @return     Phred-scaled likelihood score, or INT_MIN on failure.
+
+The reference and query sequences are coded using integers 0,1,2,3,4 for
+bases A,C,G,T,N respectively (N here is for any ambiguity code).
+
+On output, state and q are arrays of length l_query. The higher 30
+bits give the reference position the query base is matched to and the
+lower two bits can be 0 (an alignment match) or 1 (an
+insertion). q[i] gives the phred scaled posterior probability of
+state[i] being wrong.
+
+On failure, errno will be set to EINVAL if the values of l_ref or l_query
+were invalid; or ENOMEM if a memory allocation failed.
+*/
+
+HTSLIB_EXPORT
+int probaln_glocal(const uint8_t *ref, int l_ref, const uint8_t *query, int l_query, const uint8_t *iqual, const probaln_par_t *c, int *state, uint8_t *q);
+
+
+    /**********************
+     * MD5 implementation *
+     **********************/
+
+    struct hts_md5_context;
+    typedef struct hts_md5_context hts_md5_context;
+
+    /*! @abstract   Initialises an MD5 context.
+     *  @discussion
+     *    The expected use is to allocate an hts_md5_context using
+     *    hts_md5_init().  This pointer is then passed into one or more calls
+     *    of hts_md5_update() to compute successive internal portions of the
+     *    MD5 sum, which can then be externalised as a full 16-byte MD5sum
+     *    calculation by calling hts_md5_final().  This can then be turned
+     *    into ASCII via hts_md5_hex().
+     *
+     *    To dealloate any resources created by hts_md5_init() call the
+     *    hts_md5_destroy() function.
+     *
+     *  @return     hts_md5_context pointer on success, NULL otherwise.
+     */
+    HTSLIB_EXPORT
+    hts_md5_context *hts_md5_init(void);
+
+    /*! @abstract Updates the context with the MD5 of the data. */
+    HTSLIB_EXPORT
+    void hts_md5_update(hts_md5_context *ctx, const void *data, unsigned long size);
+
+    /*! @abstract Computes the final 128-bit MD5 hash from the given context */
+    HTSLIB_EXPORT
+    void hts_md5_final(unsigned char *digest, hts_md5_context *ctx);
+
+    /*! @abstract Resets an md5_context to the initial state, as returned
+     *            by hts_md5_init().
+     */
+    HTSLIB_EXPORT
+    void hts_md5_reset(hts_md5_context *ctx);
+
+    /*! @abstract Converts a 128-bit MD5 hash into a 33-byte nul-termninated
+     *            hex string.
+     */
+    HTSLIB_EXPORT
+    void hts_md5_hex(char *hex, const unsigned char *digest);
+
+    /*! @abstract Deallocates any memory allocated by hts_md5_init. */
+    HTSLIB_EXPORT
+    void hts_md5_destroy(hts_md5_context *ctx);
+
+static inline int hts_reg2bin(hts_pos_t beg, hts_pos_t end, int min_shift, int n_lvls)
+{
+    int l, s = min_shift, t = ((1<<((n_lvls<<1) + n_lvls)) - 1) / 7;
+    for (--end, l = n_lvls; l > 0; --l, s += 3, t -= 1<<((l<<1)+l))
+        if (beg>>s == end>>s) return t + (beg>>s);
+    return 0;
+}
+
+/// Compute the level of a bin in a binning index
+static inline int hts_bin_level(int bin) {
+    int l, b;
+    for (l = 0, b = bin; b; ++l, b = hts_bin_parent(b));
+    return l;
+}
+
+//! Compute the corresponding entry into the linear index of a given bin from
+//! a binning index
+/*!
+ *  @param bin    The bin number
+ *  @param n_lvls The index depth (number of levels - 0 based)
+ *  @return       The integer offset into the linear index
+ *
+ *  Explanation of the return value formula:
+ *  Each bin on level l covers exp(2, (n_lvls - l)*3 + min_shift) base pairs.
+ *  A linear index entry covers exp(2, min_shift) base pairs.
+ */
+static inline int hts_bin_bot(int bin, int n_lvls)
+{
+    int l = hts_bin_level(bin);
+    return (bin - hts_bin_first(l)) << (n_lvls - l) * 3;
+}
+
+/// Compute the (0-based exclusive) maximum position covered by a binning index
+static inline hts_pos_t hts_bin_maxpos(int min_shift, int n_lvls)
+{
+    hts_pos_t one = 1;
+    return one << (min_shift + n_lvls * 3);
+}
+
+/**************
+ * Endianness *
+ **************/
+
+static inline int ed_is_big(void)
+{
+    long one= 1;
+    return !(*((char *)(&one)));
+}
+static inline uint16_t ed_swap_2(uint16_t v)
+{
+    return (uint16_t)(((v & 0x00FF00FFU) << 8) | ((v & 0xFF00FF00U) >> 8));
+}
+static inline void *ed_swap_2p(void *x)
+{
+    *(uint16_t*)x = ed_swap_2(*(uint16_t*)x);
+    return x;
+}
+static inline uint32_t ed_swap_4(uint32_t v)
+{
+    v = ((v & 0x0000FFFFU) << 16) | (v >> 16);
+    return ((v & 0x00FF00FFU) << 8) | ((v & 0xFF00FF00U) >> 8);
+}
+static inline void *ed_swap_4p(void *x)
+{
+    *(uint32_t*)x = ed_swap_4(*(uint32_t*)x);
+    return x;
+}
+static inline uint64_t ed_swap_8(uint64_t v)
+{
+    v = ((v & 0x00000000FFFFFFFFLLU) << 32) | (v >> 32);
+    v = ((v & 0x0000FFFF0000FFFFLLU) << 16) | ((v & 0xFFFF0000FFFF0000LLU) >> 16);
+    return ((v & 0x00FF00FF00FF00FFLLU) << 8) | ((v & 0xFF00FF00FF00FF00LLU) >> 8);
+}
+static inline void *ed_swap_8p(void *x)
+{
+    *(uint64_t*)x = ed_swap_8(*(uint64_t*)x);
+    return x;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/ext/htslib/htslib/hts_defs.h b/ext/htslib/htslib/hts_defs.h
new file mode 100644
index 0000000..b5cded3
--- /dev/null
+++ b/ext/htslib/htslib/hts_defs.h
@@ -0,0 +1,135 @@
+/*  hts_defs.h -- Miscellaneous definitions.
+
+    Copyright (C) 2013-2015,2017, 2019-2020, 2024 Genome Research Ltd.
+
+    Author: John Marshall <jm18@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#ifndef HTSLIB_HTS_DEFS_H
+#define HTSLIB_HTS_DEFS_H
+
+#if defined __MINGW32__
+#include <stdio.h>     // For __MINGW_PRINTF_FORMAT macro
+#endif
+
+#ifdef __clang__
+#ifdef __has_attribute
+#define HTS_COMPILER_HAS(attribute) __has_attribute(attribute)
+#endif
+
+#elif defined __GNUC__
+#define HTS_GCC_AT_LEAST(major, minor) \
+    (__GNUC__ > (major) || (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))
+#endif
+
+#ifndef HTS_COMPILER_HAS
+#define HTS_COMPILER_HAS(attribute) 0
+#endif
+#ifndef HTS_GCC_AT_LEAST
+#define HTS_GCC_AT_LEAST(major, minor) 0
+#endif
+
+#if HTS_COMPILER_HAS(__nonstring__) || HTS_GCC_AT_LEAST(8,1)
+#define HTS_NONSTRING __attribute__ ((__nonstring__))
+#else
+#define HTS_NONSTRING
+#endif
+
+#if HTS_COMPILER_HAS(__noreturn__) || HTS_GCC_AT_LEAST(3,0)
+#define HTS_NORETURN __attribute__ ((__noreturn__))
+#else
+#define HTS_NORETURN
+#endif
+
+// Enable optimisation level 3, especially for gcc.  To be used
+// where we want to force vectorisation in hot loops and the default -O2
+// just doesn't cut it.
+#if HTS_COMPILER_HAS(optimize) || HTS_GCC_AT_LEAST(4,4)
+#define HTS_OPT3 __attribute__((optimize("O3")))
+#else
+#define HTS_OPT3
+#endif
+
+#if HTS_COMPILER_HAS(aligned) || HTS_GCC_AT_LEAST(4,3)
+#define HTS_ALIGN32 __attribute__((aligned(32)))
+#else
+#define HTS_ALIGN32
+#endif
+
+// GCC introduced warn_unused_result in 3.4 but added -Wno-unused-result later
+#if HTS_COMPILER_HAS(__warn_unused_result__) || HTS_GCC_AT_LEAST(4,5)
+#define HTS_RESULT_USED __attribute__ ((__warn_unused_result__))
+#else
+#define HTS_RESULT_USED
+#endif
+
+#if HTS_COMPILER_HAS(__unused__) || HTS_GCC_AT_LEAST(3,0)
+#define HTS_UNUSED __attribute__ ((__unused__))
+#else
+#define HTS_UNUSED
+#endif
+
+#if HTS_COMPILER_HAS(__deprecated__) || HTS_GCC_AT_LEAST(4,5)
+#define HTS_DEPRECATED(message) __attribute__ ((__deprecated__ (message)))
+#elif HTS_GCC_AT_LEAST(3,1)
+#define HTS_DEPRECATED(message) __attribute__ ((__deprecated__))
+#else
+#define HTS_DEPRECATED(message)
+#endif
+
+#if (HTS_COMPILER_HAS(__deprecated__) || HTS_GCC_AT_LEAST(6,4)) && !defined(__ICC)
+#define HTS_DEPRECATED_ENUM(message) __attribute__ ((__deprecated__ (message)))
+#else
+#define HTS_DEPRECATED_ENUM(message)
+#endif
+
+// On mingw the "printf" format type doesn't work.  It needs "gnu_printf"
+// in order to check %lld and %z, otherwise it defaults to checking against
+// the Microsoft library printf format options despite linking against the
+// GNU posix implementation of printf.  The __MINGW_PRINTF_FORMAT macro
+// expands to printf or gnu_printf as required, but obviously may not
+// exist
+#ifdef __MINGW_PRINTF_FORMAT
+#define HTS_PRINTF_FMT __MINGW_PRINTF_FORMAT
+#else
+#define HTS_PRINTF_FMT printf
+#endif
+
+#if HTS_COMPILER_HAS(__format__) || HTS_GCC_AT_LEAST(3,0)
+#define HTS_FORMAT(type, idx, first) __attribute__((__format__ (type, idx, first)))
+#else
+#define HTS_FORMAT(type, idx, first)
+#endif
+
+#if defined(_WIN32) || defined(__CYGWIN__)
+#if defined(HTS_BUILDING_LIBRARY)
+#define HTSLIB_EXPORT __declspec(dllexport)
+#else
+#define HTSLIB_EXPORT
+#endif
+#elif HTS_COMPILER_HAS(__visibility__) || HTS_GCC_AT_LEAST(4,0)
+#define HTSLIB_EXPORT __attribute__((__visibility__("default")))
+#elif defined(__SUNPRO_C) && __SUNPRO_C >= 0x550
+#define HTSLIB_EXPORT __global
+#else
+#define HTSLIB_EXPORT
+#endif
+
+#endif
diff --git a/ext/htslib/htslib/hts_endian.h b/ext/htslib/htslib/hts_endian.h
new file mode 100644
index 0000000..12effab
--- /dev/null
+++ b/ext/htslib/htslib/hts_endian.h
@@ -0,0 +1,362 @@
+/// @file hts_endian.h
+/// Byte swapping and unaligned access functions.
+/*
+   Copyright (C) 2017 Genome Research Ltd.
+
+    Author: Rob Davies <rmd@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#ifndef HTS_ENDIAN_H
+#define HTS_ENDIAN_H
+
+#include <stdint.h>
+
+/*
+ * Compile-time endianness tests.
+ *
+ * Note that these tests may fail.  They should only be used to enable
+ * faster versions of endian-neutral implementations.  The endian-neutral
+ * version should always be available as a fall-back.
+ *
+ * See https://sourceforge.net/p/predef/wiki/Endianness/
+ */
+
+/* Save typing as both endian and unaligned tests want to know about x86 */
+#if (defined(__i386__) || defined(__i386) || defined(__amd64__) || defined(__amd64) || defined(__x86_64__) || defined(__x86_64) || defined(__i686__) || defined(__i686)) && !defined(HTS_x86)
+#    define HTS_x86  /* x86 and x86_64 platform */
+#endif
+
+/** @def HTS_LITTLE_ENDIAN
+ *  @brief Defined if platform is known to be little-endian
+ */
+
+#ifndef HTS_LITTLE_ENDIAN
+#    if (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) \
+      || defined(__LITTLE_ENDIAN__) \
+      || defined(HTS_x86) \
+      || defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) \
+      || defined(_MIPSEL) || defined(__MIPSEL) || defined(__MIPSEL__)
+#        define HTS_LITTLE_ENDIAN
+#    endif
+#endif
+
+/** @def HTS_BIG_ENDIAN
+ *  @brief Defined if platform is known to be big-endian
+ */
+
+#ifndef HTS_BIG_ENDIAN
+#    if (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) \
+      || defined(__BIG_ENDIAN__) \
+      || defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AAARCHEB__) \
+      || defined(_MIPSEB) || defined(__MIPSEB) || defined(__MIPSEB__)
+#        define HTS_BIG_ENDIAN
+#    endif
+#endif
+
+/** @def HTS_ENDIAN_NEUTRAL
+ *  @brief Define this to disable any endian-specific optimizations
+ */
+
+#if defined(HTS_ENDIAN_NEUTRAL) || (defined(HTS_LITTLE_ENDIAN) && defined(HTS_BIG_ENDIAN))
+/* Disable all endian-specific code. */
+#    undef HTS_LITTLE_ENDIAN
+#    undef HTS_BIG_ENDIAN
+#endif
+
+/** @def HTS_ALLOW_UNALIGNED
+ *  @brief Control use of unaligned memory access.
+ *
+ * Defining HTS_ALLOW_UNALIGNED=1 converts shift-and-or to simple casts on
+ * little-endian platforms that can tolerate unaligned access (notably Intel
+ * x86).
+ *
+ * Defining HTS_ALLOW_UNALIGNED=0 forces shift-and-or.
+ */
+
+// Consider using AX_CHECK_ALIGNED_ACCESS_REQUIRED in autoconf.
+#ifndef HTS_ALLOW_UNALIGNED
+#    if defined(HTS_x86)
+#        define HTS_ALLOW_UNALIGNED 1
+#    else
+#        define HTS_ALLOW_UNALIGNED 0
+#    endif
+#endif
+
+#if HTS_ALLOW_UNALIGNED != 0
+#    if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) || defined(__clang__)
+// This prevents problems with gcc's vectoriser generating the wrong
+// instructions for unaligned data.
+typedef uint16_t uint16_u __attribute__ ((__aligned__ (1)));
+typedef uint32_t uint32_u __attribute__ ((__aligned__ (1)));
+typedef uint64_t uint64_u __attribute__ ((__aligned__ (1)));
+#else
+typedef uint16_t uint16_u;
+typedef uint32_t uint32_u;
+typedef uint64_t uint64_u;
+#    endif
+#endif
+
+/// Get a uint8_t value from an unsigned byte array
+/** @param buf Pointer to source byte, may be unaligned
+ *  @return An 8-bit unsigned integer
+ */
+static inline uint8_t le_to_u8(const uint8_t *buf) {
+    return *buf;
+}
+
+/// Get a uint16_t value from an unsigned byte array
+/** @param buf Pointer to source byte, may be unaligned
+ *  @return A 16 bit unsigned integer
+ *  The input is read in little-endian byte order.
+ */
+static inline uint16_t le_to_u16(const uint8_t *buf) {
+#if defined(HTS_LITTLE_ENDIAN) && HTS_ALLOW_UNALIGNED != 0
+    return *((uint16_u *) buf);
+#else
+    return (uint16_t) buf[0] | ((uint16_t) buf[1] << 8);
+#endif
+}
+
+/// Get a uint32_t value from an unsigned byte array
+/** @param buf Pointer to source byte array, may be unaligned
+ *  @return A 32 bit unsigned integer
+ *  The input is read in little-endian byte order.
+ */
+static inline uint32_t le_to_u32(const uint8_t *buf) {
+#if defined(HTS_LITTLE_ENDIAN) && HTS_ALLOW_UNALIGNED != 0
+    return *((uint32_u *) buf);
+#else
+    return ((uint32_t) buf[0] |
+            ((uint32_t) buf[1] << 8) |
+            ((uint32_t) buf[2] << 16) |
+            ((uint32_t) buf[3] << 24));
+#endif
+}
+
+/// Get a uint64_t value from an unsigned byte array
+/** @param buf Pointer to source byte array, may be unaligned
+ *  @return A 64 bit unsigned integer
+ *  The input is read in little-endian byte order.
+ */
+static inline uint64_t le_to_u64(const uint8_t *buf) {
+#if defined(HTS_LITTLE_ENDIAN) && HTS_ALLOW_UNALIGNED != 0
+    return *((uint64_u *) buf);
+#else
+    return ((uint64_t) buf[0] |
+            ((uint64_t) buf[1] << 8) |
+            ((uint64_t) buf[2] << 16) |
+            ((uint64_t) buf[3] << 24) |
+            ((uint64_t) buf[4] << 32) |
+            ((uint64_t) buf[5] << 40) |
+            ((uint64_t) buf[6] << 48) |
+            ((uint64_t) buf[7] << 56));
+#endif
+}
+
+/// Store a uint16_t value in little-endian byte order
+/** @param val The value to store
+ *  @param buf Where to store it (may be unaligned)
+ */
+static inline void u16_to_le(uint16_t val, uint8_t *buf) {
+#if defined(HTS_LITTLE_ENDIAN) && HTS_ALLOW_UNALIGNED != 0
+    *((uint16_u *) buf) = val;
+#else
+    buf[0] = val & 0xff;
+    buf[1] = (val >> 8) & 0xff;
+#endif
+}
+
+/// Store a uint32_t value in little-endian byte order
+/** @param val The value to store
+ *  @param buf Where to store it (may be unaligned)
+ */
+static inline void u32_to_le(uint32_t val, uint8_t *buf) {
+#if defined(HTS_LITTLE_ENDIAN) && HTS_ALLOW_UNALIGNED != 0
+    *((uint32_u *) buf) = val;
+#else
+    buf[0] = val & 0xff;
+    buf[1] = (val >> 8) & 0xff;
+    buf[2] = (val >> 16) & 0xff;
+    buf[3] = (val >> 24) & 0xff;
+#endif
+}
+
+/// Store a uint64_t value in little-endian byte order
+/** @param val The value to store
+ *  @param buf Where to store it (may be unaligned)
+ */
+static inline void u64_to_le(uint64_t val, uint8_t *buf) {
+#if defined(HTS_LITTLE_ENDIAN) && HTS_ALLOW_UNALIGNED != 0
+    *((uint64_u *) buf) = val;
+#else
+    buf[0] = val & 0xff;
+    buf[1] = (val >> 8) & 0xff;
+    buf[2] = (val >> 16) & 0xff;
+    buf[3] = (val >> 24) & 0xff;
+    buf[4] = (val >> 32) & 0xff;
+    buf[5] = (val >> 40) & 0xff;
+    buf[6] = (val >> 48) & 0xff;
+    buf[7] = (val >> 56) & 0xff;
+#endif
+}
+
+/* Signed values.  Grab the data as unsigned, then convert to signed without
+ * triggering undefined behaviour.  On any sensible platform, the conversion
+ * should optimise away to nothing.
+ */
+
+/// Get an int8_t value from an unsigned byte array
+/** @param buf Pointer to source byte array, may be unaligned
+ *  @return A 8 bit signed integer
+ *  The input data is interpreted as 2's complement representation.
+ */
+static inline int8_t le_to_i8(const uint8_t *buf) {
+    return *buf < 0x80 ? (int8_t) *buf : -((int8_t) (0xff - *buf)) - 1;
+}
+
+/// Get an int16_t value from an unsigned byte array
+/** @param buf Pointer to source byte array, may be unaligned
+ *  @return A 16 bit signed integer
+ *  The input data is interpreted as 2's complement representation in
+ *  little-endian byte order.
+ */
+static inline int16_t le_to_i16(const uint8_t *buf) {
+    uint16_t v = le_to_u16(buf);
+    return v < 0x8000 ? (int16_t) v : -((int16_t) (0xffff - v)) - 1;
+}
+
+/// Get an int32_t value from an unsigned byte array
+/** @param buf Pointer to source byte array, may be unaligned
+ *  @return A 32 bit signed integer
+ *  The input data is interpreted as 2's complement representation in
+ *  little-endian byte order.
+ */
+static inline int32_t le_to_i32(const uint8_t *buf) {
+    uint32_t v = le_to_u32(buf);
+    return v < 0x80000000U ? (int32_t) v : -((int32_t) (0xffffffffU - v)) - 1;
+}
+
+/// Get an int64_t value from an unsigned byte array
+/** @param buf Pointer to source byte array, may be unaligned
+ *  @return A 64 bit signed integer
+ *  The input data is interpreted as 2's complement representation in
+ *  little-endian byte order.
+ */
+static inline int64_t le_to_i64(const uint8_t *buf) {
+    uint64_t v = le_to_u64(buf);
+    return (v < 0x8000000000000000ULL
+            ? (int64_t) v : -((int64_t) (0xffffffffffffffffULL - v)) - 1);
+}
+
+// Converting the other way is easier as signed -> unsigned is well defined.
+
+/// Store a uint16_t value in little-endian byte order
+/** @param val The value to store
+ *  @param buf Where to store it (may be unaligned)
+ */
+static inline void i16_to_le(int16_t val, uint8_t *buf) {
+    u16_to_le(val, buf);
+}
+
+/// Store a uint32_t value in little-endian byte order
+/** @param val The value to store
+ *  @param buf Where to store it (may be unaligned)
+ */
+static inline void i32_to_le(int32_t val, uint8_t *buf) {
+    u32_to_le(val, buf);
+}
+
+/// Store a uint64_t value in little-endian byte order
+/** @param val The value to store
+ *  @param buf Where to store it (may be unaligned)
+ */
+static inline void i64_to_le(int64_t val, uint8_t *buf) {
+    u64_to_le(val, buf);
+}
+
+/* Floating point.  Assumptions:
+ *  Platform uses IEEE 754 format
+ *  sizeof(float) == sizeof(uint32_t)
+ *  sizeof(double) == sizeof(uint64_t)
+ *  Endian-ness is the same for both floating point and integer
+ *  Type-punning via a union is allowed
+ */
+
+/// Get a float value from an unsigned byte array
+/** @param buf Pointer to source byte array, may be unaligned
+ *  @return A 32 bit floating point value
+ *  The input is interpreted as an IEEE 754 format float in little-endian
+ *  byte order.
+ */
+static inline float le_to_float(const uint8_t *buf) {
+    union {
+        uint32_t u;
+        float   f;
+    } convert;
+
+    convert.u = le_to_u32(buf);
+    return convert.f;
+}
+
+/// Get a double value from an unsigned byte array
+/** @param buf Pointer to source byte array, may be unaligned
+ *  @return A 64 bit floating point value
+ *  The input is interpreted as an IEEE 754 format double in little-endian
+ *  byte order.
+ */
+static inline double le_to_double(const uint8_t *buf) {
+    union {
+        uint64_t u;
+        double   f;
+    } convert;
+
+    convert.u = le_to_u64(buf);
+    return convert.f;
+}
+
+/// Store a float value in little-endian byte order
+/** @param val The value to store
+ *  @param buf Where to store it (may be unaligned)
+ */
+static inline void float_to_le(float val, uint8_t *buf) {
+    union {
+        uint32_t u;
+        float f;
+    } convert;
+
+    convert.f = val;
+    u32_to_le(convert.u, buf);
+}
+
+/// Store a double value in little-endian byte order
+/** @param val The value to store
+ *  @param buf Where to store it (may be unaligned)
+ */
+static inline void double_to_le(double val, uint8_t *buf) {
+    union {
+        uint64_t u;
+        double f;
+    } convert;
+
+    convert.f = val;
+    u64_to_le(convert.u, buf);
+}
+
+#endif /* HTS_ENDIAN_H */
diff --git a/ext/htslib/htslib/hts_expr.h b/ext/htslib/htslib/hts_expr.h
new file mode 100644
index 0000000..43da89d
--- /dev/null
+++ b/ext/htslib/htslib/hts_expr.h
@@ -0,0 +1,152 @@
+/*  expr.c -- filter expression parsing and processing.
+
+    Copyright (C) 2020, 2022 Genome Research Ltd.
+
+    Author: James Bonfield <jkb@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notices and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#ifndef HTS_EXPR_H
+#define HTS_EXPR_H
+
+#include <math.h>
+#include "kstring.h"
+#include "hts_defs.h"
+
+/// Holds a filter variable.  This is also used to return the results.
+/**
+ * The expression language has 3-states of string, numeric, and unknown.
+ * The unknown state is either a NaN numeric or a null string, with both
+ * internally considered to have the same "unknown" meaning.
+ *
+ * These largely match the IEE 754 semantics for NaN comparisons: <, >, ==,
+ * != all fail, (even NaN == NaN).  Similarly arithmetic (+,-,/,*,%) with
+ * unknown values are still unknown (and false).
+ *
+ * The departure from NaN semantics though is that our unknown/null state is
+ * considered to be false while NaN in C is true.  Similarly the false nature
+ * of our unknown state meants !val becomes true, !!val is once again false,
+ * val && 1 is false, val || 0 is false, and val || 1 is true along with
+ * !val || 0 and !val && 1.
+ *
+ * Note it is possible for empty strings and zero numbers to also be true.
+ * An example of this is the aux string '[NM]' which returns true if the
+ * NM tag is found, regardless of whether it is also zero.  However the
+ * better approach added in 1.16 is 'exists([NM])'.
+ */
+typedef struct hts_expr_val_t {
+    char is_str;  // Use .s vs .d
+    char is_true; // Force true if even zero
+    kstring_t s;  // is_str and empty s permitted (eval as false)
+    double d;     // otherwise this
+} hts_expr_val_t;
+
+/// Returns true if an hts_expr_val_t is defined.
+/* An example usage of this is in the SAM expression filter where an
+ * [X0] aux tag will be the value of X0 (string or numeric) if set, or
+ * a false nul-string (not the same as an empty one) when not set.
+ */
+static inline int hts_expr_val_exists(hts_expr_val_t *v) {
+    return v && !(v->is_str == 1 && v->s.s == NULL)
+             && !(v->is_str == 0 && isnan(v->d));
+}
+
+/// Returns true if an hts_expr_val_t is defined or is undef-but-true
+static inline int hts_expr_val_existsT(hts_expr_val_t *v) {
+    return (v && v->is_true) || hts_expr_val_exists(v);
+}
+
+/// Set a value to be undefined (nan).
+static inline void hts_expr_val_undef(hts_expr_val_t *v) {
+    ks_clear(&v->s);
+    v->is_true = 0;
+    v->is_str = 0;
+    v->d = NAN;
+}
+
+/// Frees a hts_expr_val_t type.
+static inline void hts_expr_val_free(hts_expr_val_t *f) {
+    ks_free(&f->s);
+}
+
+/// Opaque hts_filter_t type.  Definition in hts_expr.c
+typedef struct hts_filter_t hts_filter_t;
+
+/// For static initialisation of hts_expr_val_t values
+#define HTS_EXPR_VAL_INIT {0, 0, KS_INITIALIZE, 0}
+
+/// Creates a filter for expression "str".
+/** @param str    The filter expression
+ *  @return       A pointer on success, NULL on failure
+ */
+HTSLIB_EXPORT
+hts_filter_t *hts_filter_init(const char *str);
+
+/// Frees an hts_filter_t created via hts_filter_init
+/** @param filt    The filter pointer.
+ */
+HTSLIB_EXPORT
+void hts_filter_free(hts_filter_t *filt);
+
+/// Type for expression symbol lookups; name -> value.
+typedef int (hts_expr_sym_func)(void *data, char *str, char **end,
+                                hts_expr_val_t *res);
+
+/// Evaluates a filter expression and returns the value
+/** @param filt      The filter, produced by hts_filter_init
+ *  @param data      Arbitrary caller data, passed into sym_func
+ *  @param sym_func  Callback function to lookup variables.
+ *  @param res       Filled out with the result of the filter evaluation
+ *  @return          Returns 0 on success, -1 on failure
+ *
+ *  sym_func and data may be NULL if the caller does not need its own data
+ *  pointer or if it has no variables to lookup.
+ *
+ *  The type of the returned result may be numeric of string, as defined by
+ *  the is_str member.  It can also be explicitly defined to be true even
+ *  for a null value.  This may be used to check for the existence of
+ *  something, irrespective of whether that something evaluates to zero.
+ *
+ *  @p res must be initialized using HTS_EXPR_VAL_INIT before passing it
+ *  to this function for the first time.
+ */
+HTSLIB_EXPORT
+int hts_filter_eval2(hts_filter_t *filt,
+                     void *data, hts_expr_sym_func *sym_func,
+                     hts_expr_val_t *res);
+
+/// Evaluate a filter expression (derecated API)
+/**
+ *  @copydetails hts_filter_eval2()
+ *
+ *  If calling this function more than once with the same @p res
+ *  parameter, hts_expr_val_free(res) must be used between invocations
+ *  to clear any allocated memory prior to reuse.
+ *
+ *  @deprecated This function has been replaced by hts_filter_eval2(),
+ *              which clears @p res properly itself.
+ */
+HTSLIB_EXPORT
+int hts_filter_eval(hts_filter_t *filt,
+                    void *data, hts_expr_sym_func *sym_func,
+                    hts_expr_val_t *res)
+    HTS_DEPRECATED("Please use hts_filter_eval2 instead");
+
+
+#endif /* HTS_EXPR_H */
diff --git a/ext/htslib/htslib/hts_log.h b/ext/htslib/htslib/hts_log.h
new file mode 100644
index 0000000..f6a50b3
--- /dev/null
+++ b/ext/htslib/htslib/hts_log.h
@@ -0,0 +1,97 @@
+/// \file htslib/hts_log.h
+/// Configuration of log levels.
+/* The MIT License
+Copyright (C) 2017 Genome Research Ltd.
+
+Author: Anders Kaplan
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+#ifndef HTS_LOG_H
+#define HTS_LOG_H
+
+#include "hts_defs.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/// Log levels.
+enum htsLogLevel {
+    HTS_LOG_OFF,            ///< All logging disabled.
+    HTS_LOG_ERROR,          ///< Logging of errors only.
+    HTS_LOG_WARNING = 3,    ///< Logging of errors and warnings.
+    HTS_LOG_INFO,           ///< Logging of errors, warnings, and normal but significant events.
+    HTS_LOG_DEBUG,          ///< Logging of all except the most detailed debug events.
+    HTS_LOG_TRACE           ///< All logging enabled.
+};
+
+/// Sets the selected log level.
+HTSLIB_EXPORT
+void hts_set_log_level(enum htsLogLevel level);
+
+/// Gets the selected log level.
+HTSLIB_EXPORT
+enum htsLogLevel hts_get_log_level(void);
+
+/// Selected log level.
+/*!
+ * One of the HTS_LOG_* values. The default is HTS_LOG_WARNING.
+ * \note Avoid direct use of this variable. Use hts_set_log_level and hts_get_log_level instead.
+ */
+HTSLIB_EXPORT
+extern int hts_verbose;
+
+/*! Logs an event.
+* \param severity      Severity of the event:
+*                      - HTS_LOG_ERROR means that something went wrong so that a task could not be completed.
+*                      - HTS_LOG_WARNING means that something unexpected happened, but that execution can continue, perhaps in a degraded mode.
+*                      - HTS_LOG_INFO means that something normal but significant happened.
+*                      - HTS_LOG_DEBUG means that something normal and insignificant happened.
+*                      - HTS_LOG_TRACE means that something happened that might be of interest when troubleshooting.
+* \param context       Context where the event occurred. Typically set to "__func__".
+* \param format        Format string with placeholders, like printf.
+*/
+HTSLIB_EXPORT
+void hts_log(enum htsLogLevel severity, const char *context, const char *format, ...)
+HTS_FORMAT(HTS_PRINTF_FMT, 3, 4);
+
+/*! Logs an event with severity HTS_LOG_ERROR and default context. Parameters: format, ... */
+#define hts_log_error(...) hts_log(HTS_LOG_ERROR, __func__, __VA_ARGS__)
+
+/*! Logs an event with severity HTS_LOG_WARNING and default context. Parameters: format, ... */
+#define hts_log_warning(...) hts_log(HTS_LOG_WARNING, __func__, __VA_ARGS__)
+
+/*! Logs an event with severity HTS_LOG_INFO and default context. Parameters: format, ... */
+#define hts_log_info(...) hts_log(HTS_LOG_INFO, __func__, __VA_ARGS__)
+
+/*! Logs an event with severity HTS_LOG_DEBUG and default context. Parameters: format, ... */
+#define hts_log_debug(...) hts_log(HTS_LOG_DEBUG, __func__, __VA_ARGS__)
+
+/*! Logs an event with severity HTS_LOG_TRACE and default context. Parameters: format, ... */
+#define hts_log_trace(...) hts_log(HTS_LOG_TRACE, __func__, __VA_ARGS__)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // #ifndef HTS_LOG_H
diff --git a/ext/htslib/htslib/hts_os.h b/ext/htslib/htslib/hts_os.h
new file mode 100644
index 0000000..c715b06
--- /dev/null
+++ b/ext/htslib/htslib/hts_os.h
@@ -0,0 +1,86 @@
+/// @file hts_os.h
+/// Operating System specific tweaks, for compatibility with POSIX.
+/*
+   Copyright (C) 2017, 2019-2020 Genome Research Ltd.
+
+    Author: James Bonfield <jkb@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#ifndef HTSLIB_HTS_OS_H
+#define HTSLIB_HTS_OS_H
+
+#include "hts_defs.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* This is srand48_deterministic() on platforms that provide it, or srand48()
+   otherwise (or our own POSIX srand48() on platforms that provide neither).
+   Hence calling hts_srand48() will always set up the same POSIX-determined
+   sequence of pseudo-random numbers on any platform, while calling srand48()
+   may (e.g., on OpenBSD) set up a different non-deterministic sequence. */
+HTSLIB_EXPORT
+void hts_srand48(long seed);
+
+HTSLIB_EXPORT
+double hts_erand48(unsigned short xseed[3]);
+
+HTSLIB_EXPORT
+double hts_drand48(void);
+
+HTSLIB_EXPORT
+long hts_lrand48(void);
+
+#if defined(_WIN32) && !defined(__CYGWIN__)
+// Windows usually lacks *rand48(), but cygwin provides them.
+#define srand48(S) hts_srand48((S))
+#define erand48(X) hts_erand48((X))
+#define drand48() hts_drand48()
+#define lrand48() hts_lrand48()
+#endif
+
+#if 0  /* def _WIN32 - disabled for now, not currently used */
+/* Check if the fd is a cygwin/msys's pty. */
+extern int is_cygpty(int fd);
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#if defined(__MINGW32__)
+#include <io.h>
+#define mkdir(filename,mode) mkdir((filename))
+#endif
+
+#ifdef _WIN32
+#include <stdlib.h>
+#define srandom srand
+#define random rand
+#endif
+
+/* MSVC does not provide ssize_t in its <sys/types.h>. This ensures the type
+   is available (unless suppressed by defining HTS_NO_SSIZE_T first). */
+#if defined _MSC_VER && defined _INTPTR_T_DEFINED && !defined _SSIZE_T_DEFINED && !defined HTS_NO_SSIZE_T && !defined ssize_t
+#define ssize_t intptr_t
+#endif
+
+#endif // HTSLIB_HTS_OS_H
diff --git a/ext/htslib/htslib/kbitset.h b/ext/htslib/htslib/kbitset.h
new file mode 100644
index 0000000..0a52958
--- /dev/null
+++ b/ext/htslib/htslib/kbitset.h
@@ -0,0 +1,203 @@
+/* The MIT License
+
+   Copyright (C) 2015, 2018 Genome Research Ltd.
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be
+   included in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+*/
+
+#ifndef KBITSET_H
+#define KBITSET_H
+
+/* Example of using kbitset_t, which represents a subset of {0,..., N-1},
+   where N is the size specified in kbs_init().
+
+	kbitset_t *bset = kbs_init(100);
+	kbs_insert(bset, 5);
+	kbs_insert(bset, 68);
+	kbs_delete(bset, 37);
+	// ...
+
+	if (kbs_exists(bset, 68)) printf("68 present\n");
+
+	kbitset_iter_t itr;
+	int i;
+	kbs_start(&itr);
+	while ((i = kbs_next(bset, &itr)) >= 0)
+		printf("%d present\n", i);
+
+	kbs_destroy(bset);
+
+   Example of declaring a kbitset_t-using function in a header file, so that
+   only source files that actually use process() need to include <kbitset.h>:
+
+	struct kbitset_t;
+	void process(struct kbitset_t *bset);
+*/
+
+#include <limits.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define KBS_ELTBITS (CHAR_BIT * sizeof (unsigned long))
+#define KBS_ELT(i)  ((i) / KBS_ELTBITS)
+#define KBS_MASK(i) (1UL << ((i) % KBS_ELTBITS))
+
+typedef struct kbitset_t {
+	size_t n, n_max;
+	unsigned long b[1];
+} kbitset_t;
+
+// (For internal use only.) Returns a mask (like 00011111) showing
+// which bits are in use in the last slot (for the given ni) set.
+static inline unsigned long kbs_last_mask(size_t ni)
+{
+	unsigned long mask = KBS_MASK(ni) - 1;
+	return mask? mask : ~0UL;
+}
+
+// Initialise a bit set capable of holding ni integers, 0 <= i < ni.
+// The set returned is empty if fill == 0, or all of [0,ni) otherwise.
+static inline kbitset_t *kbs_init2(size_t ni, int fill)
+{
+	size_t n = (ni + KBS_ELTBITS-1) / KBS_ELTBITS;
+	kbitset_t *bs =
+		(kbitset_t *) malloc(sizeof(kbitset_t) + n * sizeof(unsigned long));
+	if (bs == NULL) return NULL;
+	bs->n = bs->n_max = n;
+	memset(bs->b, fill? ~0 : 0, n * sizeof (unsigned long));
+	// b[n] is always non-zero (a fact used by kbs_next()).
+	bs->b[n] = kbs_last_mask(ni);
+	if (fill) bs->b[n-1] &= bs->b[n];
+	return bs;
+}
+
+// Initialise an empty bit set capable of holding ni integers, 0 <= i < ni.
+static inline kbitset_t *kbs_init(size_t ni)
+{
+	return kbs_init2(ni, 0);
+}
+
+// Resize an existing bit set to be capable of holding ni_new integers.
+// Elements in [ni_old,ni_new) are added to the set if fill != 0.
+static inline int kbs_resize2(kbitset_t **bsp, size_t ni_new, int fill)
+{
+	kbitset_t *bs = *bsp;
+	size_t n = bs? bs->n : 0;
+	size_t n_new = (ni_new + KBS_ELTBITS-1) / KBS_ELTBITS;
+	if (bs == NULL || n_new > bs->n_max) {
+		bs = (kbitset_t *)
+			realloc(*bsp, sizeof(kbitset_t) + n_new * sizeof(unsigned long));
+		if (bs == NULL) return -1;
+
+		bs->n_max = n_new;
+		*bsp = bs;
+	}
+
+	bs->n = n_new;
+	if (n_new >= n)
+		memset(&bs->b[n], fill? ~0 : 0, (n_new - n) * sizeof (unsigned long));
+	bs->b[n_new] = kbs_last_mask(ni_new);
+	// Need to clear excess bits when fill!=0 or n_new<n; always is simpler.
+	bs->b[n_new-1] &= bs->b[n_new];
+	return 0;
+}
+
+// Resize an existing bit set to be capable of holding ni_new integers.
+// Returns negative on error.
+static inline int kbs_resize(kbitset_t **bsp, size_t ni_new)
+{
+	return kbs_resize2(bsp, ni_new, 0);
+}
+
+// Destroy a bit set.
+static inline void kbs_destroy(kbitset_t *bs)
+{
+	free(bs);
+}
+
+// Reset the bit set to empty.
+static inline void kbs_clear(kbitset_t *bs)
+{
+	memset(bs->b, 0, bs->n * sizeof (unsigned long));
+}
+
+// Reset the bit set to all of [0,ni).
+static inline void kbs_insert_all(kbitset_t *bs)
+{
+	memset(bs->b, ~0, bs->n * sizeof (unsigned long));
+	bs->b[bs->n-1] &= bs->b[bs->n];
+}
+
+// Insert an element into the bit set.
+static inline void kbs_insert(kbitset_t *bs, int i)
+{
+	bs->b[KBS_ELT(i)] |= KBS_MASK(i);
+}
+
+// Remove an element from the bit set.
+static inline void kbs_delete(kbitset_t *bs, int i)
+{
+	bs->b[KBS_ELT(i)] &= ~KBS_MASK(i);
+}
+
+// Test whether the bit set contains the element.
+static inline int kbs_exists(const kbitset_t *bs, int i)
+{
+	return (bs->b[KBS_ELT(i)] & KBS_MASK(i)) != 0;
+}
+
+typedef struct kbitset_iter_t {
+	unsigned long mask;
+	size_t elt;
+	int i;
+} kbitset_iter_t;
+
+// Initialise or reset a bit set iterator.
+static inline void kbs_start(kbitset_iter_t *itr)
+{
+	itr->mask = 1;
+	itr->elt = 0;
+	itr->i = 0;
+}
+
+// Return the next element contained in the bit set, or -1 if there are no more.
+static inline int kbs_next(const kbitset_t *bs, kbitset_iter_t *itr)
+{
+	unsigned long b = bs->b[itr->elt];
+
+	for (;;) {
+		if (itr->mask == 0) {
+			while ((b = bs->b[++itr->elt]) == 0) itr->i += KBS_ELTBITS;
+			if (itr->elt == bs->n) return -1;
+			itr->mask = 1;
+		}
+
+		if (b & itr->mask) break;
+
+		itr->i++;
+		itr->mask <<= 1;
+	}
+
+	itr->mask <<= 1;
+	return itr->i++;
+}
+
+#endif
diff --git a/ext/htslib/htslib/kfunc.h b/ext/htslib/htslib/kfunc.h
new file mode 100644
index 0000000..34704b1
--- /dev/null
+++ b/ext/htslib/htslib/kfunc.h
@@ -0,0 +1,91 @@
+/* The MIT License
+
+   Copyright (C) 2010, 2013-2014 Genome Research Ltd.
+   Copyright (C) 2011 Attractive Chaos <attractor@live.co.uk>
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be
+   included in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+*/
+
+#ifndef HTSLIB_KFUNC_H
+#define HTSLIB_KFUNC_H
+
+#include "hts_defs.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Log gamma function
+ * \log{\Gamma(z)}
+ * AS245, 2nd algorithm, http://lib.stat.cmu.edu/apstat/245
+ */
+HTSLIB_EXPORT
+double kf_lgamma(double z);
+
+/* complementary error function
+ * \frac{2}{\sqrt{\pi}} \int_x^{\infty} e^{-t^2} dt
+ * AS66, 2nd algorithm, http://lib.stat.cmu.edu/apstat/66
+ */
+HTSLIB_EXPORT
+double kf_erfc(double x);
+
+/* The following computes regularized incomplete gamma functions.
+ * Formulas are taken from Wiki, with additional input from Numerical
+ * Recipes in C (for modified Lentz's algorithm) and AS245
+ * (http://lib.stat.cmu.edu/apstat/245).
+ *
+ * A good online calculator is available at:
+ *
+ *   http://www.danielsoper.com/statcalc/calc23.aspx
+ *
+ * It calculates upper incomplete gamma function, which equals
+ * kf_gammaq(s,z)*tgamma(s).
+ */
+
+HTSLIB_EXPORT
+double kf_gammap(double s, double z);
+HTSLIB_EXPORT
+double kf_gammaq(double s, double z);
+
+/* Regularized incomplete beta function. The method is taken from
+ * Numerical Recipe in C, 2nd edition, section 6.4. The following web
+ * page calculates the incomplete beta function, which equals
+ * kf_betai(a,b,x) * gamma(a) * gamma(b) / gamma(a+b):
+ *
+ *   http://www.danielsoper.com/statcalc/calc36.aspx
+ */
+HTSLIB_EXPORT
+double kf_betai(double a, double b, double x);
+
+/*
+ *    n11  n12  | n1_
+ *    n21  n22  | n2_
+ *   -----------+----
+ *    n_1  n_2  | n
+ */
+HTSLIB_EXPORT
+double kt_fisher_exact(int n11, int n12, int n21, int n22, double *_left, double *_right, double *two);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/ext/htslib/htslib/khash.h b/ext/htslib/htslib/khash.h
new file mode 100644
index 0000000..02e4917
--- /dev/null
+++ b/ext/htslib/htslib/khash.h
@@ -0,0 +1,749 @@
+/* The MIT License
+
+   Copyright (c) 2008, 2009, 2011 by Attractive Chaos <attractor@live.co.uk>
+   Copyright (C) 2014-2015, 2018, 2024 Genome Research Ltd.
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be
+   included in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+*/
+
+/*
+  An example:
+
+#include "khash.h"
+KHASH_MAP_INIT_INT(32, char)
+int main() {
+	int ret, is_missing;
+	khiter_t k;
+	khash_t(32) *h = kh_init(32);
+	k = kh_put(32, h, 5, &ret);
+	kh_value(h, k) = 10;
+	k = kh_get(32, h, 10);
+	is_missing = (k == kh_end(h));
+	k = kh_get(32, h, 5);
+	kh_del(32, h, k);
+	for (k = kh_begin(h); k != kh_end(h); ++k)
+		if (kh_exist(h, k)) kh_value(h, k) = 1;
+	kh_destroy(32, h);
+	return 0;
+}
+*/
+
+/*
+  2013-05-02 (0.2.8):
+
+	* Use quadratic probing. When the capacity is power of 2, stepping function
+	  i*(i+1)/2 guarantees to traverse each bucket. It is better than double
+	  hashing on cache performance and is more robust than linear probing.
+
+	  In theory, double hashing should be more robust than quadratic probing.
+	  However, my implementation is probably not for large hash tables, because
+	  the second hash function is closely tied to the first hash function,
+	  which reduce the effectiveness of double hashing.
+
+	Reference: http://research.cs.vt.edu/AVresearch/hashing/quadratic.php
+
+  2011-12-29 (0.2.7):
+
+    * Minor code clean up; no actual effect.
+
+  2011-09-16 (0.2.6):
+
+	* The capacity is a power of 2. This seems to dramatically improve the
+	  speed for simple keys. Thank Zilong Tan for the suggestion. Reference:
+
+	   - http://code.google.com/p/ulib/
+	   - http://nothings.org/computer/judy/
+
+	* Allow to optionally use linear probing which usually has better
+	  performance for random input. Double hashing is still the default as it
+	  is more robust to certain non-random input.
+
+	* Added Wang's integer hash function (not used by default). This hash
+	  function is more robust to certain non-random input.
+
+  2011-02-14 (0.2.5):
+
+    * Allow to declare global functions.
+
+  2009-09-26 (0.2.4):
+
+    * Improve portability
+
+  2008-09-19 (0.2.3):
+
+	* Corrected the example
+	* Improved interfaces
+
+  2008-09-11 (0.2.2):
+
+	* Improved speed a little in kh_put()
+
+  2008-09-10 (0.2.1):
+
+	* Added kh_clear()
+	* Fixed a compiling error
+
+  2008-09-02 (0.2.0):
+
+	* Changed to token concatenation which increases flexibility.
+
+  2008-08-31 (0.1.2):
+
+	* Fixed a bug in kh_get(), which has not been tested previously.
+
+  2008-08-31 (0.1.1):
+
+	* Added destructor
+*/
+
+
+#ifndef __AC_KHASH_H
+#define __AC_KHASH_H
+
+/*!
+  @header
+
+  Generic hash table library.
+ */
+
+#define AC_VERSION_KHASH_H "0.2.8"
+
+#include <stdlib.h>
+#include <string.h>
+#include <limits.h>
+
+#include "kstring.h"
+#include "kroundup.h"
+
+/* compiler specific configuration */
+
+#if UINT_MAX == 0xffffffffu
+typedef unsigned int khint32_t;
+#elif ULONG_MAX == 0xffffffffu
+typedef unsigned long khint32_t;
+#endif
+
+#if ULONG_MAX == ULLONG_MAX
+typedef unsigned long khint64_t;
+#else
+typedef unsigned long long khint64_t;
+#endif
+
+#ifndef kh_inline
+#ifdef _MSC_VER
+#define kh_inline __inline
+#else
+#define kh_inline inline
+#endif
+#endif /* kh_inline */
+
+#ifndef klib_unused
+#if (defined __clang__ && __clang_major__ >= 3) || (defined __GNUC__ && __GNUC__ >= 3)
+#define klib_unused __attribute__ ((__unused__))
+#else
+#define klib_unused
+#endif
+#endif /* klib_unused */
+
+typedef khint32_t khint_t;
+typedef khint_t khiter_t;
+
+#define __ac_isempty(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&2)
+#define __ac_isdel(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&1)
+#define __ac_iseither(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&3)
+#define __ac_set_isdel_false(flag, i) (flag[i>>4]&=~(1ul<<((i&0xfU)<<1)))
+#define __ac_set_isempty_false(flag, i) (flag[i>>4]&=~(2ul<<((i&0xfU)<<1)))
+#define __ac_set_isboth_false(flag, i) (flag[i>>4]&=~(3ul<<((i&0xfU)<<1)))
+#define __ac_set_isdel_true(flag, i) (flag[i>>4]|=1ul<<((i&0xfU)<<1))
+
+#define __ac_fsize(m) ((m) < 16? 1 : (m)>>4)
+
+#ifndef kroundup32
+#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
+#endif
+
+#ifndef kcalloc
+#define kcalloc(N,Z) calloc(N,Z)
+#endif
+#ifndef kmalloc
+#define kmalloc(Z) malloc(Z)
+#endif
+#ifndef krealloc
+#define krealloc(P,Z) realloc(P,Z)
+#endif
+#ifndef kfree
+#define kfree(P) free(P)
+#endif
+
+static const double __ac_HASH_UPPER = 0.77;
+
+#define __KHASH_TYPE(name, khkey_t, khval_t) \
+	typedef struct kh_##name##_s { \
+		khint_t n_buckets, size, n_occupied, upper_bound; \
+		khint32_t *flags; \
+		khkey_t *keys; \
+		khval_t *vals; \
+	} kh_##name##_t;
+
+#define __KHASH_PROTOTYPES(name, khkey_t, khval_t)	 					\
+	extern kh_##name##_t *kh_init_##name(void);							\
+	extern void kh_destroy_##name(kh_##name##_t *h);					\
+	extern void kh_clear_##name(kh_##name##_t *h);						\
+	extern khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key); 	\
+	extern int kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets); \
+	extern khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret); \
+	extern void kh_del_##name(kh_##name##_t *h, khint_t x);
+
+#define __KHASH_IMPL(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \
+	SCOPE kh_##name##_t *kh_init_##name(void) {							\
+		return (kh_##name##_t*)kcalloc(1, sizeof(kh_##name##_t));		\
+	}																	\
+	SCOPE void kh_destroy_##name(kh_##name##_t *h)						\
+	{																	\
+		if (h) {														\
+			kfree((void *)h->keys); kfree(h->flags);					\
+			kfree((void *)h->vals);										\
+			kfree(h);													\
+		}																\
+	}																	\
+	SCOPE void kh_clear_##name(kh_##name##_t *h)						\
+	{																	\
+		if (h && h->flags) {											\
+			memset(h->flags, 0xaa, __ac_fsize(h->n_buckets) * sizeof(khint32_t)); \
+			h->size = h->n_occupied = 0;								\
+		}																\
+	}																	\
+	SCOPE khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key) 	\
+	{																	\
+		if (h->n_buckets) {												\
+			khint_t k, i, last, mask, step = 0; \
+			mask = h->n_buckets - 1;									\
+			k = __hash_func(key); i = k & mask;							\
+			last = i; \
+			while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \
+				i = (i + (++step)) & mask; \
+				if (i == last) return h->n_buckets;						\
+			}															\
+			return __ac_iseither(h->flags, i)? h->n_buckets : i;		\
+		} else return 0;												\
+	}																	\
+	SCOPE int kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets) \
+	{ /* This function uses 0.25*n_buckets bytes of working space instead of [sizeof(key_t+val_t)+.25]*n_buckets. */ \
+		khint32_t *new_flags = 0;										\
+		khint_t j = 1;													\
+		{																\
+			kroundup32(new_n_buckets); 									\
+			if (new_n_buckets < 4) new_n_buckets = 4;					\
+			if (h->size >= (khint_t)(new_n_buckets * __ac_HASH_UPPER + 0.5)) j = 0;	/* requested size is too small */ \
+			else { /* hash table size to be changed (shrink or expand); rehash */ \
+				new_flags = (khint32_t*)kmalloc(__ac_fsize(new_n_buckets) * sizeof(khint32_t));	\
+				if (!new_flags) return -1;								\
+				memset(new_flags, 0xaa, __ac_fsize(new_n_buckets) * sizeof(khint32_t)); \
+				if (h->n_buckets < new_n_buckets) {	/* expand */		\
+					khkey_t *new_keys = (khkey_t*)krealloc((void *)h->keys, new_n_buckets * sizeof(khkey_t)); \
+					if (!new_keys) { kfree(new_flags); return -1; }		\
+					h->keys = new_keys;									\
+					if (kh_is_map) {									\
+						khval_t *new_vals = (khval_t*)krealloc((void *)h->vals, new_n_buckets * sizeof(khval_t)); \
+						if (!new_vals) { kfree(new_flags); return -1; }	\
+						h->vals = new_vals;								\
+					}													\
+				} /* otherwise shrink */								\
+			}															\
+		}																\
+		if (j) { /* rehashing is needed */								\
+			for (j = 0; j != h->n_buckets; ++j) {						\
+				if (__ac_iseither(h->flags, j) == 0) {					\
+					khkey_t key = h->keys[j];							\
+					khval_t val;										\
+					khint_t new_mask;									\
+					new_mask = new_n_buckets - 1; 						\
+					if (kh_is_map) val = h->vals[j];					\
+					__ac_set_isdel_true(h->flags, j);					\
+					while (1) { /* kick-out process; sort of like in Cuckoo hashing */ \
+						khint_t k, i, step = 0; \
+						k = __hash_func(key);							\
+						i = k & new_mask;								\
+						while (!__ac_isempty(new_flags, i)) i = (i + (++step)) & new_mask; \
+						__ac_set_isempty_false(new_flags, i);			\
+						if (i < h->n_buckets && __ac_iseither(h->flags, i) == 0) { /* kick out the existing element */ \
+							{ khkey_t tmp = h->keys[i]; h->keys[i] = key; key = tmp; } \
+							if (kh_is_map) { khval_t tmp = h->vals[i]; h->vals[i] = val; val = tmp; } \
+							__ac_set_isdel_true(h->flags, i); /* mark it as deleted in the old hash table */ \
+						} else { /* write the element and jump out of the loop */ \
+							h->keys[i] = key;							\
+							if (kh_is_map) h->vals[i] = val;			\
+							break;										\
+						}												\
+					}													\
+				}														\
+			}															\
+			if (h->n_buckets > new_n_buckets) { /* shrink the hash table */ \
+				h->keys = (khkey_t*)krealloc((void *)h->keys, new_n_buckets * sizeof(khkey_t)); \
+				if (kh_is_map) h->vals = (khval_t*)krealloc((void *)h->vals, new_n_buckets * sizeof(khval_t)); \
+			}															\
+			kfree(h->flags); /* free the working space */				\
+			h->flags = new_flags;										\
+			h->n_buckets = new_n_buckets;								\
+			h->n_occupied = h->size;									\
+			h->upper_bound = (khint_t)(h->n_buckets * __ac_HASH_UPPER + 0.5); \
+		}																\
+		return 0;														\
+	}																	\
+	SCOPE khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret) \
+	{																	\
+		khint_t x;														\
+		if (h->n_occupied >= h->upper_bound) { /* update the hash table */ \
+			if (h->n_buckets > (h->size<<1)) {							\
+				if (kh_resize_##name(h, h->n_buckets - 1) < 0) { /* clear "deleted" elements */ \
+					*ret = -1; return h->n_buckets;						\
+				}														\
+			} else if (kh_resize_##name(h, h->n_buckets + 1) < 0) { /* expand the hash table */ \
+				*ret = -1; return h->n_buckets;							\
+			}															\
+		} /* TODO: to implement automatically shrinking; resize() already support shrinking */ \
+		{																\
+			khint_t k, i, site, last, mask = h->n_buckets - 1, step = 0; \
+			x = site = h->n_buckets; k = __hash_func(key); i = k & mask; \
+			if (__ac_isempty(h->flags, i)) x = i; /* for speed up */	\
+			else {														\
+				last = i; \
+				while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \
+					if (__ac_isdel(h->flags, i)) site = i;				\
+					i = (i + (++step)) & mask; \
+					if (i == last) { x = site; break; }					\
+				}														\
+				if (x == h->n_buckets) {								\
+					if (__ac_isempty(h->flags, i) && site != h->n_buckets) x = site; \
+					else x = i;											\
+				}														\
+			}															\
+		}																\
+		if (__ac_isempty(h->flags, x)) { /* not present at all */		\
+			h->keys[x] = key;											\
+			__ac_set_isboth_false(h->flags, x);							\
+			++h->size; ++h->n_occupied;									\
+			*ret = 1;													\
+		} else if (__ac_isdel(h->flags, x)) { /* deleted */				\
+			h->keys[x] = key;											\
+			__ac_set_isboth_false(h->flags, x);							\
+			++h->size;													\
+			*ret = 2;													\
+		} else *ret = 0; /* Don't touch h->keys[x] if present and not deleted */ \
+		return x;														\
+	}																	\
+	SCOPE void kh_del_##name(kh_##name##_t *h, khint_t x)				\
+	{																	\
+		if (x != h->n_buckets && !__ac_iseither(h->flags, x)) {			\
+			__ac_set_isdel_true(h->flags, x);							\
+			--h->size;													\
+		}																\
+	}                                                                   \
+    SCOPE int kh_stats_##name(kh_##name##_t *h, khint_t *empty,         \
+                              khint_t *deleted, khint_t *hist_size,     \
+                              khint_t **hist_out)                       \
+    {                                                                   \
+        khint_t i, *hist = NULL, dist_max = 0, k, dist, step;           \
+        khint_t mask = h->n_buckets - 1;                                \
+        *empty = *deleted = *hist_size = 0;                             \
+        hist = (khint_t *) calloc(1, sizeof(*hist));                    \
+        if (!hist) { return -1; }                                       \
+        for (i = kh_begin(h); i < kh_end(h); ++i) {                     \
+            if (__ac_isempty(h->flags, i)) { (*empty)++; continue; }      \
+            if (__ac_isdel(h->flags, i)) { (*deleted)++; continue; }      \
+            k = __hash_func(h->keys[i]) & (h->n_buckets - 1);           \
+            dist = 0;                                                   \
+            step = 0;                                                   \
+            while (k != i) {                                            \
+                dist++;                                                 \
+                k = (k + (++step)) & mask;                              \
+            }                                                           \
+            if (dist_max <= dist) {                                     \
+                khint_t *new_hist = (khint_t *) realloc(hist, sizeof(*new_hist) * (dist + 1)); \
+                if (!new_hist) { free(hist); return -1; }               \
+                for (k = dist_max + 1; k <= dist; k++) new_hist[k] = 0; \
+                hist = new_hist;                                        \
+                dist_max = dist;                                        \
+            }                                                           \
+            hist[dist]++;                                               \
+        }                                                               \
+        *hist_out = hist;                                               \
+        *hist_size = dist_max + 1;                                      \
+        return 0;                                                       \
+    }
+
+#define KHASH_DECLARE(name, khkey_t, khval_t)		 					\
+	__KHASH_TYPE(name, khkey_t, khval_t) 								\
+	__KHASH_PROTOTYPES(name, khkey_t, khval_t)
+
+#define KHASH_INIT2(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \
+	__KHASH_TYPE(name, khkey_t, khval_t) 								\
+	__KHASH_IMPL(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal)
+
+#define KHASH_INIT(name, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \
+	KHASH_INIT2(name, static kh_inline klib_unused, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal)
+
+/* --- BEGIN OF HASH FUNCTIONS --- */
+
+/*! @function
+  @abstract     Integer hash function
+  @param  key   The integer [khint32_t]
+  @return       The hash value [khint_t]
+ */
+#define kh_int_hash_func(key) (khint32_t)(key)
+/*! @function
+  @abstract     Integer comparison function
+ */
+#define kh_int_hash_equal(a, b) ((a) == (b))
+/*! @function
+  @abstract     64-bit integer hash function
+  @param  key   The integer [khint64_t]
+  @return       The hash value [khint_t]
+ */
+#define kh_int64_hash_func(key) (khint32_t)((key)>>33^(key)^(key)<<11)
+/*! @function
+  @abstract     64-bit integer comparison function
+ */
+#define kh_int64_hash_equal(a, b) ((a) == (b))
+
+/*! @function
+  @abstract     const char* hash function
+  @param  s     Pointer to a null terminated string
+  @return       The hash value
+ */
+static kh_inline khint_t __ac_X31_hash_string(const char *s)
+{
+	khint_t h = (khint_t)*s;
+	if (h) for (++s ; *s; ++s) h = (h << 5) - h + (khint_t)*s;
+	return h;
+}
+
+/*! @function
+  @abstract     const char* FNV1a hash function
+  @param  s     Pointer to a null terminated string
+  @return       The hash value
+ */
+static kh_inline khint_t __ac_FNV1a_hash_string(const char *s)
+{
+	const khint_t offset_basis = 2166136261;
+	const khint_t FNV_prime = 16777619;
+	khint_t h = offset_basis;
+	for (; *s; ++s) h = (h ^ (uint8_t) *s) * FNV_prime;
+	return h;
+}
+
+/*! @function
+  @abstract     Another interface to const char* hash function
+  @param  key   Pointer to a nul terminated string [const char*]
+  @return       The hash value [khint_t]
+ */
+#define kh_str_hash_func(key) __ac_FNV1a_hash_string(key)
+
+/*! @function
+  @abstract     Const char* comparison function
+ */
+#define kh_str_hash_equal(a, b) (strcmp(a, b) == 0)
+
+/*! @function
+  @abstract     Kstring hash function
+  @param  s     Pointer to a kstring
+  @return       The hash value
+ */
+static kh_inline khint_t __ac_X31_hash_kstring(const kstring_t ks)
+{
+	khint_t h = 0;
+	size_t i;
+	for (i = 0; i < ks.l; i++)
+		h = (h << 5) - h + (khint_t)ks.s[i];
+	return h;
+}
+
+/*! @function
+  @abstract     Kstring hash function
+  @param  s     Pointer to a kstring
+  @return       The hash value
+ */
+static kh_inline khint_t __ac_FNV1a_hash_kstring(const kstring_t ks)
+{
+	const khint_t offset_basis = 2166136261;
+	const khint_t FNV_prime = 16777619;
+	khint_t h = offset_basis;
+	size_t i;
+	for (i = 0; i < ks.l; i++)
+		h = (h ^ (uint8_t) ks.s[i]) * FNV_prime;
+	return h;
+}
+
+/*! @function
+  @abstract     Interface to kstring hash function.
+  @param  key   Pointer to a khash; permits hashing on non-nul terminated strings.
+  @return       The hash value [khint_t]
+ */
+#define kh_kstr_hash_func(key) __ac_FNV1a_hash_kstring(key)
+/*! @function
+  @abstract     kstring comparison function
+ */
+#define kh_kstr_hash_equal(a, b) ((a).l == (b).l && strncmp((a).s, (b).s, (a).l) == 0)
+
+static kh_inline khint_t __ac_Wang_hash(khint_t key)
+{
+    key += ~(key << 15);
+    key ^=  (key >> 10);
+    key +=  (key << 3);
+    key ^=  (key >> 6);
+    key += ~(key << 11);
+    key ^=  (key >> 16);
+    return key;
+}
+#define kh_int_hash_func2(key) __ac_Wang_hash((khint_t)(key))
+
+/* --- END OF HASH FUNCTIONS --- */
+
+/* Other convenient macros... */
+
+/*!
+  @abstract Type of the hash table.
+  @param  name  Name of the hash table [symbol]
+ */
+#define khash_t(name) kh_##name##_t
+
+/*! @function
+  @abstract     Initiate a hash table.
+  @param  name  Name of the hash table [symbol]
+  @return       Pointer to the hash table [khash_t(name)*]
+ */
+#define kh_init(name) kh_init_##name()
+
+/*! @function
+  @abstract     Destroy a hash table.
+  @param  name  Name of the hash table [symbol]
+  @param  h     Pointer to the hash table [khash_t(name)*]
+ */
+#define kh_destroy(name, h) kh_destroy_##name(h)
+
+/*! @function
+  @abstract     Reset a hash table without deallocating memory.
+  @param  name  Name of the hash table [symbol]
+  @param  h     Pointer to the hash table [khash_t(name)*]
+ */
+#define kh_clear(name, h) kh_clear_##name(h)
+
+/*! @function
+  @abstract     Resize a hash table.
+  @param  name  Name of the hash table [symbol]
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @param  s     New size [khint_t]
+ */
+#define kh_resize(name, h, s) kh_resize_##name(h, s)
+
+/*! @function
+  @abstract     Insert a key to the hash table.
+  @param  name  Name of the hash table [symbol]
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @param  k     Key [type of keys]
+  @param  r     Extra return code: -1 if the operation failed;
+                0 if the key is present in the hash table;
+                1 if the bucket is empty (never used); 2 if the element in
+				the bucket has been deleted [int*]
+  @return       Iterator to the inserted element [khint_t]
+ */
+#define kh_put(name, h, k, r) kh_put_##name(h, k, r)
+
+/*! @function
+  @abstract     Retrieve a key from the hash table.
+  @param  name  Name of the hash table [symbol]
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @param  k     Key [type of keys]
+  @return       Iterator to the found element, or kh_end(h) if the element is absent [khint_t]
+ */
+#define kh_get(name, h, k) kh_get_##name(h, k)
+
+/*! @function
+  @abstract     Remove a key from the hash table.
+  @param  name  Name of the hash table [symbol]
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @param  k     Iterator to the element to be deleted [khint_t]
+ */
+#define kh_del(name, h, k) kh_del_##name(h, k)
+
+/*! @function
+  @abstract     Test whether a bucket contains data.
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @param  x     Iterator to the bucket [khint_t]
+  @return       1 if containing data; 0 otherwise [int]
+ */
+#define kh_exist(h, x) (!__ac_iseither((h)->flags, (x)))
+
+/*! @function
+  @abstract     Get key given an iterator
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @param  x     Iterator to the bucket [khint_t]
+  @return       Key [type of keys]
+ */
+#define kh_key(h, x) ((h)->keys[x])
+
+/*! @function
+  @abstract     Get value given an iterator
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @param  x     Iterator to the bucket [khint_t]
+  @return       Value [type of values]
+  @discussion   For hash sets, calling this results in segfault.
+ */
+#define kh_val(h, x) ((h)->vals[x])
+
+/*! @function
+  @abstract     Alias of kh_val()
+ */
+#define kh_value(h, x) ((h)->vals[x])
+
+/*! @function
+  @abstract     Get the start iterator
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @return       The start iterator [khint_t]
+ */
+#define kh_begin(h) (khint_t)(0)
+
+/*! @function
+  @abstract     Get the end iterator
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @return       The end iterator [khint_t]
+ */
+#define kh_end(h) ((h)->n_buckets)
+
+/*! @function
+  @abstract     Get the number of elements in the hash table
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @return       Number of elements in the hash table [khint_t]
+ */
+#define kh_size(h) ((h)->size)
+
+/*! @function
+  @abstract     Get the number of buckets in the hash table
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @return       Number of buckets in the hash table [khint_t]
+ */
+#define kh_n_buckets(h) ((h)->n_buckets)
+
+/*! @function
+  @abstract     Iterate over the entries in the hash table
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @param  kvar  Variable to which key will be assigned
+  @param  vvar  Variable to which value will be assigned
+  @param  code  Block of code to execute
+ */
+#define kh_foreach(h, kvar, vvar, code) { khint_t __i;		\
+	for (__i = kh_begin(h); __i != kh_end(h); ++__i) {		\
+		if (!kh_exist(h,__i)) continue;						\
+		(kvar) = kh_key(h,__i);								\
+		(vvar) = kh_val(h,__i);								\
+		code;												\
+	} }
+
+/*! @function
+  @abstract     Iterate over the values in the hash table
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @param  vvar  Variable to which value will be assigned
+  @param  code  Block of code to execute
+ */
+#define kh_foreach_value(h, vvar, code) { khint_t __i;		\
+	for (__i = kh_begin(h); __i != kh_end(h); ++__i) {		\
+		if (!kh_exist(h,__i)) continue;						\
+		(vvar) = kh_val(h,__i);								\
+		code;												\
+	} }
+
+/*! @function
+  @abstract  Gather hash table statistics
+  @param  name            Name of the hash table [symbol]
+  @param  h               Pointer to the hash table [khash_t(name)*]
+  @param  empty[out]      Number of empty hash bins
+  @param  deleted[out]    Number of hash bins with the deleted flag
+  @param  hist_size[out]  Size of @p hist array
+  @param  hist[out]       Probe count histogram
+  @return 0 on success; -1 on failure
+ */
+#define kh_stats(name, h, empty, deleted, hist_size, hist) \
+    kh_stats_##name(h, empty, deleted, hist_size, hist)
+
+/* More convenient interfaces */
+
+/*! @function
+  @abstract     Instantiate a hash set containing integer keys
+  @param  name  Name of the hash table [symbol]
+ */
+#define KHASH_SET_INIT_INT(name)										\
+	KHASH_INIT(name, khint32_t, char, 0, kh_int_hash_func, kh_int_hash_equal)
+
+/*! @function
+  @abstract     Instantiate a hash map containing integer keys
+  @param  name  Name of the hash table [symbol]
+  @param  khval_t  Type of values [type]
+ */
+#define KHASH_MAP_INIT_INT(name, khval_t)								\
+	KHASH_INIT(name, khint32_t, khval_t, 1, kh_int_hash_func, kh_int_hash_equal)
+
+/*! @function
+  @abstract     Instantiate a hash set containing 64-bit integer keys
+  @param  name  Name of the hash table [symbol]
+ */
+#define KHASH_SET_INIT_INT64(name)										\
+	KHASH_INIT(name, khint64_t, char, 0, kh_int64_hash_func, kh_int64_hash_equal)
+
+/*! @function
+  @abstract     Instantiate a hash map containing 64-bit integer keys
+  @param  name  Name of the hash table [symbol]
+  @param  khval_t  Type of values [type]
+ */
+#define KHASH_MAP_INIT_INT64(name, khval_t)								\
+	KHASH_INIT(name, khint64_t, khval_t, 1, kh_int64_hash_func, kh_int64_hash_equal)
+
+typedef const char *kh_cstr_t;
+/*! @function
+  @abstract     Instantiate a hash set containing const char* keys
+  @param  name  Name of the hash table [symbol]
+ */
+#define KHASH_SET_INIT_STR(name)										\
+	KHASH_INIT(name, kh_cstr_t, char, 0, kh_str_hash_func, kh_str_hash_equal)
+
+/*! @function
+  @abstract     Instantiate a hash map containing const char* keys
+  @param  name  Name of the hash table [symbol]
+  @param  khval_t  Type of values [type]
+ */
+#define KHASH_MAP_INIT_STR(name, khval_t)								\
+	KHASH_INIT(name, kh_cstr_t, khval_t, 1, kh_str_hash_func, kh_str_hash_equal)
+
+/*! @function
+  @abstract     Instantiate a hash set containing kstring_t keys
+  @param  name  Name of the hash table [symbol]
+ */
+#define KHASH_SET_INIT_KSTR(name)										\
+	KHASH_INIT(name, kstring_t, char, 0, kh_kstr_hash_func, kh_kstr_hash_equal)
+
+/*! @function
+  @abstract     Instantiate a hash map containing kstring_t keys
+  @param  name  Name of the hash table [symbol]
+  @param  khval_t  Type of values [type]
+ */
+#define KHASH_MAP_INIT_KSTR(name, khval_t)								\
+	KHASH_INIT(name, kstring_t, khval_t, 1, kh_kstr_hash_func, kh_kstr_hash_equal)
+
+#endif /* __AC_KHASH_H */
diff --git a/ext/htslib/htslib/khash_str2int.h b/ext/htslib/htslib/khash_str2int.h
new file mode 100644
index 0000000..7a5c28b
--- /dev/null
+++ b/ext/htslib/htslib/khash_str2int.h
@@ -0,0 +1,135 @@
+/*  khash_str2int.h -- C-string to integer hash table.
+
+    Copyright (C) 2013-2014,2020 Genome Research Ltd.
+
+    Author: Petr Danecek <pd3@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#ifndef HTSLIB_KHASH_STR2INT_H
+#define HTSLIB_KHASH_STR2INT_H
+
+#include "khash.h"
+
+KHASH_MAP_INIT_STR(str2int, int)
+
+/*
+ *  Wrappers for khash dictionaries used by mpileup.
+ */
+
+static inline void *khash_str2int_init(void)
+{
+    return kh_init(str2int);
+}
+
+/*
+ *  Destroy the hash structure, but not the keys
+ */
+static inline void khash_str2int_destroy(void *_hash)
+{
+    khash_t(str2int) *hash = (khash_t(str2int)*)_hash;
+    if (hash) kh_destroy(str2int, hash); // Note that strings are not freed.
+}
+
+/*
+ *  Destroys both the hash structure and the keys
+ */
+static inline void khash_str2int_destroy_free(void *_hash)
+{
+    khash_t(str2int) *hash = (khash_t(str2int)*)_hash;
+    khint_t k;
+    if (hash == 0) return;
+    for (k = 0; k < kh_end(hash); ++k)
+        if (kh_exist(hash, k)) free((char*)kh_key(hash, k));
+    kh_destroy(str2int, hash);
+}
+
+/*
+ *  Returns 1 if key exists or 0 if not
+ */
+static inline int khash_str2int_has_key(void *_hash, const char *str)
+{
+    khash_t(str2int) *hash = (khash_t(str2int)*)_hash;
+    khint_t k = kh_get(str2int, hash, str);
+    if ( k == kh_end(hash) ) return 0;
+    return 1;
+}
+
+/*
+ *  Returns 0 on success and -1 when the key is not present. On success,
+ *  *value is set, unless NULL is passed.
+ */
+static inline int khash_str2int_get(void *_hash, const char *str, int *value)
+{
+    khash_t(str2int) *hash = (khash_t(str2int)*)_hash;
+    khint_t k;
+    if ( !hash ) return -1;
+    k = kh_get(str2int, hash, str);
+    if ( k == kh_end(hash) ) return -1;
+    if ( !value ) return 0;
+    *value = kh_val(hash, k);
+    return 0;
+}
+
+/*
+ *  Add a new string to the dictionary, auto-incrementing the value.
+ *  On success returns the newly inserted integer id, on error -1
+ *  is returned. Note that the key must continue to exist throughout
+ *  the whole life of _hash.
+ */
+static inline int khash_str2int_inc(void *_hash, const char *str)
+{
+    khint_t k;
+    int ret;
+    khash_t(str2int) *hash = (khash_t(str2int)*)_hash;
+    if ( !hash ) return -1;
+    k = kh_put(str2int, hash, str, &ret);
+    if (ret < 0) return -1;
+    if (ret == 0) return kh_val(hash, k);
+    kh_val(hash, k) = kh_size(hash) - 1;
+    return kh_val(hash, k);
+}
+
+/*
+ *  Set a new key,value pair. On success returns the bin index, on
+ *  error -1 is returned. Note that the key must continue to exist
+ *  throughout the whole life of _hash.
+ */
+static inline int khash_str2int_set(void *_hash, const char *str, int value)
+{
+    khint_t k;
+    int ret;
+    khash_t(str2int) *hash = (khash_t(str2int)*)_hash;
+    if ( !hash ) return -1;
+    k = kh_put(str2int, hash, str, &ret);
+    if (ret < 0) return -1;
+    kh_val(hash,k) = value;
+    return k;
+}
+
+/*
+ *  Return the number of keys in the hash table.
+ */
+static inline int khash_str2int_size(void *_hash)
+{
+    khash_t(str2int) *hash = (khash_t(str2int)*)_hash;
+    return kh_size(hash);
+}
+
+#endif
diff --git a/ext/htslib/htslib/klist.h b/ext/htslib/htslib/klist.h
new file mode 100644
index 0000000..d008a84
--- /dev/null
+++ b/ext/htslib/htslib/klist.h
@@ -0,0 +1,136 @@
+/* The MIT License
+
+   Copyright (c) 2008-2009, by Attractive Chaos <attractor@live.co.uk>
+   Copyright (C) 2013, 2015 Genome Research Ltd.
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be
+   included in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+*/
+
+#ifndef _AC_KLIST_H
+#define _AC_KLIST_H
+
+#include <stdlib.h>
+
+#ifndef klib_unused
+#if (defined __clang__ && __clang_major__ >= 3) || (defined __GNUC__ && __GNUC__ >= 3)
+#define klib_unused __attribute__ ((__unused__))
+#else
+#define klib_unused
+#endif
+#endif /* klib_unused */
+
+#define KMEMPOOL_INIT2(SCOPE, name, kmptype_t, kmpfree_f)				\
+	typedef struct {													\
+		size_t cnt, n, max;												\
+		kmptype_t **buf;												\
+	} kmp_##name##_t;													\
+	SCOPE kmp_##name##_t *kmp_init_##name(void) {						\
+		return (kmp_##name##_t *)calloc(1, sizeof(kmp_##name##_t));		\
+	}																	\
+	SCOPE void kmp_destroy_##name(kmp_##name##_t *mp) {					\
+		size_t k;														\
+		for (k = 0; k < mp->n; ++k) {									\
+			kmpfree_f(mp->buf[k]); free(mp->buf[k]);					\
+		}																\
+		free(mp->buf); free(mp);										\
+	}																	\
+	SCOPE kmptype_t *kmp_alloc_##name(kmp_##name##_t *mp) {				\
+		++mp->cnt;														\
+		if (mp->n == 0) return (kmptype_t *)calloc(1, sizeof(kmptype_t)); \
+		return mp->buf[--mp->n];										\
+	}																	\
+	SCOPE void kmp_free_##name(kmp_##name##_t *mp, kmptype_t *p) {		\
+		--mp->cnt;														\
+		if (mp->n == mp->max) {											\
+			mp->max = mp->max? mp->max<<1 : 16;							\
+			mp->buf = (kmptype_t **)realloc(mp->buf, sizeof(kmptype_t *) * mp->max); \
+		}																\
+		mp->buf[mp->n++] = p;											\
+	}
+
+#define KMEMPOOL_INIT(name, kmptype_t, kmpfree_f)						\
+	KMEMPOOL_INIT2(static inline klib_unused, name, kmptype_t, kmpfree_f)
+
+#define kmempool_t(name) kmp_##name##_t
+#define kmp_init(name) kmp_init_##name()
+#define kmp_destroy(name, mp) kmp_destroy_##name(mp)
+#define kmp_alloc(name, mp) kmp_alloc_##name(mp)
+#define kmp_free(name, mp, p) kmp_free_##name(mp, p)
+
+#define KLIST_INIT2(SCOPE, name, kltype_t, kmpfree_t)					\
+	struct __kl1_##name {												\
+		kltype_t data;													\
+		struct __kl1_##name *next;										\
+	};																	\
+	typedef struct __kl1_##name kl1_##name;								\
+	KMEMPOOL_INIT2(SCOPE, name, kl1_##name, kmpfree_t)					\
+	typedef struct {													\
+		kl1_##name *head, *tail;										\
+		kmp_##name##_t *mp;												\
+		size_t size;													\
+	} kl_##name##_t;													\
+	SCOPE kl_##name##_t *kl_init_##name(void) {							\
+		kl_##name##_t *kl = (kl_##name##_t *)calloc(1, sizeof(kl_##name##_t)); \
+		kl->mp = kmp_init(name);										\
+		kl->head = kl->tail = kmp_alloc(name, kl->mp);					\
+		kl->head->next = 0;												\
+		return kl;														\
+	}																	\
+	SCOPE void kl_destroy_##name(kl_##name##_t *kl) {					\
+		kl1_##name *p;													\
+		for (p = kl->head; p != kl->tail; p = p->next)					\
+			kmp_free(name, kl->mp, p);									\
+		kmp_free(name, kl->mp, p);										\
+		kmp_destroy(name, kl->mp);										\
+		free(kl);														\
+	}																	\
+	SCOPE kltype_t *kl_pushp_##name(kl_##name##_t *kl) {				\
+		kl1_##name *q, *p = kmp_alloc(name, kl->mp);					\
+		q = kl->tail; p->next = 0; kl->tail->next = p; kl->tail = p;	\
+		++kl->size;														\
+		return &q->data;												\
+	}																	\
+	SCOPE int kl_shift_##name(kl_##name##_t *kl, kltype_t *d) {			\
+		kl1_##name *p;													\
+		if (kl->head->next == 0) return -1;								\
+		--kl->size;														\
+		p = kl->head; kl->head = kl->head->next;						\
+		if (d) *d = p->data;											\
+		kmp_free(name, kl->mp, p);										\
+		return 0;														\
+	}
+
+#define KLIST_INIT(name, kltype_t, kmpfree_t)							\
+	KLIST_INIT2(static inline klib_unused, name, kltype_t, kmpfree_t)
+
+#define kliter_t(name) kl1_##name
+#define klist_t(name) kl_##name##_t
+#define kl_val(iter) ((iter)->data)
+#define kl_next(iter) ((iter)->next)
+#define kl_begin(kl) ((kl)->head)
+#define kl_end(kl) ((kl)->tail)
+
+#define kl_init(name) kl_init_##name()
+#define kl_destroy(name, kl) kl_destroy_##name(kl)
+#define kl_pushp(name, kl) kl_pushp_##name(kl)
+#define kl_shift(name, kl, d) kl_shift_##name(kl, d)
+
+#endif
diff --git a/ext/htslib/htslib/knetfile.h b/ext/htslib/htslib/knetfile.h
new file mode 100644
index 0000000..0f2adec
--- /dev/null
+++ b/ext/htslib/htslib/knetfile.h
@@ -0,0 +1,117 @@
+/* The MIT License
+
+   Copyright (c) 2008, 2012, 2014, 2021-2022 Genome Research Ltd (GRL).
+                 2010 by Attractive Chaos <attractor@live.co.uk>
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be
+   included in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+*/
+
+#ifndef KNETFILE_H
+#define KNETFILE_H
+
+#include <stdint.h>
+#include <fcntl.h>
+#include <sys/types.h>
+
+#include "hts_defs.h"
+
+#ifndef _WIN32
+#define netread(fd, ptr, len) read(fd, ptr, len)
+#define netwrite(fd, ptr, len) write(fd, ptr, len)
+#define netclose(fd) close(fd)
+#else
+#include <winsock2.h>
+#define netread(fd, ptr, len) recv(fd, ptr, len, 0)
+#define netwrite(fd, ptr, len) send(fd, ptr, len, 0)
+#define netclose(fd) closesocket(fd)
+#endif
+
+// Ensure ssize_t exists within this header. All #includes must precede this,
+// and ssize_t must be undefined again at the end of this header.
+#if defined _MSC_VER && defined _INTPTR_T_DEFINED && !defined _SSIZE_T_DEFINED && !defined ssize_t
+#define HTSLIB_SSIZE_T
+#define ssize_t intptr_t
+#endif
+
+// FIXME: currently I/O is unbuffered
+
+#define KNF_TYPE_LOCAL 1
+#define KNF_TYPE_FTP   2
+#define KNF_TYPE_HTTP  3
+
+// Kept for API/ABI compatability only.  Do not use directly!
+typedef struct knetFile_s {
+        int type, fd;
+        int64_t offset;
+        char *host, *port;
+
+        // the following are for FTP only
+        int ctrl_fd, pasv_ip[4], pasv_port, max_response, no_reconnect, is_ready;
+        char *response, *retr, *size_cmd;
+        int64_t seek_offset; // for lazy seek
+        int64_t file_size;
+
+        // the following are for HTTP only
+        char *path, *http_host;
+} knetFile;
+
+#define knet_tell(fp) ((fp)->offset)
+#define knet_fileno(fp) ((fp)->fd)
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+    HTSLIB_EXPORT
+        knetFile *knet_open(const char *fn, const char *mode) HTS_DEPRECATED("Please use hopen instead");
+
+        /*
+           This only works with local files.
+         */
+    HTSLIB_EXPORT
+        knetFile *knet_dopen(int fd, const char *mode) HTS_DEPRECATED("Please use hdopen instead");
+
+        /*
+          If ->is_ready==0, this routine updates ->fd; otherwise, it simply
+          reads from ->fd.
+         */
+    HTSLIB_EXPORT
+        ssize_t knet_read(knetFile *fp, void *buf, size_t len) HTS_DEPRECATED("Please use hread instead");
+
+        /*
+          This routine only sets ->offset and ->is_ready=0. It does not
+          communicate with the FTP server.
+         */
+    HTSLIB_EXPORT
+        off_t knet_seek(knetFile *fp, off_t off, int whence) HTS_DEPRECATED("Please use hseek instead");
+    HTSLIB_EXPORT
+        int knet_close(knetFile *fp) HTS_DEPRECATED("Please use hclose instead");
+
+#ifdef __cplusplus
+}
+#endif
+
+#ifdef HTSLIB_SSIZE_T
+#undef HTSLIB_SSIZE_T
+#undef ssize_t
+#endif
+
+#endif
diff --git a/ext/htslib/htslib/kroundup.h b/ext/htslib/htslib/kroundup.h
new file mode 100644
index 0000000..1330a5b
--- /dev/null
+++ b/ext/htslib/htslib/kroundup.h
@@ -0,0 +1,76 @@
+/* The MIT License
+
+   Copyright (C) 2020 Genome Research Ltd.
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be
+   included in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+*/
+
+#ifndef KROUNDUP_H
+#define KROUNDUP_H
+
+// Value of this macro is 1 if x is a signed type; 0 if unsigned
+#define k_signed_type(x) (!(-((x) * 0 + 1) > 0))
+
+/*
+  Macro with value 1 if the highest bit in x is set for any integer type
+
+  This is written avoiding conditionals (?: operator) to reduce the likelihood
+  of gcc attempting jump thread optimisations for code paths where (x) is
+  large.  These optimisations can cause gcc to issue warnings about excessively
+  large memory allocations when the kroundup64() macro below is used with
+  malloc().  Such warnings can be misleading as they imply only the large
+  allocation happens when it's actually working fine for normal values of (x).
+
+  See https://developers.redhat.com/blog/2019/03/13/understanding-gcc-warnings-part-2/
+*/
+#define k_high_bit_set(x) ((((x) >> (sizeof(x) * 8 - 1 - k_signed_type(x))) & 1))
+
+/*! @hideinitializer
+  @abstract  Round up to next power of two
+  @discussion
+  This macro will work for unsigned types up to uint64_t.
+
+  If the next power of two does not fit in the given type, it will set
+  the largest value that does.
+ */
+#define kroundup64(x) ((x) > 0 ?                                        \
+                       (--(x),                                          \
+                        (x)|=(x)>>(sizeof(x)/8),                        \
+                        (x)|=(x)>>(sizeof(x)/4),                        \
+                        (x)|=(x)>>(sizeof(x)/2),                        \
+                        (x)|=(x)>>(sizeof(x)),                          \
+                        (x)|=(x)>>(sizeof(x)*2),                        \
+                        (x)|=(x)>>(sizeof(x)*4),                        \
+                        (x) += !k_high_bit_set(x),                      \
+                        (x))                                            \
+                       : 0)
+
+// Historic interfaces for 32-bit and size_t values.  The macro above
+// works for both (as long as size_t is no more than 64 bits).
+
+#ifndef kroundup32
+#define kroundup32(x) kroundup64(x)
+#endif
+#ifndef kroundup_size_t
+#define kroundup_size_t(x) kroundup64(x)
+#endif
+
+#endif
diff --git a/ext/htslib/htslib/kseq.h b/ext/htslib/htslib/kseq.h
new file mode 100644
index 0000000..5913f35
--- /dev/null
+++ b/ext/htslib/htslib/kseq.h
@@ -0,0 +1,255 @@
+/* The MIT License
+
+   Copyright (c) 2008, 2009, 2011 Attractive Chaos <attractor@live.co.uk>
+   Copyright (C) 2013, 2018, 2020, 2023 Genome Research Ltd.
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be
+   included in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+*/
+
+#ifndef AC_KSEQ_H
+#define AC_KSEQ_H
+
+#include <ctype.h>
+#include <string.h>
+#include <stdlib.h>
+
+#include "kstring.h"
+
+#ifndef klib_unused
+#if (defined __clang__ && __clang_major__ >= 3) || (defined __GNUC__ && __GNUC__ >= 3)
+#define klib_unused __attribute__ ((__unused__))
+#else
+#define klib_unused
+#endif
+#endif /* klib_unused */
+
+#define KS_SEP_SPACE 0 // isspace(): \t, \n, \v, \f, \r
+#define KS_SEP_TAB   1 // isspace() && !' '
+#define KS_SEP_LINE  2 // line separator: "\n" (Unix) or "\r\n" (Windows)
+#define KS_SEP_MAX   2
+
+#define __KS_TYPE(type_t) \
+	typedef struct __kstream_t { \
+		int begin, end; \
+		int is_eof:2, bufsize:30; \
+        uint64_t seek_pos; \
+		type_t f; \
+		unsigned char *buf; \
+	} kstream_t;
+
+#define ks_err(ks) ((ks)->end == -1)
+#define ks_eof(ks) ((ks)->is_eof && (ks)->begin >= (ks)->end)
+#define ks_rewind(ks) ((ks)->is_eof = (ks)->begin = (ks)->end = 0)
+
+#define __KS_BASIC(SCOPE, type_t, __bufsize) \
+	SCOPE kstream_t *ks_init(type_t f) \
+	{ \
+		kstream_t *ks = (kstream_t*)calloc(1, sizeof(kstream_t)); \
+		ks->f = f; ks->bufsize = __bufsize; \
+		ks->buf = (unsigned char*)malloc(__bufsize); \
+		return ks; \
+	} \
+	SCOPE void ks_destroy(kstream_t *ks) \
+	{ \
+		if (!ks) return; \
+		free(ks->buf); \
+		free(ks); \
+	}
+
+#define __KS_INLINED(__read) \
+	static inline klib_unused int ks_getc(kstream_t *ks) \
+	{ \
+		if (ks_err(ks)) return -3; \
+		if (ks->is_eof && ks->begin >= ks->end) return -1; \
+		if (ks->begin >= ks->end) { \
+			ks->begin = 0; \
+			ks->end = __read(ks->f, ks->buf, ks->bufsize); \
+			if (ks->end == 0) { ks->is_eof = 1; return -1; } \
+			if (ks->end == -1) { ks->is_eof = 1; return -3; } \
+		} \
+        ks->seek_pos++; \
+		return (int)ks->buf[ks->begin++]; \
+	} \
+	static inline klib_unused int ks_getuntil(kstream_t *ks, int delimiter, kstring_t *str, int *dret) \
+	{ return ks_getuntil2(ks, delimiter, str, dret, 0); }
+
+#define __KS_GETUNTIL(SCOPE, __read) \
+	SCOPE int ks_getuntil2(kstream_t *ks, int delimiter, kstring_t *str, int *dret, int append)  \
+	{ \
+		int gotany = 0; \
+		if (dret) *dret = 0; \
+		str->l = append? str->l : 0; \
+        uint64_t seek_pos = str->l; \
+		for (;;) { \
+			int i; \
+			if (ks_err(ks)) return -3; \
+			if (ks->begin >= ks->end) { \
+				if (!ks->is_eof) { \
+					ks->begin = 0; \
+					ks->end = __read(ks->f, ks->buf, ks->bufsize); \
+					if (ks->end == 0) { ks->is_eof = 1; break; } \
+					if (ks->end == -1) { ks->is_eof = 1; return -3; } \
+				} else break; \
+			} \
+			if (delimiter == KS_SEP_LINE) {  \
+				unsigned char *sep = (unsigned char *)memchr(ks->buf + ks->begin, '\n', ks->end - ks->begin); \
+				i = sep != NULL ? sep - ks->buf : ks->end; \
+			} else if (delimiter > KS_SEP_MAX) { \
+				unsigned char *sep = (unsigned char *)memchr(ks->buf + ks->begin, delimiter, ks->end - ks->begin); \
+				i = sep != NULL ? sep - ks->buf : ks->end; \
+			} else if (delimiter == KS_SEP_SPACE) { \
+				for (i = ks->begin; i < ks->end; ++i) \
+					if (isspace(ks->buf[i])) break; \
+			} else if (delimiter == KS_SEP_TAB) { \
+				for (i = ks->begin; i < ks->end; ++i) \
+					if (isspace(ks->buf[i]) && ks->buf[i] != ' ') break;  \
+			} else i = 0; /* never come to here! */ \
+			(void) ks_expand(str, i - ks->begin + 1); \
+            seek_pos += i - ks->begin; if ( i < ks->end ) seek_pos++; \
+			gotany = 1; \
+			memcpy(str->s + str->l, ks->buf + ks->begin, i - ks->begin);  \
+			str->l = str->l + (i - ks->begin); \
+			ks->begin = i + 1; \
+			if (i < ks->end) { \
+				if (dret) *dret = ks->buf[i]; \
+				break; \
+			} \
+		} \
+		if (!gotany && ks_eof(ks)) return -1; \
+        ks->seek_pos += seek_pos; \
+		if (str->s == 0) { \
+			str->m = 1; \
+			str->s = (char*)calloc(1, 1); \
+		} else if (delimiter == KS_SEP_LINE && str->l > 1 && str->s[str->l-1] == '\r') --str->l; \
+		str->s[str->l] = '\0';											\
+		return str->l; \
+	}
+
+#define KSTREAM_INIT2(SCOPE, type_t, __read, __bufsize) \
+	__KS_TYPE(type_t) \
+	__KS_BASIC(SCOPE, type_t, __bufsize) \
+	__KS_GETUNTIL(SCOPE, __read) \
+	__KS_INLINED(__read)
+
+#define KSTREAM_INIT(type_t, __read, __bufsize) KSTREAM_INIT2(static, type_t, __read, __bufsize)
+
+#define KSTREAM_DECLARE(type_t, __read) \
+	__KS_TYPE(type_t) \
+	extern int ks_getuntil2(kstream_t *ks, int delimiter, kstring_t *str, int *dret, int append); \
+	extern kstream_t *ks_init(type_t f); \
+	extern void ks_destroy(kstream_t *ks); \
+	__KS_INLINED(__read)
+
+/******************
+ * FASTA/Q parser *
+ ******************/
+
+#define kseq_rewind(ks) ((ks)->last_char = (ks)->f->is_eof = (ks)->f->begin = (ks)->f->end = 0)
+
+#define __KSEQ_BASIC(SCOPE, type_t)										\
+	SCOPE kseq_t *kseq_init(type_t fd)									\
+	{																	\
+		kseq_t *s = (kseq_t*)calloc(1, sizeof(kseq_t));					\
+		s->f = ks_init(fd);												\
+		return s;														\
+	}																	\
+	SCOPE void kseq_destroy(kseq_t *ks)									\
+	{																	\
+		if (!ks) return;												\
+		free(ks->name.s); free(ks->comment.s); free(ks->seq.s);	free(ks->qual.s); \
+		ks_destroy(ks->f);												\
+		free(ks);														\
+	}
+
+/* Return value:
+   >=0  length of the sequence (normal)
+   -1   end-of-file
+   -2   truncated quality string
+   -3   error reading stream
+   -4   overflow error
+ */
+#define __KSEQ_READ(SCOPE) \
+	SCOPE int kseq_read(kseq_t *seq) \
+	{ \
+		int c,r; \
+		kstream_t *ks = seq->f; \
+		if (seq->last_char == 0) { /* then jump to the next header line */ \
+			while ((c = ks_getc(ks)) >= 0 && c != '>' && c != '@'); \
+			if (c < 0) return c; /* end of file or error */ \
+			seq->last_char = c; \
+		} /* else: the first header char has been read in the previous call */ \
+		seq->comment.l = seq->seq.l = seq->qual.l = 0; /* reset all members */ \
+		if ((r=ks_getuntil(ks, 0, &seq->name, &c)) < 0) return r; /* normal exit: EOF or error */ \
+		if (c != '\n') ks_getuntil(ks, KS_SEP_LINE, &seq->comment, 0); /* read FASTA/Q comment */ \
+		if (seq->seq.s == 0) { /* we can do this in the loop below, but that is slower */ \
+			seq->seq.m = 256; \
+			seq->seq.s = (char*)malloc(seq->seq.m); \
+		} \
+		while ((c = ks_getc(ks)) >= 0 && c != '>' && c != '+' && c != '@') { \
+			if (c == '\n') continue; /* skip empty lines */ \
+			seq->seq.s[seq->seq.l++] = c; /* this is safe: we always have enough space for 1 char */ \
+			ks_getuntil2(ks, KS_SEP_LINE, &seq->seq, 0, 1); /* read the rest of the line */ \
+		} \
+		if (c == '>' || c == '@') seq->last_char = c; /* the first header char has been read */	\
+		if (seq->seq.l + 1 >= seq->seq.m) { /* seq->seq.s[seq->seq.l] below may be out of boundary */ \
+			seq->seq.m = seq->seq.l + 2; \
+			kroundup32(seq->seq.m); /* rounded to the next closest 2^k */ \
+			if (seq->seq.l + 1 >= seq->seq.m) return -4; /* error: adjusting m overflowed */ \
+			seq->seq.s = (char*)realloc(seq->seq.s, seq->seq.m); \
+		} \
+		seq->seq.s[seq->seq.l] = 0;	/* null terminated string */ \
+		if (c != '+') return seq->seq.l; /* FASTA */ \
+		if (seq->qual.m < seq->seq.m) {	/* allocate memory for qual in case insufficient */ \
+			seq->qual.m = seq->seq.m; \
+			seq->qual.s = (char*)realloc(seq->qual.s, seq->qual.m); \
+		} \
+		while ((c = ks_getc(ks)) >= 0 && c != '\n'); /* skip the rest of '+' line */ \
+		if (c == -1) return -2; /* error: no quality string */ \
+		while ((c = ks_getuntil2(ks, KS_SEP_LINE, &seq->qual, 0, 1)) >= 0 && seq->qual.l < seq->seq.l); \
+		if (c == -3) return -3; /* stream error */ \
+		seq->last_char = 0;	/* we have not come to the next header line */ \
+		if (seq->seq.l != seq->qual.l) return -2; /* error: qual string is of a different length */ \
+		return seq->seq.l; \
+	}
+
+#define __KSEQ_TYPE(type_t)						\
+	typedef struct {							\
+		kstring_t name, comment, seq, qual;		\
+		int last_char;							\
+		kstream_t *f;							\
+	} kseq_t;
+
+#define KSEQ_INIT2(SCOPE, type_t, __read)		\
+	KSTREAM_INIT(type_t, __read, 16384)			\
+	__KSEQ_TYPE(type_t)							\
+	__KSEQ_BASIC(SCOPE, type_t)					\
+	__KSEQ_READ(SCOPE)
+
+#define KSEQ_INIT(type_t, __read) KSEQ_INIT2(static, type_t, __read)
+
+#define KSEQ_DECLARE(type_t) \
+	__KS_TYPE(type_t) \
+	__KSEQ_TYPE(type_t) \
+	extern kseq_t *kseq_init(type_t fd); \
+	void kseq_destroy(kseq_t *ks); \
+	int kseq_read(kseq_t *seq);
+
+#endif
diff --git a/ext/htslib/htslib/ksort.h b/ext/htslib/htslib/ksort.h
new file mode 100644
index 0000000..7857d4c
--- /dev/null
+++ b/ext/htslib/htslib/ksort.h
@@ -0,0 +1,322 @@
+/* The MIT License
+
+   Copyright (c) 2008, 2012-2013, 2017-2019 Genome Research Ltd (GRL).
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be
+   included in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+*/
+
+/* Contact: Heng Li <lh3@sanger.ac.uk> */
+
+/*
+  2012-12-11 (0.1.4):
+
+    * Defined __ks_insertsort_##name as static to compile with C99.
+
+  2008-11-16 (0.1.4):
+
+    * Fixed a bug in introsort() that happens in rare cases.
+
+  2008-11-05 (0.1.3):
+
+    * Fixed a bug in introsort() for complex comparisons.
+
+	* Fixed a bug in mergesort(). The previous version is not stable.
+
+  2008-09-15 (0.1.2):
+
+	* Accelerated introsort. On my Mac (not on another Linux machine),
+	  my implementation is as fast as the C++ standard library's sort()
+	  on random input.
+
+	* Added combsort and in introsort, switch to combsort if the
+	  recursion is too deep.
+
+  2008-09-13 (0.1.1):
+
+	* Added k-small algorithm
+
+  2008-09-05 (0.1.0):
+
+	* Initial version
+
+*/
+
+#ifndef AC_KSORT_H
+#define AC_KSORT_H
+
+#include <stdlib.h>
+#include <string.h>
+#include "hts_defs.h"
+
+#ifndef klib_unused
+#if (defined __clang__ && __clang_major__ >= 3) || (defined __GNUC__ && __GNUC__ >= 3)
+#define klib_unused __attribute__ ((__unused__))
+#else
+#define klib_unused
+#endif
+#endif /* klib_unused */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Use our own drand48() symbol (used by ks_shuffle) to avoid portability
+// problems on Windows.  Don't include htslib/hts_os.h for this as it
+// may not get on with older attempts to fix this in code that includes
+// this file.
+HTSLIB_EXPORT
+extern double hts_drand48(void);
+
+typedef struct {
+	void *left, *right;
+	int depth;
+} ks_isort_stack_t;
+
+#define KSORT_SWAP(type_t, a, b) { type_t t=(a); (a)=(b); (b)=t; }
+
+#define KSORT_INIT(name, type_t, __sort_lt)	KSORT_INIT_(_ ## name, , type_t, __sort_lt)
+#define KSORT_INIT_STATIC(name, type_t, __sort_lt)	KSORT_INIT_(_ ## name, static klib_unused, type_t, __sort_lt)
+#define KSORT_INIT2(name, SCOPE, type_t, __sort_lt)	KSORT_INIT_(_ ## name, SCOPE, type_t, __sort_lt)
+
+#define KSORT_INIT_(name, SCOPE, type_t, __sort_lt)						\
+	SCOPE int ks_mergesort##name(size_t n, type_t array[], type_t temp[]) \
+	{																	\
+		type_t *a2[2], *a, *b;											\
+		int curr, shift;												\
+																		\
+		a2[0] = array;													\
+		a2[1] = temp? temp : (type_t*)malloc(sizeof(type_t) * n);		\
+		for (curr = 0, shift = 0; (1ul<<shift) < n; ++shift) {			\
+			a = a2[curr]; b = a2[1-curr];								\
+			if (shift == 0) {											\
+				type_t *p = b, *i, *eb = a + n;							\
+				for (i = a; i < eb; i += 2) {							\
+					if (i == eb - 1) *p++ = *i;							\
+					else {												\
+						if (__sort_lt(*(i+1), *i)) {					\
+							*p++ = *(i+1); *p++ = *i;					\
+						} else {										\
+							*p++ = *i; *p++ = *(i+1);					\
+						}												\
+					}													\
+				}														\
+			} else {													\
+				size_t i, step = 1ul<<shift;							\
+				for (i = 0; i < n; i += step<<1) {						\
+					type_t *p, *j, *k, *ea, *eb;						\
+					if (n < i + step) {									\
+						ea = a + n; eb = a;								\
+					} else {											\
+						ea = a + i + step;								\
+						eb = a + (n < i + (step<<1)? n : i + (step<<1)); \
+					}													\
+					j = a + i; k = a + i + step; p = b + i;				\
+					while (j < ea && k < eb) {							\
+						if (__sort_lt(*k, *j)) *p++ = *k++;				\
+						else *p++ = *j++;								\
+					}													\
+					while (j < ea) *p++ = *j++;							\
+					while (k < eb) *p++ = *k++;							\
+				}														\
+			}															\
+			curr = 1 - curr;											\
+		}																\
+		if (curr == 1) {												\
+			type_t *p = a2[0], *i = a2[1], *eb = array + n;				\
+			for (; p < eb; ++i) *p++ = *i;								\
+		}																\
+		if (temp == 0) free(a2[1]);										\
+		return 0;															\
+	}																	\
+	SCOPE void ks_heapadjust##name(size_t i, size_t n, type_t l[])		\
+	{																	\
+		size_t k = i;													\
+		type_t tmp = l[i];												\
+		while ((k = (k << 1) + 1) < n) {								\
+			if (k != n - 1 && __sort_lt(l[k], l[k+1])) ++k;				\
+			if (__sort_lt(l[k], tmp)) break;							\
+			l[i] = l[k]; i = k;											\
+		}																\
+		l[i] = tmp;														\
+	}																	\
+	SCOPE void ks_heapmake##name(size_t lsize, type_t l[])				\
+	{																	\
+		size_t i;														\
+		for (i = (lsize >> 1) - 1; i != (size_t)(-1); --i)				\
+			ks_heapadjust##name(i, lsize, l);							\
+	}																	\
+	SCOPE void ks_heapsort##name(size_t lsize, type_t l[])				\
+	{																	\
+		size_t i;														\
+		for (i = lsize - 1; i > 0; --i) {								\
+			type_t tmp;													\
+			tmp = *l; *l = l[i]; l[i] = tmp; ks_heapadjust##name(0, i, l); \
+		}																\
+	}																	\
+	static inline void __ks_insertsort##name(type_t *s, type_t *t)		\
+	{																	\
+		type_t *i, *j, swap_tmp;										\
+		for (i = s + 1; i < t; ++i)										\
+			for (j = i; j > s && __sort_lt(*j, *(j-1)); --j) {			\
+				swap_tmp = *j; *j = *(j-1); *(j-1) = swap_tmp;			\
+			}															\
+	}																	\
+	SCOPE void ks_combsort##name(size_t n, type_t a[])					\
+	{																	\
+		const double shrink_factor = 1.2473309501039786540366528676643; \
+		int do_swap;													\
+		size_t gap = n;													\
+		type_t tmp, *i, *j;												\
+		do {															\
+			if (gap > 2) {												\
+				gap = (size_t)(gap / shrink_factor);					\
+				if (gap == 9 || gap == 10) gap = 11;					\
+			}															\
+			do_swap = 0;												\
+			for (i = a; i < a + n - gap; ++i) {							\
+				j = i + gap;											\
+				if (__sort_lt(*j, *i)) {								\
+					tmp = *i; *i = *j; *j = tmp;						\
+					do_swap = 1;										\
+				}														\
+			}															\
+		} while (do_swap || gap > 2);									\
+		if (gap != 1) __ks_insertsort##name(a, a + n);					\
+	}																	\
+	SCOPE int ks_introsort##name(size_t n, type_t a[])					\
+	{																	\
+		int d;															\
+		ks_isort_stack_t *top, *stack;									\
+		type_t rp, swap_tmp;											\
+		type_t *s, *t, *i, *j, *k;										\
+																		\
+		if (n < 1) return 0;												\
+		else if (n == 2) {												\
+			if (__sort_lt(a[1], a[0])) { swap_tmp = a[0]; a[0] = a[1]; a[1] = swap_tmp; } \
+			return 0;														\
+		}																\
+		for (d = 2; 1ul<<d < n; ++d);									\
+		stack = (ks_isort_stack_t*)malloc(sizeof(ks_isort_stack_t) * ((sizeof(size_t)*d)+2)); \
+		top = stack; s = a; t = a + (n-1); d <<= 1;						\
+		while (1) {														\
+			if (s < t) {												\
+				if (--d == 0) {											\
+					ks_combsort##name(t - s + 1, s);					\
+					t = s;												\
+					continue;											\
+				}														\
+				i = s; j = t; k = i + ((j-i)>>1) + 1;					\
+				if (__sort_lt(*k, *i)) {								\
+					if (__sort_lt(*k, *j)) k = j;						\
+				} else k = __sort_lt(*j, *i)? i : j;					\
+				rp = *k;												\
+				if (k != t) { swap_tmp = *k; *k = *t; *t = swap_tmp; }	\
+				for (;;) {												\
+					do ++i; while (__sort_lt(*i, rp));					\
+					do --j; while (i <= j && __sort_lt(rp, *j));		\
+					if (j <= i) break;									\
+					swap_tmp = *i; *i = *j; *j = swap_tmp;				\
+				}														\
+				swap_tmp = *i; *i = *t; *t = swap_tmp;					\
+				if (i-s > t-i) {										\
+					if (i-s > 16) { top->left = s; top->right = i-1; top->depth = d; ++top; } \
+					s = t-i > 16? i+1 : t;								\
+				} else {												\
+					if (t-i > 16) { top->left = i+1; top->right = t; top->depth = d; ++top; } \
+					t = i-s > 16? i-1 : s;								\
+				}														\
+			} else {													\
+				if (top == stack) {										\
+					free(stack);										\
+					__ks_insertsort##name(a, a+n);						\
+					return 0;												\
+				} else { --top; s = (type_t*)top->left; t = (type_t*)top->right; d = top->depth; } \
+			}															\
+		}																\
+		return 0;															\
+	}																	\
+	/* This function is adapted from: http://ndevilla.free.fr/median/ */ \
+	/* 0 <= kk < n */													\
+	SCOPE type_t ks_ksmall##name(size_t n, type_t arr[], size_t kk)		\
+	{																	\
+		type_t *low, *high, *k, *ll, *hh, *mid;							\
+		low = arr; high = arr + n - 1; k = arr + kk;					\
+		for (;;) {														\
+			if (high <= low) return *k;									\
+			if (high == low + 1) {										\
+				if (__sort_lt(*high, *low)) KSORT_SWAP(type_t, *low, *high); \
+				return *k;												\
+			}															\
+			mid = low + (high - low) / 2;								\
+			if (__sort_lt(*high, *mid)) KSORT_SWAP(type_t, *mid, *high); \
+			if (__sort_lt(*high, *low)) KSORT_SWAP(type_t, *low, *high); \
+			if (__sort_lt(*low, *mid)) KSORT_SWAP(type_t, *mid, *low);	\
+			KSORT_SWAP(type_t, *mid, *(low+1));							\
+			ll = low + 1; hh = high;									\
+			for (;;) {													\
+				do ++ll; while (__sort_lt(*ll, *low));					\
+				do --hh; while (__sort_lt(*low, *hh));					\
+				if (hh < ll) break;										\
+				KSORT_SWAP(type_t, *ll, *hh);							\
+			}															\
+			KSORT_SWAP(type_t, *low, *hh);								\
+			if (hh <= k) low = ll;										\
+			if (hh >= k) high = hh - 1;									\
+		}																\
+	}																	\
+	SCOPE void ks_shuffle##name(size_t n, type_t a[])					\
+	{																	\
+		int i, j;														\
+		for (i = n; i > 1; --i) {										\
+			type_t tmp;													\
+			j = (int)(hts_drand48() * i);								\
+			tmp = a[j]; a[j] = a[i-1]; a[i-1] = tmp;					\
+		}																\
+	}
+
+#define ks_mergesort(name, n, a, t) ks_mergesort_##name(n, a, t)
+#define ks_introsort(name, n, a) ks_introsort_##name(n, a)
+#define ks_combsort(name, n, a) ks_combsort_##name(n, a)
+#define ks_heapsort(name, n, a) ks_heapsort_##name(n, a)
+#define ks_heapmake(name, n, a) ks_heapmake_##name(n, a)
+#define ks_heapadjust(name, i, n, a) ks_heapadjust_##name(i, n, a)
+#define ks_ksmall(name, n, a, k) ks_ksmall_##name(n, a, k)
+#define ks_shuffle(name, n, a) ks_shuffle_##name(n, a)
+
+#define ks_lt_generic(a, b) ((a) < (b))
+#define ks_lt_str(a, b) (strcmp((a), (b)) < 0)
+
+typedef const char *ksstr_t;
+
+#define KSORT_INIT_GENERIC(type_t) KSORT_INIT_(_ ## type_t, , type_t, ks_lt_generic)
+#define KSORT_INIT_STR KSORT_INIT(str, ksstr_t, ks_lt_str)
+
+#define KSORT_INIT_STATIC_GENERIC(type_t) KSORT_INIT_(_ ## type_t, static klib_unused, type_t, ks_lt_generic)
+#define KSORT_INIT_STATIC_STR KSORT_INIT_STATIC(str, ksstr_t, ks_lt_str)
+
+#define KSORT_INIT2_GENERIC(type_t, SCOPE) KSORT_INIT_(_ ## type_t, SCOPE, type_t, ks_lt_generic)
+#define KSORT_INIT2_STR KSORT_INIT2(str, SCOPE, ksstr_t, ks_lt_str)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/ext/htslib/htslib/kstring.h b/ext/htslib/htslib/kstring.h
new file mode 100644
index 0000000..ebb2f93
--- /dev/null
+++ b/ext/htslib/htslib/kstring.h
@@ -0,0 +1,457 @@
+/* The MIT License
+
+   Copyright (C) 2011 by Attractive Chaos <attractor@live.co.uk>
+   Copyright (C) 2013-2014, 2016, 2018-2020, 2022, 2024 Genome Research Ltd.
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be
+   included in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+*/
+
+#ifndef KSTRING_H
+#define KSTRING_H
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdarg.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <limits.h>
+#include <errno.h>
+#include <sys/types.h>
+
+#include "hts_defs.h"
+#include "kroundup.h"
+
+#if defined __GNUC__ && (__GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ > 4))
+#ifdef __MINGW_PRINTF_FORMAT
+#define KS_ATTR_PRINTF(fmt, arg) __attribute__((__format__ (__MINGW_PRINTF_FORMAT, fmt, arg)))
+#else
+#define KS_ATTR_PRINTF(fmt, arg) __attribute__((__format__ (__printf__, fmt, arg)))
+#endif // __MINGW_PRINTF_FORMAT
+#else
+#define KS_ATTR_PRINTF(fmt, arg)
+#endif
+
+#ifndef HAVE___BUILTIN_CLZ
+#if defined __GNUC__ && (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4))
+#define HAVE___BUILTIN_CLZ 1
+#endif
+#endif
+
+// Ensure ssize_t exists within this header. All #includes must precede this,
+// and ssize_t must be undefined again at the end of this header.
+#if defined _MSC_VER && defined _INTPTR_T_DEFINED && !defined _SSIZE_T_DEFINED && !defined ssize_t
+#define HTSLIB_SSIZE_T
+#define ssize_t intptr_t
+#endif
+
+/* kstring_t is a simple non-opaque type whose fields are likely to be
+ * used directly by user code (but see also ks_str() and ks_len() below).
+ * A kstring_t object is initialised by either of
+ *       kstring_t str = KS_INITIALIZE;
+ *       kstring_t str; ...; ks_initialize(&str);
+ * and either ownership of the underlying buffer should be given away before
+ * the object disappears (see ks_release() below) or the kstring_t should be
+ * destroyed with  ks_free(&str) or free(str.s) */
+#ifndef KSTRING_T
+#define KSTRING_T kstring_t
+typedef struct kstring_t {
+	size_t l, m;
+	char *s;
+} kstring_t;
+#endif
+
+typedef struct ks_tokaux_t {
+	uint64_t tab[4];
+	int sep, finished;
+	const char *p; // end of the current token
+} ks_tokaux_t;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+    HTSLIB_EXPORT
+	int kvsprintf(kstring_t *s, const char *fmt, va_list ap) KS_ATTR_PRINTF(2,0);
+
+    HTSLIB_EXPORT
+	int ksprintf(kstring_t *s, const char *fmt, ...) KS_ATTR_PRINTF(2,3);
+
+    HTSLIB_EXPORT
+    int kputd(double d, kstring_t *s); // custom %g only handler
+
+    HTSLIB_EXPORT
+	int ksplit_core(char *s, int delimiter, int *_max, int **_offsets);
+
+    HTSLIB_EXPORT
+	char *kstrstr(const char *str, const char *pat, int **_prep);
+
+    HTSLIB_EXPORT
+	char *kstrnstr(const char *str, const char *pat, int n, int **_prep);
+
+    HTSLIB_EXPORT
+	void *kmemmem(const void *_str, int n, const void *_pat, int m, int **_prep);
+
+	/* kstrtok() is similar to strtok_r() except that str is not
+	 * modified and both str and sep can be NULL. For efficiency, it is
+	 * actually recommended to set both to NULL in the subsequent calls
+	 * if sep is not changed. */
+    HTSLIB_EXPORT
+	char *kstrtok(const char *str, const char *sep, ks_tokaux_t *aux);
+
+    /* kgetline() uses the supplied fgets()-like function to read a "\n"-
+     * or "\r\n"-terminated line from fp.  The line read is appended to the
+     * kstring without its terminator and 0 is returned; EOF is returned at
+     * EOF or on error (determined by querying fp, as per fgets()). */
+    typedef char *kgets_func(char *, int, void *);
+    HTSLIB_EXPORT
+    int kgetline(kstring_t *s, kgets_func *fgets_fn, void *fp);
+
+    /* kgetline2() uses the supplied hgetln()-like function to read a "\n"-
+     * or "\r\n"-terminated line from fp.  The line read is appended to the
+     * ksring without its terminator and 0 is returned; EOF is returned at
+     * EOF or on error (determined by querying fp, as per fgets()). */
+    typedef ssize_t kgets_func2(char *, size_t, void *);
+    HTSLIB_EXPORT
+    int kgetline2(kstring_t *s, kgets_func2 *fgets_fn, void *fp);
+
+#ifdef __cplusplus
+}
+#endif
+
+/// kstring initializer for structure assignment
+#define KS_INITIALIZE { 0, 0, NULL }
+
+/// kstring initializer for pointers
+/**
+   @note Not to be used if the buffer has been allocated.  Use ks_release()
+   or ks_clear() instead.
+*/
+
+static inline void ks_initialize(kstring_t *s)
+{
+    s->l = s->m = 0;
+    s->s = NULL;
+}
+
+/// Resize a kstring to a given capacity
+static inline int ks_resize(kstring_t *s, size_t size)
+{
+	if (s->m < size) {
+	    char *tmp;
+	    size = (size > (SIZE_MAX>>2)) ? size : size + (size >> 1);
+	    tmp = (char*)realloc(s->s, size);
+	    if (!tmp)
+	        return -1;
+	    s->s = tmp;
+	    s->m = size;
+	}
+	return 0;
+}
+
+/// Increase kstring capacity by a given number of bytes
+static inline int ks_expand(kstring_t *s, size_t expansion)
+{
+    size_t new_size = s->l + expansion;
+
+    if (new_size < s->l) // Overflow check
+        return -1;
+    return ks_resize(s, new_size);
+}
+
+/// Returns the kstring buffer
+static inline char *ks_str(kstring_t *s)
+{
+	return s->s;
+}
+
+/// Returns the kstring buffer, or an empty string if l == 0
+/**
+ * Unlike ks_str(), this function will never return NULL.  If the kstring is
+ * empty it will return a read-only empty string.  As the returned value
+ * may be read-only, the caller should not attempt to modify it.
+ */
+static inline const char *ks_c_str(kstring_t *s)
+{
+    return s->l && s->s ? s->s : "";
+}
+
+static inline size_t ks_len(kstring_t *s)
+{
+	return s->l;
+}
+
+/// Reset kstring length to zero
+/**
+   @return The kstring itself
+
+   Example use: kputsn(string, len, ks_clear(s))
+*/
+static inline kstring_t *ks_clear(kstring_t *s)
+{
+    s->l = 0;
+    return s;
+}
+
+// Give ownership of the underlying buffer away to something else (making
+// that something else responsible for freeing it), leaving the kstring_t
+// empty and ready to be used again, or ready to go out of scope without
+// needing  free(str.s)  to prevent a memory leak.
+static inline char *ks_release(kstring_t *s)
+{
+	char *ss = s->s;
+	s->l = s->m = 0;
+	s->s = NULL;
+	return ss;
+}
+
+/// Safely free the underlying buffer in a kstring.
+static inline void ks_free(kstring_t *s)
+{
+    if (s) {
+        free(s->s);
+        ks_initialize(s);
+    }
+}
+
+static inline int kputsn(const char *p, size_t l, kstring_t *s)
+{
+	size_t new_sz = s->l + l + 2;
+	if (new_sz <= s->l || ks_resize(s, new_sz) < 0)
+		return EOF;
+	memcpy(s->s + s->l, p, l);
+	s->l += l;
+	s->s[s->l] = 0;
+	return l;
+}
+
+static inline int kputs(const char *p, kstring_t *s)
+{
+	if (!p) { errno = EFAULT; return -1; }
+	return kputsn(p, strlen(p), s);
+}
+
+static inline int kputc(int c, kstring_t *s)
+{
+	if (ks_resize(s, s->l + 2) < 0)
+		return EOF;
+	s->s[s->l++] = c;
+	s->s[s->l] = 0;
+	return (unsigned char)c;
+}
+
+static inline int kputc_(int c, kstring_t *s)
+{
+	if (ks_resize(s, s->l + 1) < 0)
+		return EOF;
+	s->s[s->l++] = c;
+	return 1;
+}
+
+static inline int kputsn_(const void *p, size_t l, kstring_t *s)
+{
+	size_t new_sz = s->l + l;
+	if (new_sz < s->l || ks_resize(s, new_sz ? new_sz : 1) < 0)
+		return EOF;
+	memcpy(s->s + s->l, p, l);
+	s->l += l;
+	return l;
+}
+
+static inline int kputuw(unsigned x, kstring_t *s)
+{
+#if HAVE___BUILTIN_CLZ && UINT_MAX == 4294967295U
+    static const unsigned int kputuw_num_digits[32] = {
+        10, 10, 10,  9,  9,  9,  8,  8,
+        8,   7,  7,  7,  7,  6,  6,  6,
+        5,   5,  5,  4,  4,  4,  4,  3,
+        3,   3,  2,  2,  2,  1,  1,  1
+    };
+    static const unsigned int kputuw_thresholds[32] = {
+        0,        0, 1000000000U, 0,       0, 100000000U,   0,      0,
+        10000000, 0,          0,  0, 1000000,         0,    0, 100000,
+        0,        0,      10000,  0,       0,         0, 1000,      0,
+        0,      100,          0,  0,      10,         0,    0,      0
+    };
+#else
+    uint64_t m;
+#endif
+    static const char kputuw_dig2r[] =
+        "00010203040506070809"
+        "10111213141516171819"
+        "20212223242526272829"
+        "30313233343536373839"
+        "40414243444546474849"
+        "50515253545556575859"
+        "60616263646566676869"
+        "70717273747576777879"
+        "80818283848586878889"
+        "90919293949596979899";
+    unsigned int l, j;
+    char *cp;
+
+    // Trivial case - also prevents __builtin_clz(0), which is undefined
+    if (x < 10) {
+        if (ks_resize(s, s->l + 2) < 0)
+            return EOF;
+        s->s[s->l++] = '0'+x;
+        s->s[s->l] = 0;
+        return 0;
+    }
+
+    // Find out how many digits are to be printed.
+#if HAVE___BUILTIN_CLZ && UINT_MAX == 4294967295U
+    /*
+     * Table method - should be quick if clz can be done in hardware.
+     * Find the most significant bit of the value to print and look
+     * up in a table to find out how many decimal digits are needed.
+     * This number needs to be adjusted by 1 for cases where the decimal
+     * length could vary for a given number of bits (for example,
+     * a four bit number could be between 8 and 15).
+     */
+
+    l = __builtin_clz(x);
+    l = kputuw_num_digits[l] - (x < kputuw_thresholds[l]);
+#else
+    // Fallback for when clz is not available
+    m = 1;
+    l = 0;
+    do {
+        l++;
+        m *= 10;
+    } while (x >= m);
+#endif
+
+    if (ks_resize(s, s->l + l + 2) < 0)
+        return EOF;
+
+    // Add digits two at a time
+    j = l;
+    cp = s->s + s->l;
+    while (x >= 10) {
+        const char *d = &kputuw_dig2r[2*(x%100)];
+        x /= 100;
+        memcpy(&cp[j-=2], d, 2);
+    }
+
+    // Last one (if necessary).  We know that x < 10 by now.
+    if (j == 1)
+        cp[0] = x + '0';
+
+    s->l += l;
+    s->s[s->l] = 0;
+    return 0;
+}
+
+static inline int kputw(int c, kstring_t *s)
+{
+    unsigned int x = c;
+    if (c < 0) {
+        x = -x;
+        if (ks_resize(s, s->l + 3) < 0)
+            return EOF;
+        s->s[s->l++] = '-';
+    }
+
+    return kputuw(x, s);
+}
+
+static inline int kputll(long long c, kstring_t *s)
+{
+    // Worst case expansion.  One check reduces function size
+    // and aids inlining chance.  Memory overhead is minimal.
+    if (ks_resize(s, s->l + 23) < 0)
+	return EOF;
+
+    unsigned long long x = c;
+    if (c < 0) {
+	x = -x;
+        s->s[s->l++] = '-';
+    }
+
+    if (x <= UINT32_MAX)
+	return kputuw(x, s);
+
+    static const char kputull_dig2r[] =
+        "00010203040506070809"
+        "10111213141516171819"
+        "20212223242526272829"
+        "30313233343536373839"
+        "40414243444546474849"
+        "50515253545556575859"
+        "60616263646566676869"
+        "70717273747576777879"
+        "80818283848586878889"
+        "90919293949596979899";
+    unsigned int l, j;
+    char *cp;
+
+    // Find out how long the number is (could consider clzll)
+    uint64_t m = 1;
+    l = 0;
+    if (sizeof(long long)==sizeof(uint64_t) && x >= 10000000000000000000ULL) {
+	// avoids overflow below
+	l = 20;
+    } else {
+	do {
+	    l++;
+	    m *= 10;
+	} while (x >= m);
+    }
+
+    // Add digits two at a time
+    j = l;
+    cp = s->s + s->l;
+    while (x >= 10) {
+        const char *d = &kputull_dig2r[2*(x%100)];
+        x /= 100;
+        memcpy(&cp[j-=2], d, 2);
+    }
+
+    // Last one (if necessary).  We know that x < 10 by now.
+    if (j == 1)
+        cp[0] = x + '0';
+
+    s->l += l;
+    s->s[s->l] = 0;
+    return 0;
+}
+
+static inline int kputl(long c, kstring_t *s) {
+    return kputll(c, s);
+}
+
+/*
+ * Returns 's' split by delimiter, with *n being the number of components;
+ *         NULL on failure.
+ */
+static inline int *ksplit(kstring_t *s, int delimiter, int *n)
+{
+	int max = 0, *offsets = 0;
+	*n = ksplit_core(s->s, delimiter, &max, &offsets);
+	return offsets;
+}
+
+#ifdef HTSLIB_SSIZE_T
+#undef HTSLIB_SSIZE_T
+#undef ssize_t
+#endif
+
+#endif
diff --git a/ext/htslib/htslib/regidx.h b/ext/htslib/htslib/regidx.h
new file mode 100644
index 0000000..cd14dbc
--- /dev/null
+++ b/ext/htslib/htslib/regidx.h
@@ -0,0 +1,242 @@
+/// @file htslib/regidx.h
+/// Region indexing.
+/*
+    Copyright (C) 2014-2019 Genome Research Ltd.
+
+    Author: Petr Danecek <pd3@sanger.ac.uk>
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in
+    all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+    THE SOFTWARE.
+*/
+
+/*
+    Region indexing with an optional payload.
+
+    Example of usage:
+
+        // Init the parser and print regions. In this example the payload is a
+        // pointer to a string. For the description of parse_custom and
+        // free_custom functions, see regidx_parse_f and regidx_free_f below,
+        // and for working example see test/test-regidx.c.
+        regidx_t *idx = regidx_init(in_fname,parse_custom,free_custom,sizeof(char*),NULL);
+
+        // Query overlap with chr:beg-end (beg,end are 1-based coordinates)
+        regitr_t *itr = regitr_init(idx);
+        if ( regidx_overlap(idx, chr,beg-1,end-1, itr) ) printf("There is an overlap!\n");
+
+        while ( regitr_overlap(itr) )
+        {
+            printf("[%"PRIhts_pos",%"PRIhts_pos"] overlaps with [%"PRIhts_pos",%"PRIhts_pos"], payload=%s\n",
+                   beg, end, itr->beg+1, itr->end+1, regitr_payload(itr,char*));
+        }
+
+        regidx_destroy(idx);
+        regitr_destroy(itr);
+
+
+    Another example, loop over all regions:
+
+        regidx_t *idx = regidx_init(in_fname,NULL,NULL,0,NULL);
+        regitr_t *itr = regitr_init(idx);
+
+        while ( regitr_loop(itr) )
+            printf("chr=%s  beg=%d  end=%d\n", itr->seq, itr->beg+1, itr->end+1);
+
+        regidx_destroy(idx);
+        regitr_destroy(itr);
+*/
+
+#ifndef HTSLIB_REGIDX_H
+#define HTSLIB_REGIDX_H
+
+#include "hts.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// maximum regidx position (0-based).  Used to represent the end point of
+// regions which do not explicitly set one.  regidx_push() also limits
+// positions passed to it to be no bigger than this.
+
+// Limit is set to ensure some internal values used by regidx keep within 32
+// bits and to stop the index from getting too big.
+
+#define REGIDX_MAX (1ULL << 35)
+
+typedef struct regidx_t regidx_t;
+typedef struct regitr_t
+{
+    hts_pos_t beg,end;
+    void *payload;
+    char *seq;
+    void *itr;
+}
+regitr_t;
+
+#define regitr_payload(itr,type_t) (*((type_t*)(itr)->payload))
+
+// Old API for backwards compatibility
+#define REGITR_START(itr) (itr).beg
+#define REGITR_END(itr)   (itr).end
+#define REGITR_PAYLOAD(itr,type_t) ((type_t*)(itr).payload)
+#define REGITR_OVERLAP(itr,from,to) regidx_overlap((itr));
+
+/*
+ *  regidx_parse_f - Function to parse one input line, such as regidx_parse_bed
+ *  or regidx_parse_tab below. The function is expected to set `chr_from` and
+ *  `chr_to` to point to first and last character of chromosome name and set
+ *  coordinates `beg` and `end` (0-based, inclusive). If regidx_init() was
+ *  called with non-zero payload_size, the `payload` points to a memory
+ *  location of the payload_size and `usr` is the data passed to regidx_init().
+ *  Any memory allocated by the function will be freed by regidx_free_f called
+ *  by regidx_destroy().
+ *
+ *  Return value: 0 on success, -1 to skip a record, -2 on fatal error.
+ */
+typedef int  (*regidx_parse_f)(const char *line, char **chr_beg, char **chr_end, hts_pos_t *beg, hts_pos_t *end, void *payload, void *usr);
+typedef void (*regidx_free_f)(void *payload);
+
+/*
+ *  A note about the parsers:
+ *      - leading spaces are ignored
+ *      - lines starting with "#" are ignored
+ */
+HTSLIB_EXPORT
+int regidx_parse_bed(const char*,char**,char**,hts_pos_t*,hts_pos_t*,void*,void*);   // CHROM or whitespace-sepatated CHROM,FROM,TO (0-based,right-open)
+HTSLIB_EXPORT
+int regidx_parse_tab(const char*,char**,char**,hts_pos_t*,hts_pos_t*,void*,void*);   // CHROM or whitespace-separated CHROM,POS (1-based, inclusive)
+HTSLIB_EXPORT
+int regidx_parse_reg(const char*,char**,char**,hts_pos_t*,hts_pos_t*,void*,void*);   // CHROM, CHROM:POS, CHROM:FROM-TO, CHROM:FROM- (1-based, inclusive)
+HTSLIB_EXPORT
+int regidx_parse_vcf(const char*,char**,char**,hts_pos_t*,hts_pos_t*,void*,void*);
+
+/*
+ *  regidx_init() - creates new index
+ *  regidx_init_string() - creates new index, from a string rather than from a file
+ *
+ *  @param fname:  input file name or NULL if regions will be added one-by-one via regidx_insert()
+ *  @param parsef: regidx_parse_bed, regidx_parse_tab or see description of regidx_parse_f. If NULL,
+ *                 the format will be autodected, currently either regidx_parse_tab (the default) or
+ *                 regidx_parse_bed (file must be named 'bed' or 'bed.gz') will be used. Note that
+ *                 the exact autodetection algorithm will change.
+ *  @param freef:  NULL or see description of regidx_parse_f
+ *  @param payload_size: 0 with regidx_parse_bed, regidx_parse_tab or see regidx_parse_f
+ *  @param usr:    optional user data passed to regidx_parse_f
+ *
+ *  Returns index on success or NULL on error.
+ *
+ *  The regidx_t index struct returned by a successful call should be freed
+ *  via regidx_destroy() when it is no longer needed.
+ */
+HTSLIB_EXPORT
+regidx_t *regidx_init(const char *fname, regidx_parse_f parsef, regidx_free_f freef, size_t payload_size, void *usr);
+HTSLIB_EXPORT
+regidx_t *regidx_init_string(const char *string, regidx_parse_f parsef, regidx_free_f freef, size_t payload_size, void *usr);
+
+/*
+ *  regidx_destroy() - free memory allocated by regidx_init
+ */
+HTSLIB_EXPORT
+void regidx_destroy(regidx_t *idx);
+
+/*
+ *  regidx_overlap() - check overlap of the location chr:from-to with regions
+ *  @param beg,end:     0-based start, end coordinate (inclusive)
+ *  @param itr:         pointer to iterator, can be NULL if regidx_loop not needed
+ *
+ *  Returns 0 if there is no overlap or 1 if overlap is found. The overlapping
+ *  regions can be iterated as shown in the example above.
+ */
+HTSLIB_EXPORT
+int regidx_overlap(regidx_t *idx, const char *chr, hts_pos_t beg, hts_pos_t end, regitr_t *itr);
+
+/*
+ *  regidx_insert() - add a new region.
+ *  regidx_insert_list() - add new regions from a list
+ *  regidx_push() - low level insertion of a new region
+ *
+ *  Returns 0 on success or -1 on error.
+ */
+HTSLIB_EXPORT
+int regidx_insert(regidx_t *idx, char *line);
+HTSLIB_EXPORT
+int regidx_insert_list(regidx_t *idx, char *line, char delim);
+HTSLIB_EXPORT
+int regidx_push(regidx_t *idx, char *chr_beg, char *chr_end, hts_pos_t beg, hts_pos_t end, void *payload);
+
+/*
+ *  regidx_seq_names() - return list of all sequence names
+ */
+HTSLIB_EXPORT
+char **regidx_seq_names(regidx_t *idx, int *n);
+
+/*
+ *  regidx_seq_nregs() - number of regions
+ *  regidx_nregs()  - total number of regions
+ */
+HTSLIB_EXPORT
+int regidx_seq_nregs(regidx_t *idx, const char *seq);
+
+HTSLIB_EXPORT
+int regidx_nregs(regidx_t *idx);
+
+/*
+ *  regitr_init() - initialize an iterator. The idx parameter is required only
+ *                  with regitr_loop. If only regitr_overlap is called, NULL
+ *                  can be given.
+ *
+ *                  The regitr_t struct returned by a successful regitr_init()
+ *                  call should be freed via regitr_destroy() when it is no
+ *                  longer needed.
+ *
+ *  regitr_reset() - initialize an iterator for a repeated regitr_loop cycle.
+ *                  Not required with regitr_overlap.
+ */
+HTSLIB_EXPORT
+regitr_t *regitr_init(regidx_t *idx);
+HTSLIB_EXPORT
+void regitr_destroy(regitr_t *itr);
+HTSLIB_EXPORT
+void regitr_reset(regidx_t *idx, regitr_t *itr);
+
+/*
+ *  regitr_overlap() - next overlapping region
+ *  Returns 0 when done or 1 when itr is set to next region
+ */
+HTSLIB_EXPORT
+int regitr_overlap(regitr_t *itr);
+
+/*
+ *  regitr_loop() - loop over all regions
+ *  Returns 0 when done or 1 when itr is set to next region
+ */
+HTSLIB_EXPORT
+int regitr_loop(regitr_t *itr);
+
+/*
+ *  regitr_copy() - create a copy of an iterator for a repeated iteration with regitr_loop
+ */
+HTSLIB_EXPORT
+void regitr_copy(regitr_t *dst, regitr_t *src);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/ext/htslib/htslib/sam.h b/ext/htslib/htslib/sam.h
new file mode 100644
index 0000000..0da5f04
--- /dev/null
+++ b/ext/htslib/htslib/sam.h
@@ -0,0 +1,2427 @@
+/// @file htslib/sam.h
+/// High-level SAM/BAM/CRAM sequence file operations.
+/*
+    Copyright (C) 2008, 2009, 2013-2023 Genome Research Ltd.
+    Copyright (C) 2010, 2012, 2013 Broad Institute.
+
+    Author: Heng Li <lh3@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#ifndef HTSLIB_SAM_H
+#define HTSLIB_SAM_H
+
+#include <errno.h>
+#include <stdint.h>
+#include <sys/types.h>
+#include "hts.h"
+#include "hts_endian.h"
+
+// Ensure ssize_t exists within this header. All #includes must precede this,
+// and ssize_t must be undefined again at the end of this header.
+#if defined _MSC_VER && defined _INTPTR_T_DEFINED && !defined _SSIZE_T_DEFINED && !defined ssize_t
+#define HTSLIB_SSIZE_T
+#define ssize_t intptr_t
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/// Highest SAM format version supported by this library
+#define SAM_FORMAT_VERSION "1.6"
+
+/***************************
+ *** SAM/BAM/CRAM header ***
+ ***************************/
+
+/*! @typedef
+ * @abstract Header extension structure, grouping a collection
+ *  of hash tables that contain the parsed header data.
+ */
+
+typedef struct sam_hrecs_t sam_hrecs_t;
+
+/*! @typedef
+ @abstract Structure for the alignment header.
+ @field n_targets   number of reference sequences
+ @field l_text      length of the plain text in the header (may be zero if
+                    the header has been edited)
+ @field target_len  lengths of the reference sequences
+ @field target_name names of the reference sequences
+ @field text        plain text (may be NULL if the header has been edited)
+ @field sdict       header dictionary
+ @field hrecs       pointer to the extended header struct (internal use only)
+ @field ref_count   reference count
+
+ @note The text and l_text fields are included for backwards compatibility.
+ These fields may be set to NULL and zero respectively as a side-effect
+ of calling some header API functions.  New code that needs to access the
+ header text should use the sam_hdr_str() and sam_hdr_length() functions
+ instead of these fields.
+ */
+
+typedef struct sam_hdr_t {
+    int32_t n_targets, ignore_sam_err;
+    size_t l_text;
+    uint32_t *target_len;
+    const int8_t *cigar_tab HTS_DEPRECATED("Use bam_cigar_table[] instead");
+    char **target_name;
+    char *text;
+    void *sdict;
+    sam_hrecs_t *hrecs;
+    uint32_t ref_count;
+} sam_hdr_t;
+
+/*! @typedef
+ * @abstract Old name for compatibility with existing code.
+ */
+typedef sam_hdr_t bam_hdr_t;
+
+/****************************
+ *** CIGAR related macros ***
+ ****************************/
+
+#define BAM_CMATCH      0
+#define BAM_CINS        1
+#define BAM_CDEL        2
+#define BAM_CREF_SKIP   3
+#define BAM_CSOFT_CLIP  4
+#define BAM_CHARD_CLIP  5
+#define BAM_CPAD        6
+#define BAM_CEQUAL      7
+#define BAM_CDIFF       8
+#define BAM_CBACK       9
+
+#define BAM_CIGAR_STR   "MIDNSHP=XB"
+#define BAM_CIGAR_SHIFT 4
+#define BAM_CIGAR_MASK  0xf
+#define BAM_CIGAR_TYPE  0x3C1A7
+
+/*! @abstract Table for converting a CIGAR operator character to BAM_CMATCH etc.
+Result is operator code or -1. Be sure to cast the index if it is a plain char:
+    int op = bam_cigar_table[(unsigned char) ch];
+*/
+HTSLIB_EXPORT
+extern const int8_t bam_cigar_table[256];
+
+#define bam_cigar_op(c) ((c)&BAM_CIGAR_MASK)
+#define bam_cigar_oplen(c) ((c)>>BAM_CIGAR_SHIFT)
+// Note that BAM_CIGAR_STR is padded to length 16 bytes below so that
+// the array look-up will not fall off the end.  '?' is chosen as the
+// padding character so it's easy to spot if one is emitted, and will
+// result in a parsing failure (in sam_parse1(), at least) if read.
+#define bam_cigar_opchr(c) (BAM_CIGAR_STR "??????" [bam_cigar_op(c)])
+#define bam_cigar_gen(l, o) ((l)<<BAM_CIGAR_SHIFT|(o))
+
+/* bam_cigar_type returns a bit flag with:
+ *   bit 1 set if the cigar operation consumes the query
+ *   bit 2 set if the cigar operation consumes the reference
+ *
+ * For reference, the unobfuscated truth table for this function is:
+ * BAM_CIGAR_TYPE  QUERY  REFERENCE
+ * --------------------------------
+ * BAM_CMATCH      1      1
+ * BAM_CINS        1      0
+ * BAM_CDEL        0      1
+ * BAM_CREF_SKIP   0      1
+ * BAM_CSOFT_CLIP  1      0
+ * BAM_CHARD_CLIP  0      0
+ * BAM_CPAD        0      0
+ * BAM_CEQUAL      1      1
+ * BAM_CDIFF       1      1
+ * BAM_CBACK       0      0
+ * --------------------------------
+ */
+#define bam_cigar_type(o) (BAM_CIGAR_TYPE>>((o)<<1)&3) // bit 1: consume query; bit 2: consume reference
+
+/*! @abstract the read is paired in sequencing, no matter whether it is mapped in a pair */
+#define BAM_FPAIRED        1
+/*! @abstract the read is mapped in a proper pair */
+#define BAM_FPROPER_PAIR   2
+/*! @abstract the read itself is unmapped; conflictive with BAM_FPROPER_PAIR */
+#define BAM_FUNMAP         4
+/*! @abstract the mate is unmapped */
+#define BAM_FMUNMAP        8
+/*! @abstract the read is mapped to the reverse strand */
+#define BAM_FREVERSE      16
+/*! @abstract the mate is mapped to the reverse strand */
+#define BAM_FMREVERSE     32
+/*! @abstract this is read1 */
+#define BAM_FREAD1        64
+/*! @abstract this is read2 */
+#define BAM_FREAD2       128
+/*! @abstract not primary alignment */
+#define BAM_FSECONDARY   256
+/*! @abstract QC failure */
+#define BAM_FQCFAIL      512
+/*! @abstract optical or PCR duplicate */
+#define BAM_FDUP        1024
+/*! @abstract supplementary alignment */
+#define BAM_FSUPPLEMENTARY 2048
+
+/*************************
+ *** Alignment records ***
+ *************************/
+
+/*
+ * Assumptions made here.  While pos can be 64-bit, no sequence
+ * itself is that long, but due to ref skip CIGAR fields it
+ * may span more than that.  (CIGAR itself is 28-bit len + 4 bit
+ * type, but in theory we can combine multiples together.)
+ *
+ * Mate position and insert size also need to be 64-bit, but
+ * we won't accept more than 32-bit for tid.
+ *
+ * The bam1_core_t structure is the *in memory* layout and not
+ * the same as the on-disk format.  64-bit changes here permit
+ * SAM to work with very long chromosomes and permit BAM and CRAM
+ * to seamlessly update in the future without further API/ABI
+ * revisions.
+ */
+
+/*! @typedef
+ @abstract Structure for core alignment information.
+ @field  pos     0-based leftmost coordinate
+ @field  tid     chromosome ID, defined by sam_hdr_t
+ @field  bin     bin calculated by bam_reg2bin()
+ @field  qual    mapping quality
+ @field  l_extranul length of extra NULs between qname & cigar (for alignment)
+ @field  flag    bitwise flag
+ @field  l_qname length of the query name
+ @field  n_cigar number of CIGAR operations
+ @field  l_qseq  length of the query sequence (read)
+ @field  mtid    chromosome ID of next read in template, defined by sam_hdr_t
+ @field  mpos    0-based leftmost coordinate of next read in template
+ @field  isize   observed template length ("insert size")
+ */
+typedef struct bam1_core_t {
+    hts_pos_t pos;
+    int32_t tid;
+    uint16_t bin; // NB: invalid on 64-bit pos
+    uint8_t qual;
+    uint8_t l_extranul;
+    uint16_t flag;
+    uint16_t l_qname;
+    uint32_t n_cigar;
+    int32_t l_qseq;
+    int32_t mtid;
+    hts_pos_t mpos;
+    hts_pos_t isize;
+} bam1_core_t;
+
+/*! @typedef
+ @abstract Structure for one alignment.
+ @field  core       core information about the alignment
+ @field  id
+ @field  data       all variable-length data, concatenated; structure: qname-cigar-seq-qual-aux
+ @field  l_data     current length of bam1_t::data
+ @field  m_data     maximum length of bam1_t::data
+ @field  mempolicy  memory handling policy, see bam_set_mempolicy()
+
+ @discussion Notes:
+
+ 1. The data blob should be accessed using bam_get_qname, bam_get_cigar,
+    bam_get_seq, bam_get_qual and bam_get_aux macros.  These returns pointers
+    to the start of each type of data.
+ 2. qname is terminated by one to four NULs, so that the following
+    cigar data is 32-bit aligned; core.l_qname includes these trailing NULs,
+    while core.l_extranul counts the excess NULs (so 0 <= l_extranul <= 3).
+ 3. Cigar data is encoded 4 bytes per CIGAR operation.
+    See the bam_cigar_* macros for manipulation.
+ 4. seq is nibble-encoded according to bam_nt16_table.
+    See the bam_seqi macro for retrieving individual bases.
+ 5. Per base qualities are stored in the Phred scale with no +33 offset.
+    Ie as per the BAM specification and not the SAM ASCII printable method.
+ */
+typedef struct bam1_t {
+    bam1_core_t core;
+    uint64_t id;
+    uint8_t *data;
+    int l_data;
+    uint32_t m_data;
+    uint32_t mempolicy:2, :30 /* Reserved */;
+} bam1_t;
+
+/*! @function
+ @abstract  Get whether the query is on the reverse strand
+ @param  b  pointer to an alignment
+ @return    boolean true if query is on the reverse strand
+ */
+#define bam_is_rev(b) (((b)->core.flag&BAM_FREVERSE) != 0)
+/*! @function
+ @abstract  Get whether the query's mate is on the reverse strand
+ @param  b  pointer to an alignment
+ @return    boolean true if query's mate on the reverse strand
+ */
+#define bam_is_mrev(b) (((b)->core.flag&BAM_FMREVERSE) != 0)
+/*! @function
+ @abstract  Get the name of the query
+ @param  b  pointer to an alignment
+ @return    pointer to the name string, null terminated
+ */
+#define bam_get_qname(b) ((char*)(b)->data)
+/*! @function
+ @abstract  Get the CIGAR array
+ @param  b  pointer to an alignment
+ @return    pointer to the CIGAR array
+
+ @discussion In the CIGAR array, each element is a 32-bit integer. The
+ lower 4 bits gives a CIGAR operation and the higher 28 bits keep the
+ length of a CIGAR.
+ */
+#define bam_get_cigar(b) ((uint32_t*)((b)->data + (b)->core.l_qname))
+/*! @function
+ @abstract  Get query sequence
+ @param  b  pointer to an alignment
+ @return    pointer to sequence
+
+ @discussion Each base is encoded in 4 bits: 1 for A, 2 for C, 4 for G,
+ 8 for T and 15 for N. Two bases are packed in one byte with the base
+ at the higher 4 bits having smaller coordinate on the read. It is
+ recommended to use bam_seqi() macro to get the base.
+ */
+#define bam_get_seq(b)   ((b)->data + ((b)->core.n_cigar<<2) + (b)->core.l_qname)
+/*! @function
+ @abstract  Get query quality
+ @param  b  pointer to an alignment
+ @return    pointer to quality string
+ */
+#define bam_get_qual(b)  ((b)->data + ((b)->core.n_cigar<<2) + (b)->core.l_qname + (((b)->core.l_qseq + 1)>>1))
+/*! @function
+ @abstract  Get auxiliary data
+ @param  b  pointer to an alignment
+ @return    pointer to the concatenated auxiliary data
+ */
+#define bam_get_aux(b)   ((b)->data + ((b)->core.n_cigar<<2) + (b)->core.l_qname + (((b)->core.l_qseq + 1)>>1) + (b)->core.l_qseq)
+/*! @function
+ @abstract  Get length of auxiliary data
+ @param  b  pointer to an alignment
+ @return    length of the concatenated auxiliary data
+ */
+#define bam_get_l_aux(b) ((b)->l_data - ((b)->core.n_cigar<<2) - (b)->core.l_qname - (b)->core.l_qseq - (((b)->core.l_qseq + 1)>>1))
+/*! @function
+ @abstract  Get a base on read
+ @param  s  Query sequence returned by bam_get_seq()
+ @param  i  The i-th position, 0-based
+ @return    4-bit integer representing the base.
+ */
+#define bam_seqi(s, i) ((s)[(i)>>1] >> ((~(i)&1)<<2) & 0xf)
+/*!
+ @abstract  Modifies a single base in the bam structure.
+ @param s   Query sequence returned by bam_get_seq()
+ @param i   The i-th position, 0-based
+ @param b   Base in nt16 nomenclature (see seq_nt16_table)
+*/
+#define bam_set_seqi(s,i,b) ((s)[(i)>>1] = ((s)[(i)>>1] & (0xf0 >> ((~(i)&1)<<2))) | ((b)<<((~(i)&1)<<2)))
+
+/**************************
+ *** Exported functions ***
+ **************************/
+
+/***************
+ *** BAM I/O ***
+ ***************/
+
+/* Header */
+
+/// Generates a new unpopulated header structure.
+/*!
+ *
+ * @return  A valid pointer to new header on success, NULL on failure
+ *
+ * The sam_hdr_t struct returned by a successful call should be freed
+ * via sam_hdr_destroy() when it is no longer needed.
+ */
+HTSLIB_EXPORT
+sam_hdr_t *sam_hdr_init(void);
+
+/// Read the header from a BAM compressed file.
+/*!
+ * @param fp  File pointer
+ * @return    A valid pointer to new header on success, NULL on failure
+ *
+ * This function only works with BAM files.  It is usually better to use
+ * sam_hdr_read(), which works on SAM, BAM and CRAM files.
+ *
+ * The sam_hdr_t struct returned by a successful call should be freed
+ * via sam_hdr_destroy() when it is no longer needed.
+ */
+HTSLIB_EXPORT
+sam_hdr_t *bam_hdr_read(BGZF *fp);
+
+/// Writes the header to a BAM file.
+/*!
+ * @param fp  File pointer
+ * @param h   Header pointer
+ * @return    0 on success, -1 on failure
+ *
+ * This function only works with BAM files.  Use sam_hdr_write() to
+ * write in any of the SAM, BAM or CRAM formats.
+ */
+HTSLIB_EXPORT
+int bam_hdr_write(BGZF *fp, const sam_hdr_t *h) HTS_RESULT_USED;
+
+/*!
+ * Frees the resources associated with a header.
+ */
+HTSLIB_EXPORT
+void sam_hdr_destroy(sam_hdr_t *h);
+
+/// Duplicate a header structure.
+/*!
+ * @return  A valid pointer to new header on success, NULL on failure
+ *
+ * The sam_hdr_t struct returned by a successful call should be freed
+ * via sam_hdr_destroy() when it is no longer needed.
+ */
+HTSLIB_EXPORT
+sam_hdr_t *sam_hdr_dup(const sam_hdr_t *h0);
+
+/*!
+ * @abstract Old names for compatibility with existing code.
+ */
+static inline sam_hdr_t *bam_hdr_init(void) { return sam_hdr_init(); }
+static inline void bam_hdr_destroy(sam_hdr_t *h) { sam_hdr_destroy(h); }
+static inline sam_hdr_t *bam_hdr_dup(const sam_hdr_t *h0) { return sam_hdr_dup(h0); }
+
+typedef htsFile samFile;
+
+/// Create a header from existing text.
+/*!
+ * @param l_text    Length of text
+ * @param text      Header text
+ * @return A populated sam_hdr_t structure on success; NULL on failure.
+ * @note The text field of the returned header will be NULL, and the l_text
+ * field will be zero.
+ *
+ * The sam_hdr_t struct returned by a successful call should be freed
+ * via sam_hdr_destroy() when it is no longer needed.
+ */
+HTSLIB_EXPORT
+sam_hdr_t *sam_hdr_parse(size_t l_text, const char *text);
+
+/// Read a header from a SAM, BAM or CRAM file.
+/*!
+ * @param fp    Pointer to a SAM, BAM or CRAM file handle
+ * @return  A populated sam_hdr_t struct on success; NULL on failure.
+ *
+ * The sam_hdr_t struct returned by a successful call should be freed
+ * via sam_hdr_destroy() when it is no longer needed.
+ */
+HTSLIB_EXPORT
+sam_hdr_t *sam_hdr_read(samFile *fp);
+
+/// Write a header to a SAM, BAM or CRAM file.
+/*!
+ * @param fp    SAM, BAM or CRAM file header
+ * @param h     Header structure to write
+ * @return  0 on success; -1 on failure
+ */
+HTSLIB_EXPORT
+int sam_hdr_write(samFile *fp, const sam_hdr_t *h) HTS_RESULT_USED;
+
+/// Returns the current length of the header text.
+/*!
+ * @return  >= 0 on success, SIZE_MAX on failure
+ */
+HTSLIB_EXPORT
+size_t sam_hdr_length(sam_hdr_t *h);
+
+/// Returns the text representation of the header.
+/*!
+ * @return  valid char pointer on success, NULL on failure
+ *
+ * The returned string is part of the header structure.  It will remain
+ * valid until a call to a header API function causes the string to be
+ * invalidated, or the header is destroyed.
+ *
+ * The caller should not attempt to free or realloc this pointer.
+ */
+HTSLIB_EXPORT
+const char *sam_hdr_str(sam_hdr_t *h);
+
+/// Returns the number of references in the header.
+/*!
+ * @return  >= 0 on success, -1 on failure
+ */
+HTSLIB_EXPORT
+int sam_hdr_nref(const sam_hdr_t *h);
+
+/* ==== Line level methods ==== */
+
+/// Add formatted lines to an existing header.
+/*!
+ * @param lines  Full SAM header record, eg "@SQ\tSN:foo\tLN:100", with
+ *               optional new-line. If it contains more than 1 line then
+ *               multiple lines will be added in order
+ * @param len    The maximum length of lines (if an early NUL is not
+ *               encountered). len may be 0 if unknown, in which case
+ *               lines must be NUL-terminated
+ * @return       0 on success, -1 on failure
+ *
+ * The lines will be appended to the end of the existing header
+ * (apart from HD, which always comes first).
+ */
+HTSLIB_EXPORT
+int sam_hdr_add_lines(sam_hdr_t *h, const char *lines, size_t len);
+
+/// Adds a single line to an existing header.
+/*!
+ * Specify type and one or more key,value pairs, ending with the NULL key.
+ * Eg. sam_hdr_add_line(h, "SQ", "SN", "foo", "LN", "100", NULL).
+ *
+ * @param type  Type of the added line. Eg. "SQ"
+ * @return      0 on success, -1 on failure
+ *
+ * The new line will be added immediately after any others of the same
+ * type, or at the end of the existing header if no lines of the
+ * given type currently exist.  The exception is HD lines, which always
+ * come first.  If an HD line already exists, it will be replaced.
+ */
+HTSLIB_EXPORT
+int sam_hdr_add_line(sam_hdr_t *h, const char *type, ...);
+
+/// Returns a complete line of formatted text for a given type and ID.
+/*!
+ * @param type      Type of the searched line. Eg. "SQ"
+ * @param ID_key    Tag key defining the line. Eg. "SN"
+ * @param ID_value  Tag value associated with the key above. Eg. "ref1"
+ * @param ks        kstring to hold the result
+ * @return          0 on success;
+ *                 -1 if no matching line is found
+ *                 -2 on other failures
+ *
+ * Puts a complete line of formatted text for a specific header type/ID
+ * combination into @p ks. If ID_key is NULL then it returns the first line of
+ * the specified type.
+ *
+ * Any existing content in @p ks will be overwritten.
+ */
+HTSLIB_EXPORT
+int sam_hdr_find_line_id(sam_hdr_t *h, const char *type,
+                      const char *ID_key, const char *ID_val, kstring_t *ks);
+
+/// Returns a complete line of formatted text for a given type and index.
+/*!
+ * @param type      Type of the searched line. Eg. "SQ"
+ * @param position  Index in lines of this type (zero-based)
+ * @param ks        kstring to hold the result
+ * @return          0 on success;
+ *                 -1 if no matching line is found
+ *                 -2 on other failures
+ *
+ * Puts a complete line of formatted text for a specific line into @p ks.
+ * The header line is selected using the @p type and @p position parameters.
+ *
+ * Any existing content in @p ks will be overwritten.
+ */
+HTSLIB_EXPORT
+int sam_hdr_find_line_pos(sam_hdr_t *h, const char *type,
+                          int pos, kstring_t *ks);
+
+/// Remove a line with given type / id from a header
+/*!
+ * @param type      Type of the searched line. Eg. "SQ"
+ * @param ID_key    Tag key defining the line. Eg. "SN"
+ * @param ID_value  Tag value associated with the key above. Eg. "ref1"
+ * @return          0 on success, -1 on error
+ *
+ * Remove a line from the header by specifying a tag:value that uniquely
+ * identifies the line, i.e. the @SQ line containing "SN:ref1".
+ *
+ * \@SQ line is uniquely identified by the SN tag.
+ * \@RG line is uniquely identified by the ID tag.
+ * \@PG line is uniquely identified by the ID tag.
+ * Eg. sam_hdr_remove_line_id(h, "SQ", "SN", "ref1")
+ *
+ * If no key:value pair is specified, the type MUST be followed by a NULL argument and
+ * the first line of the type will be removed, if any.
+ * Eg. sam_hdr_remove_line_id(h, "SQ", NULL, NULL)
+ *
+ * @note Removing \@PG lines is currently unsupported.
+ */
+HTSLIB_EXPORT
+int sam_hdr_remove_line_id(sam_hdr_t *h, const char *type, const char *ID_key, const char *ID_value);
+
+/// Remove nth line of a given type from a header
+/*!
+ * @param type     Type of the searched line. Eg. "SQ"
+ * @param position Index in lines of this type (zero-based). E.g. 3
+ * @return         0 on success, -1 on error
+ *
+ * Remove a line from the header by specifying the position in the type
+ * group, i.e. 3rd @SQ line.
+ */
+HTSLIB_EXPORT
+int sam_hdr_remove_line_pos(sam_hdr_t *h, const char *type, int position);
+
+/// Add or update tag key,value pairs in a header line.
+/*!
+ * @param type      Type of the searched line. Eg. "SQ"
+ * @param ID_key    Tag key defining the line. Eg. "SN"
+ * @param ID_value  Tag value associated with the key above. Eg. "ref1"
+ * @return          0 on success, -1 on error
+ *
+ * Adds or updates tag key,value pairs in a header line.
+ * Eg. for adding M5 tags to @SQ lines or updating sort order for the
+ * @HD line.
+ *
+ * Specify multiple key,value pairs ending in NULL. Eg.
+ * sam_hdr_update_line(h, "RG", "ID", "rg1", "DS", "description", "PG", "samtools", NULL)
+ *
+ * Attempting to update the record name (i.e. @SQ SN or @RG ID) will
+ * work as long as the new name is not already in use, however doing this
+ * on a file opened for reading may produce unexpected results.
+ *
+ * Renaming an @RG record in this way will only change the header.  Alignment
+ * records written later will not be updated automatically even if they
+ * reference the old read group name.
+ *
+ * Attempting to change an @PG ID tag is not permitted.
+ */
+HTSLIB_EXPORT
+int sam_hdr_update_line(sam_hdr_t *h, const char *type,
+        const char *ID_key, const char *ID_value, ...);
+
+/// Remove all lines of a given type from a header, except the one matching an ID
+/*!
+ * @param type      Type of the searched line. Eg. "SQ"
+ * @param ID_key    Tag key defining the line. Eg. "SN"
+ * @param ID_value  Tag value associated with the key above. Eg. "ref1"
+ * @return          0 on success, -1 on failure
+ *
+ * Remove all lines of type <type> from the header, except the one
+ * specified by tag:value, i.e. the @SQ line containing "SN:ref1".
+ *
+ * If no line matches the key:value ID, all lines of the given type are removed.
+ * To remove all lines of a given type, use NULL for both ID_key and ID_value.
+ */
+HTSLIB_EXPORT
+int sam_hdr_remove_except(sam_hdr_t *h, const char *type, const char *ID_key, const char *ID_value);
+
+/// Remove header lines of a given type, except those in a given ID set
+/*!
+ * @param type  Type of the searched line. Eg. "RG"
+ * @param id    Tag key defining the line. Eg. "ID"
+ * @param rh    Hash set initialised by the caller with the values to be kept.
+ *              See description for how to create this. If @p rh is NULL, all
+ *              lines of this type will be removed.
+ * @return      0 on success, -1 on failure
+ *
+ * Remove all lines of type @p type from the header, except the ones
+ * specified in the hash set @p rh. If @p rh is NULL, all lines of
+ * this type will be removed.
+ * Declaration of @p rh is done using KHASH_SET_INIT_STR macro. Eg.
+ * @code{.c}
+ *              #include "htslib/khash.h"
+ *              KHASH_SET_INIT_STR(keep)
+ *              typedef khash_t(keep) *keephash_t;
+ *
+ *              void your_method() {
+ *                  samFile *sf = sam_open("alignment.bam", "r");
+ *                  sam_hdr_t *h = sam_hdr_read(sf);
+ *                  keephash_t rh = kh_init(keep);
+ *                  int ret = 0;
+ *                  kh_put(keep, rh, strdup("chr2"), &ret);
+ *                  kh_put(keep, rh, strdup("chr3"), &ret);
+ *                  if (sam_hdr_remove_lines(h, "SQ", "SN", rh) == -1)
+ *                      fprintf(stderr, "Error removing lines\n");
+ *                  khint_t k;
+ *                  for (k = 0; k < kh_end(rh); ++k)
+ *                     if (kh_exist(rh, k)) free((char*)kh_key(rh, k));
+ *                  kh_destroy(keep, rh);
+ *                  sam_hdr_destroy(h);
+ *                  sam_close(sf);
+ *              }
+ * @endcode
+ *
+ */
+HTSLIB_EXPORT
+int sam_hdr_remove_lines(sam_hdr_t *h, const char *type, const char *id, void *rh);
+
+/// Count the number of lines for a given header type
+/*!
+ * @param h     BAM header
+ * @param type  Header type to count. Eg. "RG"
+ * @return  Number of lines of this type on success; -1 on failure
+ */
+HTSLIB_EXPORT
+int sam_hdr_count_lines(sam_hdr_t *h, const char *type);
+
+/// Index of the line for the types that have dedicated look-up tables (SQ, RG, PG)
+/*!
+ * @param h     BAM header
+ * @param type  Type of the searched line. Eg. "RG"
+ * @param key   The value of the identifying key. Eg. "rg1"
+ * @return  0-based index on success; -1 if line does not exist; -2 on failure
+ */
+HTSLIB_EXPORT
+int sam_hdr_line_index(sam_hdr_t *bh, const char *type, const char *key);
+
+/// Id key of the line for the types that have dedicated look-up tables (SQ, RG, PG)
+/*!
+ * @param h     BAM header
+ * @param type  Type of the searched line. Eg. "RG"
+ * @param pos   Zero-based index inside the type group. Eg. 2 (for the third RG line)
+ * @return  Valid key string on success; NULL on failure
+ */
+HTSLIB_EXPORT
+const char *sam_hdr_line_name(sam_hdr_t *bh, const char *type, int pos);
+
+/* ==== Key:val level methods ==== */
+
+/// Return the value associated with a key for a header line identified by ID_key:ID_val
+/*!
+ * @param type      Type of the line to which the tag belongs. Eg. "SQ"
+ * @param ID_key    Tag key defining the line. Eg. "SN". Can be NULL, if looking for the first line.
+ * @param ID_value  Tag value associated with the key above. Eg. "ref1". Can be NULL, if ID_key is NULL.
+ * @param key       Key of the searched tag. Eg. "LN"
+ * @param ks        kstring where the value will be written
+ * @return          0 on success
+ *                 -1 if the requested tag does not exist
+ *                 -2 on other errors
+ *
+ * Looks for a specific key in a single SAM header line and writes the
+ * associated value into @p ks.  The header line is selected using the ID_key
+ * and ID_value parameters.  Any pre-existing content in @p ks will be
+ * overwritten.
+ */
+HTSLIB_EXPORT
+int sam_hdr_find_tag_id(sam_hdr_t *h, const char *type, const char *ID_key, const char *ID_value, const char *key, kstring_t *ks);
+
+/// Return the value associated with a key for a header line identified by position
+/*!
+ * @param type      Type of the line to which the tag belongs. Eg. "SQ"
+ * @param position  Index in lines of this type (zero-based). E.g. 3
+ * @param key       Key of the searched tag. Eg. "LN"
+ * @param ks        kstring where the value will be written
+ * @return          0 on success
+ *                 -1 if the requested tag does not exist
+ *                 -2 on other errors
+ *
+ * Looks for a specific key in a single SAM header line and writes the
+ * associated value into @p ks.  The header line is selected using the @p type
+ * and @p position parameters.  Any pre-existing content in @p ks will be
+ * overwritten.
+ */
+HTSLIB_EXPORT
+int sam_hdr_find_tag_pos(sam_hdr_t *h, const char *type, int pos, const char *key, kstring_t *ks);
+
+/// Remove the key from the line identified by type, ID_key and ID_value.
+/*!
+ * @param type      Type of the line to which the tag belongs. Eg. "SQ"
+ * @param ID_key    Tag key defining the line. Eg. "SN"
+ * @param ID_value  Tag value associated with the key above. Eg. "ref1"
+ * @param key       Key of the targeted tag. Eg. "M5"
+ * @return          1 if the key was removed; 0 if it was not present; -1 on error
+ */
+HTSLIB_EXPORT
+int sam_hdr_remove_tag_id(sam_hdr_t *h, const char *type, const char *ID_key, const char *ID_value, const char *key);
+
+/// Get the target id for a given reference sequence name
+/*!
+ * @param ref  Reference name
+ * @return     Positive value on success,
+ *             -1 if unknown reference,
+ *             -2 if the header could not be parsed
+ *
+ * Looks up a reference sequence by name in the reference hash table
+ * and returns the numerical target id.
+ */
+HTSLIB_EXPORT
+int sam_hdr_name2tid(sam_hdr_t *h, const char *ref);
+
+/// Get the reference sequence name from a target index
+/*!
+ * @param tid  Target index
+ * @return     Valid reference name on success, NULL on failure
+ *
+ * Fetch the reference sequence name from the target name array,
+ * using the numerical target id.
+ */
+HTSLIB_EXPORT
+const char *sam_hdr_tid2name(const sam_hdr_t *h, int tid);
+
+/// Get the reference sequence length from a target index
+/*!
+ * @param tid  Target index
+ * @return     Strictly positive value on success, 0 on failure
+ *
+ * Fetch the reference sequence length from the target length array,
+ * using the numerical target id.
+ */
+HTSLIB_EXPORT
+hts_pos_t sam_hdr_tid2len(const sam_hdr_t *h, int tid);
+
+/// Alias of sam_hdr_name2tid(), for backwards compatibility.
+/*!
+ * @param ref  Reference name
+ * @return     Positive value on success,
+ *             -1 if unknown reference,
+ *             -2 if the header could not be parsed
+ */
+static inline int bam_name2id(sam_hdr_t *h, const char *ref) { return sam_hdr_name2tid(h, ref); }
+
+/// Generate a unique \@PG ID: value
+/*!
+ * @param name  Name of the program. Eg. samtools
+ * @return      Valid ID on success, NULL on failure
+ *
+ * Returns a unique ID from a base name.  The string returned will remain
+ * valid until the next call to this function, or the header is destroyed.
+ * The caller should not attempt to free() or realloc() it.
+ */
+HTSLIB_EXPORT
+const char *sam_hdr_pg_id(sam_hdr_t *h, const char *name);
+
+/// Add an \@PG line.
+/*!
+ * @param name  Name of the program. Eg. samtools
+ * @return      0 on success, -1 on failure
+ *
+ * If we wish complete control over this use sam_hdr_add_line() directly. This
+ * function uses that, but attempts to do a lot of tedious house work for
+ * you too.
+ *
+ * - It will generate a suitable ID if the supplied one clashes.
+ * - It will generate multiple \@PG records if we have multiple PG chains.
+ *
+ * Call it as per sam_hdr_add_line() with a series of key,value pairs ending
+ * in NULL.
+ */
+HTSLIB_EXPORT
+int sam_hdr_add_pg(sam_hdr_t *h, const char *name, ...);
+
+/*!
+ * A function to help with construction of CL tags in @PG records.
+ * Takes an argc, argv pair and returns a single space-separated string.
+ * This string should be deallocated by the calling function.
+ *
+ * @return
+ * Returns malloced char * on success;
+ *         NULL on failure
+ */
+HTSLIB_EXPORT
+char *stringify_argv(int argc, char *argv[]);
+
+/// Increments the reference count on a header
+/*!
+ * This permits multiple files to share the same header, all calling
+ * sam_hdr_destroy when done, without causing errors for other open files.
+ */
+HTSLIB_EXPORT
+void sam_hdr_incr_ref(sam_hdr_t *h);
+
+/*
+ * Macros for changing the \@HD line. They eliminate the need to use NULL method arguments.
+ */
+
+/// Returns the SAM formatted text of the \@HD header line
+#define sam_hdr_find_hd(h, ks) sam_hdr_find_line_id((h), "HD", NULL, NULL, (ks))
+/// Returns the value associated with a given \@HD line tag
+#define sam_hdr_find_tag_hd(h, key, ks) sam_hdr_find_tag_id((h), "HD", NULL, NULL, (key), (ks))
+/// Adds or updates tags on the header \@HD line
+#define sam_hdr_update_hd(h, ...) sam_hdr_update_line((h), "HD", NULL, NULL, __VA_ARGS__, NULL)
+/// Removes the \@HD line tag with the given key
+#define sam_hdr_remove_tag_hd(h, key) sam_hdr_remove_tag_id((h), "HD", NULL, NULL, (key))
+
+/* Alignment */
+
+/// Create a new bam1_t alignment structure
+/**
+   @return An empty bam1_t structure on success, NULL on failure
+
+   The bam1_t struct returned by a successful call should be freed
+   via bam_destroy1() when it is no longer needed.
+ */
+HTSLIB_EXPORT
+bam1_t *bam_init1(void);
+
+/// Destroy a bam1_t structure
+/**
+   @param b  structure to destroy
+
+   Does nothing if @p b is NULL.  If not, all memory associated with @p b
+   will be freed, along with the structure itself.  @p b should not be
+   accessed after calling this function.
+ */
+HTSLIB_EXPORT
+void bam_destroy1(bam1_t *b);
+
+#define BAM_USER_OWNS_STRUCT 1
+#define BAM_USER_OWNS_DATA   2
+
+/// Set alignment record memory policy
+/**
+   @param b       Alignment record
+   @param policy  Desired policy
+
+   Allows the way HTSlib reallocates and frees bam1_t data to be
+   changed.  @policy can be set to the bitwise-or of the following
+   values:
+
+   \li \c BAM_USER_OWNS_STRUCT
+   If this is set then bam_destroy1() will not try to free the bam1_t struct.
+
+   \li \c BAM_USER_OWNS_DATA
+   If this is set, bam_destroy1() will not free the bam1_t::data pointer.
+   Also, functions which need to expand bam1_t::data memory will change
+   behaviour.  Instead of calling realloc() on the pointer, they will
+   allocate a new data buffer and copy any existing content in to it.
+   The existing memory will \b not be freed.  bam1_t::data will be
+   set to point to the new memory and the BAM_USER_OWNS_DATA flag will be
+   cleared.
+
+   BAM_USER_OWNS_STRUCT allows bam_destroy1() to be called on bam1_t
+   structures that are members of an array.
+
+   BAM_USER_OWNS_DATA can be used by applications that want more control
+   over where the variable-length parts of the bam record will be stored.
+   By preventing calls to free() and realloc(), it allows bam1_t::data
+   to hold pointers to memory that cannot be passed to those functions.
+
+   Example:  Read a block of alignment records, storing the variable-length
+   data in a single buffer and the records in an array.  Stop when either
+   the array or the buffer is full.
+
+   \code{.c}
+   #define MAX_RECS 1000
+   #define REC_LENGTH 400  // Average length estimate, to get buffer size
+   size_t bufsz = MAX_RECS * REC_LENGTH, nrecs, buff_used = 0;
+   bam1_t *recs = calloc(MAX_RECS, sizeof(bam1_t));
+   uint8_t *buffer = malloc(bufsz);
+   int res = 0, result = EXIT_FAILURE;
+   uint32_t new_m_data;
+
+   if (!recs || !buffer) goto cleanup;
+   for (nrecs = 0; nrecs < MAX_RECS; nrecs++) {
+      bam_set_mempolicy(&recs[nrecs], BAM_USER_OWNS_STRUCT|BAM_USER_OWNS_DATA);
+
+      // Set data pointer to unused part of buffer
+      recs[nrecs].data = &buffer[buff_used];
+
+      // Set m_data to size of unused part of buffer.  On 64-bit platforms it
+      // will be necessary to limit this to UINT32_MAX due to the size of
+      // bam1_t::m_data (not done here as our buffer is only 400K).
+      recs[nrecs].m_data = bufsz - buff_used;
+
+      // Read the record
+      res = sam_read1(file_handle, header, &recs[nrecs]);
+      if (res <= 0) break; // EOF or error
+
+      // Check if the record data didn't fit - if not, stop reading
+      if ((bam_get_mempolicy(&recs[nrecs]) & BAM_USER_OWNS_DATA) == 0) {
+         nrecs++; // Include last record in count
+         break;
+      }
+
+      // Adjust m_data to the space actually used.  If space is available,
+      // round up to eight bytes so the next record aligns nicely.
+      new_m_data = ((uint32_t) recs[nrecs].l_data + 7) & (~7U);
+      if (new_m_data < recs[nrecs].m_data) recs[nrecs].m_data = new_m_data;
+
+      buff_used += recs[nrecs].m_data;
+   }
+   if (res < 0) goto cleanup;
+   result = EXIT_SUCCESS;
+
+   // ... use data ...
+
+ cleanup:
+   if (recs) {
+      for (size_t i = 0; i < nrecs; i++)
+         bam_destroy1(&recs[i]);
+      free(recs);
+   }
+   free(buffer);
+
+   \endcode
+*/
+static inline void bam_set_mempolicy(bam1_t *b, uint32_t policy) {
+    b->mempolicy = policy;
+}
+
+/// Get alignment record memory policy
+/** @param b    Alignment record
+
+    See bam_set_mempolicy()
+ */
+static inline uint32_t bam_get_mempolicy(bam1_t *b) {
+    return b->mempolicy;
+}
+
+/// Read a BAM format alignment record
+/**
+   @param fp   BGZF file being read
+   @param b    Destination for the alignment data
+   @return number of bytes read on success
+           -1 at end of file
+           < -1 on failure
+
+   This function can only read BAM format files.  Most code should use
+   sam_read1() instead, which can be used with BAM, SAM and CRAM formats.
+*/
+HTSLIB_EXPORT
+int bam_read1(BGZF *fp, bam1_t *b) HTS_RESULT_USED;
+
+/// Write a BAM format alignment record
+/**
+   @param fp  BGZF file being written
+   @param b   Alignment record to write
+   @return number of bytes written on success
+           -1 on error
+
+   This function can only write BAM format files.  Most code should use
+   sam_write1() instead, which can be used with BAM, SAM and CRAM formats.
+*/
+HTSLIB_EXPORT
+int bam_write1(BGZF *fp, const bam1_t *b) HTS_RESULT_USED;
+
+/// Copy alignment record data
+/**
+   @param bdst  Destination alignment record
+   @param bsrc  Source alignment record
+   @return bdst on success; NULL on failure
+ */
+HTSLIB_EXPORT
+bam1_t *bam_copy1(bam1_t *bdst, const bam1_t *bsrc) HTS_RESULT_USED;
+
+/// Create a duplicate alignment record
+/**
+   @param bsrc  Source alignment record
+   @return Pointer to a new alignment record on success; NULL on failure
+
+   The bam1_t struct returned by a successful call should be freed
+   via bam_destroy1() when it is no longer needed.
+ */
+HTSLIB_EXPORT
+bam1_t *bam_dup1(const bam1_t *bsrc);
+
+/// Sets all components of an alignment structure
+/**
+   @param bam      Target alignment structure. Must be initialized by a call to bam_init1().
+                   The data field will be reallocated automatically as needed.
+   @param l_qname  Length of the query name. If set to 0, the placeholder query name "*" will be used.
+   @param qname    Query name, may be NULL if l_qname = 0
+   @param flag     Bitwise flag, a combination of the BAM_F* constants.
+   @param tid      Chromosome ID, defined by sam_hdr_t (a.k.a. RNAME).
+   @param pos      0-based leftmost coordinate.
+   @param mapq     Mapping quality.
+   @param n_cigar  Number of CIGAR operations.
+   @param cigar    CIGAR data, may be NULL if n_cigar = 0.
+   @param mtid     Chromosome ID of next read in template, defined by sam_hdr_t (a.k.a. RNEXT).
+   @param mpos     0-based leftmost coordinate of next read in template (a.k.a. PNEXT).
+   @param isize    Observed template length ("insert size") (a.k.a. TLEN).
+   @param l_seq    Length of the query sequence (read) and sequence quality string.
+   @param seq      Sequence, may be NULL if l_seq = 0.
+   @param qual     Sequence quality, may be NULL.
+   @param l_aux    Length to be reserved for auxiliary field data, may be 0.
+
+   @return >= 0 on success (number of bytes written to bam->data), negative (with errno set) on failure.
+*/
+HTSLIB_EXPORT
+int bam_set1(bam1_t *bam,
+             size_t l_qname, const char *qname,
+             uint16_t flag, int32_t tid, hts_pos_t pos, uint8_t mapq,
+             size_t n_cigar, const uint32_t *cigar,
+             int32_t mtid, hts_pos_t mpos, hts_pos_t isize,
+             size_t l_seq, const char *seq, const char *qual,
+             size_t l_aux);
+
+/// Calculate query length from CIGAR data
+/**
+   @param n_cigar   Number of items in @p cigar
+   @param cigar     CIGAR data
+   @return Query length
+
+   CIGAR data is stored as in the BAM format, i.e. (op_len << 4) | op
+   where op_len is the length in bases and op is a value between 0 and 8
+   representing one of the operations "MIDNSHP=X" (M = 0; X = 8)
+
+   This function returns the sum of the lengths of the M, I, S, = and X
+   operations in @p cigar (these are the operations that "consume" query
+   bases).  All other operations (including invalid ones) are ignored.
+
+   @note This return type of this function is hts_pos_t so that it can
+   correctly return the length of CIGAR sequences including many long
+   operations without overflow. However, other restrictions (notably the sizes
+   of bam1_core_t::l_qseq and bam1_t::data) limit the maximum query sequence
+   length supported by HTSlib to fewer than INT_MAX bases.
+ */
+HTSLIB_EXPORT
+hts_pos_t bam_cigar2qlen(int n_cigar, const uint32_t *cigar);
+
+/// Calculate reference length from CIGAR data
+/**
+   @param n_cigar   Number of items in @p cigar
+   @param cigar     CIGAR data
+   @return Reference length
+
+   CIGAR data is stored as in the BAM format, i.e. (op_len << 4) | op
+   where op_len is the length in bases and op is a value between 0 and 8
+   representing one of the operations "MIDNSHP=X" (M = 0; X = 8)
+
+   This function returns the sum of the lengths of the M, D, N, = and X
+   operations in @p cigar (these are the operations that "consume" reference
+   bases).  All other operations (including invalid ones) are ignored.
+ */
+HTSLIB_EXPORT
+hts_pos_t bam_cigar2rlen(int n_cigar, const uint32_t *cigar);
+
+/*!
+      @abstract Calculate the rightmost base position of an alignment on the
+      reference genome.
+
+      @param  b  pointer to an alignment
+      @return    the coordinate of the first base after the alignment, 0-based
+
+      @discussion For a mapped read, this is just b->core.pos + bam_cigar2rlen.
+      For an unmapped read (either according to its flags or if it has no cigar
+      string) or a read whose cigar string consumes no reference bases at all,
+      we return b->core.pos + 1 by convention.
+ */
+HTSLIB_EXPORT
+hts_pos_t bam_endpos(const bam1_t *b);
+
+HTSLIB_EXPORT
+int   bam_str2flag(const char *str);    /** returns negative value on error */
+
+HTSLIB_EXPORT
+char *bam_flag2str(int flag);   /** The string must be freed by the user */
+
+/*! @function
+ @abstract  Set the name of the query
+ @param  b  pointer to an alignment
+ @return    0 on success, -1 on failure
+ */
+HTSLIB_EXPORT
+int bam_set_qname(bam1_t *b, const char *qname);
+
+/*! @function
+ @abstract  Parse a CIGAR string into a uint32_t array
+ @param  in      [in]  pointer to the source string
+ @param  end     [out] address of the pointer to the new end of the input string
+                       can be NULL
+ @param  a_cigar [in/out]  address of the destination uint32_t buffer
+ @param  a_mem   [in/out]  address of the allocated number of buffer elements
+ @return         number of processed CIGAR operators; -1 on error
+ */
+HTSLIB_EXPORT
+ssize_t sam_parse_cigar(const char *in, char **end, uint32_t **a_cigar, size_t *a_mem);
+
+/*! @function
+ @abstract  Parse a CIGAR string into a bam1_t struct
+ @param  in      [in]  pointer to the source string
+ @param  end     [out] address of the pointer to the new end of the input string
+                       can be NULL
+ @param  b       [in/out]  address of the destination bam1_t struct
+ @return         number of processed CIGAR operators; -1 on error
+
+ @discussion The BAM record may be partial and empty of existing cigar, seq
+ and quality, as is the case during SAM parsing, or it may be an existing
+ BAM record in which case this function replaces the existing CIGAR field
+ and shuffles data accordingly.  A CIGAR of "*" will remove the CIGAR,
+ returning zero.
+ */
+HTSLIB_EXPORT
+ssize_t bam_parse_cigar(const char *in, char **end, bam1_t *b);
+
+/*************************
+ *** BAM/CRAM indexing ***
+ *************************/
+
+// These BAM iterator functions work only on BAM files.  To work with either
+// BAM or CRAM files use the sam_index_load() & sam_itr_*() functions.
+#define bam_itr_destroy(iter) hts_itr_destroy(iter)
+#define bam_itr_queryi(idx, tid, beg, end) sam_itr_queryi(idx, tid, beg, end)
+#define bam_itr_querys(idx, hdr, region) sam_itr_querys(idx, hdr, region)
+#define bam_itr_next(htsfp, itr, r) sam_itr_next((htsfp), (itr), (r))
+
+// Load/build .csi or .bai BAM index file.  Does not work with CRAM.
+// It is recommended to use the sam_index_* functions below instead.
+#define bam_index_load(fn) hts_idx_load((fn), HTS_FMT_BAI)
+#define bam_index_build(fn, min_shift) (sam_index_build((fn), (min_shift)))
+
+/// Initialise fp->idx for the current format type for SAM, BAM and CRAM types .
+/** @param fp        File handle for the data file being written.
+    @param h         Bam header structured (needed for BAI and CSI).
+    @param min_shift 0 for BAI, or larger for CSI (CSI defaults to 14).
+    @param fnidx     Filename to write index to.  This pointer must remain valid
+                     until after sam_idx_save is called.
+    @return          0 on success, <0 on failure.
+
+    @note This must be called after the header has been written, but before
+          any other data.
+*/
+HTSLIB_EXPORT
+int sam_idx_init(htsFile *fp, sam_hdr_t *h, int min_shift, const char *fnidx);
+
+/// Writes the index initialised with sam_idx_init to disk.
+/** @param fp        File handle for the data file being written.
+    @return          0 on success, <0 on failure.
+*/
+HTSLIB_EXPORT
+int sam_idx_save(htsFile *fp) HTS_RESULT_USED;
+
+/// Load a BAM (.csi or .bai) or CRAM (.crai) index file
+/** @param fp  File handle of the data file whose index is being opened
+    @param fn  BAM/CRAM/etc filename to search alongside for the index file
+    @return  The index, or NULL if an error occurred.
+
+Equivalent to sam_index_load3(fp, fn, NULL, HTS_IDX_SAVE_REMOTE);
+*/
+HTSLIB_EXPORT
+hts_idx_t *sam_index_load(htsFile *fp, const char *fn);
+
+/// Load a specific BAM (.csi or .bai) or CRAM (.crai) index file
+/** @param fp     File handle of the data file whose index is being opened
+    @param fn     BAM/CRAM/etc data file filename
+    @param fnidx  Index filename, or NULL to search alongside @a fn
+    @return  The index, or NULL if an error occurred.
+
+Equivalent to sam_index_load3(fp, fn, fnidx, HTS_IDX_SAVE_REMOTE);
+*/
+HTSLIB_EXPORT
+hts_idx_t *sam_index_load2(htsFile *fp, const char *fn, const char *fnidx);
+
+/// Load or stream a BAM (.csi or .bai) or CRAM (.crai) index file
+/** @param fp     File handle of the data file whose index is being opened
+    @param fn     BAM/CRAM/etc data file filename
+    @param fnidx  Index filename, or NULL to search alongside @a fn
+    @param flags  Flags to alter behaviour (see description)
+    @return  The index, or NULL if an error occurred.
+
+The @p flags parameter can be set to a combination of the following values:
+
+        HTS_IDX_SAVE_REMOTE   Save a local copy of any remote indexes
+        HTS_IDX_SILENT_FAIL   Fail silently if the index is not present
+
+Note that HTS_IDX_SAVE_REMOTE has no effect for remote CRAM indexes.  They
+are always downloaded and never cached locally.
+
+The index struct returned by a successful call should be freed
+via hts_idx_destroy() when it is no longer needed.
+*/
+HTSLIB_EXPORT
+hts_idx_t *sam_index_load3(htsFile *fp, const char *fn, const char *fnidx, int flags);
+
+/// Generate and save an index file
+/** @param fn        Input BAM/etc filename, to which .csi/etc will be added
+    @param min_shift Positive to generate CSI, or 0 to generate BAI
+    @return  0 if successful, or negative if an error occurred (usually -1; or
+             -2: opening fn failed; -3: format not indexable; -4:
+             failed to create and/or save the index)
+*/
+HTSLIB_EXPORT
+int sam_index_build(const char *fn, int min_shift) HTS_RESULT_USED;
+
+/// Generate and save an index to a specific file
+/** @param fn        Input BAM/CRAM/etc filename
+    @param fnidx     Output filename, or NULL to add .bai/.csi/etc to @a fn
+    @param min_shift Positive to generate CSI, or 0 to generate BAI
+    @return  0 if successful, or negative if an error occurred (see
+             sam_index_build for error codes)
+*/
+HTSLIB_EXPORT
+int sam_index_build2(const char *fn, const char *fnidx, int min_shift) HTS_RESULT_USED;
+
+/// Generate and save an index to a specific file
+/** @param fn        Input BAM/CRAM/etc filename
+    @param fnidx     Output filename, or NULL to add .bai/.csi/etc to @a fn
+    @param min_shift Positive to generate CSI, or 0 to generate BAI
+    @param nthreads  Number of threads to use when building the index
+    @return  0 if successful, or negative if an error occurred (see
+             sam_index_build for error codes)
+*/
+HTSLIB_EXPORT
+int sam_index_build3(const char *fn, const char *fnidx, int min_shift, int nthreads) HTS_RESULT_USED;
+
+/// Free a SAM iterator
+/// @param iter     Iterator to free
+#define sam_itr_destroy(iter) hts_itr_destroy(iter)
+
+/// Create a BAM/CRAM iterator
+/** @param idx     Index
+    @param tid     Target id
+    @param beg     Start position in target
+    @param end     End position in target
+    @return An iterator on success; NULL on failure
+
+The following special values (defined in htslib/hts.h)can be used for @p tid.
+When using one of these values, @p beg and @p end are ignored.
+
+  HTS_IDX_NOCOOR iterates over unmapped reads sorted at the end of the file
+  HTS_IDX_START  iterates over the entire file
+  HTS_IDX_REST   iterates from the current position to the end of the file
+  HTS_IDX_NONE   always returns "no more alignment records"
+
+When using HTS_IDX_REST or HTS_IDX_NONE, NULL can be passed in to @p idx.
+ */
+HTSLIB_EXPORT
+hts_itr_t *sam_itr_queryi(const hts_idx_t *idx, int tid, hts_pos_t beg, hts_pos_t end);
+
+/// Create a SAM/BAM/CRAM iterator
+/** @param idx     Index
+    @param hdr     Header
+    @param region  Region specification
+    @return An iterator on success; NULL on failure
+
+Regions are parsed by hts_parse_reg(), and take one of the following forms:
+
+region          | Outputs
+--------------- | -------------
+REF             | All reads with RNAME REF
+REF:            | All reads with RNAME REF
+REF:START       | Reads with RNAME REF overlapping START to end of REF
+REF:-END        | Reads with RNAME REF overlapping start of REF to END
+REF:START-END   | Reads with RNAME REF overlapping START to END
+.               | All reads from the start of the file
+*               | Unmapped reads at the end of the file (RNAME '*' in SAM)
+
+The form `REF:` should be used when the reference name itself contains a colon.
+
+Note that SAM files must be bgzf-compressed for iterators to work.
+ */
+HTSLIB_EXPORT
+hts_itr_t *sam_itr_querys(const hts_idx_t *idx, sam_hdr_t *hdr, const char *region);
+
+/// Create a multi-region iterator
+/** @param idx       Index
+    @param hdr       Header
+    @param reglist   Array of regions to iterate over
+    @param regcount  Number of items in reglist
+
+Each @p reglist entry should have the reference name in the `reg` field, an
+array of regions for that reference in `intervals` and the number of items
+in `intervals` should be stored in `count`.  No other fields need to be filled
+in.
+
+The iterator will return all reads overlapping the given regions.  If a read
+overlaps more than one region, it will only be returned once.
+ */
+HTSLIB_EXPORT
+hts_itr_t *sam_itr_regions(const hts_idx_t *idx, sam_hdr_t *hdr, hts_reglist_t *reglist, unsigned int regcount);
+
+/// Create a multi-region iterator
+/** @param idx       Index
+    @param hdr       Header
+    @param regarray  Array of ref:interval region specifiers
+    @param regcount  Number of items in regarray
+
+Each @p regarray entry is parsed by hts_parse_reg(), and takes one of the
+following forms:
+
+region          | Outputs
+--------------- | -------------
+REF             | All reads with RNAME REF
+REF:            | All reads with RNAME REF
+REF:START       | Reads with RNAME REF overlapping START to end of REF
+REF:-END        | Reads with RNAME REF overlapping start of REF to END
+REF:START-END   | Reads with RNAME REF overlapping START to END
+.               | All reads from the start of the file
+*               | Unmapped reads at the end of the file (RNAME '*' in SAM)
+
+The form `REF:` should be used when the reference name itself contains a colon.
+
+The iterator will return all reads overlapping the given regions.  If a read
+overlaps more than one region, it will only be returned once.
+ */
+HTSLIB_EXPORT
+hts_itr_t *sam_itr_regarray(const hts_idx_t *idx, sam_hdr_t *hdr, char **regarray, unsigned int regcount);
+
+/// Get the next read from a SAM/BAM/CRAM iterator
+/** @param htsfp       Htsfile pointer for the input file
+    @param itr         Iterator
+    @param r           Pointer to a bam1_t struct
+    @return >= 0 on success; -1 when there is no more data; < -1 on error
+ */
+static inline int sam_itr_next(htsFile *htsfp, hts_itr_t *itr, bam1_t *r) {
+    if (!htsfp->is_bgzf && !htsfp->is_cram) {
+        hts_log_error("%s not BGZF compressed", htsfp->fn ? htsfp->fn : "File");
+        return -2;
+    }
+    if (!itr) {
+        hts_log_error("Null iterator");
+        return -2;
+    }
+
+    if (itr->multi)
+        return hts_itr_multi_next(htsfp, itr, r);
+    else
+        return hts_itr_next(htsfp->is_bgzf ? htsfp->fp.bgzf : NULL, itr, r, htsfp);
+}
+
+/// Get the next read from a BAM/CRAM multi-iterator
+/** @param htsfp       Htsfile pointer for the input file
+    @param itr         Iterator
+    @param r           Pointer to a bam1_t struct
+    @return >= 0 on success; -1 when there is no more data; < -1 on error
+ */
+#define sam_itr_multi_next(htsfp, itr, r) sam_itr_next(htsfp, itr, r)
+
+HTSLIB_EXPORT
+const char *sam_parse_region(sam_hdr_t *h, const char *s, int *tid,
+                             hts_pos_t *beg, hts_pos_t *end, int flags);
+
+    /***************
+     *** SAM I/O ***
+     ***************/
+
+    #define sam_open(fn, mode) (hts_open((fn), (mode)))
+    #define sam_open_format(fn, mode, fmt) (hts_open_format((fn), (mode), (fmt)))
+    #define sam_flush(fp) hts_flush((fp))
+    #define sam_close(fp) hts_close(fp)
+
+    HTSLIB_EXPORT
+    int sam_open_mode(char *mode, const char *fn, const char *format);
+
+    // A version of sam_open_mode that can handle ,key=value options.
+    // The format string is allocated and returned, to be freed by the caller.
+    // Prefix should be "r" or "w",
+    HTSLIB_EXPORT
+    char *sam_open_mode_opts(const char *fn,
+                             const char *mode,
+                             const char *format);
+
+    HTSLIB_EXPORT
+    int sam_hdr_change_HD(sam_hdr_t *h, const char *key, const char *val);
+
+    HTSLIB_EXPORT
+    int sam_parse1(kstring_t *s, sam_hdr_t *h, bam1_t *b) HTS_RESULT_USED;
+    HTSLIB_EXPORT
+    int sam_format1(const sam_hdr_t *h, const bam1_t *b, kstring_t *str) HTS_RESULT_USED;
+
+/// sam_read1 - Read a record from a file
+/** @param fp   Pointer to the source file
+ *  @param h    Pointer to the header previously read (fully or partially)
+ *  @param b    Pointer to the record placeholder
+ *  @return >= 0 on successfully reading a new record, -1 on end of stream, < -1 on error
+ */
+    HTSLIB_EXPORT
+    int sam_read1(samFile *fp, sam_hdr_t *h, bam1_t *b) HTS_RESULT_USED;
+/// sam_write1 - Write a record to a file
+/** @param fp    Pointer to the destination file
+ *  @param h     Pointer to the header structure previously read
+ *  @param b     Pointer to the record to be written
+ *  @return >= 0 on successfully writing the record, -ve on error
+ */
+    HTSLIB_EXPORT
+    int sam_write1(samFile *fp, const sam_hdr_t *h, const bam1_t *b) HTS_RESULT_USED;
+
+// Forward declaration, see hts_expr.h for full.
+struct hts_filter_t;
+
+/// sam_passes_filter - Checks whether a record passes an hts_filter.
+/** @param h      Pointer to the header structure previously read
+ *  @param b      Pointer to the BAM record to be checked
+ *  @param filt   Pointer to the filter, created from hts_filter_init.
+ *  @return       1 if passes, 0 if not, and <0 on error.
+ */
+HTSLIB_EXPORT
+int sam_passes_filter(const sam_hdr_t *h, const bam1_t *b,
+                      struct hts_filter_t *filt);
+
+    /*************************************
+     *** Manipulating auxiliary fields ***
+     *************************************/
+
+/// Converts a BAM aux tag to SAM format
+/*
+ * @param key  Two letter tag key
+ * @param type Single letter type code: ACcSsIifHZB.
+ * @param tag  Tag data pointer, in BAM format
+ * @param end  Pointer to end of bam record (largest extent of tag)
+ * @param ks   kstring to write the formatted tag to
+ *
+ * @return pointer to end of tag on success,
+ *         NULL on failure.
+ *
+ * @discussion The three separate parameters key, type, tag may be
+ * derived from a s=bam_aux_get() query as s-2, *s and s+1.  However
+ * it is recommended to use bam_aux_get_str in this situation.
+ * The desire to split these parameters up is for potential processing
+ * of non-BAM formats that encode using a BAM type mechanism
+ * (such as the internal CRAM representation).
+ */
+static inline const uint8_t *sam_format_aux1(const uint8_t *key,
+                                             const uint8_t type,
+                                             const uint8_t *tag,
+                                             const uint8_t *end,
+                                             kstring_t *ks) {
+    int r = 0;
+    const uint8_t *s = tag; // brevity and consistency with other code.
+    r |= kputsn_((char*)key, 2, ks) < 0;
+    r |= kputc_(':', ks) < 0;
+    if (type == 'C') {
+        r |= kputsn_("i:", 2, ks) < 0;
+        r |= kputw(*s, ks) < 0;
+        ++s;
+    } else if (type == 'c') {
+        r |= kputsn_("i:", 2, ks) < 0;
+        r |= kputw(*(int8_t*)s, ks) < 0;
+        ++s;
+    } else if (type == 'S') {
+        if (end - s >= 2) {
+            r |= kputsn_("i:", 2, ks) < 0;
+            r |= kputuw(le_to_u16(s), ks) < 0;
+            s += 2;
+        } else goto bad_aux;
+    } else if (type == 's') {
+        if (end - s >= 2) {
+            r |= kputsn_("i:", 2, ks) < 0;
+            r |= kputw(le_to_i16(s), ks) < 0;
+            s += 2;
+        } else goto bad_aux;
+    } else if (type == 'I') {
+        if (end - s >= 4) {
+            r |= kputsn_("i:", 2, ks) < 0;
+            r |= kputuw(le_to_u32(s), ks) < 0;
+            s += 4;
+        } else goto bad_aux;
+    } else if (type == 'i') {
+        if (end - s >= 4) {
+            r |= kputsn_("i:", 2, ks) < 0;
+            r |= kputw(le_to_i32(s), ks) < 0;
+            s += 4;
+        } else goto bad_aux;
+    } else if (type == 'A') {
+        r |= kputsn_("A:", 2, ks) < 0;
+        r |= kputc_(*s, ks) < 0;
+        ++s;
+    } else if (type == 'f') {
+        if (end - s >= 4) {
+            // cast to avoid triggering -Wdouble-promotion
+            ksprintf(ks, "f:%g", (double)le_to_float(s));
+            s += 4;
+        } else goto bad_aux;
+
+    } else if (type == 'd') {
+        // NB: "d" is not an official type in the SAM spec.
+        // However for unknown reasons samtools has always supported this.
+        // We believe, HOPE, it is not in general usage and we do not
+        // encourage it.
+        if (end - s >= 8) {
+            ksprintf(ks, "d:%g", le_to_double(s));
+            s += 8;
+        } else goto bad_aux;
+    } else if (type == 'Z' || type == 'H') {
+        r |= kputc_(type, ks) < 0;
+        r |= kputc_(':', ks) < 0;
+        while (s < end && *s) r |= kputc_(*s++, ks) < 0;
+        r |= kputsn("", 0, ks) < 0;     //ensures NUL termination
+        if (s >= end)
+            goto bad_aux;
+        ++s;
+    } else if (type == 'B') {
+        uint8_t sub_type = *(s++);
+        unsigned sub_type_size;
+
+        // or externalise sam.c's aux_type2size function?
+        switch (sub_type) {
+        case 'A': case 'c': case 'C':
+            sub_type_size = 1;
+            break;
+        case 's': case 'S':
+            sub_type_size = 2;
+            break;
+        case 'i': case 'I': case 'f':
+            sub_type_size = 4;
+            break;
+        default:
+            sub_type_size = 0;
+            break;
+        }
+
+        uint32_t i, n;
+        if (sub_type_size == 0 || end - s < 4)
+            goto bad_aux;
+        n = le_to_u32(s);
+        s += 4; // now points to the start of the array
+        if ((size_t)(end - s) / sub_type_size < n)
+            goto bad_aux;
+        r |= kputsn_("B:", 2, ks) < 0;
+        r |= kputc(sub_type, ks) < 0; // write the type
+        switch (sub_type) {
+        case 'c':
+            if (ks_expand(ks, n*2) < 0) goto mem_err;
+            for (i = 0; i < n; ++i) {
+                ks->s[ks->l++] = ',';
+                r |= kputw(*(int8_t*)s, ks) < 0;
+                ++s;
+            }
+            break;
+        case 'C':
+            if (ks_expand(ks, n*2) < 0) goto mem_err;
+            for (i = 0; i < n; ++i) {
+                ks->s[ks->l++] = ',';
+                r |= kputuw(*(uint8_t*)s, ks) < 0;
+                ++s;
+            }
+            break;
+        case 's':
+            if (ks_expand(ks, n*4) < 0) goto mem_err;
+            for (i = 0; i < n; ++i) {
+                ks->s[ks->l++] = ',';
+                r |= kputw(le_to_i16(s), ks) < 0;
+                s += 2;
+            }
+            break;
+        case 'S':
+            if (ks_expand(ks, n*4) < 0) goto mem_err;
+            for (i = 0; i < n; ++i) {
+                ks->s[ks->l++] = ',';
+                r |= kputuw(le_to_u16(s), ks) < 0;
+                s += 2;
+            }
+            break;
+        case 'i':
+            if (ks_expand(ks, n*6) < 0) goto mem_err;
+            for (i = 0; i < n; ++i) {
+                ks->s[ks->l++] = ',';
+                r |= kputw(le_to_i32(s), ks) < 0;
+                s += 4;
+            }
+            break;
+        case 'I':
+            if (ks_expand(ks, n*6) < 0) goto mem_err;
+            for (i = 0; i < n; ++i) {
+                ks->s[ks->l++] = ',';
+                r |= kputuw(le_to_u32(s), ks) < 0;
+                s += 4;
+            }
+            break;
+        case 'f':
+            if (ks_expand(ks, n*8) < 0) goto mem_err;
+            for (i = 0; i < n; ++i) {
+                ks->s[ks->l++] = ',';
+                // cast to avoid triggering -Wdouble-promotion
+                r |= kputd((double)le_to_float(s), ks) < 0;
+                s += 4;
+            }
+            break;
+        default:
+            goto bad_aux;
+        }
+    } else { // Unknown type
+        goto bad_aux;
+    }
+    return r ? NULL : s;
+
+ bad_aux:
+    errno = EINVAL;
+    return NULL;
+
+ mem_err:
+    hts_log_error("Out of memory");
+    errno = ENOMEM;
+    return NULL;
+}
+
+/// Return a pointer to a BAM record's first aux field
+/** @param b   Pointer to the BAM record
+    @return    Aux field pointer, or NULL if the record has none
+
+When NULL is returned, errno will also be set to ENOENT. ("Aux field pointers"
+point to the TYPE byte within the auxiliary data for that field; but in general
+it is unnecessary for user code to be aware of this.)
+ */
+HTSLIB_EXPORT
+uint8_t *bam_aux_first(const bam1_t *b);
+
+/// Return a pointer to a BAM record's next aux field
+/** @param b   Pointer to the BAM record
+    @param s   Aux field pointer, as returned by bam_aux_first()/_next()/_get()
+    @return    Pointer to the next aux field, or NULL if no next field or error
+
+Whenever NULL is returned, errno will also be set: ENOENT if @p s was the
+record's last aux field; otherwise EINVAL, indicating that the BAM record's
+aux data is corrupt.
+ */
+HTSLIB_EXPORT
+uint8_t *bam_aux_next(const bam1_t *b, const uint8_t *s);
+
+/// Return a pointer to an aux record
+/** @param b   Pointer to the bam record
+    @param tag Desired aux tag
+    @return Pointer to the tag data, or NULL if tag is not present or on error
+    If the tag is not present, this function returns NULL and sets errno to
+    ENOENT.  If the bam record's aux data is corrupt (either a tag has an
+    invalid type, or the last record is incomplete) then errno is set to
+    EINVAL and NULL is returned.
+ */
+HTSLIB_EXPORT
+uint8_t *bam_aux_get(const bam1_t *b, const char tag[2]);
+
+/// Return the aux field's 2-character tag
+/** @param s   Aux field pointer, as returned by bam_aux_first()/_next()/_get()
+    @return    Pointer to the tag characters, NOT NUL-terminated
+ */
+static inline
+const char *bam_aux_tag(const uint8_t *s) { return (const char *) (s-2); }
+
+/// Return the aux field's type character
+/** @param s   Aux field pointer, as returned by bam_aux_first()/_next()/_get()
+    @return    The type character: one of cCsSiI/fd/A/Z/H/B
+ */
+static inline char bam_aux_type(const uint8_t *s) { return *s; }
+
+/// Return a SAM formatting string containing a BAM tag
+/** @param b   Pointer to the bam record
+    @param tag Desired aux tag
+    @param s   The kstring to write to.
+
+    @return 1 on success,
+            0 on no tag found with errno = ENOENT,
+           -1 on error (errno will be either EINVAL or ENOMEM).
+ */
+static inline int bam_aux_get_str(const bam1_t *b,
+                                  const char tag[2],
+                                  kstring_t *s) {
+    const uint8_t *t = bam_aux_get(b, tag);
+    if (!t)
+        return errno == ENOENT ? 0 : -1;
+
+    if (!sam_format_aux1(t-2, *t, t+1, b->data + b->l_data, s))
+        return -1;
+
+    return 1;
+}
+
+/// Get an integer aux value
+/** @param s Pointer to the tag data, as returned by bam_aux_get()
+    @return The value, or 0 if the tag was not an integer type
+    If the tag is not an integer type, errno is set to EINVAL.  This function
+    will not return the value of floating-point tags.
+*/
+HTSLIB_EXPORT
+int64_t bam_aux2i(const uint8_t *s);
+
+/// Get a float aux value
+/** @param s Pointer to the tag data, as returned by bam_aux_get()
+    @return The value, or 0 if the tag was not a float type
+    If the tag is not an numeric type, errno is set to EINVAL.  The value of
+    the float will be returned cast to a double.
+*/
+HTSLIB_EXPORT
+double bam_aux2f(const uint8_t *s);
+
+/// Get a character aux value
+/** @param s Pointer to the tag data, as returned by bam_aux_get().
+    @return The value, or 0 if the tag was not a character ('A') type
+    If the tag is not a character type, errno is set to EINVAL.
+*/
+HTSLIB_EXPORT
+char bam_aux2A(const uint8_t *s);
+
+/// Get a string aux value
+/** @param s Pointer to the tag data, as returned by bam_aux_get().
+    @return Pointer to the string, or NULL if the tag was not a string type
+    If the tag is not a string type ('Z' or 'H'), errno is set to EINVAL.
+*/
+HTSLIB_EXPORT
+char *bam_aux2Z(const uint8_t *s);
+
+/// Get the length of an array-type ('B') tag
+/** @param s Pointer to the tag data, as returned by bam_aux_get().
+    @return The length of the array, or 0 if the tag is not an array type.
+    If the tag is not an array type, errno is set to EINVAL.
+ */
+HTSLIB_EXPORT
+uint32_t bam_auxB_len(const uint8_t *s);
+
+/// Get an integer value from an array-type tag
+/** @param s   Pointer to the tag data, as returned by bam_aux_get().
+    @param idx 0-based Index into the array
+    @return The idx'th value, or 0 on error.
+    If the array is not an integer type, errno is set to EINVAL.  If idx
+    is greater than or equal to  the value returned by bam_auxB_len(s),
+    errno is set to ERANGE.  In both cases, 0 will be returned.
+ */
+HTSLIB_EXPORT
+int64_t bam_auxB2i(const uint8_t *s, uint32_t idx);
+
+/// Get a floating-point value from an array-type tag
+/** @param s   Pointer to the tag data, as returned by bam_aux_get().
+    @param idx 0-based Index into the array
+    @return The idx'th value, or 0.0 on error.
+    If the array is not a numeric type, errno is set to EINVAL.  This can
+    only actually happen if the input record has an invalid type field.  If
+    idx is greater than or equal to  the value returned by bam_auxB_len(s),
+    errno is set to ERANGE.  In both cases, 0.0 will be returned.
+ */
+HTSLIB_EXPORT
+double bam_auxB2f(const uint8_t *s, uint32_t idx);
+
+/// Append tag data to a bam record
+/* @param b    The bam record to append to.
+   @param tag  Tag identifier
+   @param type Tag data type
+   @param len  Length of the data in bytes
+   @param data The data to append
+   @return 0 on success; -1 on failure.
+If there is not enough space to store the additional tag, errno is set to
+ENOMEM.  If the type is invalid, errno may be set to EINVAL.  errno is
+also set to EINVAL if the bam record's aux data is corrupt.
+*/
+HTSLIB_EXPORT
+int bam_aux_append(bam1_t *b, const char tag[2], char type, int len, const uint8_t *data);
+
+/// Delete tag data from a bam record
+/** @param b   The BAM record to update
+    @param s   Pointer to the aux field to delete, as returned by bam_aux_get()
+               Must not be NULL
+    @return    0 on success; -1 on failure
+
+If the BAM record's aux data is corrupt, errno is set to EINVAL and this
+function returns -1.
+*/
+HTSLIB_EXPORT
+int bam_aux_del(bam1_t *b, uint8_t *s);
+
+/// Delete an aux field from a BAM record
+/** @param b   The BAM record to update
+    @param s   Pointer to the aux field to delete, as returned by
+               bam_aux_first()/_next()/_get(); must not be NULL
+    @return    Pointer to the following aux field, or NULL if none or on error
+
+Identical to @c bam_aux_del() apart from the return value, which is an
+aux iterator suitable for use with @c bam_aux_next()/etc.
+
+Whenever NULL is returned, errno will also be set: ENOENT if the aux field
+deleted was the record's last one; otherwise EINVAL, indicating that the
+BAM record's aux data is corrupt.
+ */
+HTSLIB_EXPORT
+uint8_t *bam_aux_remove(bam1_t *b, uint8_t *s);
+
+/// Update or add a string-type tag
+/* @param b    The bam record to update
+   @param tag  Tag identifier
+   @param len  The length of the new string
+   @param data The new string
+   @return 0 on success, -1 on failure
+   This function will not change the ordering of tags in the bam record.
+   New tags will be appended to any existing aux records.
+
+   If @p len is less than zero, the length of the input string will be
+   calculated using strlen().  Otherwise exactly @p len bytes will be
+   copied from @p data to make the new tag.  If these bytes do not
+   include a terminating NUL character, one will be added.  (Note that
+   versions of HTSlib up to 1.10.2 had different behaviour here and
+   simply copied @p len bytes from data.  To generate a valid tag it
+   was necessary to ensure the last character was a NUL, and include
+   it in @p len.)
+
+   On failure, errno may be set to one of the following values:
+
+   EINVAL: The bam record's aux data is corrupt or an existing tag with the
+   given ID is not of type 'Z'.
+
+   ENOMEM: The bam data needs to be expanded and either the attempt to
+   reallocate the data buffer failed or the resulting buffer would be
+   longer than the maximum size allowed in a bam record (2Gbytes).
+*/
+HTSLIB_EXPORT
+int bam_aux_update_str(bam1_t *b, const char tag[2], int len, const char *data);
+
+/// Update or add an integer tag
+/* @param b    The bam record to update
+   @param tag  Tag identifier
+   @param val  The new value
+   @return 0 on success, -1 on failure
+   This function will not change the ordering of tags in the bam record.
+   New tags will be appended to any existing aux records.
+
+   On failure, errno may be set to one of the following values:
+
+   EINVAL: The bam record's aux data is corrupt or an existing tag with the
+   given ID is not of an integer type (c, C, s, S, i or I).
+
+   EOVERFLOW (or ERANGE on systems that do not have EOVERFLOW): val is
+   outside the range that can be stored in an integer bam tag (-2147483647
+   to 4294967295).
+
+   ENOMEM: The bam data needs to be expanded and either the attempt to
+   reallocate the data buffer failed or the resulting buffer would be
+   longer than the maximum size allowed in a bam record (2Gbytes).
+*/
+HTSLIB_EXPORT
+int bam_aux_update_int(bam1_t *b, const char tag[2], int64_t val);
+
+/// Update or add a floating-point tag
+/* @param b    The bam record to update
+   @param tag  Tag identifier
+   @param val  The new value
+   @return 0 on success, -1 on failure
+   This function will not change the ordering of tags in the bam record.
+   New tags will be appended to any existing aux records.
+
+   On failure, errno may be set to one of the following values:
+
+   EINVAL: The bam record's aux data is corrupt or an existing tag with the
+   given ID is not of a float type.
+
+   ENOMEM: The bam data needs to be expanded and either the attempt to
+   reallocate the data buffer failed or the resulting buffer would be
+   longer than the maximum size allowed in a bam record (2Gbytes).
+*/
+HTSLIB_EXPORT
+int bam_aux_update_float(bam1_t *b, const char tag[2], float val);
+
+/// Update or add an array tag
+/* @param b     The bam record to update
+   @param tag   Tag identifier
+   @param type  Data type (one of c, C, s, S, i, I or f)
+   @param items Number of items
+   @param data  Pointer to data
+   @return 0 on success, -1 on failure
+   The type parameter indicates the how the data is interpreted:
+
+   Letter code | Data type | Item Size (bytes)
+   ----------- | --------- | -----------------
+   c           | int8_t    | 1
+   C           | uint8_t   | 1
+   s           | int16_t   | 2
+   S           | uint16_t  | 2
+   i           | int32_t   | 4
+   I           | uint32_t  | 4
+   f           | float     | 4
+
+   This function will not change the ordering of tags in the bam record.
+   New tags will be appended to any existing aux records.  The bam record
+   will grow or shrink in order to accommodate the new data.
+
+   The data parameter must not point to any data in the bam record itself or
+   undefined behaviour may result.
+
+   On failure, errno may be set to one of the following values:
+
+   EINVAL: The bam record's aux data is corrupt, an existing tag with the
+   given ID is not of an array type or the type parameter is not one of
+   the values listed above.
+
+   ENOMEM: The bam data needs to be expanded and either the attempt to
+   reallocate the data buffer failed or the resulting buffer would be
+   longer than the maximum size allowed in a bam record (2Gbytes).
+*/
+HTSLIB_EXPORT
+int bam_aux_update_array(bam1_t *b, const char tag[2],
+                         uint8_t type, uint32_t items, void *data);
+
+/**************************
+ *** Pileup and Mpileup ***
+ **************************/
+
+#if !defined(BAM_NO_PILEUP)
+
+/*! @typedef
+ @abstract Generic pileup 'client data'.
+
+ @discussion The pileup iterator allows setting a constructor and
+ destructor function, which will be called every time a sequence is
+ fetched and discarded.  This permits caching of per-sequence data in
+ a tidy manner during the pileup process.  This union is the cached
+ data to be manipulated by the "client" (the caller of pileup).
+*/
+typedef union {
+    void *p;
+    int64_t i;
+    double f;
+} bam_pileup_cd;
+
+/*! @typedef
+ @abstract Structure for one alignment covering the pileup position.
+ @field  b          pointer to the alignment
+ @field  qpos       position of the read base at the pileup site, 0-based
+ @field  indel      indel length; 0 for no indel, positive for ins and negative for del
+ @field  level      the level of the read in the "viewer" mode
+ @field  is_del     1 iff the base on the padded read is a deletion
+ @field  is_head    1 iff this is the first base in the query sequence
+ @field  is_tail    1 iff this is the last base in the query sequence
+ @field  is_refskip 1 iff the base on the padded read is part of CIGAR N op
+ @field  aux        (used by bcf_call_gap_prep())
+ @field  cigar_ind  index of the CIGAR operator that has just been processed
+
+ @discussion See also bam_plbuf_push() and bam_lplbuf_push(). The
+ difference between the two functions is that the former does not
+ set bam_pileup1_t::level, while the later does. Level helps the
+ implementation of alignment viewers, but calculating this has some
+ overhead.
+ */
+typedef struct bam_pileup1_t {
+    bam1_t *b;
+    int32_t qpos;
+    int indel, level;
+    uint32_t is_del:1, is_head:1, is_tail:1, is_refskip:1, /* reserved */ :1, aux:27;
+    bam_pileup_cd cd; // generic per-struct data, owned by caller.
+    int cigar_ind;
+} bam_pileup1_t;
+
+typedef int (*bam_plp_auto_f)(void *data, bam1_t *b);
+
+struct bam_plp_s;
+typedef struct bam_plp_s *bam_plp_t;
+
+struct bam_mplp_s;
+typedef struct bam_mplp_s *bam_mplp_t;
+
+    /**
+     *  bam_plp_init() - sets an iterator over multiple
+     *  @func:      see mplp_func in bam_plcmd.c in samtools for an example. Expected return
+     *              status: 0 on success, -1 on end, < -1 on non-recoverable errors
+     *  @data:      user data to pass to @func
+     *
+     *  The struct returned by a successful call should be freed
+     *  via bam_plp_destroy() when it is no longer needed.
+     */
+    HTSLIB_EXPORT
+    bam_plp_t bam_plp_init(bam_plp_auto_f func, void *data);
+
+    HTSLIB_EXPORT
+    void bam_plp_destroy(bam_plp_t iter);
+
+    HTSLIB_EXPORT
+    int bam_plp_push(bam_plp_t iter, const bam1_t *b);
+
+    HTSLIB_EXPORT
+    const bam_pileup1_t *bam_plp_next(bam_plp_t iter, int *_tid, int *_pos, int *_n_plp);
+
+    HTSLIB_EXPORT
+    const bam_pileup1_t *bam_plp_auto(bam_plp_t iter, int *_tid, int *_pos, int *_n_plp);
+
+    HTSLIB_EXPORT
+    const bam_pileup1_t *bam_plp64_next(bam_plp_t iter, int *_tid, hts_pos_t *_pos, int *_n_plp);
+
+    HTSLIB_EXPORT
+    const bam_pileup1_t *bam_plp64_auto(bam_plp_t iter, int *_tid, hts_pos_t *_pos, int *_n_plp);
+
+    HTSLIB_EXPORT
+    void bam_plp_set_maxcnt(bam_plp_t iter, int maxcnt);
+
+    HTSLIB_EXPORT
+    void bam_plp_reset(bam_plp_t iter);
+
+    /**
+     *  bam_plp_constructor() - sets a callback to initialise any per-pileup1_t fields.
+     *  @plp:       The bam_plp_t initialised using bam_plp_init.
+     *  @func:      The callback function itself.  When called, it is given
+     *              the data argument (specified in bam_plp_init), the bam
+     *              structure and a pointer to a locally allocated
+     *              bam_pileup_cd union.  This union will also be present in
+     *              each bam_pileup1_t created.
+     *              The callback function should have a negative return
+     *              value to indicate an error. (Similarly for destructor.)
+     */
+    HTSLIB_EXPORT
+    void bam_plp_constructor(bam_plp_t plp,
+                             int (*func)(void *data, const bam1_t *b, bam_pileup_cd *cd));
+    HTSLIB_EXPORT
+    void bam_plp_destructor(bam_plp_t plp,
+                            int (*func)(void *data, const bam1_t *b, bam_pileup_cd *cd));
+
+    /// Get pileup padded insertion sequence
+    /**
+     * @param p       pileup data
+     * @param ins     the kstring where the insertion sequence will be written
+     * @param del_len location for deletion length
+     * @return the length of insertion string on success; -1 on failure.
+     *
+     * Fills out the kstring with the padded insertion sequence for the current
+     * location in 'p'.  If this is not an insertion site, the string is blank.
+     *
+     * If del_len is not NULL, the location pointed to is set to the length of
+     * any deletion immediately following the insertion, or zero if none.
+     */
+    HTSLIB_EXPORT
+    int bam_plp_insertion(const bam_pileup1_t *p, kstring_t *ins, int *del_len) HTS_RESULT_USED;
+
+
+    /*! @typedef
+     @abstract An opaque type used for caching base modification state between
+     successive calls to bam_mods_* functions.
+    */
+    typedef struct hts_base_mod_state hts_base_mod_state;
+
+    /// Get pileup padded insertion sequence, including base modifications
+    /**
+     * @param p       pileup data
+     * @param m       state data for the base modification finder
+     * @param ins     the kstring where the insertion sequence will be written
+     * @param del_len location for deletion length
+     * @return the number of insertion string on success, with string length
+     *         being accessable via ins->l; -1 on failure.
+     *
+     * Fills out the kstring with the padded insertion sequence for the current
+     * location in 'p'.  If this is not an insertion site, the string is blank.
+     *
+     * The modification state needs to have been previously initialised using
+     * bam_parse_basemod.  It is permitted to be passed in as NULL, in which
+     * case this function outputs identically to bam_plp_insertion.
+     *
+     * If del_len is not NULL, the location pointed to is set to the length of
+     * any deletion immediately following the insertion, or zero if none.
+     */
+    HTSLIB_EXPORT
+    int bam_plp_insertion_mod(const bam_pileup1_t *p, hts_base_mod_state *m,
+                              kstring_t *ins, int *del_len) HTS_RESULT_USED;
+
+    /// Create a new bam_mplp_t structure
+    /** The struct returned by a successful call should be freed
+     *  via bam_mplp_destroy() when it is no longer needed.
+     */
+    HTSLIB_EXPORT
+    bam_mplp_t bam_mplp_init(int n, bam_plp_auto_f func, void **data);
+
+    /// Set up mpileup overlap detection
+    /**
+     * @param iter    mpileup iterator
+     * @return 0 on success; a negative value on error
+     *
+     *  If called, mpileup will detect overlapping
+     *  read pairs and for each base pair set the base quality of the
+     *  lower-quality base to zero, thus effectively discarding it from
+     *  calling. If the two bases are identical, the quality of the other base
+     *  is increased to the sum of their qualities (capped at 200), otherwise
+     *  it is multiplied by 0.8.
+     */
+    HTSLIB_EXPORT
+    int bam_mplp_init_overlaps(bam_mplp_t iter);
+
+    HTSLIB_EXPORT
+    void bam_mplp_destroy(bam_mplp_t iter);
+
+    HTSLIB_EXPORT
+    void bam_mplp_set_maxcnt(bam_mplp_t iter, int maxcnt);
+
+    HTSLIB_EXPORT
+    int bam_mplp_auto(bam_mplp_t iter, int *_tid, int *_pos, int *n_plp, const bam_pileup1_t **plp);
+
+    HTSLIB_EXPORT
+    int bam_mplp64_auto(bam_mplp_t iter, int *_tid, hts_pos_t *_pos, int *n_plp, const bam_pileup1_t **plp);
+
+    HTSLIB_EXPORT
+    void bam_mplp_reset(bam_mplp_t iter);
+
+    HTSLIB_EXPORT
+    void bam_mplp_constructor(bam_mplp_t iter,
+                              int (*func)(void *data, const bam1_t *b, bam_pileup_cd *cd));
+
+    HTSLIB_EXPORT
+    void bam_mplp_destructor(bam_mplp_t iter,
+                             int (*func)(void *data, const bam1_t *b, bam_pileup_cd *cd));
+
+#endif // ~!defined(BAM_NO_PILEUP)
+
+
+/***********************************
+ * BAQ calculation and realignment *
+ ***********************************/
+
+HTSLIB_EXPORT
+int sam_cap_mapq(bam1_t *b, const char *ref, hts_pos_t ref_len, int thres);
+
+// Used as flag parameter in sam_prob_realn.
+enum htsRealnFlags {
+    BAQ_APPLY = 1,
+    BAQ_EXTEND = 2,
+    BAQ_REDO = 4,
+
+    // Platform subfield, in bit position 3 onwards
+    BAQ_AUTO = 0<<3,
+    BAQ_ILLUMINA = 1<<3,
+    BAQ_PACBIOCCS = 2<<3,
+    BAQ_PACBIO = 3<<3,
+    BAQ_ONT = 4<<3,
+    BAQ_GENAPSYS = 5<<3
+};
+
+/// Calculate BAQ scores
+/** @param b   BAM record
+    @param ref     Reference sequence
+    @param ref_len Reference sequence length
+    @param flag    Flags, see description
+    @return 0 on success \n
+           -1 if the read was unmapped, zero length, had no quality values, did not have at least one M, X or = CIGAR operator, or included a reference skip. \n
+           -3 if BAQ alignment has already been done and does not need to be applied, or has already been applied. \n
+           -4 if alignment failed (most likely due to running out of memory)
+
+This function calculates base alignment quality (BAQ) values using the method
+described in "Improving SNP discovery by base alignment quality", Heng Li,
+Bioinformatics, Volume 27, Issue 8 (https://doi.org/10.1093/bioinformatics/btr076).
+
+The @param flag value can be generated using the htsRealnFlags enum, but for
+backwards compatibilty reasons is retained as an "int".  An example usage
+of the enum could be this, equivalent to flag 19:
+
+    sam_prob_realn(b, ref, len, BAQ_APPLY | BAQ_EXTEND | BAQ_PACBIOCCS);
+
+The following @param flag bits can be used:
+
+Bit 0 (BAQ_APPLY): Adjust the quality values using the BAQ values
+
+ If set, the data in the BQ:Z tag is used to adjust the quality values, and
+ the BQ:Z tag is renamed to ZQ:Z.
+
+ If clear, and a ZQ:Z tag is present, the quality values are reverted using
+ the data in the tag, and the tag is renamed to BQ:Z.
+
+Bit 1 (BAQ_EXTEND): Use "extended" BAQ.
+
+ Changes the BAQ calculation to increase sensitivity at the expense of
+ reduced specificity.
+
+Bit 2 (BAQ_REDO): Recalculate BAQ, even if a BQ tag is present.
+
+ Force BAQ to be recalculated.  Note that a ZQ:Z tag will always disable
+ recalculation.
+
+Bits 3-10: Choose parameters tailored to a specific instrument type.
+
+ One of BAQ_AUTO, BAQ_ILLUMINA, BAQ_PACBIOCCS, BAQ_PACBIO, BAQ_ONT and
+ BAQ_GENAPSYS.  The BAQ parameter tuning are still a work in progress and
+ at the time of writing mainly consist of Illumina vs long-read technology
+ adjustments.
+
+@bug
+If the input read has both BQ:Z and ZQ:Z tags, the ZQ:Z one will be removed.
+Depending on what previous processing happened, this may or may not be the
+correct thing to do.  It would be wise to avoid this situation if possible.
+*/
+HTSLIB_EXPORT
+int sam_prob_realn(bam1_t *b, const char *ref, hts_pos_t ref_len, int flag);
+
+// ---------------------------
+// Base modification retrieval
+
+/*! @typedef
+ @abstract Holds a single base modification.
+ @field modified_base     The short base code (m, h, etc) or -ChEBI (negative)
+ @field canonical_base    The canonical base referred to in the MM tag.
+                          One of A, C, G, T or N.  Note this may not be the
+                          explicit base recorded in the SEQ column (esp. if N).
+ @field strand            0 or 1, indicating + or - strand from MM tag.
+ @field qual              Quality code (256*probability), or -1 if unknown
+
+ @discussion
+ Note this doesn't hold any location data or information on which other
+ modifications may be possible at this site.
+*/
+typedef struct hts_base_mod {
+    int modified_base;
+    int canonical_base;
+    int strand;
+    int qual;
+} hts_base_mod;
+
+#define HTS_MOD_UNKNOWN   -1  // In MM but not ML
+#define HTS_MOD_UNCHECKED -2  // Not in MM and in explicit mode
+
+// Flags for bam_parse_basemod2
+#define HTS_MOD_REPORT_UNCHECKED 1
+
+/// Allocates an hts_base_mode_state.
+/**
+ * @return An hts_base_mode_state pointer on success,
+ *         NULL on failure.
+ *
+ * This just allocates the memory.  The initialisation of the contents is
+ * done using bam_parse_basemod.  Successive calls may be made to that
+ * without the need to free and allocate a new state.
+ *
+ * The state be destroyed using the hts_base_mode_state_free function.
+ */
+HTSLIB_EXPORT
+hts_base_mod_state *hts_base_mod_state_alloc(void);
+
+/// Destroys an  hts_base_mode_state.
+/**
+ * @param state    The base modification state pointer.
+ *
+ * The should have previously been created by hts_base_mode_state_alloc.
+ */
+HTSLIB_EXPORT
+void hts_base_mod_state_free(hts_base_mod_state *state);
+
+/// Parses the MM and ML tags out of a bam record.
+/**
+ * @param b        BAM alignment record
+ * @param state    The base modification state pointer.
+ * @return 0 on success,
+ *         -1 on failure.
+ *
+ * This fills out the contents of the modification state, resetting the
+ * iterator location to the first sequence base.
+ * (Parses the draft Mm/Ml tags instead if MM and/or ML are not present.)
+ */
+HTSLIB_EXPORT
+int bam_parse_basemod(const bam1_t *b, hts_base_mod_state *state);
+
+/// Parses the MM and ML tags out of a bam record.
+/**
+ * @param b        BAM alignment record
+ * @param state    The base modification state pointer.
+ * @param flags    A bit-field controlling base modification processing
+ *
+ * @return 0 on success,
+ *         -1 on failure.
+ *
+ * This fills out the contents of the modification state, resetting the
+ * iterator location to the first sequence base.
+ * (Parses the draft Mm/Ml tags instead if MM and/or ML are not present.)
+ */
+HTSLIB_EXPORT
+int bam_parse_basemod2(const bam1_t *b, hts_base_mod_state *state,
+                       uint32_t flags);
+
+/// Returns modification status for the next base position in the query seq.
+/**
+ * @param b        BAM alignment record
+ * @param state    The base modification state pointer.
+ * @param mods     A supplied array for returning base modifications
+ * @param n_mods   The size of the mods array
+ * @return The number of modifications found on success,
+ *         -1 on failure.
+ *
+ * This is intended to be used as an iterator, with one call per location
+ * along the query sequence.
+ *
+ * If no modifications are found, the returned value is zero.
+ * If more than n_mods modifications are found, the total found is returned.
+ * Note this means the caller needs to check whether this is higher than
+ * n_mods.
+ */
+HTSLIB_EXPORT
+int bam_mods_at_next_pos(const bam1_t *b, hts_base_mod_state *state,
+                         hts_base_mod *mods, int n_mods);
+
+/// Finds the next location containing base modifications and returns them
+/**
+ * @param b        BAM alignment record
+ * @param state    The base modification state pointer.
+ * @param mods     A supplied array for returning base modifications
+ * @param n_mods   The size of the mods array
+ * @param pos      Pointer holding position of modification in sequence
+ * @return The number of modifications found on success,
+ *         0 if no more modifications are present,
+ *         -1 on failure.
+ *
+ * Unlike bam_mods_at_next_pos this skips ahead to the next site
+ * with modifications.
+ *
+ * If more than n_mods modifications are found, the total found is returned.
+ * Note this means the caller needs to check whether this is higher than
+ * n_mods.
+ */
+HTSLIB_EXPORT
+int bam_next_basemod(const bam1_t *b, hts_base_mod_state *state,
+                     hts_base_mod *mods, int n_mods, int *pos);
+
+/// Returns modification status for a specific query position.
+/**
+ * @param b        BAM alignment record
+ * @param state    The base modification state pointer.
+ * @param mods     A supplied array for returning base modifications
+ * @param n_mods   The size of the mods array
+ * @return The number of modifications found on success,
+ *         -1 on failure.
+ *
+ * Note if called multipled times, qpos must be higher than the previous call.
+ * Hence this is suitable for use from a pileup iterator.  If more random
+ * access is required, bam_parse_basemod must be called each time to reset
+ * the state although this has an efficiency cost.
+ *
+ * If no modifications are found, the returned value is zero.
+ * If more than n_mods modifications are found, the total found is returned.
+ * Note this means the caller needs to check whether this is higher than
+ * n_mods.
+ */
+HTSLIB_EXPORT
+int bam_mods_at_qpos(const bam1_t *b, int qpos, hts_base_mod_state *state,
+                     hts_base_mod *mods, int n_mods);
+
+
+/// Returns data about a specific modification type for the alignment record.
+/**
+ * @param b          BAM alignment record
+ * @param state      The base modification state pointer.
+ * @param code       Modification code.  If positive this is a character code,
+ *                   if negative it is a -ChEBI code.
+ *
+ * @param strand     Boolean for top (0) or bottom (1) strand
+ * @param implicit   Boolean for whether unlisted positions should be
+ *                   implicitly assumed to be unmodified, or require an
+ *                   explicit score and should be considered as unknown.
+ *                   Returned.
+ * @param canonical  Canonical base type associated with this modification
+ *                   Returned.
+ *
+ * @return 0 on success or -1 if not found.  The strand, implicit and canonical
+ * fields are filled out if passed in as non-NULL pointers.
+ */
+HTSLIB_EXPORT
+int bam_mods_query_type(hts_base_mod_state *state, int code,
+                        int *strand, int *implicit, char *canonical);
+
+/// Returns data about the i^th modification type for the alignment record.
+/**
+ * @param b          BAM alignment record
+ * @param state      The base modification state pointer.
+ * @param i          Modification index, from 0 to ntype-1
+ * @param strand     Boolean for top (0) or bottom (1) strand
+ * @param implicit   Boolean for whether unlisted positions should be
+ *                   implicitly assumed to be unmodified, or require an
+ *                   explicit score and should be considered as unknown.
+ *                   Returned.
+ * @param canonical  Canonical base type associated with this modification
+ *                   Returned.
+ *
+ * @return 0 on success or -1 if not found.  The strand, implicit and canonical
+ * fields are filled out if passed in as non-NULL pointers.
+ */
+HTSLIB_EXPORT
+int bam_mods_queryi(hts_base_mod_state *state, int i,
+                    int *strand, int *implicit, char *canonical);
+
+/// Returns the list of base modification codes provided for this
+/// alignment record as an array of character codes (+ve) or ChEBI numbers
+/// (negative).
+/*
+ * @param b          BAM alignment record
+ * @param state      The base modification state pointer.
+ * @param ntype      Filled out with the number of array elements returned
+ *
+ * @return the type array, with *ntype filled out with the size.
+ *         The array returned should not be freed.
+ *         It is a valid pointer until the state is freed using
+ *         hts_base_mod_free().
+ */
+HTSLIB_EXPORT
+int *bam_mods_recorded(hts_base_mod_state *state, int *ntype);
+
+#ifdef __cplusplus
+}
+#endif
+
+#ifdef HTSLIB_SSIZE_T
+#undef HTSLIB_SSIZE_T
+#undef ssize_t
+#endif
+
+#endif
diff --git a/ext/htslib/htslib/synced_bcf_reader.h b/ext/htslib/htslib/synced_bcf_reader.h
new file mode 100644
index 0000000..9a6b484
--- /dev/null
+++ b/ext/htslib/htslib/synced_bcf_reader.h
@@ -0,0 +1,396 @@
+/// @file htslib/synced_bcf_reader.h
+/// Stream through multiple VCF files.
+/*
+    Copyright (C) 2012-2017, 2019-2023 Genome Research Ltd.
+
+    Author: Petr Danecek <pd3@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+/*
+    The synced_bcf_reader allows to keep multiple VCFs open and stream them
+    using the next_line iterator in a seamless matter without worrying about
+    chromosomes and synchronizing the sites. This is used by vcfcheck to
+    compare multiple VCFs simultaneously and is used also for merging,
+    creating intersections, etc.
+
+    The synced_bcf_reader also provides API for reading indexed BCF/VCF,
+    hiding differences in BCF/VCF opening, indexing and reading.
+
+
+    Example of usage:
+
+        bcf_srs_t *sr = bcf_sr_init();
+        bcf_sr_set_opt(sr, BCF_SR_PAIR_LOGIC, BCF_SR_PAIR_BOTH_REF);
+        bcf_sr_set_opt(sr, BCF_SR_REQUIRE_IDX);
+        for (i=0; i<nfiles; i++)
+            bcf_sr_add_reader(sr,files[i]);
+        while ( bcf_sr_next_line(sr) )
+        {
+            for (i=0; i<nfiles; i++)
+            {
+                bcf1_t *line = bcf_sr_get_line(sr,i);
+                ...
+            }
+        }
+        if ( sr->errnum ) error("Error: %s\n", bcf_sr_strerror(sr->errnum));
+        bcf_sr_destroy(sr);
+*/
+
+#ifndef HTSLIB_SYNCED_BCF_READER_H
+#define HTSLIB_SYNCED_BCF_READER_H
+
+#include "hts.h"
+#include "vcf.h"
+#include "tbx.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+    When reading multiple files in parallel, duplicate records within each
+    file will be reordered and offered in intuitive order. For example,
+    when reading two files, each with unsorted SNP and indel record, the
+    reader should return the SNP records together and the indel records
+    together. The logic of compatible records can vary depending on the
+    application and can be set using the PAIR_* defined below.
+
+    The COLLAPSE_* definitions will be deprecated in future versions, please
+    use the PAIR_* definitions instead.
+*/
+#define COLLAPSE_NONE   0   // require the exact same set of alleles in all files
+#define COLLAPSE_SNPS   1   // allow different alleles, as long as they all are SNPs
+#define COLLAPSE_INDELS 2   // the same as above, but with indels
+#define COLLAPSE_ANY    4   // any combination of alleles can be returned by bcf_sr_next_line()
+#define COLLAPSE_SOME   8   // at least some of the ALTs must match
+#define COLLAPSE_BOTH  (COLLAPSE_SNPS|COLLAPSE_INDELS)
+
+#define BCF_SR_PAIR_SNPS       (1<<0)  // allow different alleles, as long as they all are SNPs
+#define BCF_SR_PAIR_INDELS     (1<<1)  // the same as above, but with indels
+#define BCF_SR_PAIR_ANY        (1<<2)  // any combination of alleles can be returned by bcf_sr_next_line()
+#define BCF_SR_PAIR_SOME       (1<<3)  // at least some of multiallelic ALTs must match. Implied by all the others with the exception of EXACT
+#define BCF_SR_PAIR_SNP_REF    (1<<4)  // allow REF-only records with SNPs
+#define BCF_SR_PAIR_INDEL_REF  (1<<5)  // allow REF-only records with indels
+#define BCF_SR_PAIR_EXACT      (1<<6)  // require the exact same set of alleles in all files
+#define BCF_SR_PAIR_BOTH       (BCF_SR_PAIR_SNPS|BCF_SR_PAIR_INDELS)
+#define BCF_SR_PAIR_BOTH_REF   (BCF_SR_PAIR_SNPS|BCF_SR_PAIR_INDELS|BCF_SR_PAIR_SNP_REF|BCF_SR_PAIR_INDEL_REF)
+
+typedef enum
+{
+    BCF_SR_REQUIRE_IDX,
+    BCF_SR_PAIR_LOGIC,          // combination of the PAIR_* values above
+    BCF_SR_ALLOW_NO_IDX,        // allow to proceed even if required index is not present (at the user's risk)
+    BCF_SR_REGIONS_OVERLAP,     // include overlapping records with POS outside the regions: 0=no, 1=VCF line overlap, 2=true variant overlap [1]
+    BCF_SR_TARGETS_OVERLAP      // include overlapping records with POS outside the targets: 0=no, 1=VCF line overlap, 2=true variant overlap [0]
+}
+bcf_sr_opt_t;
+
+struct bcf_sr_region_t;
+
+typedef struct bcf_sr_regions_t
+{
+    // for reading from tabix-indexed file (big data)
+    tbx_t *tbx;             // tabix index
+    hts_itr_t *itr;         // tabix iterator
+    kstring_t line;         // holder of the current line, set only when reading from tabix-indexed files
+    htsFile *file;
+    char *fname;
+    int is_bin;             // is open in binary mode (tabix access)
+    char **als;             // parsed alleles if targets_als set and _regions_match_alleles called
+    kstring_t als_str;      // block of parsed alleles
+    int nals, mals;         // number of set alleles and the size of allocated array
+    int als_type;           // alleles type, currently VCF_SNP or VCF_INDEL
+
+    // user handler to deal with skipped regions without a counterpart in VCFs
+    void (*missed_reg_handler)(struct bcf_sr_regions_t *, void *);
+    void *missed_reg_data;
+
+    // for in-memory regions (small data)
+    struct bcf_sr_region_t *regs; // the regions
+
+    // shared by both tabix-index and in-memory regions
+    void *seq_hash;         // keys: sequence names, values: index to seqs
+    char **seq_names;       // sequence names
+    int nseqs;              // number of sequences (chromosomes) in the file
+    int iseq;               // current position: chr name, index to snames
+    hts_pos_t start, end;   // current position: start, end of the region (0-based)
+    int prev_seq;
+    hts_pos_t prev_start, prev_end;
+    int overlap;            // see BCF_SR_REGIONS_OVERLAP/BCF_SR_TARGETS_OVERLAP
+}
+bcf_sr_regions_t;
+
+typedef struct bcf_sr_t
+{
+    htsFile *file;
+    tbx_t *tbx_idx;
+    hts_idx_t *bcf_idx;
+    bcf_hdr_t *header;
+    hts_itr_t *itr;
+    char *fname;
+    bcf1_t **buffer;                // cached VCF records. First is the current record synced across the reader
+    int nbuffer, mbuffer;           // number of cached records (including the current record); number of allocated records
+    int nfilter_ids, *filter_ids;   // -1 for ".", otherwise filter id as returned by bcf_hdr_id2int
+    int *samples, n_smpl;   // list of columns in the order consistent with bcf_srs_t.samples
+}
+bcf_sr_t;
+
+typedef enum
+{
+    open_failed, not_bgzf, idx_load_failed, file_type_error, api_usage_error,
+    header_error, no_eof, no_memory, vcf_parse_error, bcf_read_error, noidx_error
+}
+bcf_sr_error;
+
+typedef struct bcf_srs_t
+{
+    // Parameters controlling the logic
+    int collapse;           // Do not access directly, use bcf_sr_set_pairing_logic() instead
+    char *apply_filters;    // If set, sites where none of the FILTER strings is listed
+                            // will be skipped. Active only at the time of
+                            // initialization, that is during the add_reader()
+                            // calls. Therefore, each reader can be initialized with different
+                            // filters.
+    int require_index;  // Some tools do not need random access
+    int max_unpack;     // When reading VCFs and knowing some fields will not be needed, boost performance of vcf_parse1
+    int *has_line;      // Corresponds to return value of bcf_sr_next_line but is not limited by sizeof(int). Use bcf_sr_has_line macro to query.
+    bcf_sr_error errnum;
+
+    // Auxiliary data
+    bcf_sr_t *readers;
+    int nreaders;
+    int streaming;      // reading mode: index-jumping or streaming
+    int explicit_regs;  // was the list of regions se by bcf_sr_set_regions or guessed from tabix index?
+    char **samples; // List of samples
+    bcf_sr_regions_t *regions, *targets;    // see bcf_sr_set_[targets|regions] for description
+    int targets_als;    // subset to targets not only by position but also by alleles?
+    int targets_exclude;
+    kstring_t tmps;
+    int n_smpl;
+
+    int n_threads;      // Simple multi-threaded decoding / encoding.
+    htsThreadPool *p;   // Our pool, but it can be used by others if needed.
+    void *aux;          // Opaque auxiliary data
+}
+bcf_srs_t;
+
+/** Allocate and initialize a bcf_srs_t struct.
+ *
+ *  The bcf_srs_t struct returned by a successful call should be freed
+ *  via bcf_sr_destroy() when it is no longer needed.
+ */
+HTSLIB_EXPORT
+bcf_srs_t *bcf_sr_init(void);
+
+/** Destroy a bcf_srs_t struct */
+HTSLIB_EXPORT
+void bcf_sr_destroy(bcf_srs_t *readers);
+
+HTSLIB_EXPORT
+char *bcf_sr_strerror(int errnum);
+
+HTSLIB_EXPORT
+int bcf_sr_set_opt(bcf_srs_t *readers, bcf_sr_opt_t opt, ...);
+
+
+/**
+ * bcf_sr_set_threads() - allocates a thread-pool for use by the synced reader.
+ * @n_threads: size of thread pool
+ *
+ * Returns 0 if the call succeeded, or <0 on error.
+ */
+HTSLIB_EXPORT
+int bcf_sr_set_threads(bcf_srs_t *files, int n_threads);
+
+/** Deallocates thread memory, if owned by us. */
+HTSLIB_EXPORT
+void bcf_sr_destroy_threads(bcf_srs_t *files);
+
+/**
+ *  bcf_sr_add_reader() - open new reader
+ *  @readers: holder of the open readers
+ *  @fname:   the VCF file
+ *
+ *  Returns 1 if the call succeeded, or 0 on error.
+ *
+ *  See also the bcf_srs_t data structure for parameters controlling
+ *  the reader's logic.
+ */
+HTSLIB_EXPORT
+int bcf_sr_add_reader(bcf_srs_t *readers, const char *fname);
+
+HTSLIB_EXPORT
+void bcf_sr_remove_reader(bcf_srs_t *files, int i);
+
+/**
+ * bcf_sr_next_line() - the iterator
+ * @readers:    holder of the open readers
+ *
+ * Returns the number of readers which have the current line
+ * (bcf_sr_t.buffer[0]) set at this position. Use the bcf_sr_has_line macro to
+ * determine which of the readers are set.
+ */
+HTSLIB_EXPORT
+int bcf_sr_next_line(bcf_srs_t *readers);
+
+#define bcf_sr_has_line(readers, i) (readers)->has_line[i]
+#define bcf_sr_get_line(_readers, i) ((_readers)->has_line[i] ? ((_readers)->readers[i].buffer[0]) : (bcf1_t *) NULL)
+#define bcf_sr_swap_line(_readers, i, lieu) { bcf1_t *tmp = lieu; lieu = (_readers)->readers[i].buffer[0]; (_readers)->readers[i].buffer[0] = tmp; }
+#define bcf_sr_region_done(_readers,i) (!(_readers)->has_line[i] && !(_readers)->readers[i].nbuffer ? 1 : 0)
+#define bcf_sr_get_header(_readers, i) (_readers)->readers[i].header
+#define bcf_sr_get_reader(_readers, i) &((_readers)->readers[i])
+
+
+/**
+ *  bcf_sr_seek() - set all readers to selected position
+ *  @seq:  sequence name; NULL to seek to start
+ *  @pos:  0-based coordinate
+ */
+HTSLIB_EXPORT
+int bcf_sr_seek(bcf_srs_t *readers, const char *seq, hts_pos_t pos);
+
+/**
+ * bcf_sr_set_samples() - sets active samples
+ * @readers: holder of the open readers
+ * @samples: this can be one of: file name with one sample per line;
+ *           or column-separated list of samples; or '-' for a list of
+ *           samples shared by all files. If first character is the
+ *           exclamation mark, all but the listed samples are included.
+ * @is_file: 0: list of samples; 1: file with sample names
+ *
+ * Returns 1 if the call succeeded, or 0 on error.
+ */
+HTSLIB_EXPORT
+int bcf_sr_set_samples(bcf_srs_t *readers, const char *samples, int is_file);
+
+/**
+ *  bcf_sr_set_targets(), bcf_sr_set_regions() - init targets/regions
+ *  @readers:   holder of the open readers
+ *  @targets:   list of regions, one-based and inclusive.
+ *  @is_fname:  0: targets is a comma-separated list of regions (chr,chr:from-to)
+ *              1: targets is a tabix indexed file with a list of regions
+ *              (<chr,pos> or <chr,from,to>)
+ *
+ *  Returns 0 if the call succeeded, or -1 on error.
+ *
+ *  Both functions behave the same way, unlisted positions will be skipped by
+ *  bcf_sr_next_line(). However, there is an important difference: regions use
+ *  index to jump to desired positions while targets streams the whole files
+ *  and merely skip unlisted positions.
+ *
+ *  Moreover, bcf_sr_set_targets() accepts an optional parameter $alleles which
+ *  is interpreted as a 1-based column index in the tab-delimited file where
+ *  alleles are listed. This in principle enables to perform the COLLAPSE_*
+ *  logic also with tab-delimited files. However, the current implementation
+ *  considers the alleles merely as a suggestion for prioritizing one of possibly
+ *  duplicate VCF lines. It is up to the caller to examine targets->als if
+ *  perfect match is sought after. Note that the duplicate positions in targets
+ *  file are currently not supported.
+ *  Targets (but not regions) can be prefixed with "^" to request logical complement,
+ *  for example "^X,Y,MT" indicates that sequences X, Y and MT should be skipped.
+ *
+ *  API notes:
+ *  - bcf_sr_set_targets MUST be called before the first call to bcf_sr_add_reader()
+ *  - calling bcf_sr_set_regions AFTER readers have been initialized will
+ *    reposition the readers and discard all previous regions.
+ */
+HTSLIB_EXPORT
+int bcf_sr_set_targets(bcf_srs_t *readers, const char *targets, int is_file, int alleles);
+
+HTSLIB_EXPORT
+int bcf_sr_set_regions(bcf_srs_t *readers, const char *regions, int is_file);
+
+
+
+/*
+ *  bcf_sr_regions_init()
+ *  @regions:   regions can be either a comma-separated list of regions
+ *              (chr|chr:pos|chr:from-to|chr:from-) or VCF, BED, or
+ *              tab-delimited file (the default). Uncompressed files
+ *              are stored in memory while bgzip-compressed and tabix-indexed
+ *              region files are streamed.
+ *  @is_file:   0: regions is a comma-separated list of regions
+ *                  (chr|chr:pos|chr:from-to|chr:from-)
+ *              1: VCF, BED or tab-delimited file
+ *  @chr, from, to:
+ *              Column indexes of chromosome, start position and end position
+ *              in the tab-delimited file. The positions are 1-based and
+ *              inclusive.
+ *              These parameters are ignored when reading from VCF, BED or
+ *              tabix-indexed files. When end position column is not present,
+ *              supply 'from' in place of 'to'. When 'to' is negative, first
+ *              abs(to) will be attempted and if that fails, 'from' will be used
+ *              instead.
+ *              If chromosome name contains the characters ':' or '-', it should
+ *              be put in curly brackets, for example as "{weird-chr-name:1-2}:1000-2000"
+ *
+ *  The bcf_sr_regions_t struct returned by a successful call should be freed
+ *  via bcf_sr_regions_destroy() when it is no longer needed.
+ */
+HTSLIB_EXPORT
+bcf_sr_regions_t *bcf_sr_regions_init(const char *regions, int is_file, int chr, int from, int to);
+
+HTSLIB_EXPORT
+void bcf_sr_regions_destroy(bcf_sr_regions_t *regions);
+
+/*
+ *  bcf_sr_regions_seek() - seek to the chromosome block
+ *
+ *  Returns 0 on success or -1 on failure. Sets reg->seq appropriately and
+ *  reg->start,reg->end to -1.
+ */
+HTSLIB_EXPORT
+int bcf_sr_regions_seek(bcf_sr_regions_t *regions, const char *chr);
+
+/*
+ *  bcf_sr_regions_next() - retrieves next region. Returns 0 on success and -1
+ *  when all regions have been read. The fields reg->seq, reg->start and
+ *  reg->end are filled with the genomic coordinates on success or with
+ *  NULL,-1,-1 when no region is available. The coordinates are 0-based,
+ *  inclusive.
+ */
+HTSLIB_EXPORT
+int bcf_sr_regions_next(bcf_sr_regions_t *reg);
+
+/*
+ *  bcf_sr_regions_overlap() - checks if the interval <start,end> overlaps any of
+ *  the regions, the coordinates are 0-based, inclusive. The coordinate queries
+ *  must come in ascending order.
+ *
+ *  Returns 0 if the position is in regions; -1 if the position is not in the
+ *  regions and more regions exist; -2 if not in the regions and there are no more
+ *  regions left.
+ */
+HTSLIB_EXPORT
+int bcf_sr_regions_overlap(bcf_sr_regions_t *reg, const char *seq, hts_pos_t start, hts_pos_t end);
+
+/*
+ *  bcf_sr_regions_flush() - calls repeatedly regs->missed_reg_handler() until
+ *  all remaining records are processed.
+ *  Returns 0 on success, <0 on error.
+ */
+HTSLIB_EXPORT
+int bcf_sr_regions_flush(bcf_sr_regions_t *regs);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/ext/htslib/htslib/tbx.h b/ext/htslib/htslib/tbx.h
new file mode 100644
index 0000000..f4b5bd8
--- /dev/null
+++ b/ext/htslib/htslib/tbx.h
@@ -0,0 +1,144 @@
+/// @file htslib/tbx.h
+/// Tabix API functions.
+/*
+    Copyright (C) 2009, 2012-2015, 2019 Genome Research Ltd.
+    Copyright (C) 2010, 2012 Broad Institute.
+
+    Author: Heng Li <lh3@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#ifndef HTSLIB_TBX_H
+#define HTSLIB_TBX_H
+
+#include "hts.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define TBX_MAX_SHIFT 31
+
+#define TBX_GENERIC 0
+#define TBX_SAM     1
+#define TBX_VCF     2
+#define TBX_GAF     3
+#define TBX_UCSC    0x10000
+
+typedef struct tbx_conf_t {
+    int32_t preset;
+    int32_t sc, bc, ec; // seq col., beg col. and end col.
+    int32_t meta_char, line_skip;
+} tbx_conf_t;
+
+typedef struct tbx_t {
+    tbx_conf_t conf;
+    hts_idx_t *idx;
+    void *dict;
+} tbx_t;
+
+HTSLIB_EXPORT
+extern const tbx_conf_t tbx_conf_gff, tbx_conf_bed, tbx_conf_psltbl, tbx_conf_sam, tbx_conf_vcf, tbx_conf_gaf;
+
+    #define tbx_itr_destroy(iter) hts_itr_destroy(iter)
+    #define tbx_itr_queryi(tbx, tid, beg, end) hts_itr_query((tbx)->idx, (tid), (beg), (end), tbx_readrec)
+    #define tbx_itr_querys(tbx, s) hts_itr_querys((tbx)->idx, (s), (hts_name2id_f)(tbx_name2id), (tbx), hts_itr_query, tbx_readrec)
+    #define tbx_itr_next(htsfp, tbx, itr, r) hts_itr_next(hts_get_bgzfp(htsfp), (itr), (r), (tbx))
+    #define tbx_bgzf_itr_next(bgzfp, tbx, itr, r) hts_itr_next((bgzfp), (itr), (r), (tbx))
+
+    HTSLIB_EXPORT
+    int tbx_name2id(tbx_t *tbx, const char *ss);
+
+    /* Internal helper function used by tbx_itr_next() */
+    HTSLIB_EXPORT
+    BGZF *hts_get_bgzfp(htsFile *fp);
+
+    HTSLIB_EXPORT
+    int tbx_readrec(BGZF *fp, void *tbxv, void *sv, int *tid, hts_pos_t *beg, hts_pos_t *end);
+
+/// Build an index of the lines in a BGZF-compressed file
+/** The index struct returned by a successful call should be freed
+    via tbx_destroy() when it is no longer needed.
+*/
+    HTSLIB_EXPORT
+    tbx_t *tbx_index(BGZF *fp, int min_shift, const tbx_conf_t *conf);
+/*
+ * All tbx_index_build* methods return: 0 (success), -1 (general failure) or -2 (compression not BGZF)
+ */
+    HTSLIB_EXPORT
+    int tbx_index_build(const char *fn, int min_shift, const tbx_conf_t *conf);
+
+    HTSLIB_EXPORT
+    int tbx_index_build2(const char *fn, const char *fnidx, int min_shift, const tbx_conf_t *conf);
+
+    HTSLIB_EXPORT
+    int tbx_index_build3(const char *fn, const char *fnidx, int min_shift, int n_threads, const tbx_conf_t *conf);
+
+
+/// Load or stream a .tbi or .csi index
+/** @param fn     Name of the data file corresponding to the index
+
+    Equivalent to tbx_index_load3(fn, NULL, HTS_IDX_SAVE_REMOTE);
+*/
+    HTSLIB_EXPORT
+    tbx_t *tbx_index_load(const char *fn);
+
+/// Load or stream a .tbi or .csi index
+/** @param fn     Name of the data file corresponding to the index
+    @param fnidx  Name of the indexed file
+    @return The index, or NULL if an error occurred
+
+    If @p fnidx is NULL, the index name will be derived from @p fn.
+
+    Equivalent to tbx_index_load3(fn, fnidx, HTS_IDX_SAVE_REMOTE);
+*/
+    HTSLIB_EXPORT
+    tbx_t *tbx_index_load2(const char *fn, const char *fnidx);
+
+/// Load or stream a .tbi or .csi index
+/** @param fn     Name of the data file corresponding to the index
+    @param fnidx  Name of the indexed file
+    @param flags  Flags to alter behaviour (see description)
+    @return The index, or NULL if an error occurred
+
+    If @p fnidx is NULL, the index name will be derived from @p fn.
+
+    The @p flags parameter can be set to a combination of the following
+    values:
+
+        HTS_IDX_SAVE_REMOTE   Save a local copy of any remote indexes
+        HTS_IDX_SILENT_FAIL   Fail silently if the index is not present
+
+    The index struct returned by a successful call should be freed
+    via tbx_destroy() when it is no longer needed.
+*/
+    HTSLIB_EXPORT
+    tbx_t *tbx_index_load3(const char *fn, const char *fnidx, int flags);
+
+    HTSLIB_EXPORT
+    const char **tbx_seqnames(tbx_t *tbx, int *n);  // free the array but not the values
+
+    HTSLIB_EXPORT
+    void tbx_destroy(tbx_t *tbx);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/ext/htslib/htslib/thread_pool.h b/ext/htslib/htslib/thread_pool.h
new file mode 100644
index 0000000..b13ccb7
--- /dev/null
+++ b/ext/htslib/htslib/thread_pool.h
@@ -0,0 +1,385 @@
+/// @file htslib/thread_pool.h
+/// Thread pool for multi-threading applications.
+/*
+    Copyright (c) 2013-2017, 2019, 2020 Genome Research Ltd.
+
+    Author: James Bonfield <jkb@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+/*
+ * This file implements a thread pool for multi-threading applications.  It
+ * consists of two distinct interfaces: thread pools and thread process
+ * queues (a queue of both jobs to-do and of the results of completed jobs).
+ * Do not confuse "process" here with a unix PID; rather it is analogous to a
+ * program reading a stream of data blocks, processing them in some manner,
+ * and outputting a stream of new data blocks.
+ *
+ * The pool of threads is given a function pointer and void* data to pass in.
+ * This means the pool can run jobs of multiple types, albeit first come
+ * first served with no job scheduling except to pick tasks for the
+ * processes that have room to store the result.
+ *
+ * Upon completion, the return value from the function pointer is
+ * added to back to the process result queue if required.  We may have
+ * multiple "processes" in use for the one pool.
+ *
+ * To see example usage, please look at the #ifdef TEST_MAIN code in
+ * thread_pool.c.
+ */
+
+#ifndef HTSLIB_THREAD_POOL_H
+#define HTSLIB_THREAD_POOL_H
+
+#include "hts_defs.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*-----------------------------------------------------------------------------
+ * Opaque data types.
+ *
+ * Actual definitions are in thread_pool_internal.h, but these should only
+ * be used by thread_pool.c itself.
+ */
+
+/*
+ * An hts_tpool_process implements a queue of input jobs to process and a
+ * queue of resulting output  post-processing.  Internally it consists of two
+ * buffered queues, analogous to the pipes in a unix pipeline:
+ *    ...input | process |  output...
+ *
+ * Both input and output queues have size limits to prevent either queue from
+ * growing too large and serial numbers to ensure sequential consumption of
+ * the output.
+ *
+ * The thread pool may have many heterogeneous tasks, each using its own
+ * process mixed into the same thread pool.
+ */
+typedef struct hts_tpool_process hts_tpool_process;
+
+/*
+ * The single pool structure itself.
+ *
+ * This knows nothing about the nature of the jobs or where their output is
+ * going, but it maintains a list of process-queues associated with this pool
+ * from which the jobs are taken.
+ */
+typedef struct hts_tpool hts_tpool;
+
+/*
+ * An output, after job has executed.
+ */
+typedef struct hts_tpool_result hts_tpool_result;
+
+
+/*-----------------------------------------------------------------------------
+ * Thread pool external functions
+ */
+
+
+/*
+ * Creates a worker pool with n worker threads.
+ *
+ * Returns pool pointer on success;
+ *         NULL on failure
+ *
+ * The hts_tpool struct returned by a successful call should be freed
+ * via hts_tpool_destroy() when it is no longer needed.
+ */
+HTSLIB_EXPORT
+hts_tpool *hts_tpool_init(int n);
+
+
+/*
+ * Returns the number of requested threads for a pool.
+ */
+HTSLIB_EXPORT
+int hts_tpool_size(hts_tpool *p);
+
+
+/// Add an item to the work pool.
+/**
+ * @param p     Thread pool
+ * @param q     Process queue
+ * @param func  Function run by the thread pool
+ * @param arg   Data for use by func()
+ * @return 0 on success
+ *        -1 on failure
+ */
+// FIXME: should this drop the hts_tpool*p argument? It's just q->p
+HTSLIB_EXPORT
+int hts_tpool_dispatch(hts_tpool *p, hts_tpool_process *q,
+                       void *(*func)(void *arg), void *arg);
+
+/// Add an item to the work pool, with nonblocking option.
+/**
+ * @param p         Thread pool
+ * @param q         Process queue
+ * @param func      Function run by the thread pool
+ * @param arg       Data for use by func()
+ * @param nonblock  Non-blocking flag (see description)
+ * @return 0 on success
+ *        -1 on failure
+ *
+ * The @p nonblock parameter can take one of the following values:
+ *      0 => block if input queue is full
+ *     +1 => don't block if input queue is full, but do not add task
+ *     -1 => add task regardless of whether queue is full (over-size)
+ *
+ * If @p nonblock is +1 and the queue is full, -1 will be returned and
+ * `errno` is set to `EAGAIN`.
+ */
+HTSLIB_EXPORT
+int hts_tpool_dispatch2(hts_tpool *p, hts_tpool_process *q,
+                        void *(*func)(void *arg), void *arg, int nonblock);
+
+/// Add an item to the work pool, with nonblocking and cleanup callbacks.
+/**
+ * @param p               Thread pool
+ * @param q               Process queue
+ * @param exec_func       Function run by the thread pool
+ * @param arg             Data for use by func()
+ * @param job_cleanup     Callback to clean up when discarding jobs
+ * @param result_cleanup  Callback to clean up when discarding result data
+ * @param nonblock        Non-blocking flag (see description)
+ * @return 0 on success
+ *        -1 on failure
+ *
+ * The @p nonblock parameter can take one of the following values:
+ *      0 => block if input queue is full
+ *     +1 => don't block if input queue is full, but do not add task
+ *     -1 => add task regardless of whether queue is full (over-size)
+ *
+ * If @p nonblock is +1 and the queue is full, -1 will be returned and
+ * `errno` is set to `EAGAIN`.
+ *
+ * The job_cleanup() and result_cleanup() callbacks are used when discarding
+ * data from a queue, for example when calling hts_tpool_process_reset()
+ * or hts_tpool_process_destroy().
+ *
+ * If not NULL, job_cleanup() will be called for each pending job with the
+ * value of @p arg that was set for that job.  This can be used to free
+ * any data associated with @p arg, and also @p arg itself.
+ *
+ * Similarly, result_cleanup() can be used to free any results left by
+ * jobs that had started before hts_tpool_process_reset() was called.
+ * The argument passed to result_cleanup() is the pointer that would
+ * have been returned by calling hts_tpool_result_data() on the result
+ * when pulled from the queue.
+ *
+ * job_cleanup() and result_cleanup() are only called when discarding jobs.
+ * For jobs that are processed normally, it is the responsibility of
+ * exec_func() and / or consumers of any results to do any cleaning up
+ * necessary.
+ */
+HTSLIB_EXPORT
+int hts_tpool_dispatch3(hts_tpool *p, hts_tpool_process *q,
+                        void *(*exec_func)(void *arg), void *arg,
+                        void (*job_cleanup)(void *arg),
+                        void (*result_cleanup)(void *data),
+                        int nonblock);
+
+/*
+ * Wakes up a single thread stuck in dispatch and make it return with
+ * errno EAGAIN.
+ */
+HTSLIB_EXPORT
+void hts_tpool_wake_dispatch(hts_tpool_process *q);
+
+/*
+ * Flushes the process-queue, but doesn't exit. This simply drains the queue
+ * and ensures all worker threads have finished their current tasks
+ * associated with this process.
+ *
+ * NOT: This does not mean the worker threads are not executing jobs in
+ * another process-queue.
+ *
+ * Returns 0 on success;
+ *        -1 on failure
+ */
+HTSLIB_EXPORT
+int hts_tpool_process_flush(hts_tpool_process *q);
+
+/*
+ * Resets a process to the initial state.
+ *
+ * This removes any queued up input jobs, disables any notification of
+ * new results/output, flushes what is left and then discards any
+ * queued output.  Anything consumer stuck in a wait on results to
+ * appear should stay stuck and will only wake up when new data is
+ * pushed through the queue.
+ *
+ * Returns 0 on success;
+ *        -1 on failure
+ */
+HTSLIB_EXPORT
+int hts_tpool_process_reset(hts_tpool_process *q, int free_results);
+
+/* Returns the process queue size */
+HTSLIB_EXPORT
+int hts_tpool_process_qsize(hts_tpool_process *q);
+
+
+/*
+ * Destroys a thread pool.  The threads are joined into the main
+ * thread so they will finish their current work load.
+ */
+HTSLIB_EXPORT
+void hts_tpool_destroy(hts_tpool *p);
+
+/*
+ * Destroys a thread pool without waiting on jobs to complete.
+ * Use hts_tpool_kill(p) to quickly exit after a fatal error.
+ */
+HTSLIB_EXPORT
+void hts_tpool_kill(hts_tpool *p);
+
+/*
+ * Pulls the next item off the process result queue.  The caller should free
+ * it (and any internals as appropriate) after use.  This doesn't wait for a
+ * result to be present.
+ *
+ * Results will be returned in strict order.
+ *
+ * Returns hts_tpool_result pointer if a result is ready.
+ *         NULL if not.
+ */
+HTSLIB_EXPORT
+hts_tpool_result *hts_tpool_next_result(hts_tpool_process *q);
+
+/*
+ * Pulls the next item off the process result queue.  The caller should free
+ * it (and any internals as appropriate) after use.  This will wait for
+ * a result to be present if none are currently available.
+ *
+ * Results will be returned in strict order.
+ *
+ * Returns hts_tpool_result pointer if a result is ready.
+ *         NULL on error or during shutdown.
+ */
+HTSLIB_EXPORT
+hts_tpool_result *hts_tpool_next_result_wait(hts_tpool_process *q);
+
+/*
+ * Frees a result 'r' and if free_data is true also frees
+ * the internal r->data result too.
+ */
+HTSLIB_EXPORT
+void hts_tpool_delete_result(hts_tpool_result *r, int free_data);
+
+/*
+ * Returns the data portion of a hts_tpool_result, corresponding
+ * to the actual "result" itself.
+ */
+HTSLIB_EXPORT
+void *hts_tpool_result_data(hts_tpool_result *r);
+
+/*
+ * Initialises a thread process-queue.
+ *
+ * In_only, if true, indicates that the process generates does not need to
+ * hold any output.  Otherwise an output queue is used to store the results
+ * of processing each input job.
+ *
+ * Results hts_tpool_process pointer on success;
+ *         NULL on failure
+ *
+ * The hts_tpool_process struct returned by a successful call should be freed
+ * via hts_tpool_process_destroy() when it is no longer needed.
+ */
+HTSLIB_EXPORT
+hts_tpool_process *hts_tpool_process_init(hts_tpool *p, int qsize, int in_only);
+
+
+/* Deallocates memory for a thread process-queue.
+ * Must be called before the thread pool is destroyed.
+ */
+HTSLIB_EXPORT
+void hts_tpool_process_destroy(hts_tpool_process *q);
+
+/*
+ * Returns true if there are no items in the process results queue and
+ * also none still pending.
+ */
+HTSLIB_EXPORT
+int hts_tpool_process_empty(hts_tpool_process *q);
+
+/*
+ * Returns the number of completed jobs in the process results queue.
+ */
+HTSLIB_EXPORT
+int hts_tpool_process_len(hts_tpool_process *q);
+
+/*
+ * Returns the number of completed jobs in the process results queue plus the
+ * number running and queued up to run.
+ */
+HTSLIB_EXPORT
+int hts_tpool_process_sz(hts_tpool_process *q);
+
+/*
+ * Shutdown a process.
+ *
+ * This sets the shutdown flag and wakes any threads waiting on process
+ * condition variables.
+ */
+HTSLIB_EXPORT
+void hts_tpool_process_shutdown(hts_tpool_process *q);
+
+/*
+ * Returns whether this process queue has been shutdown.
+ * Return value of 1 signifies normal shutdown while >1 signifies it
+ * was shutdown due to an error condition.
+ */
+HTSLIB_EXPORT
+int hts_tpool_process_is_shutdown(hts_tpool_process *q);
+
+/*
+ * Attach and detach a thread process-queue with / from the thread pool
+ * scheduler.
+ *
+ * We need to do attach after making a thread process, but may also wish
+ * to temporarily detach if we wish to stop running jobs on a specific
+ * process while permitting other process to continue.
+ */
+HTSLIB_EXPORT
+void hts_tpool_process_attach(hts_tpool *p, hts_tpool_process *q);
+
+HTSLIB_EXPORT
+void hts_tpool_process_detach(hts_tpool *p, hts_tpool_process *q);
+
+/*
+ * Increment and decrement the reference count in a process-queue.
+ * If the queue is being driven from two external (non thread-pool)
+ * threads, eg "main" and a "reader", this permits each end to
+ * decrement its use of the process-queue independently.
+ */
+HTSLIB_EXPORT
+void hts_tpool_process_ref_incr(hts_tpool_process *q);
+
+HTSLIB_EXPORT
+void hts_tpool_process_ref_decr(hts_tpool_process *q);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/ext/htslib/htslib/vcf.h b/ext/htslib/htslib/vcf.h
new file mode 100644
index 0000000..9a36cab
--- /dev/null
+++ b/ext/htslib/htslib/vcf.h
@@ -0,0 +1,1674 @@
+/// @file htslib/vcf.h
+/// High-level VCF/BCF variant calling file operations.
+/*
+    Copyright (C) 2012, 2013 Broad Institute.
+    Copyright (C) 2012-2020, 2022-2023 Genome Research Ltd.
+
+    Author: Heng Li <lh3@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+/*
+    todo:
+        - make the function names consistent
+        - provide calls to abstract away structs as much as possible
+ */
+
+#ifndef HTSLIB_VCF_H
+#define HTSLIB_VCF_H
+
+#include <stdint.h>
+#include <limits.h>
+#include <errno.h>
+#include "hts.h"
+#include "kstring.h"
+#include "hts_defs.h"
+#include "hts_endian.h"
+
+/* Included only for backwards compatibility with e.g. bcftools 1.10 */
+#include <assert.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*****************
+ * Header struct *
+ *****************/
+
+#define BCF_HL_FLT  0 // header line
+#define BCF_HL_INFO 1
+#define BCF_HL_FMT  2
+#define BCF_HL_CTG  3
+#define BCF_HL_STR  4 // structured header line TAG=<A=..,B=..>
+#define BCF_HL_GEN  5 // generic header line
+
+#define BCF_HT_FLAG 0 // header type
+#define BCF_HT_INT  1
+#define BCF_HT_REAL 2
+#define BCF_HT_STR  3
+#define BCF_HT_LONG (BCF_HT_INT | 0x100) // BCF_HT_INT, but for int64_t values; VCF only!
+
+#define BCF_VL_FIXED 0 // variable length
+#define BCF_VL_VAR   1
+#define BCF_VL_A     2
+#define BCF_VL_G     3
+#define BCF_VL_R     4
+
+/* === Dictionary ===
+
+   The header keeps three dictionaries. The first keeps IDs in the
+   "FILTER/INFO/FORMAT" lines, the second keeps the sequence names and lengths
+   in the "contig" lines and the last keeps the sample names. bcf_hdr_t::dict[]
+   is the actual hash table, which is opaque to the end users. In the hash
+   table, the key is the ID or sample name as a C string and the value is a
+   bcf_idinfo_t struct. bcf_hdr_t::id[] points to key-value pairs in the hash
+   table in the order that they appear in the VCF header. bcf_hdr_t::n[] is the
+   size of the hash table or, equivalently, the length of the id[] arrays.
+*/
+
+#define BCF_DT_ID       0 // dictionary type
+#define BCF_DT_CTG      1
+#define BCF_DT_SAMPLE   2
+
+// Complete textual representation of a header line
+typedef struct bcf_hrec_t {
+    int type;       // One of the BCF_HL_* type
+    char *key;      // The part before '=', i.e. FILTER/INFO/FORMAT/contig/fileformat etc.
+    char *value;    // Set only for generic lines, NULL for FILTER/INFO, etc.
+    int nkeys;              // Number of structured fields
+    char **keys, **vals;    // The key=value pairs
+} bcf_hrec_t;
+
+typedef struct bcf_idinfo_t {
+    uint64_t info[3];  // stores Number:20, var:4, Type:4, ColType:4 in info[0..2]
+                       // for BCF_HL_FLT,INFO,FMT and contig length in info[0] for BCF_HL_CTG
+    bcf_hrec_t *hrec[3];
+    int id;
+} bcf_idinfo_t;
+
+typedef struct bcf_idpair_t {
+    const char *key;
+    const bcf_idinfo_t *val;
+} bcf_idpair_t;
+
+// Note that bcf_hdr_t structs must always be created via bcf_hdr_init()
+typedef struct bcf_hdr_t {
+    int32_t n[3];           // n:the size of the dictionary block in use, (allocated size, m, is below to preserve ABI)
+    bcf_idpair_t *id[3];
+    void *dict[3];          // ID dictionary, contig dict and sample dict
+    char **samples;
+    bcf_hrec_t **hrec;
+    int nhrec, dirty;
+    int ntransl, *transl[2];    // for bcf_translate()
+    int nsamples_ori;           // for bcf_hdr_set_samples()
+    uint8_t *keep_samples;
+    kstring_t mem;
+    int32_t m[3];          // m: allocated size of the dictionary block in use (see n above)
+} bcf_hdr_t;
+
+HTSLIB_EXPORT
+extern uint8_t bcf_type_shift[];
+
+/**************
+ * VCF record *
+ **************/
+
+#define BCF_BT_NULL     0
+#define BCF_BT_INT8     1
+#define BCF_BT_INT16    2
+#define BCF_BT_INT32    3
+#define BCF_BT_INT64    4  // Unofficial, for internal use only.
+#define BCF_BT_FLOAT    5
+#define BCF_BT_CHAR     7
+
+#define VCF_REF         0
+#define VCF_SNP     (1<<0)
+#define VCF_MNP     (1<<1)
+#define VCF_INDEL   (1<<2)
+#define VCF_OTHER   (1<<3)
+#define VCF_BND     (1<<4)      // breakend
+#define VCF_OVERLAP (1<<5)      // overlapping deletion, ALT=*
+#define VCF_INS     (1<<6)      // implies VCF_INDEL
+#define VCF_DEL     (1<<7)      // implies VCF_INDEL
+#define VCF_ANY     (VCF_SNP|VCF_MNP|VCF_INDEL|VCF_OTHER|VCF_BND|VCF_OVERLAP|VCF_INS|VCF_DEL)       // any variant type (but not VCF_REF)
+
+typedef struct bcf_variant_t {
+    int type, n;    // variant type and the number of bases affected, negative for deletions
+} bcf_variant_t;
+
+typedef struct bcf_fmt_t {
+    int id;             // id: numeric tag id, the corresponding string is bcf_hdr_t::id[BCF_DT_ID][$id].key
+    int n, size, type;  // n: number of values per-sample; size: number of bytes per-sample; type: one of BCF_BT_* types
+    uint8_t *p;         // same as vptr and vptr_* in bcf_info_t below
+    uint32_t p_len;
+    uint32_t p_off:31, p_free:1;
+} bcf_fmt_t;
+
+typedef struct bcf_info_t {
+    int key;        // key: numeric tag id, the corresponding string is bcf_hdr_t::id[BCF_DT_ID][$key].key
+    int type;  // type: one of BCF_BT_* types
+    union {
+        int64_t i; // integer value
+        float f;   // float value
+    } v1; // only set if $len==1; for easier access
+    uint8_t *vptr;          // pointer to data array in bcf1_t->shared.s, excluding the size+type and tag id bytes
+    uint32_t vptr_len;      // length of the vptr block or, when set, of the vptr_mod block, excluding offset
+    uint32_t vptr_off:31,   // vptr offset, i.e., the size of the INFO key plus size+type bytes
+            vptr_free:1;    // indicates that vptr-vptr_off must be freed; set only when modified and the new
+                            //    data block is bigger than the original
+    int len;                // vector length, 1 for scalars
+} bcf_info_t;
+
+
+#define BCF1_DIRTY_ID  1
+#define BCF1_DIRTY_ALS 2
+#define BCF1_DIRTY_FLT 4
+#define BCF1_DIRTY_INF 8
+
+typedef struct bcf_dec_t {
+    int m_fmt, m_info, m_id, m_als, m_allele, m_flt; // allocated size (high-water mark); do not change
+    int n_flt;  // Number of FILTER fields
+    int *flt;   // FILTER keys in the dictionary
+    char *id, *als;     // ID and REF+ALT block (\0-separated)
+    char **allele;      // allele[0] is the REF (allele[] pointers to the als block); all null terminated
+    bcf_info_t *info;   // INFO
+    bcf_fmt_t *fmt;     // FORMAT and individual sample
+    bcf_variant_t *var; // $var and $var_type set only when set_variant_types called
+    int n_var, var_type;
+    int shared_dirty;   // if set, shared.s must be recreated on BCF output
+    int indiv_dirty;    // if set, indiv.s must be recreated on BCF output
+} bcf_dec_t;
+
+
+#define BCF_ERR_CTG_UNDEF 1
+#define BCF_ERR_TAG_UNDEF 2
+#define BCF_ERR_NCOLS     4
+#define BCF_ERR_LIMITS    8
+#define BCF_ERR_CHAR     16
+#define BCF_ERR_CTG_INVALID   32
+#define BCF_ERR_TAG_INVALID   64
+
+/// Get error description for bcf error code
+/** @param errorcode  The error code which is to be described
+    @param buffer     The buffer in which description to be added
+    @param maxbuffer  The size of buffer passed
+    @return NULL on invalid buffer; buffer on other cases
+
+The buffer will be an empty string when @p errorcode is 0.
+Description of errors present in code will be appended to @p buffer with ',' separation.
+The buffer has to be at least 4 characters long. NULL will be returned if it is smaller or when buffer is NULL.
+
+'...' will be appended if the description doesn't fit in the given buffer.
+ */
+
+HTSLIB_EXPORT
+const char *bcf_strerror(int errorcode, char *buffer, size_t maxbuffer);
+
+/*
+    The bcf1_t structure corresponds to one VCF/BCF line. Reading from VCF file
+    is slower because the string is first to be parsed, packed into BCF line
+    (done in vcf_parse), then unpacked into internal bcf1_t structure. If it
+    is known in advance that some of the fields will not be required (notably
+    the sample columns), parsing of these can be skipped by setting max_unpack
+    appropriately.
+    Similarly, it is fast to output a BCF line because the columns (kept in
+    shared.s, indiv.s, etc.) are written directly by bcf_write, whereas a VCF
+    line must be formatted in vcf_format.
+ */
+typedef struct bcf1_t {
+    hts_pos_t pos;  // POS
+    hts_pos_t rlen; // length of REF
+    int32_t rid;  // CHROM
+    float qual;   // QUAL
+    uint32_t n_info:16, n_allele:16;
+    uint32_t n_fmt:8, n_sample:24;
+    kstring_t shared, indiv;
+    bcf_dec_t d; // lazy evaluation: $d is not generated by bcf_read(), but by explicitly calling bcf_unpack()
+    int max_unpack;         // Set to BCF_UN_STR, BCF_UN_FLT, or BCF_UN_INFO to boost performance of vcf_parse when some of the fields won't be needed
+    int unpacked;           // remember what has been unpacked to allow calling bcf_unpack() repeatedly without redoing the work
+    int unpack_size[3];     // the original block size of ID, REF+ALT and FILTER
+    int errcode;    // one of BCF_ERR_* codes
+} bcf1_t;
+
+/*******
+ * API *
+ *******/
+
+    /***********************************************************************
+     *  BCF and VCF I/O
+     *
+     *  A note about naming conventions: htslib internally represents VCF
+     *  records as bcf1_t data structures, therefore most functions are
+     *  prefixed with bcf_. There are a few exceptions where the functions must
+     *  be aware of both BCF and VCF worlds, such as bcf_parse vs vcf_parse. In
+     *  these cases, functions prefixed with bcf_ are more general and work
+     *  with both BCF and VCF.
+     *
+     ***********************************************************************/
+
+    /** These macros are defined only for consistency with other parts of htslib */
+    #define bcf_init1()         bcf_init()
+    #define bcf_read1(fp,h,v)   bcf_read((fp),(h),(v))
+    #define vcf_read1(fp,h,v)   vcf_read((fp),(h),(v))
+    #define bcf_write1(fp,h,v)  bcf_write((fp),(h),(v))
+    #define vcf_write1(fp,h,v)  vcf_write((fp),(h),(v))
+    #define bcf_destroy1(v)     bcf_destroy(v)
+    #define bcf_empty1(v)       bcf_empty(v)
+    #define vcf_parse1(s,h,v)   vcf_parse((s),(h),(v))
+    #define bcf_clear1(v)       bcf_clear(v)
+    #define vcf_format1(h,v,s)  vcf_format((h),(v),(s))
+
+    /**
+     *  bcf_hdr_init() - create an empty BCF header.
+     *  @param mode    "r" or "w"
+     *
+     *  When opened for writing, the mandatory fileFormat and
+     *  FILTER=PASS lines are added automatically.
+     *
+     * The bcf_hdr_t struct returned by a successful call should be freed
+     * via bcf_hdr_destroy() when it is no longer needed.
+     */
+    HTSLIB_EXPORT
+    bcf_hdr_t *bcf_hdr_init(const char *mode);
+
+    /** Destroy a BCF header struct */
+    HTSLIB_EXPORT
+    void bcf_hdr_destroy(bcf_hdr_t *h);
+
+    /** Allocate and initialize a bcf1_t object.
+     *
+     * The bcf1_t struct returned by a successful call should be freed
+     * via bcf_destroy() when it is no longer needed.
+     */
+    HTSLIB_EXPORT
+    bcf1_t *bcf_init(void);
+
+    /** Deallocate a bcf1_t object */
+    HTSLIB_EXPORT
+    void bcf_destroy(bcf1_t *v);
+
+    /**
+     *  Same as bcf_destroy() but frees only the memory allocated by bcf1_t,
+     *  not the bcf1_t object itself.
+     */
+    HTSLIB_EXPORT
+    void bcf_empty(bcf1_t *v);
+
+    /**
+     *  Make the bcf1_t object ready for next read. Intended mostly for
+     *  internal use, the user should rarely need to call this function
+     *  directly.
+     */
+    HTSLIB_EXPORT
+    void bcf_clear(bcf1_t *v);
+
+
+    /** bcf_open and vcf_open mode: please see hts_open() in hts.h */
+    typedef htsFile vcfFile;
+    #define bcf_open(fn, mode) hts_open((fn), (mode))
+    #define vcf_open(fn, mode) hts_open((fn), (mode))
+    #define bcf_flush(fp) hts_flush((fp))
+    #define bcf_close(fp) hts_close(fp)
+    #define vcf_close(fp) hts_close(fp)
+
+    /// Read a VCF or BCF header
+    /** @param  fp  The file to read the header from
+        @return Pointer to a populated header structure on success;
+                NULL on failure
+
+        The bcf_hdr_t struct returned by a successful call should be freed
+        via bcf_hdr_destroy() when it is no longer needed.
+    */
+    HTSLIB_EXPORT
+    bcf_hdr_t *bcf_hdr_read(htsFile *fp) HTS_RESULT_USED;
+
+    /**
+     *  bcf_hdr_set_samples() - for more efficient VCF parsing when only one/few samples are needed
+     *  @param samples  samples to include or exclude from file or as a comma-separated string.
+     *              LIST|FILE   .. select samples in list/file
+     *              ^LIST|FILE  .. exclude samples from list/file
+     *              -           .. include all samples
+     *              NULL        .. exclude all samples
+     *  @param is_file  @p samples is a file (1) or a comma-separated list (0)
+     *
+     *  The bottleneck of VCF reading is parsing of genotype fields. If the
+     *  reader knows in advance that only subset of samples is needed (possibly
+     *  no samples at all), the performance of bcf_read() can be significantly
+     *  improved by calling bcf_hdr_set_samples after bcf_hdr_read().
+     *  The function bcf_read() will subset the VCF/BCF records automatically
+     *  with the notable exception when reading records via bcf_itr_next().
+     *  In this case, bcf_subset_format() must be called explicitly, because
+     *  bcf_readrec() does not see the header.
+     *
+     *  Returns 0 on success, -1 on error or a positive integer if the list
+     *  contains samples not present in the VCF header. In such a case, the
+     *  return value is the index of the offending sample.
+     */
+    HTSLIB_EXPORT
+    int bcf_hdr_set_samples(bcf_hdr_t *hdr, const char *samples, int is_file) HTS_RESULT_USED;
+
+    HTSLIB_EXPORT
+    int bcf_subset_format(const bcf_hdr_t *hdr, bcf1_t *rec);
+
+    /// Write a VCF or BCF header
+    /** @param  fp  Output file
+        @param  h   The header to write
+        @return 0 on success; -1 on failure
+     */
+    HTSLIB_EXPORT
+    int bcf_hdr_write(htsFile *fp, bcf_hdr_t *h) HTS_RESULT_USED;
+
+    /**
+     * Parse VCF line contained in kstring and populate the bcf1_t struct
+     * The line must not end with \n or \r characters.
+     */
+    HTSLIB_EXPORT
+    int vcf_parse(kstring_t *s, const bcf_hdr_t *h, bcf1_t *v);
+
+    /**
+     * Complete the file opening mode, according to its extension.
+     * @param mode      Preallocated mode string to be completed.
+     * @param fn        File name to be opened.
+     * @param format    Format string (vcf|bcf|vcf.gz)
+     * @return          0 on success; -1 on failure
+     */
+    HTSLIB_EXPORT
+    int vcf_open_mode(char *mode, const char *fn, const char *format);
+
+    /** The opposite of vcf_parse. It should rarely be called directly, see vcf_write */
+    HTSLIB_EXPORT
+    int vcf_format(const bcf_hdr_t *h, const bcf1_t *v, kstring_t *s);
+
+    /// Read next VCF or BCF record
+    /** @param fp  The file to read the record from
+        @param h   The header for the vcf/bcf file
+        @param v   The bcf1_t structure to populate
+        @return 0 on success; -1 on end of file; < -1 on critical error
+
+On errors which are not critical for reading, such as missing header
+definitions in vcf files, zero will be returned but v->errcode will have been
+set to one of BCF_ERR* codes and must be checked before calling bcf_write().
+     */
+    HTSLIB_EXPORT
+    int bcf_read(htsFile *fp, const bcf_hdr_t *h, bcf1_t *v) HTS_RESULT_USED;
+
+    /**
+     *  bcf_unpack() - unpack/decode a BCF record (fills the bcf1_t::d field)
+     *
+     *  Note that bcf_unpack() must be called even when reading VCF. It is safe
+     *  to call the function repeatedly, it will not unpack the same field
+     *  twice.
+     */
+    #define BCF_UN_STR  1       // up to ALT inclusive
+    #define BCF_UN_FLT  2       // up to FILTER
+    #define BCF_UN_INFO 4       // up to INFO
+    #define BCF_UN_SHR  (BCF_UN_STR|BCF_UN_FLT|BCF_UN_INFO) // all shared information
+    #define BCF_UN_FMT  8                           // unpack format and each sample
+    #define BCF_UN_IND  BCF_UN_FMT                  // a synonym of BCF_UN_FMT
+    #define BCF_UN_ALL  (BCF_UN_SHR|BCF_UN_FMT)     // everything
+    HTSLIB_EXPORT
+    int bcf_unpack(bcf1_t *b, int which);
+
+    /*
+     *  bcf_dup() - create a copy of BCF record.
+     *
+     *  Note that bcf_unpack() must be called on the returned copy as if it was
+     *  obtained from bcf_read(). Also note that bcf_dup() calls bcf_sync1(src)
+     *  internally to reflect any changes made by bcf_update_* functions.
+     *
+     *  The bcf1_t struct returned by a successful call should be freed
+     *  via bcf_destroy() when it is no longer needed.
+     */
+    HTSLIB_EXPORT
+    bcf1_t *bcf_dup(bcf1_t *src);
+
+    HTSLIB_EXPORT
+    bcf1_t *bcf_copy(bcf1_t *dst, bcf1_t *src);
+
+    /// Write one VCF or BCF record. The type is determined at the open() call.
+    /** @param  fp  The file to write to
+        @param  h   The header for the vcf/bcf file
+        @param  v   The bcf1_t structure to write
+        @return 0 on success; -1 on error
+     */
+    HTSLIB_EXPORT
+    int bcf_write(htsFile *fp, bcf_hdr_t *h, bcf1_t *v) HTS_RESULT_USED;
+
+    /**
+     *  The following functions work only with VCFs and should rarely be called
+     *  directly. Usually one wants to use their bcf_* alternatives, which work
+     *  transparently with both VCFs and BCFs.
+     */
+    /// Read a VCF format header
+    /** @param  fp  The file to read the header from
+        @return Pointer to a populated header structure on success;
+                NULL on failure
+
+        Use bcf_hdr_read() instead.
+
+        The bcf_hdr_t struct returned by a successful call should be freed
+        via bcf_hdr_destroy() when it is no longer needed.
+    */
+    HTSLIB_EXPORT
+    bcf_hdr_t *vcf_hdr_read(htsFile *fp) HTS_RESULT_USED;
+
+    /// Write a VCF format header
+    /** @param  fp  Output file
+        @param  h   The header to write
+        @return 0 on success; -1 on failure
+
+        Use bcf_hdr_write() instead
+    */
+    HTSLIB_EXPORT
+    int vcf_hdr_write(htsFile *fp, const bcf_hdr_t *h) HTS_RESULT_USED;
+
+    /// Read a record from a VCF file
+    /** @param fp  The file to read the record from
+        @param h   The header for the vcf file
+        @param v   The bcf1_t structure to populate
+        @return 0 on success; -1 on end of file; < -1 on error
+
+        Use bcf_read() instead
+    */
+    HTSLIB_EXPORT
+    int vcf_read(htsFile *fp, const bcf_hdr_t *h, bcf1_t *v) HTS_RESULT_USED;
+
+    /// Write a record to a VCF file
+    /** @param  fp  The file to write to
+        @param h   The header for the vcf file
+        @param v   The bcf1_t structure to write
+        @return 0 on success; -1 on error
+
+        Use bcf_write() instead
+    */
+    HTSLIB_EXPORT
+    int vcf_write(htsFile *fp, const bcf_hdr_t *h, bcf1_t *v) HTS_RESULT_USED;
+
+    /** Helper function for the bcf_itr_next() macro; internal use, ignore it */
+    HTSLIB_EXPORT
+    int bcf_readrec(BGZF *fp, void *null, void *v, int *tid, hts_pos_t *beg, hts_pos_t *end);
+
+    /// Write a line to a VCF file
+    /** @param line   Line to write
+        @param fp     File to write it to
+        @return 0 on success; -1 on failure
+
+        @note No checks are done on the line being added, apart from
+              ensuring that it ends with a newline.  This function
+              should therefore be used with care.
+    */
+    HTSLIB_EXPORT
+    int vcf_write_line(htsFile *fp, kstring_t *line);
+
+    /**************************************************************************
+     *  Header querying and manipulation routines
+     **************************************************************************/
+
+    /** Create a new header using the supplied template
+     *
+     *  The bcf_hdr_t struct returned by a successful call should be freed
+     *  via bcf_hdr_destroy() when it is no longer needed.
+     *  @return NULL on failure, header otherwise
+     */
+    HTSLIB_EXPORT
+    bcf_hdr_t *bcf_hdr_dup(const bcf_hdr_t *hdr);
+
+    /**
+     *  Copy header lines from src to dst if not already present in dst. See also bcf_translate().
+     *  Returns 0 on success or sets a bit on error:
+     *      1 .. conflicting definitions of tag length
+     *      // todo
+     */
+    HTSLIB_EXPORT
+    int bcf_hdr_combine(bcf_hdr_t *dst, const bcf_hdr_t *src) HTS_DEPRECATED("Please use bcf_hdr_merge instead");
+
+    /**
+     *  bcf_hdr_merge() - copy header lines from src to dst, see also bcf_translate()
+     *  @param dst: the destination header to be merged into, NULL on the first pass
+     *  @param src: the source header
+     *  @return NULL on failure, header otherwise
+     *
+     *  Notes:
+     *      - use as:
+     *          bcf_hdr_t *dst = NULL;
+     *          for (i=0; i<nsrc; i++) dst = bcf_hdr_merge(dst,src[i]);
+     *
+     *      - bcf_hdr_merge() replaces bcf_hdr_combine() which had a problem when
+     *      combining multiple BCF headers. The current bcf_hdr_combine()
+     *      does not have this problem, but became slow when used for many files.
+     */
+    HTSLIB_EXPORT
+    bcf_hdr_t *bcf_hdr_merge(bcf_hdr_t *dst, const bcf_hdr_t *src);
+
+    /**
+     *  bcf_hdr_add_sample() - add a new sample.
+     *  @param sample:  sample name to be added
+     *
+     *  Note:
+     *      After all samples have been added, the internal header structure must be updated
+     *      by calling bcf_hdr_sync(). This is normally done automatically by the first bcf_hdr_write()
+     *      or bcf_write() call. Otherwise, the caller must force the update by calling bcf_hdr_sync()
+     *      explicitly.
+     */
+    HTSLIB_EXPORT
+    int bcf_hdr_add_sample(bcf_hdr_t *hdr, const char *sample);
+
+    /** Read VCF header from a file and update the header */
+    HTSLIB_EXPORT
+    int bcf_hdr_set(bcf_hdr_t *hdr, const char *fname);
+
+    /// Appends formatted header text to _str_.
+    /** If _is_bcf_ is zero, `IDX` fields are discarded.
+     *  @return 0 if successful, or negative if an error occurred
+     *  @since 1.4
+     */
+    HTSLIB_EXPORT
+    int bcf_hdr_format(const bcf_hdr_t *hdr, int is_bcf, kstring_t *str);
+
+    /** Returns formatted header (newly allocated string) and its length,
+     *  excluding the terminating \0. If is_bcf parameter is unset, IDX
+     *  fields are discarded.
+     *  @deprecated Use bcf_hdr_format() instead as it can handle huge headers.
+     */
+    HTSLIB_EXPORT
+    char *bcf_hdr_fmt_text(const bcf_hdr_t *hdr, int is_bcf, int *len)
+        HTS_DEPRECATED("use bcf_hdr_format() instead");
+
+    /** Append new VCF header line, returns 0 on success */
+    HTSLIB_EXPORT
+    int bcf_hdr_append(bcf_hdr_t *h, const char *line);
+
+    HTSLIB_EXPORT
+    int bcf_hdr_printf(bcf_hdr_t *h, const char *format, ...)
+    HTS_FORMAT(HTS_PRINTF_FMT, 2, 3);
+
+    /** VCF version, e.g. VCFv4.2 */
+    HTSLIB_EXPORT
+    const char *bcf_hdr_get_version(const bcf_hdr_t *hdr);
+
+    /// Set version in bcf header
+    /**
+       @param hdr     BCF header struct
+       @param version Version to set, e.g. "VCFv4.3"
+       @return 0 on success; < 0 on error
+     */
+    HTSLIB_EXPORT
+    int bcf_hdr_set_version(bcf_hdr_t *hdr, const char *version);
+
+    /**
+     *  bcf_hdr_remove() - remove VCF header tag
+     *  @param type:      one of BCF_HL_*
+     *  @param key:       tag name or NULL to remove all tags of the given type
+     */
+    HTSLIB_EXPORT
+    void bcf_hdr_remove(bcf_hdr_t *h, int type, const char *key);
+
+    /**
+     *  bcf_hdr_subset() - creates a new copy of the header removing unwanted samples
+     *  @param n:        number of samples to keep
+     *  @param samples:  names of the samples to keep
+     *  @param imap:     mapping from index in @samples to the sample index in the original file
+     *  @return NULL on failure, header otherwise
+     *
+     *  Sample names not present in h0 are ignored. The number of unmatched samples can be checked
+     *  by comparing n and bcf_hdr_nsamples(out_hdr).
+     *  This function can be used to reorder samples.
+     *  See also bcf_subset() which subsets individual records.
+     *  The bcf_hdr_t struct returned by a successful call should be freed
+     *  via bcf_hdr_destroy() when it is no longer needed.
+     */
+    HTSLIB_EXPORT
+    bcf_hdr_t *bcf_hdr_subset(const bcf_hdr_t *h0, int n, char *const* samples, int *imap);
+
+    /**
+     *  Creates a list of sequence names. It is up to the caller to free the list (but not the sequence names).
+     *  NB: sequence name indexes returned by bcf_hdr_seqnames() may not correspond to bcf1_t.rid, use
+     *  bcf_hdr_id2name() or bcf_seqname() instead.
+     */
+    HTSLIB_EXPORT
+    const char **bcf_hdr_seqnames(const bcf_hdr_t *h, int *nseqs);
+
+    /** Get number of samples */
+    #define bcf_hdr_nsamples(hdr) (hdr)->n[BCF_DT_SAMPLE]
+
+
+    /** The following functions are for internal use and should rarely be called directly */
+    HTSLIB_EXPORT
+    int bcf_hdr_parse(bcf_hdr_t *hdr, char *htxt);
+
+    /// Synchronize internal header structures
+    /** @param h  Header
+        @return 0 on success, -1 on failure
+
+        This function updates the id, sample and contig arrays in the
+        bcf_hdr_t structure so that they point to the same locations as
+        the id, sample and contig dictionaries.
+    */
+    HTSLIB_EXPORT
+    int bcf_hdr_sync(bcf_hdr_t *h) HTS_RESULT_USED;
+
+    /**
+     * bcf_hdr_parse_line() - parse a single line of VCF textual header
+     * @param h     BCF header struct
+     * @param line  One or more lines of header text
+     * @param len   Filled out with length data parsed from 'line'.
+     * @return bcf_hrec_t* on success;
+     *         NULL on error or on end of header text.
+     *         NB: to distinguish error from end-of-header, check *len:
+     *           *len == 0 indicates @p line did not start with "##"
+     *           *len == -1 indicates failure, likely due to out of memory
+     *           *len > 0 indicates a malformed header line
+     *
+     * If *len > 0 on exit, it will contain the full length of the line
+     * including any trailing newline (this includes cases where NULL was
+     * returned due to a malformed line).  Callers can use this to skip to
+     * the next header line.
+     */
+    HTSLIB_EXPORT
+    bcf_hrec_t *bcf_hdr_parse_line(const bcf_hdr_t *h, const char *line, int *len);
+    /// Convert a bcf header record to string form
+    /**
+     * @param hrec    Header record
+     * @param str     Destination kstring
+     * @return 0 on success; < 0 on error
+     */
+    HTSLIB_EXPORT
+    int bcf_hrec_format(const bcf_hrec_t *hrec, kstring_t *str);
+
+    /// Add a header record into a header
+    /**
+     *  @param hdr  Destination header
+     *  @param hrec Header record
+     *  @return 0 on success, -1 on failure
+     *
+     *  If this function returns success, ownership of @p hrec will have
+     *  been transferred to the header structure.  It may also have been
+     *  freed if it was a duplicate of a record already in the header.
+     *  Therefore the @p hrec pointer should not be used after a successful
+     *  return from this function.
+     *
+     *  If this function returns failure, ownership will not have been taken
+     *  and the caller is responsible for cleaning up @p hrec.
+     */
+
+    HTSLIB_EXPORT
+    int bcf_hdr_add_hrec(bcf_hdr_t *hdr, bcf_hrec_t *hrec);
+
+    /**
+     *  bcf_hdr_get_hrec() - get header line info
+     *  @param type:  one of the BCF_HL_* types: FLT,INFO,FMT,CTG,STR,GEN
+     *  @param key:   the header key for generic lines (e.g. "fileformat"), any field
+     *                  for structured lines, typically "ID".
+     *  @param value: the value which pairs with key. Can be be NULL for BCF_HL_GEN
+     *  @param str_class: the class of BCF_HL_STR line (e.g. "ALT" or "SAMPLE"), otherwise NULL
+     */
+    HTSLIB_EXPORT
+    bcf_hrec_t *bcf_hdr_get_hrec(const bcf_hdr_t *hdr, int type, const char *key, const char *value, const char *str_class);
+
+    /// Duplicate a header record
+    /** @param hrec   Header record to copy
+        @return A new header record on success; NULL on failure
+
+        The bcf_hrec_t struct returned by a successful call should be freed
+        via bcf_hrec_destroy() when it is no longer needed.
+    */
+    HTSLIB_EXPORT
+    bcf_hrec_t *bcf_hrec_dup(bcf_hrec_t *hrec);
+
+    /// Add a new header record key
+    /** @param hrec  Header record
+        @param str   Key name
+        @param len   Length of @p str
+        @return 0 on success; -1 on failure
+    */
+    HTSLIB_EXPORT
+    int bcf_hrec_add_key(bcf_hrec_t *hrec, const char *str, size_t len) HTS_RESULT_USED;
+
+    /// Set a header record value
+    /** @param hrec      Header record
+        @param i         Index of value
+        @param str       Value to set
+        @param len       Length of @p str
+        @param is_quoted Value should be quoted
+        @return 0 on success; -1 on failure
+    */
+    HTSLIB_EXPORT
+    int bcf_hrec_set_val(bcf_hrec_t *hrec, int i, const char *str, size_t len, int is_quoted) HTS_RESULT_USED;
+
+    HTSLIB_EXPORT
+    int bcf_hrec_find_key(bcf_hrec_t *hrec, const char *key);
+
+
+    /// Add an IDX header record
+    /** @param hrec   Header record
+        @param idx    IDX value to add
+        @return 0 on success; -1 on failure
+    */
+    HTSLIB_EXPORT
+    int hrec_add_idx(bcf_hrec_t *hrec, int idx) HTS_RESULT_USED;
+
+    /// Free up a header record and associated structures
+    /** @param hrec  Header record
+     */
+    HTSLIB_EXPORT
+    void bcf_hrec_destroy(bcf_hrec_t *hrec);
+
+
+
+    /**************************************************************************
+     *  Individual record querying and manipulation routines
+     **************************************************************************/
+
+    /** See the description of bcf_hdr_subset() */
+    HTSLIB_EXPORT
+    int bcf_subset(const bcf_hdr_t *h, bcf1_t *v, int n, int *imap);
+
+    /**
+     *  bcf_translate() - translate tags ids to be consistent with different header. This function
+     *                    is useful when lines from multiple VCF need to be combined.
+     *  @dst_hdr:   the destination header, to be used in bcf_write(), see also bcf_hdr_combine()
+     *  @src_hdr:   the source header, used in bcf_read()
+     *  @src_line:  line obtained by bcf_read()
+     */
+    HTSLIB_EXPORT
+    int bcf_translate(const bcf_hdr_t *dst_hdr, bcf_hdr_t *src_hdr, bcf1_t *src_line);
+
+    /// Get variant types in a BCF record
+    /**
+     *  @param rec   BCF/VCF record
+     *  @return Types of variant present
+     *
+     *  The return value will be a bitwise-or of VCF_SNP, VCF_MNP,
+     *  VCF_INDEL, VCF_OTHER, VCF_BND or VCF_OVERLAP.  If will return
+     *  VCF_REF (i.e. 0) if none of the other types is present.
+     *  @deprecated Please use bcf_has_variant_types() instead
+     */
+    HTSLIB_EXPORT
+    int bcf_get_variant_types(bcf1_t *rec);
+
+    /// Get variant type in a BCF record, for a given allele
+    /**
+     *  @param  rec        BCF/VCF record
+     *  @param  ith_allele Allele to check
+     *  @return Type of variant present
+     *
+     *  The return value will be one of VCF_REF, VCF_SNP, VCF_MNP,
+     *  VCF_INDEL, VCF_OTHER, VCF_BND or VCF_OVERLAP.
+     *  @deprecated Please use bcf_has_variant_type() instead
+     */
+    HTSLIB_EXPORT
+    int bcf_get_variant_type(bcf1_t *rec, int ith_allele);
+
+    /// Match mode for bcf_has_variant_types()
+    enum bcf_variant_match {
+        bcf_match_exact,   ///< Types present exactly match tested for
+        bcf_match_overlap, ///< At least one variant type in common
+        bcf_match_subset,  ///< Test set is a subset of types present
+    };
+
+    /// Check for presence of variant types in a BCF record
+    /**
+     *  @param rec      BCF/VCF record
+     *  @param bitmask  Set of variant types to test for
+     *  @param mode     Match mode
+     *  @return >0 if the variant types are present,
+     *           0 if not present,
+     *          -1 on error
+     *
+     *  @p bitmask should be the bitwise-or of the variant types (VCF_SNP,
+     *     VCF_MNP, etc.) to test for.
+     *
+     *  The return value is the bitwise-and of the set of types present
+     *  and @p bitmask.  Callers that want to check for the presence of more
+     *  than one type can avoid function call overhead by passing all the
+     *  types to be checked for in a single call to this function, in
+     *  bcf_match_overlap mode, and then check for them individually in the
+     *  returned value.
+     *
+     *  As VCF_REF is represented by 0 (i.e. the absence of other variants)
+     *  it should be tested for using
+     *    bcf_has_variant_types(rec, VCF_REF, bcf_match_exact)
+     *  which will return 1 if no other variant type is present, otherwise 0.
+     */
+    HTSLIB_EXPORT
+    int bcf_has_variant_types(bcf1_t *rec, uint32_t bitmask, enum bcf_variant_match mode);
+
+    /// Check for presence of variant types in a BCF record, for a given allele
+    /**
+     *  @param rec         BCF/VCF record
+     *  @param ith_allele  Allele to check
+     *  @param bitmask     Set of variant types to test for
+     *  @return >0 if one of the variant types is present,
+     *           0 if not present,
+     *          -1 on error
+     *
+     *  @p bitmask should be the bitwise-or of the variant types (VCF_SNP,
+     *     VCF_MNP, etc.) to test for, or VCF_REF on its own.
+     *
+     *  The return value is the bitwise-and of the set of types present
+     *  and @p bitmask.  Callers that want to check for the presence of more
+     *  than one type can avoid function call overhead by passing all the
+     *  types to be checked for in a single call to this function, and then
+     *  check for them individually in the returned value.
+     *
+     *  As a special case, if @p bitmask is VCF_REF (i.e. 0), the function
+     *  tests for an exact match.  The return value will be 1 if the
+     *  variant type calculated for the allele is VCF_REF, otherwise if
+     *  any other type is present it will be 0.
+     */
+    HTSLIB_EXPORT
+    int bcf_has_variant_type(bcf1_t *rec, int ith_allele, uint32_t bitmask);
+
+    /// Return the number of bases affected by a variant, for a given allele
+    /**
+     *  @param rec         BCF/VCF record
+     *  @param ith_allele  Allele index
+     *  @return The number of bases affected (negative for deletions),
+     *          or bcf_int32_missing on error.
+     */
+    HTSLIB_EXPORT
+    int bcf_variant_length(bcf1_t *rec, int ith_allele);
+
+    HTSLIB_EXPORT
+    int bcf_is_snp(bcf1_t *v);
+
+    /**
+     *  bcf_update_filter() - sets the FILTER column
+     *  @flt_ids:  The filter IDs to set, numeric IDs returned by bcf_hdr_id2int(hdr, BCF_DT_ID, "PASS")
+     *  @n:        Number of filters. If n==0, all filters are removed
+     */
+    HTSLIB_EXPORT
+    int bcf_update_filter(const bcf_hdr_t *hdr, bcf1_t *line, int *flt_ids, int n);
+    /**
+     *  bcf_add_filter() - adds to the FILTER column
+     *  @flt_id:   filter ID to add, numeric ID returned by bcf_hdr_id2int(hdr, BCF_DT_ID, "PASS")
+     *
+     *  If flt_id is PASS, all existing filters are removed first. If other than PASS, existing PASS is removed.
+     */
+    HTSLIB_EXPORT
+    int bcf_add_filter(const bcf_hdr_t *hdr, bcf1_t *line, int flt_id);
+    /**
+     *  bcf_remove_filter() - removes from the FILTER column
+     *  @flt_id:   filter ID to remove, numeric ID returned by bcf_hdr_id2int(hdr, BCF_DT_ID, "PASS")
+     *  @pass:     when set to 1 and no filters are present, set to PASS
+     */
+    HTSLIB_EXPORT
+    int bcf_remove_filter(const bcf_hdr_t *hdr, bcf1_t *line, int flt_id, int pass);
+    /**
+     *  Returns 1 if present, 0 if absent, or -1 if filter does not exist. "PASS" and "." can be used interchangeably.
+     */
+    HTSLIB_EXPORT
+    int bcf_has_filter(const bcf_hdr_t *hdr, bcf1_t *line, char *filter);
+    /**
+     *  bcf_update_alleles() and bcf_update_alleles_str() - update REF and ALT column
+     *  @alleles:           Array of alleles
+     *  @nals:              Number of alleles
+     *  @alleles_string:    Comma-separated alleles, starting with the REF allele
+     */
+    HTSLIB_EXPORT
+    int bcf_update_alleles(const bcf_hdr_t *hdr, bcf1_t *line, const char **alleles, int nals);
+
+    HTSLIB_EXPORT
+    int bcf_update_alleles_str(const bcf_hdr_t *hdr, bcf1_t *line, const char *alleles_string);
+
+    /**
+      *  bcf_update_id() - sets new ID string
+      *  bcf_add_id() - adds to the ID string checking for duplicates
+      */
+    HTSLIB_EXPORT
+    int bcf_update_id(const bcf_hdr_t *hdr, bcf1_t *line, const char *id);
+
+    HTSLIB_EXPORT
+    int bcf_add_id(const bcf_hdr_t *hdr, bcf1_t *line, const char *id);
+
+    /**
+     *  bcf_update_info_*() - functions for updating INFO fields
+     *  @param hdr:       the BCF header
+     *  @param line:      VCF line to be edited
+     *  @param key:       the INFO tag to be updated
+     *  @param values:    pointer to the array of values. Pass NULL to remove the tag.
+     *  @param n:         number of values in the array. When set to 0, the INFO tag is removed
+     *  @return 0 on success or negative value on error.
+     *
+     *  The @p string in bcf_update_info_flag() is optional,
+     *  @p n indicates whether the flag is set or removed.
+     *
+     *  Note that updating an END info tag will cause line->rlen to be
+     *  updated as a side-effect (removing the tag will set it to the
+     *  string length of the REF allele). If line->pos is being changed as
+     *  well, it is important that this is done before calling
+     *  bcf_update_info_int32() to update the END tag, otherwise rlen will be
+     *  set incorrectly.  If the new END value is less than or equal to
+     *  line->pos, a warning will be printed and line->rlen will be set to
+     *  the length of the REF allele.
+     */
+    #define bcf_update_info_int32(hdr,line,key,values,n)   bcf_update_info((hdr),(line),(key),(values),(n),BCF_HT_INT)
+    #define bcf_update_info_float(hdr,line,key,values,n)   bcf_update_info((hdr),(line),(key),(values),(n),BCF_HT_REAL)
+    #define bcf_update_info_flag(hdr,line,key,string,n)    bcf_update_info((hdr),(line),(key),(string),(n),BCF_HT_FLAG)
+    #define bcf_update_info_string(hdr,line,key,string)    bcf_update_info((hdr),(line),(key),(string),1,BCF_HT_STR)
+    HTSLIB_EXPORT
+    int bcf_update_info(const bcf_hdr_t *hdr, bcf1_t *line, const char *key, const void *values, int n, int type);
+
+    /// Set or update 64-bit integer INFO values
+    /**
+     *  @param hdr:       the BCF header
+     *  @param line:      VCF line to be edited
+     *  @param key:       the INFO tag to be updated
+     *  @param values:    pointer to the array of values. Pass NULL to remove the tag.
+     *  @param n:         number of values in the array. When set to 0, the INFO tag is removed
+     *  @return 0 on success or negative value on error.
+     *
+     *  This function takes an int64_t values array as input.  The data
+     *  actually stored will be shrunk to the minimum size that can
+     *  accept all of the values.
+     *
+     *  INFO values outside of the range BCF_MIN_BT_INT32 to BCF_MAX_BT_INT32
+     *  can only be written to VCF files.
+     */
+    static inline int bcf_update_info_int64(const bcf_hdr_t *hdr, bcf1_t *line,
+                                            const char *key,
+                                            const int64_t *values, int n)
+    {
+        return bcf_update_info(hdr, line, key, values, n, BCF_HT_LONG);
+    }
+
+    /*
+     *  bcf_update_format_*() - functions for updating FORMAT fields
+     *  @values:    pointer to the array of values, the same number of elements
+     *              is expected for each sample. Missing values must be padded
+     *              with bcf_*_missing or bcf_*_vector_end values.
+     *  @n:         number of values in the array. If n==0, existing tag is removed.
+     *
+     *  The function bcf_update_format_string() is a higher-level (slower) variant of
+     *  bcf_update_format_char(). The former accepts array of \0-terminated strings
+     *  whereas the latter requires that the strings are collapsed into a single array
+     *  of fixed-length strings. In case of strings with variable length, shorter strings
+     *  can be \0-padded. Note that the collapsed strings passed to bcf_update_format_char()
+     *  are not \0-terminated.
+     *
+     *  Returns 0 on success or negative value on error.
+     */
+    #define bcf_update_format_int32(hdr,line,key,values,n) bcf_update_format((hdr),(line),(key),(values),(n),BCF_HT_INT)
+    #define bcf_update_format_float(hdr,line,key,values,n) bcf_update_format((hdr),(line),(key),(values),(n),BCF_HT_REAL)
+    #define bcf_update_format_char(hdr,line,key,values,n) bcf_update_format((hdr),(line),(key),(values),(n),BCF_HT_STR)
+    #define bcf_update_genotypes(hdr,line,gts,n) bcf_update_format((hdr),(line),"GT",(gts),(n),BCF_HT_INT)     // See bcf_gt_ macros below
+
+    HTSLIB_EXPORT
+    int bcf_update_format_string(const bcf_hdr_t *hdr, bcf1_t *line, const char *key, const char **values, int n);
+
+    HTSLIB_EXPORT
+    int bcf_update_format(const bcf_hdr_t *hdr, bcf1_t *line, const char *key, const void *values, int n, int type);
+
+    // Macros for setting genotypes correctly, for use with bcf_update_genotypes only; idx corresponds
+    // to VCF's GT (1-based index to ALT or 0 for the reference allele) and val is the opposite, obtained
+    // from bcf_get_genotypes() below.
+    #define bcf_gt_phased(idx)      (((idx)+1)<<1|1)
+    #define bcf_gt_unphased(idx)    (((idx)+1)<<1)
+    #define bcf_gt_missing          0
+    #define bcf_gt_is_missing(val)  ((val)>>1 ? 0 : 1)
+    #define bcf_gt_is_phased(idx)   ((idx)&1)
+    #define bcf_gt_allele(val)      (((val)>>1)-1)
+
+    /** Conversion between alleles indexes to Number=G genotype index (assuming diploid, all 0-based) */
+    #define bcf_alleles2gt(a,b) ((a)>(b)?((a)*((a)+1)/2+(b)):((b)*((b)+1)/2+(a)))
+    static inline void bcf_gt2alleles(int igt, int *a, int *b)
+    {
+        int k = 0, dk = 1;
+        while ( k<igt ) { dk++; k += dk; }
+        *b = dk - 1; *a = igt - k + *b;
+    }
+
+    /**
+     * bcf_get_fmt() - returns pointer to FORMAT's field data
+     * @header: for access to BCF_DT_ID dictionary
+     * @line:   VCF line obtained from vcf_parse1
+     * @fmt:    one of GT,PL,...
+     *
+     * Returns bcf_fmt_t* if the call succeeded, or returns NULL when the field
+     * is not available.
+     */
+    HTSLIB_EXPORT
+    bcf_fmt_t *bcf_get_fmt(const bcf_hdr_t *hdr, bcf1_t *line, const char *key);
+
+    HTSLIB_EXPORT
+    bcf_info_t *bcf_get_info(const bcf_hdr_t *hdr, bcf1_t *line, const char *key);
+
+    /**
+     * bcf_get_*_id() - returns pointer to FORMAT/INFO field data given the header index instead of the string ID
+     * @line: VCF line obtained from vcf_parse1
+     * @id:  The header index for the tag, obtained from bcf_hdr_id2int()
+     *
+     * Returns bcf_fmt_t* / bcf_info_t*. These functions do not check if the index is valid
+     * as their goal is to avoid the header lookup.
+     */
+    HTSLIB_EXPORT
+    bcf_fmt_t *bcf_get_fmt_id(bcf1_t *line, const int id);
+
+    HTSLIB_EXPORT
+    bcf_info_t *bcf_get_info_id(bcf1_t *line, const int id);
+
+    /**
+     *  bcf_get_info_*() - get INFO values, integers or floats
+     *  @param hdr:    BCF header
+     *  @param line:   BCF record
+     *  @param tag:    INFO tag to retrieve
+     *  @param dst:    *dst is pointer to a memory location, can point to NULL
+     *  @param ndst:   pointer to the size of allocated memory
+     *  @return  >=0 on success
+     *          -1 .. no such INFO tag defined in the header
+     *          -2 .. clash between types defined in the header and encountered in the VCF record
+     *          -3 .. tag is not present in the VCF record
+     *          -4 .. the operation could not be completed (e.g. out of memory)
+     *
+     *  Returns negative value on error or the number of values (including
+     *  missing values) put in *dst on success. bcf_get_info_string() returns
+     *  on success the number of characters stored excluding the nul-
+     *  terminating byte. bcf_get_info_flag() does not store anything in *dst
+     *  but returns 1 if the flag is set or 0 if not.
+     *
+     *  *dst will be reallocated if it is not big enough (i.e. *ndst is too
+     *  small) or NULL on entry.  The new size will be stored in *ndst.
+     */
+    #define bcf_get_info_int32(hdr,line,tag,dst,ndst)  bcf_get_info_values(hdr,line,tag,(void**)(dst),ndst,BCF_HT_INT)
+    #define bcf_get_info_float(hdr,line,tag,dst,ndst)  bcf_get_info_values(hdr,line,tag,(void**)(dst),ndst,BCF_HT_REAL)
+    #define bcf_get_info_string(hdr,line,tag,dst,ndst) bcf_get_info_values(hdr,line,tag,(void**)(dst),ndst,BCF_HT_STR)
+    #define bcf_get_info_flag(hdr,line,tag,dst,ndst)   bcf_get_info_values(hdr,line,tag,(void**)(dst),ndst,BCF_HT_FLAG)
+
+    HTSLIB_EXPORT
+    int bcf_get_info_values(const bcf_hdr_t *hdr, bcf1_t *line, const char *tag, void **dst, int *ndst, int type);
+
+    /// Put integer INFO values into an int64_t array
+    /**
+     *  @param hdr:    BCF header
+     *  @param line:   BCF record
+     *  @param tag:    INFO tag to retrieve
+     *  @param dst:    *dst is pointer to a memory location, can point to NULL
+     *  @param ndst:   pointer to the size of allocated memory
+     *  @return  >=0 on success
+     *          -1 .. no such INFO tag defined in the header
+     *          -2 .. clash between types defined in the header and encountered in the VCF record
+     *          -3 .. tag is not present in the VCF record
+     *          -4 .. the operation could not be completed (e.g. out of memory)
+     *
+     *  Returns negative value on error or the number of values (including
+     *  missing values) put in *dst on success.
+     *
+     *  *dst will be reallocated if it is not big enough (i.e. *ndst is too
+     *  small) or NULL on entry.  The new size will be stored in *ndst.
+     */
+    static inline int bcf_get_info_int64(const bcf_hdr_t *hdr, bcf1_t *line,
+                                         const char *tag, int64_t **dst,
+                                         int *ndst)
+    {
+        return bcf_get_info_values(hdr, line, tag,
+                                   (void **) dst, ndst, BCF_HT_LONG);
+    }
+
+    /**
+     *  bcf_get_format_*() - same as bcf_get_info*() above
+     *
+     *  The function bcf_get_format_string() is a higher-level (slower) variant of bcf_get_format_char().
+     *  see the description of bcf_update_format_string() and bcf_update_format_char() above.
+     *  Unlike other bcf_get_format__*() functions, bcf_get_format_string() allocates two arrays:
+     *  a single block of \0-terminated strings collapsed into a single array and an array of pointers
+     *  to these strings. Both arrays must be cleaned by the user.
+     *
+     *  Returns negative value on error or the number of written values on success.
+     *
+     *  Use the returned number of written values for accessing valid entries of dst, as ndst is only a
+     *  watermark that can be higher than the returned value, i.e. the end of dst can contain carry-over
+     *  values from previous calls to bcf_get_format_*() on lines with more values per sample.
+     *
+     *  Example:
+     *      int ndst = 0; char **dst = NULL;
+     *      if ( bcf_get_format_string(hdr, line, "XX", &dst, &ndst) > 0 )
+     *          for (i=0; i<bcf_hdr_nsamples(hdr); i++) printf("%s\n", dst[i]);
+     *      free(dst[0]); free(dst);
+     *
+     *  Example:
+     *      int i, j, ngt, nsmpl = bcf_hdr_nsamples(hdr);
+     *      int32_t *gt_arr = NULL, ngt_arr = 0;
+     *
+     *      ngt = bcf_get_genotypes(hdr, line, &gt_arr, &ngt_arr);
+     *      if ( ngt<=0 ) return; // GT not present
+     *
+     *      int max_ploidy = ngt/nsmpl;
+     *      for (i=0; i<nsmpl; i++)
+     *      {
+     *        int32_t *ptr = gt_arr + i*max_ploidy;
+     *        for (j=0; j<max_ploidy; j++)
+     *        {
+     *           // if true, the sample has smaller ploidy
+     *           if ( ptr[j]==bcf_int32_vector_end ) break;
+     *
+     *           // missing allele
+     *           if ( bcf_gt_is_missing(ptr[j]) ) continue;
+     *
+     *           // the VCF 0-based allele index
+     *           int allele_index = bcf_gt_allele(ptr[j]);
+     *
+     *           // is phased?
+     *           int is_phased = bcf_gt_is_phased(ptr[j]);
+     *
+     *           // .. do something ..
+     *         }
+     *      }
+     *      free(gt_arr);
+     *
+     */
+    #define bcf_get_format_int32(hdr,line,tag,dst,ndst)  bcf_get_format_values(hdr,line,tag,(void**)(dst),ndst,BCF_HT_INT)
+    #define bcf_get_format_float(hdr,line,tag,dst,ndst)  bcf_get_format_values(hdr,line,tag,(void**)(dst),ndst,BCF_HT_REAL)
+    #define bcf_get_format_char(hdr,line,tag,dst,ndst)   bcf_get_format_values(hdr,line,tag,(void**)(dst),ndst,BCF_HT_STR)
+    #define bcf_get_genotypes(hdr,line,dst,ndst)         bcf_get_format_values(hdr,line,"GT",(void**)(dst),ndst,BCF_HT_INT)
+
+    HTSLIB_EXPORT
+    int bcf_get_format_string(const bcf_hdr_t *hdr, bcf1_t *line, const char *tag, char ***dst, int *ndst);
+
+    HTSLIB_EXPORT
+    int bcf_get_format_values(const bcf_hdr_t *hdr, bcf1_t *line, const char *tag, void **dst, int *ndst, int type);
+
+
+
+    /**************************************************************************
+     *  Helper functions
+     **************************************************************************/
+
+    /**
+     *  bcf_hdr_id2int() - Translates string into numeric ID
+     *  bcf_hdr_int2id() - Translates numeric ID into string
+     *  @type:     one of BCF_DT_ID, BCF_DT_CTG, BCF_DT_SAMPLE
+     *  @id:       tag name, such as: PL, DP, GT, etc.
+     *
+     *  Returns -1 if string is not in dictionary, otherwise numeric ID which identifies
+     *  fields in BCF records.
+     */
+    HTSLIB_EXPORT
+    int bcf_hdr_id2int(const bcf_hdr_t *hdr, int type, const char *id);
+    #define bcf_hdr_int2id(hdr,type,int_id) ((hdr)->id[type][int_id].key)
+
+    /**
+     *  bcf_hdr_name2id() - Translates sequence names (chromosomes) into numeric ID
+     *  bcf_hdr_id2name() - Translates numeric ID to sequence name
+     */
+    static inline int bcf_hdr_name2id(const bcf_hdr_t *hdr, const char *id) { return bcf_hdr_id2int(hdr, BCF_DT_CTG, id); }
+    static inline const char *bcf_hdr_id2name(const bcf_hdr_t *hdr, int rid)
+    {
+        if ( !hdr || rid<0 || rid>=hdr->n[BCF_DT_CTG] ) return NULL;
+        return hdr->id[BCF_DT_CTG][rid].key;
+    }
+    static inline const char *bcf_seqname(const bcf_hdr_t *hdr, const bcf1_t *rec) {
+        return bcf_hdr_id2name(hdr, rec ? rec->rid : -1);
+    }
+
+    /** Return CONTIG name, or "(unknown)"
+
+        Like bcf_seqname(), but this function will never return NULL.  If
+        the contig name cannot be found (either because @p hdr was not
+        supplied or rec->rid was out of range) it returns the string
+        "(unknown)".
+    */
+    static inline const char *bcf_seqname_safe(const bcf_hdr_t *hdr, const bcf1_t *rec) {
+        const char *name = bcf_seqname(hdr, rec);
+        return name ? name : "(unknown)";
+    }
+
+    /**
+     *  bcf_hdr_id2*() - Macros for accessing bcf_idinfo_t
+     *  @type:      one of BCF_HL_FLT, BCF_HL_INFO, BCF_HL_FMT
+     *  @int_id:    return value of bcf_hdr_id2int, must be >=0
+     *
+     *  The returned values are:
+     *     bcf_hdr_id2length   ..  whether the number of values is fixed or variable, one of BCF_VL_*
+     *     bcf_hdr_id2number   ..  the number of values, 0xfffff for variable length fields
+     *     bcf_hdr_id2type     ..  the field type, one of BCF_HT_*
+     *     bcf_hdr_id2coltype  ..  the column type, one of BCF_HL_*
+     *
+     *  Notes: Prior to using the macros, the presence of the info should be
+     *  tested with bcf_hdr_idinfo_exists().
+     */
+    #define bcf_hdr_id2length(hdr,type,int_id)  ((hdr)->id[BCF_DT_ID][int_id].val->info[type]>>8 & 0xf)
+    #define bcf_hdr_id2number(hdr,type,int_id)  ((hdr)->id[BCF_DT_ID][int_id].val->info[type]>>12)
+    #define bcf_hdr_id2type(hdr,type,int_id)    (uint32_t)((hdr)->id[BCF_DT_ID][int_id].val->info[type]>>4 & 0xf)
+    #define bcf_hdr_id2coltype(hdr,type,int_id) (uint32_t)((hdr)->id[BCF_DT_ID][int_id].val->info[type] & 0xf)
+    #define bcf_hdr_idinfo_exists(hdr,type,int_id)  ((int_id)>=0 && (int_id)<(hdr)->n[BCF_DT_ID] && (hdr)->id[BCF_DT_ID][int_id].val && bcf_hdr_id2coltype((hdr),(type),(int_id))!=0xf)
+    #define bcf_hdr_id2hrec(hdr,dict_type,col_type,int_id)    ((hdr)->id[(dict_type)==BCF_DT_CTG?BCF_DT_CTG:BCF_DT_ID][int_id].val->hrec[(dict_type)==BCF_DT_CTG?0:(col_type)])
+    /// Convert BCF FORMAT data to string form
+    /**
+     * @param s    kstring to write into
+     * @param n    number of items in @p data
+     * @param type type of items in @p data
+     * @param data BCF format data
+     * @return  0 on success
+     *         -1 if out of memory
+     */
+    HTSLIB_EXPORT
+    int bcf_fmt_array(kstring_t *s, int n, int type, void *data);
+
+    HTSLIB_EXPORT
+    uint8_t *bcf_fmt_sized_array(kstring_t *s, uint8_t *ptr);
+
+    /// Encode a variable-length char array in BCF format
+    /**
+     * @param s    kstring to write into
+     * @param l    length of input
+     * @param a    input data to encode
+     * @return 0 on success; < 0 on error
+     */
+    HTSLIB_EXPORT
+    int bcf_enc_vchar(kstring_t *s, int l, const char *a);
+
+    /// Encode a variable-length integer array in BCF format
+    /**
+     * @param s      kstring to write into
+     * @param n      total number of items in @p a (<= 0 to encode BCF_BT_NULL)
+     * @param a      input data to encode
+     * @param wsize  vector length (<= 0 is equivalent to @p n)
+     * @return 0 on success; < 0 on error
+     * @note @p n should be an exact multiple of @p wsize
+     */
+    HTSLIB_EXPORT
+    int bcf_enc_vint(kstring_t *s, int n, int32_t *a, int wsize);
+
+    /// Encode a variable-length float array in BCF format
+    /**
+     * @param s      kstring to write into
+     * @param n      total number of items in @p a (<= 0 to encode BCF_BT_NULL)
+     * @param a      input data to encode
+     * @return 0 on success; < 0 on error
+     */
+    HTSLIB_EXPORT
+    int bcf_enc_vfloat(kstring_t *s, int n, float *a);
+
+
+    /**************************************************************************
+     *  BCF index
+     *
+     *  Note that these functions work with BCFs only. See synced_bcf_reader.h
+     *  which provides (amongst other things) an API to work transparently with
+     *  both indexed BCFs and VCFs.
+     **************************************************************************/
+
+    #define bcf_itr_destroy(iter) hts_itr_destroy(iter)
+    #define bcf_itr_queryi(idx, tid, beg, end) hts_itr_query((idx), (tid), (beg), (end), bcf_readrec)
+    #define bcf_itr_querys(idx, hdr, s) hts_itr_querys((idx), (s), (hts_name2id_f)(bcf_hdr_name2id), (hdr), hts_itr_query, bcf_readrec)
+
+    static inline int bcf_itr_next(htsFile *htsfp, hts_itr_t *itr, void *r) {
+        if (htsfp->is_bgzf)
+            return hts_itr_next(htsfp->fp.bgzf, itr, r, 0);
+
+        hts_log_error("Only bgzf compressed files can be used with iterators");
+        errno = EINVAL;
+        return -2;
+    }
+/// Load a BCF index
+/** @param fn   BCF file name
+    @return The index, or NULL if an error occurred.
+     @note This only works for BCF files.  Consider synced_bcf_reader instead
+which works for both BCF and VCF.
+*/
+    #define bcf_index_load(fn) hts_idx_load(fn, HTS_FMT_CSI)
+    #define bcf_index_seqnames(idx, hdr, nptr) hts_idx_seqnames((idx),(nptr),(hts_id2name_f)(bcf_hdr_id2name),(hdr))
+
+/// Load a BCF index from a given index file name
+/**  @param fn     Input BAM/BCF/etc filename
+     @param fnidx  The input index filename
+     @return  The index, or NULL if an error occurred.
+     @note This only works for BCF files.  Consider synced_bcf_reader instead
+which works for both BCF and VCF.
+*/
+    HTSLIB_EXPORT
+    hts_idx_t *bcf_index_load2(const char *fn, const char *fnidx);
+
+/// Load a BCF index from a given index file name
+/**  @param fn     Input BAM/BCF/etc filename
+     @param fnidx  The input index filename
+     @param flags  Flags to alter behaviour (see description)
+     @return  The index, or NULL if an error occurred.
+     @note This only works for BCF files.  Consider synced_bcf_reader instead
+which works for both BCF and VCF.
+
+     The @p flags parameter can be set to a combination of the following
+     values:
+
+        HTS_IDX_SAVE_REMOTE   Save a local copy of any remote indexes
+        HTS_IDX_SILENT_FAIL   Fail silently if the index is not present
+
+     Equivalent to hts_idx_load3(fn, fnidx, HTS_FMT_CSI, flags);
+*/
+    HTSLIB_EXPORT
+    hts_idx_t *bcf_index_load3(const char *fn, const char *fnidx, int flags);
+
+    /**
+     *  bcf_index_build() - Generate and save an index file
+     *  @fn:         Input VCF(compressed)/BCF filename
+     *  @min_shift:  log2(width of the smallest bin), e.g. a value of 14
+     *  imposes a 16k base lower limit on the width of index bins.
+     *  Positive to generate CSI, or 0 to generate TBI. However, a small
+     *  value of min_shift would create a large index, which would lead to
+     *  reduced performance when using the index. A recommended value is 14.
+     *  For BCF files, only the CSI index can be generated.
+     *
+     *  Returns 0 if successful, or negative if an error occurred.
+     *
+     *  List of error codes:
+     *      -1 .. indexing failed
+     *      -2 .. opening @fn failed
+     *      -3 .. format not indexable
+     *      -4 .. failed to create and/or save the index
+     */
+    HTSLIB_EXPORT
+    int bcf_index_build(const char *fn, int min_shift);
+
+    /**
+     *  bcf_index_build2() - Generate and save an index to a specific file
+     *  @fn:         Input VCF/BCF filename
+     *  @fnidx:      Output filename, or NULL to add .csi/.tbi to @fn
+     *  @min_shift:  Positive to generate CSI, or 0 to generate TBI
+     *
+     *  Returns 0 if successful, or negative if an error occurred.
+     *
+     *  List of error codes:
+     *      -1 .. indexing failed
+     *      -2 .. opening @fn failed
+     *      -3 .. format not indexable
+     *      -4 .. failed to create and/or save the index
+     */
+    HTSLIB_EXPORT
+    int bcf_index_build2(const char *fn, const char *fnidx, int min_shift);
+
+    /**
+     *  bcf_index_build3() - Generate and save an index to a specific file
+     *  @fn:         Input VCF/BCF filename
+     *  @fnidx:      Output filename, or NULL to add .csi/.tbi to @fn
+     *  @min_shift:  Positive to generate CSI, or 0 to generate TBI
+     *  @n_threads:  Number of VCF/BCF decoder threads
+     *
+     *  Returns 0 if successful, or negative if an error occurred.
+     *
+     *  List of error codes:
+     *      -1 .. indexing failed
+     *      -2 .. opening @fn failed
+     *      -3 .. format not indexable
+     *      -4 .. failed to create and/or save the index
+     */
+     HTSLIB_EXPORT
+     int bcf_index_build3(const char *fn, const char *fnidx, int min_shift, int n_threads);
+
+     /// Initialise fp->idx for the current format type, for VCF and BCF files.
+     /** @param fp        File handle for the data file being written.
+         @param h         BCF header structured (needed for BAI and CSI).
+         @param min_shift CSI bin size (CSI default is 14).
+         @param fnidx     Filename to write index to.  This pointer must remain valid
+                          until after bcf_idx_save is called.
+         @return          0 on success, <0 on failure.
+         @note This must be called after the header has been written, but before
+               any other data.
+     */
+     HTSLIB_EXPORT
+     int bcf_idx_init(htsFile *fp, bcf_hdr_t *h, int min_shift, const char *fnidx);
+
+     /// Writes the index initialised with bcf_idx_init to disk.
+     /** @param fp        File handle for the data file being written.
+         @return          0 on success, <0 on failure.
+     */
+     HTSLIB_EXPORT
+     int bcf_idx_save(htsFile *fp);
+
+/*******************
+ * Typed value I/O *
+ *******************/
+
+/*
+    Note that in contrast with BCFv2.1 specification, HTSlib implementation
+    allows missing values in vectors. For integer types, the values 0x80,
+    0x8000, 0x80000000 are interpreted as missing values and 0x81, 0x8001,
+    0x80000001 as end-of-vector indicators.  Similarly for floats, the value of
+    0x7F800001 is interpreted as a missing value and 0x7F800002 as an
+    end-of-vector indicator.
+    Note that the end-of-vector byte is not part of the vector.
+
+    This trial BCF version (v2.2) is compatible with the VCF specification and
+    enables to handle correctly vectors with different ploidy in presence of
+    missing values.
+ */
+#define bcf_int8_vector_end  (-127)         /* INT8_MIN  + 1 */
+#define bcf_int16_vector_end (-32767)       /* INT16_MIN + 1 */
+#define bcf_int32_vector_end (-2147483647)  /* INT32_MIN + 1 */
+#define bcf_int64_vector_end (-9223372036854775807LL)  /* INT64_MIN + 1 */
+#define bcf_str_vector_end   0
+#define bcf_int8_missing     (-128)          /* INT8_MIN  */
+#define bcf_int16_missing    (-32767-1)      /* INT16_MIN */
+#define bcf_int32_missing    (-2147483647-1) /* INT32_MIN */
+#define bcf_int64_missing    (-9223372036854775807LL - 1LL)  /* INT64_MIN */
+
+// All of the above are values, which may occur multiple times in lists of
+// integers or lists of floating point.  Strings in VCF don't have
+// lists - a list of strings is just another (comma-separated) string.
+//
+// Hence bcf_str_missing is the whole string being missing rather than
+// an element of a list.  Ie a string of length zero: (0<<4)|BCF_BT_CHAR.
+#define bcf_str_missing      BCF_BT_CHAR
+
+// Limits on BCF values stored in given types.  Max values are the same
+// as for the underlying type.  Min values are slightly different as
+// the last 8 values for each type were reserved by BCFv2.2.
+#define BCF_MAX_BT_INT8  (0x7f)        /* INT8_MAX  */
+#define BCF_MAX_BT_INT16 (0x7fff)      /* INT16_MAX */
+#define BCF_MAX_BT_INT32 (0x7fffffff)  /* INT32_MAX */
+#define BCF_MIN_BT_INT8  (-120)        /* INT8_MIN  + 8 */
+#define BCF_MIN_BT_INT16 (-32760)      /* INT16_MIN + 8 */
+#define BCF_MIN_BT_INT32 (-2147483640) /* INT32_MIN + 8 */
+
+HTSLIB_EXPORT
+extern uint32_t bcf_float_vector_end;
+HTSLIB_EXPORT
+extern uint32_t bcf_float_missing;
+static inline void bcf_float_set(float *ptr, uint32_t value)
+{
+    union { uint32_t i; float f; } u;
+    u.i = value;
+    *ptr = u.f;
+}
+#define bcf_float_set_vector_end(x) bcf_float_set(&(x),bcf_float_vector_end)
+#define bcf_float_set_missing(x)    bcf_float_set(&(x),bcf_float_missing)
+static inline int bcf_float_is_missing(float f)
+{
+    union { uint32_t i; float f; } u;
+    u.f = f;
+    return u.i==bcf_float_missing ? 1 : 0;
+}
+static inline int bcf_float_is_vector_end(float f)
+{
+    union { uint32_t i; float f; } u;
+    u.f = f;
+    return u.i==bcf_float_vector_end ? 1 : 0;
+}
+
+static inline int bcf_format_gt(bcf_fmt_t *fmt, int isample, kstring_t *str)
+{
+    uint32_t e = 0;
+    #define BRANCH(type_t, convert, missing, vector_end) { \
+        uint8_t *ptr = fmt->p + isample*fmt->size; \
+        int i; \
+        for (i=0; i<fmt->n; i++, ptr += sizeof(type_t)) \
+        { \
+            type_t val = convert(ptr); \
+            if ( val == vector_end ) break; \
+            if ( i ) e |= kputc("/|"[val&1], str) < 0; \
+            if ( !(val>>1) ) e |= kputc('.', str) < 0; \
+            else e |= kputw((val>>1) - 1, str) < 0; \
+        } \
+        if (i == 0) e |= kputc('.', str) < 0; \
+    }
+    switch (fmt->type) {
+        case BCF_BT_INT8:  BRANCH(int8_t,  le_to_i8,  bcf_int8_missing, bcf_int8_vector_end); break;
+        case BCF_BT_INT16: BRANCH(int16_t, le_to_i16, bcf_int16_missing, bcf_int16_vector_end); break;
+        case BCF_BT_INT32: BRANCH(int32_t, le_to_i32, bcf_int32_missing, bcf_int32_vector_end); break;
+        case BCF_BT_NULL:  e |= kputc('.', str) < 0; break;
+        default: hts_log_error("Unexpected type %d", fmt->type); return -2;
+    }
+    #undef BRANCH
+    return e == 0 ? 0 : -1;
+}
+
+static inline int bcf_enc_size(kstring_t *s, int size, int type)
+{
+    // Most common case is first
+    if (size < 15) {
+        if (ks_resize(s, s->l + 1) < 0)
+            return -1;
+        uint8_t *p = (uint8_t *)s->s + s->l;
+        *p++ = (size<<4) | type;
+        s->l++;
+        return 0;
+    }
+
+    if (ks_resize(s, s->l + 6) < 0)
+        return -1;
+    uint8_t *p = (uint8_t *)s->s + s->l;
+    *p++ = 15<<4|type;
+
+    if (size < 128) {
+        *p++ = 1<<4|BCF_BT_INT8;
+        *p++ = size;
+        s->l += 3;
+    } else {
+        if (size < 32768) {
+            *p++ = 1<<4|BCF_BT_INT16;
+            i16_to_le(size, p);
+            s->l += 4;
+        } else {
+            *p++ = 1<<4|BCF_BT_INT32;
+            i32_to_le(size, p);
+            s->l += 6;
+        }
+    }
+    return 0;
+}
+
+static inline int bcf_enc_inttype(long x)
+{
+    if (x <= BCF_MAX_BT_INT8 && x >= BCF_MIN_BT_INT8) return BCF_BT_INT8;
+    if (x <= BCF_MAX_BT_INT16 && x >= BCF_MIN_BT_INT16) return BCF_BT_INT16;
+    return BCF_BT_INT32;
+}
+
+static inline int bcf_enc_int1(kstring_t *s, int32_t x)
+{
+    if (ks_resize(s, s->l + 5) < 0)
+        return -1;
+    uint8_t *p = (uint8_t *)s->s + s->l;
+
+    if (x == bcf_int32_vector_end) {
+        // An inline implementation of bcf_enc_size with size==1 and
+        // memory allocation already accounted for.
+        *p = (1<<4) | BCF_BT_INT8;
+        p[1] = bcf_int8_vector_end;
+        s->l+=2;
+    } else if (x == bcf_int32_missing) {
+        *p = (1<<4) | BCF_BT_INT8;
+        p[1] = bcf_int8_missing;
+        s->l+=2;
+    } else if (x <= BCF_MAX_BT_INT8 && x >= BCF_MIN_BT_INT8) {
+        *p = (1<<4) | BCF_BT_INT8;
+        p[1] = x;
+        s->l+=2;
+    } else if (x <= BCF_MAX_BT_INT16 && x >= BCF_MIN_BT_INT16) {
+        *p = (1<<4) | BCF_BT_INT16;
+        i16_to_le(x, p+1);
+        s->l+=3;
+    } else {
+        *p = (1<<4) | BCF_BT_INT32;
+        i32_to_le(x, p+1);
+        s->l+=5;
+    }
+
+    return 0;
+}
+
+/// Return the value of a single typed integer.
+/** @param      p    Pointer to input data block.
+    @param      type One of the BCF_BT_INT* type codes
+    @param[out] q    Location to store an updated value for p
+    @return The integer value, or zero if @p type is not valid.
+
+If @p type is not one of BCF_BT_INT8, BCF_BT_INT16, BCF_BT_INT32 or
+BCF_BT_INT64, zero will be returned and @p *q will not be updated.
+Otherwise, the integer value will be returned and @p *q will be set
+to the memory location immediately following the integer value.
+
+Cautious callers can detect invalid type codes by checking that *q has
+actually been updated.
+*/
+
+static inline int64_t bcf_dec_int1(const uint8_t *p, int type, uint8_t **q)
+{
+    if (type == BCF_BT_INT8) {
+        *q = (uint8_t*)p + 1;
+        return le_to_i8(p);
+    } else if (type == BCF_BT_INT16) {
+        *q = (uint8_t*)p + 2;
+        return le_to_i16(p);
+    } else if (type == BCF_BT_INT32) {
+        *q = (uint8_t*)p + 4;
+        return le_to_i32(p);
+    } else if (type == BCF_BT_INT64) {
+        *q = (uint8_t*)p + 8;
+        return le_to_i64(p);
+    } else { // Invalid type.
+        return 0;
+    }
+}
+
+/// Return the value of a single typed integer from a byte stream.
+/** @param      p    Pointer to input data block.
+    @param[out] q    Location to store an updated value for p
+    @return The integer value, or zero if the type code was not valid.
+
+Reads a one-byte type code from @p p, and uses it to decode an integer
+value from the following bytes in @p p.
+
+If the type is not one of BCF_BT_INT8, BCF_BT_INT16 or BCF_BT_INT32, zero
+will be returned and @p *q will unchanged.  Otherwise, the integer value will
+be returned and @p *q will be set to the memory location immediately following
+the integer value.
+
+Cautious callers can detect invalid type codes by checking that *q has
+actually been updated.
+*/
+static inline int64_t bcf_dec_typed_int1(const uint8_t *p, uint8_t **q)
+{
+    return bcf_dec_int1(p + 1, *p&0xf, q);
+}
+
+static inline int32_t bcf_dec_size(const uint8_t *p, uint8_t **q, int *type)
+{
+    *type = *p & 0xf;
+    if (*p>>4 != 15) {
+        *q = (uint8_t*)p + 1;
+        return *p>>4;
+    } else return bcf_dec_typed_int1(p + 1, q);
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/ext/htslib/htslib/vcf_sweep.h b/ext/htslib/htslib/vcf_sweep.h
new file mode 100644
index 0000000..9200554
--- /dev/null
+++ b/ext/htslib/htslib/vcf_sweep.h
@@ -0,0 +1,57 @@
+/// @file htslib/vcf_sweep.h
+/// Forward/reverse sweep API.
+/*
+    Copyright (C) 2013-2015, 2019 Genome Research Ltd.
+
+    Author: Petr Danecek <pd3@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#ifndef HTSLIB_VCF_SWEEP_H
+#define HTSLIB_VCF_SWEEP_H
+
+#include "hts.h"
+#include "vcf.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct bcf_sweep_t bcf_sweep_t;
+
+HTSLIB_EXPORT
+bcf_sweep_t *bcf_sweep_init(const char *fname);
+
+HTSLIB_EXPORT
+void bcf_sweep_destroy(bcf_sweep_t *sw);
+
+HTSLIB_EXPORT
+bcf_hdr_t *bcf_sweep_hdr(bcf_sweep_t *sw);
+
+HTSLIB_EXPORT
+bcf1_t *bcf_sweep_fwd(bcf_sweep_t *sw);
+
+HTSLIB_EXPORT
+bcf1_t *bcf_sweep_bwd(bcf_sweep_t *sw);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/ext/htslib/htslib/vcfutils.h b/ext/htslib/htslib/vcfutils.h
new file mode 100644
index 0000000..dd69edd
--- /dev/null
+++ b/ext/htslib/htslib/vcfutils.h
@@ -0,0 +1,143 @@
+/// @file htslib/vcfutils.h
+/// Allele-related utility functions.
+/*
+    Copyright (C) 2012, 2013, 2015-2016 Genome Research Ltd.
+
+    Author: Petr Danecek <pd3@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#ifndef HTSLIB_VCFUTILS_H
+#define HTSLIB_VCFUTILS_H
+
+#include "vcf.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct kbitset_t;
+
+/**
+ *  bcf_trim_alleles() - remove ALT alleles unused in genotype fields
+ *  @header:  for access to BCF_DT_ID dictionary
+ *  @line:    VCF line obtain from vcf_parse1
+ *
+ *  Returns the number of removed alleles on success or negative
+ *  on error:
+ *      -1 .. some allele index is out of bounds
+ *      -2 .. could not remove alleles
+ */
+HTSLIB_EXPORT
+int bcf_trim_alleles(const bcf_hdr_t *header, bcf1_t *line);
+
+/**
+ *  bcf_remove_alleles() - remove ALT alleles according to bitmask @mask
+ *  @header:  for access to BCF_DT_ID dictionary
+ *  @line:    VCF line obtained from vcf_parse1
+ *  @mask:    alleles to remove
+ *
+ *  If you have more than 31 alleles, then the integer bit mask will
+ *  overflow, so use bcf_remove_allele_set instead
+ *  Returns 0 on success, <0 on error
+ */
+HTSLIB_EXPORT
+int bcf_remove_alleles(const bcf_hdr_t *header, bcf1_t *line, int mask) HTS_DEPRECATED("Please use bcf_remove_allele_set instead");
+
+/**
+ *  bcf_remove_allele_set() - remove ALT alleles according to bitset @rm_set
+ *  @header:  for access to BCF_DT_ID dictionary
+ *  @line:    VCF line obtained from vcf_parse1
+ *  @rm_set:  pointer to kbitset_t object with bits set for allele
+ *            indexes to remove
+ *
+ *  Returns 0 on success or -1 on failure
+ *
+ *  Number=A,R,G INFO and FORMAT fields will be updated accordingly.
+ */
+HTSLIB_EXPORT
+int bcf_remove_allele_set(const bcf_hdr_t *header, bcf1_t *line, const struct kbitset_t *rm_set);
+
+/**
+ *  bcf_calc_ac() - calculate the number of REF and ALT alleles
+ *  @header:  for access to BCF_DT_ID dictionary
+ *  @line:    VCF line obtained from vcf_parse1
+ *  @ac:      array of length line->n_allele
+ *  @which:   determine if INFO/AN,AC and indv fields be used
+ *
+ *  Returns 1 if the call succeeded, or 0 if the value could not
+ *  be determined.
+ *
+ *  The value of @which determines if existing INFO/AC,AN can be
+ *  used (BCF_UN_INFO) and and if indv fields can be split (BCF_UN_FMT).
+ */
+HTSLIB_EXPORT
+int bcf_calc_ac(const bcf_hdr_t *header, bcf1_t *line, int *ac, int which);
+
+
+/**
+ * bcf_gt_type() - determines type of the genotype
+ * @fmt_ptr:  the GT format field as set for example by set_fmt_ptr
+ * @isample:  sample index (starting from 0)
+ * @ial:      index of the 1st non-reference allele (starting from 1)
+ * @jal:      index of the 2nd non-reference allele (starting from 1)
+ *
+ * Returns the type of the genotype (one of GT_HOM_RR, GT_HET_RA,
+ * GT_HOM_AA, GT_HET_AA, GT_HAPL_R, GT_HAPL_A or GT_UNKN). If $ial
+ * is not NULL and the genotype has one or more non-reference
+ * alleles, $ial will be set. In case of GT_HET_AA, $ial is the
+ * position of the allele which appeared first in ALT. If $jal is
+ * not null and the genotype is GT_HET_AA, $jal will be set and is
+ * the position of the second allele in ALT.
+ */
+#define GT_HOM_RR 0 // note: the actual value of GT_* matters, used in dosage r2 calculation
+#define GT_HOM_AA 1
+#define GT_HET_RA 2
+#define GT_HET_AA 3
+#define GT_HAPL_R 4
+#define GT_HAPL_A 5
+#define GT_UNKN   6
+HTSLIB_EXPORT
+int bcf_gt_type(bcf_fmt_t *fmt_ptr, int isample, int *ial, int *jal);
+
+static inline int bcf_acgt2int(char c)
+{
+    if ( (int)c>96 ) c -= 32;
+    if ( c=='A' ) return 0;
+    if ( c=='C' ) return 1;
+    if ( c=='G' ) return 2;
+    if ( c=='T' ) return 3;
+    return -1;
+}
+
+#define bcf_int2acgt(i) "ACGT"[i]
+
+/**
+  * bcf_ij2G() - common task: allele indexes to Number=G index (diploid)
+  * @i,j:  allele indexes, 0-based, i<=j
+  *
+  * Returns index to the Number=G diploid array
+  */
+#define bcf_ij2G(i, j) ((j)*((j)+1)/2+(i))
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/ext/htslib/htslib_vars.mk b/ext/htslib/htslib_vars.mk
new file mode 100644
index 0000000..6af7186
--- /dev/null
+++ b/ext/htslib/htslib_vars.mk
@@ -0,0 +1,54 @@
+# Makefile variables useful for third-party code using htslib's public API.
+#
+#    Copyright (C) 2013-2017, 2019-2020 Genome Research Ltd.
+#
+#    Author: John Marshall <jm18@sanger.ac.uk>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+# These variables can be used to express dependencies on htslib headers.
+# See htslib.mk for details.
+
+htslib_bgzf_h = $(HTSPREFIX)htslib/bgzf.h $(htslib_hts_defs_h)
+htslib_cram_h = $(HTSPREFIX)htslib/cram.h $(htslib_hts_defs_h) $(htslib_hts_h) $(htslib_sam_h)
+htslib_faidx_h = $(HTSPREFIX)htslib/faidx.h $(htslib_hts_defs_h) $(htslib_hts_h)
+htslib_hfile_h = $(HTSPREFIX)htslib/hfile.h $(htslib_hts_defs_h)
+htslib_hts_h = $(HTSPREFIX)htslib/hts.h $(htslib_hts_defs_h) $(htslib_hts_log_h) $(htslib_kstring_h) $(htslib_kroundup_h)
+htslib_hts_defs_h = $(HTSPREFIX)htslib/hts_defs.h
+htslib_hts_endian_h = $(HTSPREFIX)htslib/hts_endian.h
+htslib_hts_expr_h = $(HTSPREFIX)htslib/hts_expr.h $(htslib_kstring_h) $(htslib_hts_defs_h)
+htslib_hts_log_h = $(HTSPREFIX)htslib/hts_log.h $(htslib_hts_defs_h)
+htslib_hts_os_h = $(HTSPREFIX)htslib/hts_os.h $(htslib_hts_defs_h)
+htslib_kbitset_h = $(HTSPREFIX)htslib/kbitset.h
+htslib_kfunc_h = $(HTSPREFIX)htslib/kfunc.h $(htslib_hts_defs_h)
+htslib_khash_h = $(HTSPREFIX)htslib/khash.h $(htslib_kstring_h) $(htslib_kroundup_h)
+htslib_khash_str2int_h = $(HTSPREFIX)htslib/khash_str2int.h $(htslib_khash_h)
+htslib_klist_h = $(HTSPREFIX)htslib/klist.h
+htslib_kroundup_h = $(HTSPREFIX)htslib/kroundup.h
+htslib_kseq_h = $(HTSPREFIX)htslib/kseq.h
+htslib_ksort_h = $(HTSPREFIX)htslib/ksort.h $(htslib_hts_defs_h)
+htslib_kstring_h = $(HTSPREFIX)htslib/kstring.h $(htslib_hts_defs_h) $(htslib_kroundup_h)
+htslib_regidx_h = $(HTSPREFIX)htslib/regidx.h $(htslib_hts_h)
+htslib_sam_h = $(HTSPREFIX)htslib/sam.h $(htslib_hts_h) $(htslib_hts_endian_h)
+htslib_synced_bcf_reader_h = $(HTSPREFIX)htslib/synced_bcf_reader.h $(htslib_hts_h) $(htslib_vcf_h) $(htslib_tbx_h)
+htslib_tbx_h = $(HTSPREFIX)htslib/tbx.h $(htslib_hts_h)
+htslib_thread_pool_h = $(HTSPREFIX)htslib/thread_pool.h $(htslib_hts_defs_h)
+htslib_vcf_h = $(HTSPREFIX)htslib/vcf.h $(htslib_hts_h) $(htslib_kstring_h) $(htslib_hts_defs_h) $(htslib_hts_endian_h)
+htslib_vcf_sweep_h = $(HTSPREFIX)htslib/vcf_sweep.h $(htslib_hts_h) $(htslib_vcf_h)
+htslib_vcfutils_h = $(HTSPREFIX)htslib/vcfutils.h $(htslib_vcf_h)
diff --git a/ext/htslib/kfunc.c b/ext/htslib/kfunc.c
new file mode 100644
index 0000000..bf15cdf
--- /dev/null
+++ b/ext/htslib/kfunc.c
@@ -0,0 +1,313 @@
+/* The MIT License
+
+   Copyright (C) 2010, 2013-2014, 2020 Genome Research Ltd.
+   Copyright (C) 2011 Attractive Chaos <attractor@live.co.uk>
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be
+   included in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+*/
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+
+#include <math.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include "htslib/kfunc.h"
+
+/* Log gamma function
+ * \log{\Gamma(z)}
+ * AS245, 2nd algorithm, http://lib.stat.cmu.edu/apstat/245
+ */
+double kf_lgamma(double z)
+{
+	double x = 0;
+	x += 0.1659470187408462e-06 / (z+7);
+	x += 0.9934937113930748e-05 / (z+6);
+	x -= 0.1385710331296526     / (z+5);
+	x += 12.50734324009056      / (z+4);
+	x -= 176.6150291498386      / (z+3);
+	x += 771.3234287757674      / (z+2);
+	x -= 1259.139216722289      / (z+1);
+	x += 676.5203681218835      / z;
+	x += 0.9999999999995183;
+	return log(x) - 5.58106146679532777 - z + (z-0.5) * log(z+6.5);
+}
+
+/* complementary error function
+ * \frac{2}{\sqrt{\pi}} \int_x^{\infty} e^{-t^2} dt
+ * AS66, 2nd algorithm, http://lib.stat.cmu.edu/apstat/66
+ */
+double kf_erfc(double x)
+{
+	const double p0 = 220.2068679123761;
+	const double p1 = 221.2135961699311;
+	const double p2 = 112.0792914978709;
+	const double p3 = 33.912866078383;
+	const double p4 = 6.37396220353165;
+	const double p5 = .7003830644436881;
+	const double p6 = .03526249659989109;
+	const double q0 = 440.4137358247522;
+	const double q1 = 793.8265125199484;
+	const double q2 = 637.3336333788311;
+	const double q3 = 296.5642487796737;
+	const double q4 = 86.78073220294608;
+	const double q5 = 16.06417757920695;
+	const double q6 = 1.755667163182642;
+	const double q7 = .08838834764831844;
+	double expntl, z, p;
+	z = fabs(x) * M_SQRT2;
+	if (z > 37.) return x > 0.? 0. : 2.;
+	expntl = exp(z * z * - .5);
+	if (z < 10. / M_SQRT2) // for small z
+	    p = expntl * ((((((p6 * z + p5) * z + p4) * z + p3) * z + p2) * z + p1) * z + p0)
+			/ (((((((q7 * z + q6) * z + q5) * z + q4) * z + q3) * z + q2) * z + q1) * z + q0);
+	else p = expntl / 2.506628274631001 / (z + 1. / (z + 2. / (z + 3. / (z + 4. / (z + .65)))));
+	return x > 0.? 2. * p : 2. * (1. - p);
+}
+
+/* The following computes regularized incomplete gamma functions.
+ * Formulas are taken from Wiki, with additional input from Numerical
+ * Recipes in C (for modified Lentz's algorithm) and AS245
+ * (http://lib.stat.cmu.edu/apstat/245).
+ *
+ * A good online calculator is available at:
+ *
+ *   http://www.danielsoper.com/statcalc/calc23.aspx
+ *
+ * It calculates upper incomplete gamma function, which equals
+ * kf_gammaq(s,z)*tgamma(s).
+ */
+
+#define KF_GAMMA_EPS 1e-14
+#define KF_TINY 1e-290
+
+// regularized lower incomplete gamma function, by series expansion
+static double _kf_gammap(double s, double z)
+{
+	double sum, x;
+	int k;
+	for (k = 1, sum = x = 1.; k < 100; ++k) {
+		sum += (x *= z / (s + k));
+		if (x / sum < KF_GAMMA_EPS) break;
+	}
+	return exp(s * log(z) - z - kf_lgamma(s + 1.) + log(sum));
+}
+// regularized upper incomplete gamma function, by continued fraction
+static double _kf_gammaq(double s, double z)
+{
+	int j;
+	double C, D, f;
+	f = 1. + z - s; C = f; D = 0.;
+	// Modified Lentz's algorithm for computing continued fraction
+	// See Numerical Recipes in C, 2nd edition, section 5.2
+	for (j = 1; j < 100; ++j) {
+		double a = j * (s - j), b = (j<<1) + 1 + z - s, d;
+		D = b + a * D;
+		if (D < KF_TINY) D = KF_TINY;
+		C = b + a / C;
+		if (C < KF_TINY) C = KF_TINY;
+		D = 1. / D;
+		d = C * D;
+		f *= d;
+		if (fabs(d - 1.) < KF_GAMMA_EPS) break;
+	}
+	return exp(s * log(z) - z - kf_lgamma(s) - log(f));
+}
+
+double kf_gammap(double s, double z)
+{
+	return z <= 1. || z < s? _kf_gammap(s, z) : 1. - _kf_gammaq(s, z);
+}
+
+double kf_gammaq(double s, double z)
+{
+	return z <= 1. || z < s? 1. - _kf_gammap(s, z) : _kf_gammaq(s, z);
+}
+
+/* Regularized incomplete beta function. The method is taken from
+ * Numerical Recipe in C, 2nd edition, section 6.4. The following web
+ * page calculates the incomplete beta function, which equals
+ * kf_betai(a,b,x) * gamma(a) * gamma(b) / gamma(a+b):
+ *
+ *   http://www.danielsoper.com/statcalc/calc36.aspx
+ */
+static double kf_betai_aux(double a, double b, double x)
+{
+	double C, D, f;
+	int j;
+	if (x == 0.) return 0.;
+	if (x == 1.) return 1.;
+	f = 1.; C = f; D = 0.;
+	// Modified Lentz's algorithm for computing continued fraction
+	for (j = 1; j < 200; ++j) {
+		double aa, d;
+		int m = j>>1;
+		aa = (j&1)? -(a + m) * (a + b + m) * x / ((a + 2*m) * (a + 2*m + 1))
+			: m * (b - m) * x / ((a + 2*m - 1) * (a + 2*m));
+		D = 1. + aa * D;
+		if (D < KF_TINY) D = KF_TINY;
+		C = 1. + aa / C;
+		if (C < KF_TINY) C = KF_TINY;
+		D = 1. / D;
+		d = C * D;
+		f *= d;
+		if (fabs(d - 1.) < KF_GAMMA_EPS) break;
+	}
+	return exp(kf_lgamma(a+b) - kf_lgamma(a) - kf_lgamma(b) + a * log(x) + b * log(1.-x)) / a / f;
+}
+double kf_betai(double a, double b, double x)
+{
+	return x < (a + 1.) / (a + b + 2.)? kf_betai_aux(a, b, x) : 1. - kf_betai_aux(b, a, 1. - x);
+}
+
+#ifdef KF_MAIN
+#include <stdio.h>
+int main(int argc, char *argv[])
+{
+	double x = 5.5, y = 3;
+	double a, b;
+	printf("erfc(%lg): %lg, %lg\n", x, erfc(x), kf_erfc(x));
+	printf("upper-gamma(%lg,%lg): %lg\n", x, y, kf_gammaq(y, x)*tgamma(y));
+	a = 2; b = 2; x = 0.5;
+	printf("incomplete-beta(%lg,%lg,%lg): %lg\n", a, b, x, kf_betai(a, b, x) / exp(kf_lgamma(a+b) - kf_lgamma(a) - kf_lgamma(b)));
+	return 0;
+}
+#endif
+
+
+// log\binom{n}{k}
+static double lbinom(int n, int k)
+{
+    if (k == 0 || n == k) return 0;
+    return lgamma(n+1) - lgamma(k+1) - lgamma(n-k+1);
+}
+
+// n11  n12  | n1_
+// n21  n22  | n2_
+//-----------+----
+// n_1  n_2  | n
+
+// hypergeometric distribution
+static double hypergeo(int n11, int n1_, int n_1, int n)
+{
+    return exp(lbinom(n1_, n11) + lbinom(n-n1_, n_1-n11) - lbinom(n, n_1));
+}
+
+typedef struct {
+    int n11, n1_, n_1, n;
+    double p;
+} hgacc_t;
+
+// incremental version of hypergenometric distribution
+static double hypergeo_acc(int n11, int n1_, int n_1, int n, hgacc_t *aux)
+{
+    if (n1_ || n_1 || n) {
+        aux->n11 = n11; aux->n1_ = n1_; aux->n_1 = n_1; aux->n = n;
+    } else { // then only n11 changed; the rest fixed
+        if (n11%11 && n11 + aux->n - aux->n1_ - aux->n_1) {
+            if (n11 == aux->n11 + 1) { // incremental
+                aux->p *= (double)(aux->n1_ - aux->n11) / n11
+                    * (aux->n_1 - aux->n11) / (n11 + aux->n - aux->n1_ - aux->n_1);
+                aux->n11 = n11;
+                return aux->p;
+            }
+            if (n11 == aux->n11 - 1) { // incremental
+                aux->p *= (double)aux->n11 / (aux->n1_ - n11)
+                    * (aux->n11 + aux->n - aux->n1_ - aux->n_1) / (aux->n_1 - n11);
+                aux->n11 = n11;
+                return aux->p;
+            }
+        }
+        aux->n11 = n11;
+    }
+    aux->p = hypergeo(aux->n11, aux->n1_, aux->n_1, aux->n);
+    return aux->p;
+}
+
+double kt_fisher_exact(int n11, int n12, int n21, int n22, double *_left, double *_right, double *two)
+{
+    int i, j, max, min;
+    double p, q, left, right;
+    hgacc_t aux;
+    int n1_, n_1, n;
+
+    n1_ = n11 + n12; n_1 = n11 + n21; n = n11 + n12 + n21 + n22; // calculate n1_, n_1 and n
+    max = (n_1 < n1_) ? n_1 : n1_; // max n11, for right tail
+    min = n1_ + n_1 - n;    // not sure why n11-n22 is used instead of min(n_1,n1_)
+    if (min < 0) min = 0; // min n11, for left tail
+    *two = *_left = *_right = 1.;
+    if (min == max) return 1.; // no need to do test
+    q = hypergeo_acc(n11, n1_, n_1, n, &aux); // the probability of the current table
+
+    if (q == 0.0) {
+        /*
+          If here, the calculated probablility is so small it can't be stored
+          in a double, which is possible when the table contains fairly large
+          numbers.  If this happens, most of the calculation can be skipped
+          as 'left', 'right' and '*two' will be (to a good approximation) 0.0.
+          The returned values '*_left' and '*_right' depend on which side
+          of the hypergeometric PDF 'n11' sits.  This can be found by
+          comparing with the mode of the distribution, the formula for which
+          can be found at:
+          https://en.wikipedia.org/wiki/Hypergeometric_distribution
+          Note that in the comparison we multiply through by the denominator
+          of the mode (n + 2) to avoid a division.
+        */
+        if ((int64_t) n11 * ((int64_t) n + 2) < ((int64_t) n_1 + 1) * ((int64_t) n1_ + 1)) {
+            // Peak to right of n11, so probability will be lower for all
+            // of the region from min to n11 and higher for at least some
+            // of the region from n11 to max; hence abs(i-n11) will be 0,
+            // abs(j-n11) will be > 0 and:
+            *_left = 0.0; *_right = 1.0; *two = 0.0;
+            return 0.0;
+        } else {
+            // Peak to left of n11, so probability will be lower for all
+            // of the region from n11 to max and higher for at least some
+            // of the region from min to n11; hence abs(i-n11) will be > 0,
+            // abs(j-n11) will be 0 and:
+            *_left = 1.0; *_right = 0.0; *two = 0.0;
+            return 0.0;
+        }
+    }
+
+    // left tail
+    p = hypergeo_acc(min, 0, 0, 0, &aux);
+    for (left = 0., i = min + 1; p < 0.99999999 * q && i<=max; ++i) // loop until underflow
+        left += p, p = hypergeo_acc(i, 0, 0, 0, &aux);
+    --i;
+    if (p < 1.00000001 * q) left += p;
+    else --i;
+    // right tail
+    p = hypergeo_acc(max, 0, 0, 0, &aux);
+    for (right = 0., j = max - 1; p < 0.99999999 * q && j>=0; --j) // loop until underflow
+        right += p, p = hypergeo_acc(j, 0, 0, 0, &aux);
+    ++j;
+    if (p < 1.00000001 * q) right += p;
+    else ++j;
+    // two-tail
+    *two = left + right;
+    if (*two > 1.) *two = 1.;
+    // adjust left and right
+    if (abs(i - n11) < abs(j - n11)) right = 1. - left + q;
+    else left = 1.0 - right + q;
+    *_left = left; *_right = right;
+    return q;
+}
diff --git a/ext/htslib/kstring.c b/ext/htslib/kstring.c
new file mode 100644
index 0000000..9a6142e
--- /dev/null
+++ b/ext/htslib/kstring.c
@@ -0,0 +1,452 @@
+/* The MIT License
+
+   Copyright (C) 2011 by Attractive Chaos <attractor@live.co.uk>
+   Copyright (C) 2013-2018, 2020-2021, 2023 Genome Research Ltd.
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be
+   included in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+*/
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+
+#include <stdarg.h>
+#include <stdio.h>
+#include <ctype.h>
+#include <string.h>
+#include <stdint.h>
+#include <math.h>
+#include "htslib/kstring.h"
+
+int kputd(double d, kstring_t *s) {
+	int len = 0;
+	char buf[21], *cp = buf+20, *ep;
+	if (d == 0) {
+		if (signbit(d)) {
+			kputsn("-0",2,s);
+			return 2;
+		} else {
+			kputsn("0",1,s);
+			return 1;
+		}
+	}
+
+	if (d < 0) {
+		kputc('-',s);
+		len = 1;
+		d=-d;
+	}
+	if (!(d >= 0.0001 && d <= 999999)) {
+		if (ks_resize(s, s->l + 50) < 0)
+			return EOF;
+		// We let stdio handle the exponent cases
+		int s2 = snprintf(s->s + s->l, s->m - s->l, "%g", d);
+		len += s2;
+		s->l += s2;
+		return len;
+	}
+
+	// Correction for rounding - rather ugly
+	// Optimised for small numbers.
+
+	uint32_t i;
+	if (d<0.001)         i = rint(d*1000000000), cp -= 1;
+	else if (d < 0.01)   i = rint(d*100000000),  cp -= 2;
+	else if (d < 0.1)    i = rint(d*10000000),   cp -= 3;
+	else if (d < 1)      i = rint(d*1000000),    cp -= 4;
+	else if (d < 10)     i = rint(d*100000),     cp -= 5;
+	else if (d < 100)    i = rint(d*10000),      cp -= 6;
+	else if (d < 1000)   i = rint(d*1000),       cp -= 7;
+	else if (d < 10000)  i = rint(d*100),        cp -= 8;
+	else if (d < 100000) i = rint(d*10),         cp -= 9;
+	else                 i = rint(d),            cp -= 10;
+
+	// integer i is always 6 digits, so print it 2 at a time.
+	static const char kputuw_dig2r[] =
+		"00010203040506070809"
+		"10111213141516171819"
+		"20212223242526272829"
+		"30313233343536373839"
+		"40414243444546474849"
+		"50515253545556575859"
+		"60616263646566676869"
+		"70717273747576777879"
+		"80818283848586878889"
+		"90919293949596979899";
+
+	memcpy(cp-=2, &kputuw_dig2r[2*(i%100)], 2); i /= 100;
+	memcpy(cp-=2, &kputuw_dig2r[2*(i%100)], 2); i /= 100;
+	memcpy(cp-=2, &kputuw_dig2r[2*(i%100)], 2);
+
+	// Except when it rounds up (d=0.009999999 is i=1000000)
+	if (i >= 100)
+		*--cp = '0' + (i/100);
+
+
+	int p = buf+20-cp;
+	if (p <= 10) { /* d < 1 */
+		// 0.00123 is 123, so add leading zeros and 0.
+		ep = cp+5; // 6 precision
+		while (p < 10) { // aka d < 1
+			*--cp = '0';
+			p++;
+		}
+		*--cp = '.';
+		*--cp = '0';
+	} else {
+		// 123.001 is 123001 with p==13, so move 123 down and add "."
+		// Equiv to memmove(cp-1, cp, p-10); cp--;
+		char *xp = --cp;
+		ep = cp+6;
+		while (p > 10) {
+			xp[0] = xp[1];
+			xp++;
+			p--;
+		}
+		xp[0] = '.';
+	}
+
+	// Cull trailing zeros
+	while (*ep == '0' && ep > cp)
+		ep--;
+
+	// End can be 1 out due to the mostly-6 but occasionally 7 (i==1) case.
+	// Also code with "123." which should be "123"
+	if (*ep && *ep != '.')
+		ep++;
+	*ep = 0;
+
+	int sl = ep-cp;
+	len += sl;
+	kputsn(cp, sl, s);
+	return len;
+}
+
+int kvsprintf(kstring_t *s, const char *fmt, va_list ap)
+{
+	va_list args;
+	int l;
+	va_copy(args, ap);
+
+	if (fmt[0] == '%' && fmt[1] == 'g' && fmt[2] == 0) {
+		double d = va_arg(args, double);
+		l = kputd(d, s);
+		va_end(args);
+		return l;
+	}
+
+	if (!s->s) {
+		const size_t sz = 64;
+		s->s = malloc(sz);
+		if (!s->s)
+			return -1;
+		s->m = sz;
+		s->l = 0;
+	}
+
+	l = vsnprintf(s->s + s->l, s->m - s->l, fmt, args); // This line does not work with glibc 2.0. See `man snprintf'.
+	va_end(args);
+	if (l + 1 > s->m - s->l) {
+		if (ks_resize(s, s->l + l + 2) < 0)
+			return -1;
+		va_copy(args, ap);
+		l = vsnprintf(s->s + s->l, s->m - s->l, fmt, args);
+		va_end(args);
+	}
+	s->l += l;
+	return l;
+}
+
+int ksprintf(kstring_t *s, const char *fmt, ...)
+{
+	va_list ap;
+	int l;
+	va_start(ap, fmt);
+	l = kvsprintf(s, fmt, ap);
+	va_end(ap);
+	return l;
+}
+
+char *kstrtok(const char *str, const char *sep_in, ks_tokaux_t *aux)
+{
+	const unsigned char *p, *start, *sep = (unsigned char *) sep_in;
+	if (sep) { // set up the table
+		if (str == 0 && aux->finished) return 0; // no need to set up if we have finished
+		aux->finished = 0;
+		if (sep[0] && sep[1]) {
+			aux->sep = -1;
+			aux->tab[0] = aux->tab[1] = aux->tab[2] = aux->tab[3] = 0;
+			for (p = sep; *p; ++p) aux->tab[*p>>6] |= 1ull<<(*p&0x3f);
+		} else aux->sep = sep[0];
+	}
+	if (aux->finished) return 0;
+	else if (str) start = (unsigned char *) str, aux->finished = 0;
+	else start = (unsigned char *) aux->p + 1;
+	if (aux->sep < 0) {
+		for (p = start; *p; ++p)
+			if (aux->tab[*p>>6]>>(*p&0x3f)&1) break;
+	} else {
+		// Using strchr is fast for next token, but slower for
+		// last token due to extra pass from strlen.  Overall
+		// on a VCF parse this func was 146% faster with // strchr.
+		// Equiv to:
+		// for (p = start; *p; ++p) if (*p == aux->sep) break;
+
+		// NB: We could use strchrnul() here from glibc if detected,
+		// which is ~40% faster again, but it's not so portable.
+		// i.e.   p = (uint8_t *)strchrnul((char *)start, aux->sep);
+		uint8_t *p2 = (uint8_t *)strchr((char *)start, aux->sep);
+		p = p2 ? p2 : start + strlen((char *)start);
+	}
+	aux->p = (const char *) p; // end of token
+	if (*p == 0) aux->finished = 1; // no more tokens
+	return (char*)start;
+}
+
+// s MUST BE a null terminated string; l = strlen(s)
+int ksplit_core(char *s, int delimiter, int *_max, int **_offsets)
+{
+	int i, n, max, last_char, last_start, *offsets, l;
+	n = 0; max = *_max; offsets = *_offsets;
+	l = strlen(s);
+
+#define __ksplit_aux do {						\
+		if (_offsets) {						\
+			s[i] = 0;					\
+			if (n == max) {					\
+				int *tmp;				\
+				max = max? max<<1 : 2;			\
+				if ((tmp = (int*)realloc(offsets, sizeof(int) * max))) {  \
+					offsets = tmp;			\
+				} else	{				\
+					free(offsets);			\
+					*_offsets = NULL;		\
+					return 0;			\
+				}					\
+			}						\
+			offsets[n++] = last_start;			\
+		} else ++n;						\
+	} while (0)
+
+	for (i = 0, last_char = last_start = 0; i <= l; ++i) {
+		if (delimiter == 0) {
+			if (isspace((int)((unsigned char) s[i])) || s[i] == 0) {
+				if (isgraph(last_char))
+                    __ksplit_aux; // the end of a field
+			} else {
+				if (isspace(last_char) || last_char == 0)
+                    last_start = i;
+			}
+		} else {
+			if (s[i] == delimiter || s[i] == 0) {
+				if (last_char != 0 && last_char != delimiter) __ksplit_aux; // the end of a field
+			} else {
+				if (last_char == delimiter || last_char == 0) last_start = i;
+			}
+		}
+		last_char = (int)((unsigned char)s[i]);
+	}
+	*_max = max; *_offsets = offsets;
+	return n;
+}
+
+int kgetline(kstring_t *s, kgets_func *fgets_fn, void *fp)
+{
+	size_t l0 = s->l;
+
+	while (s->l == l0 || s->s[s->l-1] != '\n') {
+		if (s->m - s->l < 200) {
+			if (ks_resize(s, s->m + 200) < 0)
+				return EOF;
+		}
+		if (fgets_fn(s->s + s->l, s->m - s->l, fp) == NULL) break;
+		s->l += strlen(s->s + s->l);
+	}
+
+	if (s->l == l0) return EOF;
+
+	if (s->l > l0 && s->s[s->l-1] == '\n') {
+		s->l--;
+		if (s->l > l0 && s->s[s->l-1] == '\r') s->l--;
+	}
+	s->s[s->l] = '\0';
+	return 0;
+}
+
+int kgetline2(kstring_t *s, kgets_func2 *fgets_fn, void *fp)
+{
+	size_t l0 = s->l;
+
+	while (s->l == l0 || s->s[s->l-1] != '\n') {
+		if (s->m - s->l < 200) {
+			// We return EOF for both EOF and error and the caller
+			// needs to check for errors in fp, and we haven't
+			// even got there yet.
+			//
+			// The only way of propagating memory errors is to
+			// deliberately call something that we know triggers
+			// and error so fp is also set.  This works for
+			// hgets, but not for gets where reading <= 0 bytes
+			// isn't an error.
+			if (ks_resize(s, s->m + 200) < 0) {
+				fgets_fn(s->s + s->l, 0, fp);
+				return EOF;
+			}
+		}
+		ssize_t len = fgets_fn(s->s + s->l, s->m - s->l, fp);
+		if (len <= 0) break;
+		s->l += len;
+	}
+
+	if (s->l == l0) return EOF;
+
+	if (s->l > l0 && s->s[s->l-1] == '\n') {
+		s->l--;
+		if (s->l > l0 && s->s[s->l-1] == '\r') s->l--;
+	}
+	s->s[s->l] = '\0';
+	return 0;
+}
+
+/**********************
+ * Boyer-Moore search *
+ **********************/
+
+typedef unsigned char ubyte_t;
+
+// reference: http://www-igm.univ-mlv.fr/~lecroq/string/node14.html
+static int *ksBM_prep(const ubyte_t *pat, int m)
+{
+	int i, *suff, *prep, *bmGs, *bmBc;
+	prep = (int*)calloc(m + 256, sizeof(int));
+    if (!prep) return NULL;
+	bmGs = prep; bmBc = prep + m;
+	{ // preBmBc()
+		for (i = 0; i < 256; ++i) bmBc[i] = m;
+		for (i = 0; i < m - 1; ++i) bmBc[pat[i]] = m - i - 1;
+	}
+	suff = (int*)calloc(m, sizeof(int));
+    if (!suff) { free(prep); return NULL; }
+	{ // suffixes()
+		int f = 0, g;
+		suff[m - 1] = m;
+		g = m - 1;
+		for (i = m - 2; i >= 0; --i) {
+			if (i > g && suff[i + m - 1 - f] < i - g)
+				suff[i] = suff[i + m - 1 - f];
+			else {
+				if (i < g) g = i;
+				f = i;
+				while (g >= 0 && pat[g] == pat[g + m - 1 - f]) --g;
+				suff[i] = f - g;
+			}
+		}
+	}
+	{ // preBmGs()
+		int j = 0;
+		for (i = 0; i < m; ++i) bmGs[i] = m;
+		for (i = m - 1; i >= 0; --i)
+			if (suff[i] == i + 1)
+				for (; j < m - 1 - i; ++j)
+					if (bmGs[j] == m)
+						bmGs[j] = m - 1 - i;
+		for (i = 0; i <= m - 2; ++i)
+			bmGs[m - 1 - suff[i]] = m - 1 - i;
+	}
+	free(suff);
+	return prep;
+}
+
+void *kmemmem(const void *_str, int n, const void *_pat, int m, int **_prep)
+{
+	int i, j, *prep = 0, *bmGs, *bmBc;
+	const ubyte_t *str, *pat;
+	str = (const ubyte_t*)_str; pat = (const ubyte_t*)_pat;
+	prep = (_prep == 0 || *_prep == 0)? ksBM_prep(pat, m) : *_prep;
+    if (!prep) return NULL;
+	if (_prep && *_prep == 0) *_prep = prep;
+	bmGs = prep; bmBc = prep + m;
+	j = 0;
+	while (j <= n - m) {
+		for (i = m - 1; i >= 0 && pat[i] == str[i+j]; --i);
+		if (i >= 0) {
+			int max = bmBc[str[i+j]] - m + 1 + i;
+			if (max < bmGs[i]) max = bmGs[i];
+			j += max;
+		} else return (void*)(str + j);
+	}
+	if (_prep == 0) free(prep);
+	return 0;
+}
+
+char *kstrstr(const char *str, const char *pat, int **_prep)
+{
+	return (char*)kmemmem(str, strlen(str), pat, strlen(pat), _prep);
+}
+
+char *kstrnstr(const char *str, const char *pat, int n, int **_prep)
+{
+	return (char*)kmemmem(str, n, pat, strlen(pat), _prep);
+}
+
+/***********************
+ * The main() function *
+ ***********************/
+
+#ifdef KSTRING_MAIN
+#include <stdio.h>
+int main()
+{
+	kstring_t *s;
+	int *fields, n, i;
+	ks_tokaux_t aux;
+	char *p;
+	s = (kstring_t*)calloc(1, sizeof(kstring_t));
+	// test ksprintf()
+	ksprintf(s, " abcdefg:    %d ", 100);
+	printf("'%s'\n", s->s);
+	// test ksplit()
+	fields = ksplit(s, 0, &n);
+	for (i = 0; i < n; ++i)
+		printf("field[%d] = '%s'\n", i, s->s + fields[i]);
+	// test kstrtok()
+	s->l = 0;
+	for (p = kstrtok("ab:cde:fg/hij::k", ":/", &aux); p; p = kstrtok(0, 0, &aux)) {
+		kputsn(p, aux.p - p, s);
+		kputc('\n', s);
+	}
+	printf("%s", s->s);
+	// free
+	free(s->s); free(s); free(fields);
+
+	{
+		static char *str = "abcdefgcdgcagtcakcdcd";
+		static char *pat = "cd";
+		char *ret, *s = str;
+		int *prep = 0;
+		while ((ret = kstrstr(s, pat, &prep)) != 0) {
+			printf("match: %s\n", ret);
+			s = ret + prep[0];
+		}
+		free(prep);
+	}
+	return 0;
+}
+#endif
diff --git a/ext/htslib/m4/hts_check_compile_flags_needed.m4 b/ext/htslib/m4/hts_check_compile_flags_needed.m4
new file mode 100644
index 0000000..7c1b6de
--- /dev/null
+++ b/ext/htslib/m4/hts_check_compile_flags_needed.m4
@@ -0,0 +1,63 @@
+# hts_check_compile_flags_needed.m4
+#
+# SYNOPSIS
+#
+#   HTS_CHECK_COMPILE_FLAGS_NEEDED(FEATURE, FLAGS, [INPUT], [ACTION-SUCCESS], [ACTION-FAILURE], [EXTRA-FLAGS])
+#
+# DESCRIPTION
+#
+#   Check whether the given FLAGS are required to build and link INPUT with
+#   the current language's compiler.  Compilation and linking are first
+#   tries without FLAGS.  If that fails it then tries to compile and
+#   link again with FLAGS.
+#
+#   FEATURE describes the feature being tested, and is used when printing
+#   messages and to name the cache entry (along with the tested flags).
+#
+#   ACTION-SUCCESS/ACTION-FAILURE are shell commands to execute on
+#   success/failure.  In ACTION-SUCCESS, $flags_needed will be set to
+#   either an empty string or FLAGS depending on the test results.
+#
+#   If EXTRA-FLAGS is defined, it is added to the current language's default
+#   flags (e.g. CFLAGS) when the check is done.  The check is thus made with
+#   the flags: "CFLAGS EXTRA-FLAGS FLAG".  This can for example be used to
+#   force the compiler to issue an error when a bad flag is given.
+#
+#   If omitted, INPUT defaults to AC_LANG_PROGRAM(), although that probably
+#   isn't very useful.
+#
+#   NOTE: Implementation based on AX_CHECK_COMPILE_FLAG.
+#
+# LICENSE
+#
+#   Copyright (c) 2008 Guido U. Draheim <guidod@gmx.de>
+#   Copyright (c) 2011 Maarten Bosmans <mkbosmans@gmail.com>
+#   Copyright (c) 2023 Robert Davies <rmd@sanger.ac.uk>
+#
+#   Copying and distribution of this file, with or without modification, are
+#   permitted in any medium without royalty provided the copyright notice
+#   and this notice are preserved.  This file is offered as-is, without any
+#   warranty.
+
+#   HTS_CHECK_COMPILE_FLAGS_NEEDED(FEATURE, FLAGS, [INPUT], [ACTION-SUCCESS], [ACTION-FAILURE], [EXTRA-FLAGS])
+
+AC_DEFUN([HTS_CHECK_COMPILE_FLAGS_NEEDED],
+[AC_PREREQ(2.64)dnl for _AC_LANG_PREFIX and AS_VAR_IF
+AS_VAR_PUSHDEF([CACHEVAR],[hts_cv_check_[]_AC_LANG_ABBREV[]flags_needed_$1_$6_$2])dnl
+AC_CACHE_CHECK([_AC_LANG compiler flags needed for $1], CACHEVAR, [
+  AC_LINK_IFELSE([m4_default([$3],[AC_LANG_PROGRAM()])],
+    [AS_VAR_SET(CACHEVAR,[none])],
+    [ax_check_save_flags=$[]_AC_LANG_PREFIX[]FLAGS
+     _AC_LANG_PREFIX[]FLAGS="$[]_AC_LANG_PREFIX[]FLAGS $6 $2"
+     AC_LINK_IFELSE([m4_default([$3],[AC_LANG_PROGRAM()])],
+       [AS_VAR_SET(CACHEVAR,["$2"])],
+       [AS_VAR_SET(CACHEVAR,[unsupported])])
+     _AC_LANG_PREFIX[]FLAGS=$ax_check_save_flags])])
+AS_VAR_IF(CACHEVAR,unsupported, [
+  m4_default([$5], :)
+], [
+  AS_VAR_IF(CACHEVAR,none,[flags_needed=""], [flags_needed="$CACHEVAR"])
+  m4_default([$4], :)
+])
+AS_VAR_POPDEF([CACHEVAR])dnl
+])dnl HTS_CHECK_COMPILE_FLAGS_NEEDED
diff --git a/ext/htslib/m4/hts_hide_dynamic_syms.m4 b/ext/htslib/m4/hts_hide_dynamic_syms.m4
new file mode 100644
index 0000000..62ccb8e
--- /dev/null
+++ b/ext/htslib/m4/hts_hide_dynamic_syms.m4
@@ -0,0 +1,65 @@
+dnl @synopsis HTS_HIDE_DYNAMIC_SYMBOLS
+dnl
+dnl Turn on compiler options that prevent unwanted symbols from being exported
+dnl by shared libraries.
+dnl
+dnl @author Rob Davies <rmd@sanger.ac.uk>
+dnl @license MIT/Expat
+dnl
+dnl Copyright (C) 2018 Genome Research Ltd.
+dnl
+dnl Permission is hereby granted, free of charge, to any person obtaining a copy
+dnl of this software and associated documentation files (the "Software"), to
+dnl deal in the Software without restriction, including without limitation the
+dnl rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+dnl  sell copies of the Software, and to permit persons to whom the Software is
+dnl furnished to do so, subject to the following conditions:
+dnl
+dnl The above copyright notice and this permission notice shall be included in
+dnl all copies or substantial portions of the Software.
+dnl
+dnl THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+dnl IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+dnl FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+dnl THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+dnl LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+dnl FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+dnl DEALINGS IN THE SOFTWARE.
+
+# SYNOPSIS
+#
+# HTS_TEST_CC_C_LD_FLAG(FLAG, FOUND_VAR)
+#
+# Test if FLAG can be used on both CFLAGS and LDFLAGS.  It it works,
+# variable FOUND_VAR is set to FLAG.
+
+AC_DEFUN([HTS_TEST_CC_C_LD_FLAG],
+ [AS_VAR_PUSHDEF([hts_cv_check_flag],[hts_cv_check_$1])dnl
+  AC_CACHE_CHECK([whether the compiler accepts $1],
+   [hts_cv_check_flag],
+   [ac_check_save_cflags=$CFLAGS
+    ac_check_save_ldflags=$LDFLAGS
+    CFLAGS="$CFLAGS $1"
+    LDFLAGS="$LDFLAGS $1"
+    AC_LINK_IFELSE([AC_LANG_PROGRAM()],
+      [AS_VAR_SET([hts_cv_check_flag],[yes])
+       AS_IF([test "x$2" != x],[eval AS_TR_SH([$2])="$1"])],
+      [AS_VAR_SET([hts_cv_check_flag],[no])])
+    CFLAGS=$ac_check_save_cflags
+    LDFLAGS=$ac_check_save_ldflags])
+  AS_VAR_POPDEF([hts_cv_check_flag])dnl
+])
+
+AC_DEFUN([HTS_HIDE_DYNAMIC_SYMBOLS], [
+  # Test for flags to set default shared library visibility to hidden
+  # -fvisibility=hidden : GCC compatible
+  # -xldscope=hidden    : SunStudio
+  ac_opt_found=no
+  m4_foreach_w([ac_opt],[-fvisibility=hidden -xldscope=hidden],
+   [AS_IF([test "x$ac_opt_found" = "xno"],
+     [HTS_TEST_CC_C_LD_FLAG(ac_opt,[ac_opt_found])])
+   ])
+  AS_IF([test "x$ac_opt_found" != "xno"],
+   [CFLAGS="$CFLAGS $ac_opt_found"
+    LDFLAGS="$LDFLAGS $ac_opt_found"])
+])
diff --git a/ext/htslib/m4/hts_prog_cc_warnings.m4 b/ext/htslib/m4/hts_prog_cc_warnings.m4
new file mode 100644
index 0000000..f2aed93
--- /dev/null
+++ b/ext/htslib/m4/hts_prog_cc_warnings.m4
@@ -0,0 +1,208 @@
+dnl @synopsis HTS_PROG_CC_WARNINGS([ANSI])
+dnl
+dnl Derived from
+dnl     http://ac-archive.sourceforge.net/ac-archive/vl_prog_cc_warnings.html
+dnl
+dnl Enables a reasonable set of warnings for the C compiler.
+dnl Optionally, if the first argument is nonempty, turns on flags which
+dnl enforce and/or enable proper ANSI C if such are known with the
+dnl compiler used.
+dnl
+dnl Currently this macro knows about GCC, Solaris C compiler, Digital
+dnl Unix C compiler, C for AIX Compiler, HP-UX C compiler, IRIX C
+dnl compiler, NEC SX-5 (Super-UX 10) C compiler, and Cray J90 (Unicos
+dnl 10.0.0.8) C compiler.
+dnl
+dnl @category C
+dnl @author Ville Laurikari <vl@iki.fi>
+dnl Updated by Rob Davies <rmd@sanger.ac.uk> for HTSlib
+dnl @license AllPermissive
+dnl Copying and distribution of this file, with or without modification,
+dnl are permitted in any medium without royalty provided the copyright notice
+dnl and this notice are preserved. Users of this software should generally
+dnl follow the principles of the MIT License including its disclaimer.
+dnl Original Copyright (c) Ville Laurikari 2002
+dnl Modifications Copyright (c) Genome Research Limited 2015,2017
+
+AC_DEFUN([HTS_PROG_CC_WARNINGS], [
+  AC_ARG_ENABLE([warnings],
+    [AS_HELP_STRING([--disable-warnings], [turn off compiler warnings])],
+    [],
+    [enable_warnings=yes])
+
+  AS_IF([test "x$enable_warnings" != xno],[
+    AC_REQUIRE([AC_PROG_GREP])
+
+    ansi="$1"
+    AS_IF([test "x$ansi" = "x"],
+          [msg="for C compiler warning flags"],
+          [msg="for C compiler warning and ANSI conformance flags"])
+
+    AC_MSG_CHECKING($msg)
+    AC_CACHE_VAL(hts_cv_prog_cc_warnings, [dnl
+      hts_cv_prog_cc_warnings=""
+      AS_IF([test "x$CC" != "x"],[
+        cat > conftest.c <<EOF
+int main(int argc, char **argv) { return 0; }
+EOF
+
+dnl Most compilers print some kind of a version string with some command
+dnl line options (often "-V").  The version string should be checked
+dnl before doing a test compilation run with compiler-specific flags.
+dnl This is because some compilers (like the Cray compiler) only
+dnl produce a warning message for unknown flags instead of returning
+dnl an error, resulting in a false positive.  Also, compilers may do
+dnl erratic things when invoked with flags meant for a different
+dnl compiler.
+
+dnl We attempt to strip out any flags that are already on CFLAGS.
+dnl If an option needs more than one word (e.g. see Cray below) then
+dnl they should be separated by hash signs (#), which will be converted
+dnl to spaces before comparing and possibly adding to CFLAGS.
+dnl This separator will need to be changed if a new compiler ever needs
+dnl an option that includes a hash sign...
+
+        # Tests for flags to enable C compiler warnings
+        # GCC compatible
+        AS_IF([test "x$GCC" = "xyes" &&
+               "$CC" -c -Wall conftest.c > /dev/null 2>&1 &&
+               test -f conftest.o],[dnl
+          AS_IF([test "x$ansi" = "x"],
+                [hts_cv_prog_cc_warnings="-Wall"],
+                [hts_cv_prog_cc_warnings="-Wall -ansi -pedantic"])
+        ],
+        # Sun Studio or Solaris C compiler
+        ["$CC" -V 2>&1 | $GREP -i -E "WorkShop|Sun C" > /dev/null 2>&1 &&
+         "$CC" -c -v -Xc conftest.c > /dev/null 2>&1 &&
+         test -f conftest.o],[dnl
+        AS_IF([test "x$ansi" = "x"],
+              [hts_cv_prog_cc_warnings="-v"],
+              [hts_cv_prog_cc_warnings="-v -Xc"])
+        ],
+        # Digital Unix C compiler
+        ["$CC" -V 2>&1 | $GREP -i "Digital UNIX Compiler" > /dev/null 2>&1 &&
+         "$CC" -c -verbose -w0 -warnprotos -std1 conftest.c > /dev/null 2>&1 &&
+         test -f conftest.o], [dnl
+           AS_IF([test "x$ansi" = "x"],
+                 [hts_cv_prog_cc_warnings="-verbose -w0 -warnprotos"],
+                 [hts_cv_prog_cc_warnings="-verbose -w0 -warnprotos -std1"])
+           ],
+        # C for AIX Compiler
+        ["$CC" 2>&1 | $GREP -i "C for AIX Compiler" > /dev/null 2>&1 &&
+         "$CC" -c -qlanglvl=ansi -qinfo=all conftest.c > /dev/null 2>&1 &&
+         test -f conftest.o],[dnl
+        AS_IF([test "x$ansi" = "x"],
+              [hts_cv_prog_cc_warnings="-qsrcmsg -qinfo=all:noppt:noppc:noobs:nocnd"],
+              [hts_cv_prog_cc_warnings="-qsrcmsg -qinfo=all:noppt:noppc:noobs:nocnd -qlanglvl=ansi"])
+        ],
+        # IRIX C compiler
+        ["$CC" -version 2>&1 | $GREP -i "MIPSpro Compilers" > /dev/null 2>&1 &&
+         "$CC" -c -fullwarn -ansi -ansiE conftest.c > /dev/null 2>&1 &&
+         test -f conftest.o],[dnl
+           AS_IF([test "x$ansi" = "x"],
+                 [hts_cv_prog_cc_warnings="-fullwarn"],
+                 [hts_cv_prog_cc_warnings="-fullwarn -ansi -ansiE"])
+          ],
+        # HP-UX C compiler
+        [what "$CC" 2>&1 | $GREP -i "HP C Compiler" > /dev/null 2>&1 &&
+         "$CC" -c -Aa +w1 conftest.c > /dev/null 2>&1 &&
+         test -f conftest.o],[dnl
+        AS_IF([test "x$ansi" = "x"],
+              [hts_cv_prog_cc_warnings="+w1"],
+              [hts_cv_prog_cc_warnings="+w1 -Aa"])
+        ],
+        # The NEC SX series (Super-UX 10) C compiler
+        ["$CC" -V 2>&1 | $GREP "/SX" > /dev/null 2>&1 &&
+         "$CC" -c -pvctl[,]fullmsg -Xc conftest.c > /dev/null 2>&1 &&
+         test -f conftest.o],[
+        AS_IF([test "x$ansi" = "x"],
+              [hts_cv_prog_cc_warnings="-pvctl[,]fullmsg"],
+              [hts_cv_prog_cc_warnings="-pvctl[,]fullmsg -Xc"])
+        ],
+        # The Cray C compiler (Unicos)
+        ["$CC" -V 2>&1 | $GREP -i "Cray" > /dev/null 2>&1 &&
+         "$CC" -c -h msglevel_2 conftest.c > /dev/null 2>&1 &&
+         test -f conftest.o],[dnl
+        AS_IF([test "x$ansi" = "x"],
+              [hts_cv_prog_cc_warnings="-h#msglevel_2"],
+              [hts_cv_prog_cc_warnings="-h#msglevel_2,conform"])
+        ],
+	# The Tiny C Compiler
+        ["$CC" -v 2>&1 | $GREP "tcc version" > /dev/null &&
+         "$CC" -Wall -c conftest.c > /dev/null 2>&1 &&
+         test -f conftest.o],[dnl
+         hts_cv_prog_cc_warnings="-Wall"
+        ])
+        rm -f conftest.*
+      ])
+    ])
+
+    AS_IF([test "x$hts_cv_prog_cc_warnings" != "x"],[
+dnl Print result, with underscores as spaces
+ac_arg_result=`echo "$hts_cv_prog_cc_warnings" | tr '#' ' '`
+AC_MSG_RESULT($ac_arg_result)
+
+dnl Add options to CFLAGS only if they are not already present
+ac_arg_needed=""
+for ac_arg in $hts_cv_prog_cc_warnings
+do
+  ac_arg_sp=`echo "$ac_arg" | tr '#' ' '`
+  AS_CASE([" $CFLAGS "],
+[*" $ac_arg_sp "*], [],
+[ac_arg_needed="$ac_arg_all $ac_arg_sp"])
+done
+CFLAGS="$ac_arg_needed $CFLAGS"],[dnl
+      AC_MSG_RESULT(unknown)
+    ])
+  ])
+])dnl HTS_PROG_CC_WARNINGS
+
+# SYNOPSIS
+#
+# HTS_PROG_CC_WERROR(FLAGS_VAR)
+#
+# Set FLAGS_VAR to the flags needed to make the C compiler treat warnings
+# as errors.
+
+AC_DEFUN([HTS_PROG_CC_WERROR], [
+  AC_ARG_ENABLE([werror],
+    [AS_HELP_STRING([--enable-werror], [change warnings into errors, where supported])],
+    [],
+    [enable_werror=no])
+
+  AS_IF([test "x$enable_werror" != xno],[
+    AC_MSG_CHECKING([for C compiler flags to error on warnings])
+    AC_CACHE_VAL(hts_cv_prog_cc_werror, [dnl
+      hts_cv_prog_cc_werror=""
+      AS_IF([test "x$CC" != "x"],[
+        cat > conftest.c <<EOF
+int main(int argc, char **argv) { return 0; }
+EOF
+
+        AS_IF(dnl
+	 # Tests for flags to make the C compiler treat warnings as errors
+         # GCC compatible
+         [test "x$GCC" = "xyes" &&
+          "$CC" -c -Werror conftest.c > /dev/null 2>&1 &&
+          test -f conftest.o],[hts_cv_prog_cc_werror="-Werror"],
+         # Sun Studio or Solaris C compiler
+         ["$CC" -V 2>&1 | $GREP -i -E "WorkShop|Sun C" > /dev/null 2>&1 &&
+          "$CC" -c -errwarn=%all conftest.c > /dev/null 2>&1 &&
+          test -f conftest.o],[hts_cv_prog_cc_werror="-errwarn=%all"],
+	 # The Tiny C Compiler
+         ["$CC" -v 2>&1 | $GREP "tcc version" > /dev/null &&
+          "$CC" -Wall -c conftest.c > /dev/null 2>&1 &&
+          test -f conftest.o],[hts_cv_prog_cc_werror="-Werror"]
+         dnl TODO: Add more compilers
+        )
+        rm -f conftest.*
+      ])
+    ])
+    AS_IF([test "x$hts_cv_prog_cc_werror" != x],[
+      AC_MSG_RESULT($hts_cv_prog_cc_werror)
+      AS_IF([test "x$1" != x],[eval AS_TR_SH([$1])="$hts_cv_prog_cc_werror"])
+    ],[dnl
+      AC_MSG_RESULT(unknown)
+    ])
+  ])
+])dnl HTS_PROG_CC_WERROR
diff --git a/ext/htslib/m4/pkg.m4 b/ext/htslib/m4/pkg.m4
new file mode 100644
index 0000000..4b95a02
--- /dev/null
+++ b/ext/htslib/m4/pkg.m4
@@ -0,0 +1,275 @@
+# pkg.m4 - Macros to locate and use pkg-config.   -*- Autoconf -*-
+# serial 12 (pkg-config-0.29.2)
+
+dnl Copyright © 2004 Scott James Remnant <scott@netsplit.com>.
+dnl Copyright © 2012-2015 Dan Nicholson <dbn.lists@gmail.com>
+dnl
+dnl This program is free software; you can redistribute it and/or modify
+dnl it under the terms of the GNU General Public License as published by
+dnl the Free Software Foundation; either version 2 of the License, or
+dnl (at your option) any later version.
+dnl
+dnl This program is distributed in the hope that it will be useful, but
+dnl WITHOUT ANY WARRANTY; without even the implied warranty of
+dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+dnl General Public License for more details.
+dnl
+dnl You should have received a copy of the GNU General Public License
+dnl along with this program; if not, write to the Free Software
+dnl Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
+dnl 02111-1307, USA.
+dnl
+dnl As a special exception to the GNU General Public License, if you
+dnl distribute this file as part of a program that contains a
+dnl configuration script generated by Autoconf, you may include it under
+dnl the same distribution terms that you use for the rest of that
+dnl program.
+
+dnl PKG_PREREQ(MIN-VERSION)
+dnl -----------------------
+dnl Since: 0.29
+dnl
+dnl Verify that the version of the pkg-config macros are at least
+dnl MIN-VERSION. Unlike PKG_PROG_PKG_CONFIG, which checks the user's
+dnl installed version of pkg-config, this checks the developer's version
+dnl of pkg.m4 when generating configure.
+dnl
+dnl To ensure that this macro is defined, also add:
+dnl m4_ifndef([PKG_PREREQ],
+dnl     [m4_fatal([must install pkg-config 0.29 or later before running autoconf/autogen])])
+dnl
+dnl See the "Since" comment for each macro you use to see what version
+dnl of the macros you require.
+m4_defun([PKG_PREREQ],
+[m4_define([PKG_MACROS_VERSION], [0.29.2])
+m4_if(m4_version_compare(PKG_MACROS_VERSION, [$1]), -1,
+    [m4_fatal([pkg.m4 version $1 or higher is required but ]PKG_MACROS_VERSION[ found])])
+])dnl PKG_PREREQ
+
+dnl PKG_PROG_PKG_CONFIG([MIN-VERSION])
+dnl ----------------------------------
+dnl Since: 0.16
+dnl
+dnl Search for the pkg-config tool and set the PKG_CONFIG variable to
+dnl first found in the path. Checks that the version of pkg-config found
+dnl is at least MIN-VERSION. If MIN-VERSION is not specified, 0.9.0 is
+dnl used since that's the first version where most current features of
+dnl pkg-config existed.
+AC_DEFUN([PKG_PROG_PKG_CONFIG],
+[m4_pattern_forbid([^_?PKG_[A-Z_]+$])
+m4_pattern_allow([^PKG_CONFIG(_(PATH|LIBDIR|SYSROOT_DIR|ALLOW_SYSTEM_(CFLAGS|LIBS)))?$])
+m4_pattern_allow([^PKG_CONFIG_(DISABLE_UNINSTALLED|TOP_BUILD_DIR|DEBUG_SPEW)$])
+AC_ARG_VAR([PKG_CONFIG], [path to pkg-config utility])
+AC_ARG_VAR([PKG_CONFIG_PATH], [directories to add to pkg-config's search path])
+AC_ARG_VAR([PKG_CONFIG_LIBDIR], [path overriding pkg-config's built-in search path])
+
+if test "x$ac_cv_env_PKG_CONFIG_set" != "xset"; then
+	AC_PATH_TOOL([PKG_CONFIG], [pkg-config])
+fi
+if test -n "$PKG_CONFIG"; then
+	_pkg_min_version=m4_default([$1], [0.9.0])
+	AC_MSG_CHECKING([pkg-config is at least version $_pkg_min_version])
+	if $PKG_CONFIG --atleast-pkgconfig-version $_pkg_min_version; then
+		AC_MSG_RESULT([yes])
+	else
+		AC_MSG_RESULT([no])
+		PKG_CONFIG=""
+	fi
+fi[]dnl
+])dnl PKG_PROG_PKG_CONFIG
+
+dnl PKG_CHECK_EXISTS(MODULES, [ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND])
+dnl -------------------------------------------------------------------
+dnl Since: 0.18
+dnl
+dnl Check to see whether a particular set of modules exists. Similar to
+dnl PKG_CHECK_MODULES(), but does not set variables or print errors.
+dnl
+dnl Please remember that m4 expands AC_REQUIRE([PKG_PROG_PKG_CONFIG])
+dnl only at the first occurrence in configure.ac, so if the first place
+dnl it's called might be skipped (such as if it is within an "if", you
+dnl have to call PKG_CHECK_EXISTS manually
+AC_DEFUN([PKG_CHECK_EXISTS],
+[AC_REQUIRE([PKG_PROG_PKG_CONFIG])dnl
+if test -n "$PKG_CONFIG" && \
+    AC_RUN_LOG([$PKG_CONFIG --exists --print-errors "$1"]); then
+  m4_default([$2], [:])
+m4_ifvaln([$3], [else
+  $3])dnl
+fi])
+
+dnl _PKG_CONFIG([VARIABLE], [COMMAND], [MODULES])
+dnl ---------------------------------------------
+dnl Internal wrapper calling pkg-config via PKG_CONFIG and setting
+dnl pkg_failed based on the result.
+m4_define([_PKG_CONFIG],
+[if test -n "$$1"; then
+    pkg_cv_[]$1="$$1"
+ elif test -n "$PKG_CONFIG"; then
+    PKG_CHECK_EXISTS([$3],
+                     [pkg_cv_[]$1=`$PKG_CONFIG --[]$2 "$3" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes ],
+		     [pkg_failed=yes])
+ else
+    pkg_failed=untried
+fi[]dnl
+])dnl _PKG_CONFIG
+
+dnl _PKG_SHORT_ERRORS_SUPPORTED
+dnl ---------------------------
+dnl Internal check to see if pkg-config supports short errors.
+AC_DEFUN([_PKG_SHORT_ERRORS_SUPPORTED],
+[AC_REQUIRE([PKG_PROG_PKG_CONFIG])
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+        _pkg_short_errors_supported=yes
+else
+        _pkg_short_errors_supported=no
+fi[]dnl
+])dnl _PKG_SHORT_ERRORS_SUPPORTED
+
+
+dnl PKG_CHECK_MODULES(VARIABLE-PREFIX, MODULES, [ACTION-IF-FOUND],
+dnl   [ACTION-IF-NOT-FOUND])
+dnl --------------------------------------------------------------
+dnl Since: 0.4.0
+dnl
+dnl Note that if there is a possibility the first call to
+dnl PKG_CHECK_MODULES might not happen, you should be sure to include an
+dnl explicit call to PKG_PROG_PKG_CONFIG in your configure.ac
+AC_DEFUN([PKG_CHECK_MODULES],
+[AC_REQUIRE([PKG_PROG_PKG_CONFIG])dnl
+AC_ARG_VAR([$1][_CFLAGS], [C compiler flags for $1, overriding pkg-config])dnl
+AC_ARG_VAR([$1][_LIBS], [linker flags for $1, overriding pkg-config])dnl
+
+pkg_failed=no
+AC_MSG_CHECKING([for $2])
+
+_PKG_CONFIG([$1][_CFLAGS], [cflags], [$2])
+_PKG_CONFIG([$1][_LIBS], [libs], [$2])
+
+m4_define([_PKG_TEXT], [Alternatively, you may set the environment variables $1[]_CFLAGS
+and $1[]_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.])
+
+if test $pkg_failed = yes; then
+        AC_MSG_RESULT([no])
+        _PKG_SHORT_ERRORS_SUPPORTED
+        if test $_pkg_short_errors_supported = yes; then
+	        $1[]_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "$2" 2>&1`
+        else
+	        $1[]_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "$2" 2>&1`
+        fi
+	# Put the nasty error message in config.log where it belongs
+	echo "$$1[]_PKG_ERRORS" >&AS_MESSAGE_LOG_FD
+
+	m4_default([$4], [AC_MSG_ERROR(
+[Package requirements ($2) were not met:
+
+$$1_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+_PKG_TEXT])[]dnl
+        ])
+elif test $pkg_failed = untried; then
+        AC_MSG_RESULT([no])
+	m4_default([$4], [AC_MSG_FAILURE(
+[The pkg-config script could not be found or is too old.  Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+_PKG_TEXT
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/>.])[]dnl
+        ])
+else
+	$1[]_CFLAGS=$pkg_cv_[]$1[]_CFLAGS
+	$1[]_LIBS=$pkg_cv_[]$1[]_LIBS
+        AC_MSG_RESULT([yes])
+	$3
+fi[]dnl
+])dnl PKG_CHECK_MODULES
+
+
+dnl PKG_CHECK_MODULES_STATIC(VARIABLE-PREFIX, MODULES, [ACTION-IF-FOUND],
+dnl   [ACTION-IF-NOT-FOUND])
+dnl ---------------------------------------------------------------------
+dnl Since: 0.29
+dnl
+dnl Checks for existence of MODULES and gathers its build flags with
+dnl static libraries enabled. Sets VARIABLE-PREFIX_CFLAGS from --cflags
+dnl and VARIABLE-PREFIX_LIBS from --libs.
+dnl
+dnl Note that if there is a possibility the first call to
+dnl PKG_CHECK_MODULES_STATIC might not happen, you should be sure to
+dnl include an explicit call to PKG_PROG_PKG_CONFIG in your
+dnl configure.ac.
+AC_DEFUN([PKG_CHECK_MODULES_STATIC],
+[AC_REQUIRE([PKG_PROG_PKG_CONFIG])dnl
+_save_PKG_CONFIG=$PKG_CONFIG
+PKG_CONFIG="$PKG_CONFIG --static"
+PKG_CHECK_MODULES($@)
+PKG_CONFIG=$_save_PKG_CONFIG[]dnl
+])dnl PKG_CHECK_MODULES_STATIC
+
+
+dnl PKG_INSTALLDIR([DIRECTORY])
+dnl -------------------------
+dnl Since: 0.27
+dnl
+dnl Substitutes the variable pkgconfigdir as the location where a module
+dnl should install pkg-config .pc files. By default the directory is
+dnl $libdir/pkgconfig, but the default can be changed by passing
+dnl DIRECTORY. The user can override through the --with-pkgconfigdir
+dnl parameter.
+AC_DEFUN([PKG_INSTALLDIR],
+[m4_pushdef([pkg_default], [m4_default([$1], ['${libdir}/pkgconfig'])])
+m4_pushdef([pkg_description],
+    [pkg-config installation directory @<:@]pkg_default[@:>@])
+AC_ARG_WITH([pkgconfigdir],
+    [AS_HELP_STRING([--with-pkgconfigdir], pkg_description)],,
+    [with_pkgconfigdir=]pkg_default)
+AC_SUBST([pkgconfigdir], [$with_pkgconfigdir])
+m4_popdef([pkg_default])
+m4_popdef([pkg_description])
+])dnl PKG_INSTALLDIR
+
+
+dnl PKG_NOARCH_INSTALLDIR([DIRECTORY])
+dnl --------------------------------
+dnl Since: 0.27
+dnl
+dnl Substitutes the variable noarch_pkgconfigdir as the location where a
+dnl module should install arch-independent pkg-config .pc files. By
+dnl default the directory is $datadir/pkgconfig, but the default can be
+dnl changed by passing DIRECTORY. The user can override through the
+dnl --with-noarch-pkgconfigdir parameter.
+AC_DEFUN([PKG_NOARCH_INSTALLDIR],
+[m4_pushdef([pkg_default], [m4_default([$1], ['${datadir}/pkgconfig'])])
+m4_pushdef([pkg_description],
+    [pkg-config arch-independent installation directory @<:@]pkg_default[@:>@])
+AC_ARG_WITH([noarch-pkgconfigdir],
+    [AS_HELP_STRING([--with-noarch-pkgconfigdir], pkg_description)],,
+    [with_noarch_pkgconfigdir=]pkg_default)
+AC_SUBST([noarch_pkgconfigdir], [$with_noarch_pkgconfigdir])
+m4_popdef([pkg_default])
+m4_popdef([pkg_description])
+])dnl PKG_NOARCH_INSTALLDIR
+
+
+dnl PKG_CHECK_VAR(VARIABLE, MODULE, CONFIG-VARIABLE,
+dnl [ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND])
+dnl -------------------------------------------
+dnl Since: 0.28
+dnl
+dnl Retrieves the value of the pkg-config variable for the given module.
+AC_DEFUN([PKG_CHECK_VAR],
+[AC_REQUIRE([PKG_PROG_PKG_CONFIG])dnl
+AC_ARG_VAR([$1], [value of $3 for $2, overriding pkg-config])dnl
+
+_PKG_CONFIG([$1], [variable="][$3]["], [$2])
+AS_VAR_COPY([$1], [pkg_cv_][$1])
+
+AS_VAR_IF([$1], [""], [$5], [$4])dnl
+])dnl PKG_CHECK_VAR
diff --git a/ext/htslib/md5.c b/ext/htslib/md5.c
new file mode 100644
index 0000000..1a43da5
--- /dev/null
+++ b/ext/htslib/md5.c
@@ -0,0 +1,388 @@
+/*
+ * Trivial amendments by James Bonfield <jkb@sanger.ac.uk> to provide an
+ * HTSlib interface. 2015.
+ *
+ * Externally our API uses an opaque hts_md5_context structure.
+ *
+ * Internally either this gets defined and used with the routines here
+ * or it remains incomplete and is cast to the OpenSSL MD5_CTX structure
+ * and used by routines from OpenSSL.
+ */
+
+/*
+ * This is an OpenSSL-compatible implementation of the RSA Data Security, Inc.
+ * MD5 Message-Digest Algorithm (RFC 1321).
+ *
+ * Homepage:
+ * http://openwall.info/wiki/people/solar/software/public-domain-source-code/md5
+ *
+ * Author:
+ * Alexander Peslyak, better known as Solar Designer <solar at openwall.com>
+ *
+ * This software was written by Alexander Peslyak in 2001.  No copyright is
+ * claimed, and the software is hereby placed in the public domain.
+ * In case this attempt to disclaim copyright and place the software in the
+ * public domain is deemed null and void, then the software is
+ * Copyright (c) 2001 Alexander Peslyak and it is hereby released to the
+ * general public under the following terms:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted.
+ *
+ * There's ABSOLUTELY NO WARRANTY, express or implied.
+ *
+ * (This is a heavily cut-down "BSD license".)
+ *
+ * This differs from Colin Plumb's older public domain implementation in that
+ * no exactly 32-bit integer data type is required (any 32-bit or wider
+ * unsigned integer data type will do), there's no compile-time endianness
+ * configuration, and the function prototypes match OpenSSL's.  No code from
+ * Colin Plumb's implementation has been reused; this comment merely compares
+ * the properties of the two independent implementations.
+ *
+ * The primary goals of this implementation are portability and ease of use.
+ * It is meant to be fast, but not as fast as possible.  Some known
+ * optimizations are not included to reduce source code size and avoid
+ * compile-time configuration.
+ */
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+
+#include <stdlib.h>
+#include "htslib/hts.h"
+#include "htslib/hts_endian.h"
+
+#ifndef HAVE_OPENSSL
+
+#include <string.h>
+
+/* Any 32-bit or wider unsigned integer data type will do */
+typedef unsigned int hts_md5_u32plus;
+
+struct hts_md5_context {
+	hts_md5_u32plus lo, hi;
+	hts_md5_u32plus a, b, c, d;
+	unsigned char buffer[64];
+	hts_md5_u32plus block[16];
+};
+
+/*
+ * The basic MD5 functions.
+ *
+ * F and G are optimized compared to their RFC 1321 definitions for
+ * architectures that lack an AND-NOT instruction, just like in Colin Plumb's
+ * implementation.
+ */
+#define F(x, y, z)			((z) ^ ((x) & ((y) ^ (z))))
+#define G(x, y, z)			((y) ^ ((z) & ((x) ^ (y))))
+#define H(x, y, z)			(((x) ^ (y)) ^ (z))
+#define H2(x, y, z)			((x) ^ ((y) ^ (z)))
+#define I(x, y, z)			((y) ^ ((x) | ~(z)))
+
+/*
+ * The MD5 transformation for all four rounds.
+ */
+#define STEP(f, a, b, c, d, x, t, s) \
+	(a) += f((b), (c), (d)) + (x) + (t); \
+	(a) = (((a) << (s)) | (((a) & 0xffffffff) >> (32 - (s)))); \
+	(a) += (b);
+
+/*
+ * SET reads 4 input bytes in little-endian byte order and stores them
+ * in a properly aligned word in host byte order.
+ *
+ * The check for little-endian architectures that tolerate unaligned
+ * memory accesses is just an optimization.  Nothing will break if it
+ * doesn't work.
+ */
+#if defined(HTS_LITTLE_ENDIAN) && HTS_ALLOW_UNALIGNED != 0
+#define SET(n) \
+	(*(hts_md5_u32plus *)&ptr[(n) * 4])
+#define GET(n) \
+	SET(n)
+#else
+#define SET(n) \
+	(ctx->block[(n)] = \
+	(hts_md5_u32plus)ptr[(n) * 4] | \
+	((hts_md5_u32plus)ptr[(n) * 4 + 1] << 8) | \
+	((hts_md5_u32plus)ptr[(n) * 4 + 2] << 16) | \
+	((hts_md5_u32plus)ptr[(n) * 4 + 3] << 24))
+#define GET(n) \
+	(ctx->block[(n)])
+#endif
+
+/*
+ * This processes one or more 64-byte data blocks, but does NOT update
+ * the bit counters.  There are no alignment requirements.
+ */
+static const void *body(hts_md5_context *ctx, const void *data, unsigned long size)
+{
+	const unsigned char *ptr;
+	hts_md5_u32plus a, b, c, d;
+	hts_md5_u32plus saved_a, saved_b, saved_c, saved_d;
+
+	ptr = (const unsigned char *)data;
+
+	a = ctx->a;
+	b = ctx->b;
+	c = ctx->c;
+	d = ctx->d;
+
+	do {
+		saved_a = a;
+		saved_b = b;
+		saved_c = c;
+		saved_d = d;
+
+/* Round 1 */
+		STEP(F, a, b, c, d, SET(0), 0xd76aa478, 7)
+		STEP(F, d, a, b, c, SET(1), 0xe8c7b756, 12)
+		STEP(F, c, d, a, b, SET(2), 0x242070db, 17)
+		STEP(F, b, c, d, a, SET(3), 0xc1bdceee, 22)
+		STEP(F, a, b, c, d, SET(4), 0xf57c0faf, 7)
+		STEP(F, d, a, b, c, SET(5), 0x4787c62a, 12)
+		STEP(F, c, d, a, b, SET(6), 0xa8304613, 17)
+		STEP(F, b, c, d, a, SET(7), 0xfd469501, 22)
+		STEP(F, a, b, c, d, SET(8), 0x698098d8, 7)
+		STEP(F, d, a, b, c, SET(9), 0x8b44f7af, 12)
+		STEP(F, c, d, a, b, SET(10), 0xffff5bb1, 17)
+		STEP(F, b, c, d, a, SET(11), 0x895cd7be, 22)
+		STEP(F, a, b, c, d, SET(12), 0x6b901122, 7)
+		STEP(F, d, a, b, c, SET(13), 0xfd987193, 12)
+		STEP(F, c, d, a, b, SET(14), 0xa679438e, 17)
+		STEP(F, b, c, d, a, SET(15), 0x49b40821, 22)
+
+/* Round 2 */
+		STEP(G, a, b, c, d, GET(1), 0xf61e2562, 5)
+		STEP(G, d, a, b, c, GET(6), 0xc040b340, 9)
+		STEP(G, c, d, a, b, GET(11), 0x265e5a51, 14)
+		STEP(G, b, c, d, a, GET(0), 0xe9b6c7aa, 20)
+		STEP(G, a, b, c, d, GET(5), 0xd62f105d, 5)
+		STEP(G, d, a, b, c, GET(10), 0x02441453, 9)
+		STEP(G, c, d, a, b, GET(15), 0xd8a1e681, 14)
+		STEP(G, b, c, d, a, GET(4), 0xe7d3fbc8, 20)
+		STEP(G, a, b, c, d, GET(9), 0x21e1cde6, 5)
+		STEP(G, d, a, b, c, GET(14), 0xc33707d6, 9)
+		STEP(G, c, d, a, b, GET(3), 0xf4d50d87, 14)
+		STEP(G, b, c, d, a, GET(8), 0x455a14ed, 20)
+		STEP(G, a, b, c, d, GET(13), 0xa9e3e905, 5)
+		STEP(G, d, a, b, c, GET(2), 0xfcefa3f8, 9)
+		STEP(G, c, d, a, b, GET(7), 0x676f02d9, 14)
+		STEP(G, b, c, d, a, GET(12), 0x8d2a4c8a, 20)
+
+/* Round 3 */
+		STEP(H, a, b, c, d, GET(5), 0xfffa3942, 4)
+		STEP(H2, d, a, b, c, GET(8), 0x8771f681, 11)
+		STEP(H, c, d, a, b, GET(11), 0x6d9d6122, 16)
+		STEP(H2, b, c, d, a, GET(14), 0xfde5380c, 23)
+		STEP(H, a, b, c, d, GET(1), 0xa4beea44, 4)
+		STEP(H2, d, a, b, c, GET(4), 0x4bdecfa9, 11)
+		STEP(H, c, d, a, b, GET(7), 0xf6bb4b60, 16)
+		STEP(H2, b, c, d, a, GET(10), 0xbebfbc70, 23)
+		STEP(H, a, b, c, d, GET(13), 0x289b7ec6, 4)
+		STEP(H2, d, a, b, c, GET(0), 0xeaa127fa, 11)
+		STEP(H, c, d, a, b, GET(3), 0xd4ef3085, 16)
+		STEP(H2, b, c, d, a, GET(6), 0x04881d05, 23)
+		STEP(H, a, b, c, d, GET(9), 0xd9d4d039, 4)
+		STEP(H2, d, a, b, c, GET(12), 0xe6db99e5, 11)
+		STEP(H, c, d, a, b, GET(15), 0x1fa27cf8, 16)
+		STEP(H2, b, c, d, a, GET(2), 0xc4ac5665, 23)
+
+/* Round 4 */
+		STEP(I, a, b, c, d, GET(0), 0xf4292244, 6)
+		STEP(I, d, a, b, c, GET(7), 0x432aff97, 10)
+		STEP(I, c, d, a, b, GET(14), 0xab9423a7, 15)
+		STEP(I, b, c, d, a, GET(5), 0xfc93a039, 21)
+		STEP(I, a, b, c, d, GET(12), 0x655b59c3, 6)
+		STEP(I, d, a, b, c, GET(3), 0x8f0ccc92, 10)
+		STEP(I, c, d, a, b, GET(10), 0xffeff47d, 15)
+		STEP(I, b, c, d, a, GET(1), 0x85845dd1, 21)
+		STEP(I, a, b, c, d, GET(8), 0x6fa87e4f, 6)
+		STEP(I, d, a, b, c, GET(15), 0xfe2ce6e0, 10)
+		STEP(I, c, d, a, b, GET(6), 0xa3014314, 15)
+		STEP(I, b, c, d, a, GET(13), 0x4e0811a1, 21)
+		STEP(I, a, b, c, d, GET(4), 0xf7537e82, 6)
+		STEP(I, d, a, b, c, GET(11), 0xbd3af235, 10)
+		STEP(I, c, d, a, b, GET(2), 0x2ad7d2bb, 15)
+		STEP(I, b, c, d, a, GET(9), 0xeb86d391, 21)
+
+		a += saved_a;
+		b += saved_b;
+		c += saved_c;
+		d += saved_d;
+
+		ptr += 64;
+	} while (size -= 64);
+
+	ctx->a = a;
+	ctx->b = b;
+	ctx->c = c;
+	ctx->d = d;
+
+	return ptr;
+}
+
+void hts_md5_reset(hts_md5_context *ctx)
+{
+	ctx->a = 0x67452301;
+	ctx->b = 0xefcdab89;
+	ctx->c = 0x98badcfe;
+	ctx->d = 0x10325476;
+
+	ctx->lo = 0;
+	ctx->hi = 0;
+}
+
+void hts_md5_update(hts_md5_context *ctx, const void *data, unsigned long size)
+{
+	hts_md5_u32plus saved_lo;
+	unsigned long used, available;
+
+	saved_lo = ctx->lo;
+	if ((ctx->lo = (saved_lo + size) & 0x1fffffff) < saved_lo)
+		ctx->hi++;
+	ctx->hi += size >> 29;
+
+	used = saved_lo & 0x3f;
+
+	if (used) {
+		available = 64 - used;
+
+		if (size < available) {
+			memcpy(&ctx->buffer[used], data, size);
+			return;
+		}
+
+		memcpy(&ctx->buffer[used], data, available);
+		data = (const unsigned char *)data + available;
+		size -= available;
+		body(ctx, ctx->buffer, 64);
+	}
+
+	if (size >= 64) {
+		data = body(ctx, data, size & ~(unsigned long)0x3f);
+		size &= 0x3f;
+	}
+
+	memcpy(ctx->buffer, data, size);
+}
+
+void hts_md5_final(unsigned char *result, hts_md5_context *ctx)
+{
+	unsigned long used, available;
+
+	used = ctx->lo & 0x3f;
+
+	ctx->buffer[used++] = 0x80;
+
+	available = 64 - used;
+
+	if (available < 8) {
+		memset(&ctx->buffer[used], 0, available);
+		body(ctx, ctx->buffer, 64);
+		used = 0;
+		available = 64;
+	}
+
+	memset(&ctx->buffer[used], 0, available - 8);
+
+	ctx->lo <<= 3;
+	ctx->buffer[56] = ctx->lo;
+	ctx->buffer[57] = ctx->lo >> 8;
+	ctx->buffer[58] = ctx->lo >> 16;
+	ctx->buffer[59] = ctx->lo >> 24;
+	ctx->buffer[60] = ctx->hi;
+	ctx->buffer[61] = ctx->hi >> 8;
+	ctx->buffer[62] = ctx->hi >> 16;
+	ctx->buffer[63] = ctx->hi >> 24;
+
+	body(ctx, ctx->buffer, 64);
+
+	result[0] = ctx->a;
+	result[1] = ctx->a >> 8;
+	result[2] = ctx->a >> 16;
+	result[3] = ctx->a >> 24;
+	result[4] = ctx->b;
+	result[5] = ctx->b >> 8;
+	result[6] = ctx->b >> 16;
+	result[7] = ctx->b >> 24;
+	result[8] = ctx->c;
+	result[9] = ctx->c >> 8;
+	result[10] = ctx->c >> 16;
+	result[11] = ctx->c >> 24;
+	result[12] = ctx->d;
+	result[13] = ctx->d >> 8;
+	result[14] = ctx->d >> 16;
+	result[15] = ctx->d >> 24;
+
+	memset(ctx, 0, sizeof(*ctx));
+}
+
+
+hts_md5_context *hts_md5_init(void)
+{
+    hts_md5_context *ctx = malloc(sizeof(*ctx));
+    if (!ctx)
+        return NULL;
+
+    hts_md5_reset(ctx);
+    return ctx;
+}
+
+#else
+
+#include <openssl/md5.h>
+#include <assert.h>
+
+/*
+ * Wrappers around the OpenSSL libcrypto.so MD5 implementation.
+ *
+ * These are here to ensure they end up in the symbol table of the
+ * library regardless of the static inline in the headers.
+ */
+hts_md5_context *hts_md5_init(void)
+{
+    MD5_CTX *ctx = malloc(sizeof(*ctx));
+    if (!ctx)
+        return NULL;
+
+    MD5_Init(ctx);
+
+    return (hts_md5_context *)ctx;
+}
+
+void hts_md5_reset(hts_md5_context *ctx)
+{
+    MD5_Init((MD5_CTX *)ctx);
+}
+
+void hts_md5_update(hts_md5_context *ctx, const void *data, unsigned long size)
+{
+    MD5_Update((MD5_CTX *)ctx, data, size);
+}
+
+void hts_md5_final(unsigned char *result, hts_md5_context *ctx)
+{
+    MD5_Final(result, (MD5_CTX *)ctx);
+}
+
+#endif
+
+void hts_md5_destroy(hts_md5_context *ctx)
+{
+    if (!ctx)
+        return;
+
+    free(ctx);
+}
+
+void hts_md5_hex(char *hex, const unsigned char *digest)
+{
+    int i;
+    for (i = 0; i < 16; i++) {
+        hex[i*2+0] = "0123456789abcdef"[(digest[i]>>4)&0xf];
+        hex[i*2+1] = "0123456789abcdef"[digest[i]&0xf];
+    }
+    hex[32] = 0;
+}
diff --git a/ext/htslib/multipart.c b/ext/htslib/multipart.c
new file mode 100644
index 0000000..12d0df2
--- /dev/null
+++ b/ext/htslib/multipart.c
@@ -0,0 +1,267 @@
+/*  multipart.c -- GA4GH redirection and multipart backend for file streams.
+
+    Copyright (C) 2016-2017 Genome Research Ltd.
+
+    Author: John Marshall <jm18@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+
+#include "htslib/kstring.h"
+
+#include "hts_internal.h"
+#include "hfile_internal.h"
+
+#ifndef EPROTO
+#define EPROTO ENOEXEC
+#endif
+
+typedef struct hfile_part {
+    char *url;
+    char **headers;
+} hfile_part;
+
+typedef struct {
+    hFILE base;
+    hfile_part *parts;
+    size_t nparts, maxparts, current;
+    hFILE *currentfp;
+} hFILE_multipart;
+
+static void free_part(hfile_part *p)
+{
+    free(p->url);
+    if (p->headers) {
+        char **hdr;
+        for (hdr = p->headers; *hdr; hdr++) free(*hdr);
+        free(p->headers);
+    }
+
+    p->url = NULL;
+    p->headers = NULL;
+}
+
+static void free_all_parts(hFILE_multipart *fp)
+{
+    size_t i;
+    for (i = 0; i < fp->nparts; i++) free_part(&fp->parts[i]);
+    free(fp->parts);
+}
+
+static ssize_t multipart_read(hFILE *fpv, void *buffer, size_t nbytes)
+{
+    hFILE_multipart *fp = (hFILE_multipart *) fpv;
+    size_t n;
+
+open_next:
+    if (fp->currentfp == NULL) {
+        if (fp->current < fp->nparts) {
+            const hfile_part *p = &fp->parts[fp->current];
+            hts_log_debug("Opening part #%zu of %zu: \"%.120s%s\"",
+                fp->current+1, fp->nparts, p->url,
+                (strlen(p->url) > 120)? "..." : "");
+
+            fp->currentfp = p->headers?
+                  hopen(p->url, "r:",
+                        "httphdr:v", p->headers,
+                        "auth_token_enabled", "false", NULL)
+                : hopen(p->url, "r:", "auth_token_enabled", "false", NULL);
+
+            if (fp->currentfp == NULL) return -1;
+        }
+        else return 0;  // No more parts, so we're truly at EOF
+    }
+
+    n = fp->currentfp->mobile?
+          fp->currentfp->backend->read(fp->currentfp, buffer, nbytes)
+        : hread(fp->currentfp, buffer, nbytes);
+
+    if (n == 0) {
+        // We're at EOF on this part, so set up the next part
+        hFILE *prevfp = fp->currentfp;
+        free_part(&fp->parts[fp->current]);
+        fp->current++;
+        fp->currentfp = NULL;
+        if (hclose(prevfp) < 0) return -1;
+        goto open_next;
+    }
+
+    return n;  // Number of bytes read by (or an error from) fp->currentfp
+}
+
+static ssize_t multipart_write(hFILE *fpv, const void *buffer, size_t nbytes)
+{
+    errno = EROFS;
+    return -1;
+}
+
+static off_t multipart_seek(hFILE *fpv, off_t offset, int whence)
+{
+    errno = ESPIPE;
+    return -1;
+}
+
+static int multipart_close(hFILE *fpv)
+{
+    hFILE_multipart *fp = (hFILE_multipart *) fpv;
+
+    free_all_parts(fp);
+    if (fp->currentfp) {
+        if (hclose(fp->currentfp) < 0) return -1;
+    }
+
+    return 0;
+}
+
+static const struct hFILE_backend multipart_backend =
+{
+    multipart_read, multipart_write, multipart_seek, NULL, multipart_close
+};
+
+// Returns 'v' (valid value), 'i' (invalid; required GA4GH field missing),
+// or upon encountering an unexpected token, that token's type.
+// Explicit `return '?'` means a JSON parsing error, typically a member key
+// that is not a string.  An unexpected token may be a valid token that was
+// not the type expected for a particular GA4GH field, or it may be '?' or
+// '\0' which should be propagated.
+static char
+parse_ga4gh_body_json(hFILE_multipart *fp, hFILE *json,
+                      kstring_t *b, kstring_t *header)
+{
+    hts_json_token t;
+
+    if (hts_json_fnext(json, &t, b) != '{') return t.type;
+    while (hts_json_fnext(json, &t, b) != '}') {
+        if (t.type != 's') return '?';
+
+        if (strcmp(t.str, "urls") == 0) {
+            if (hts_json_fnext(json, &t, b) != '[') return t.type;
+
+            while (hts_json_fnext(json, &t, b) != ']') {
+                hfile_part *part;
+                size_t n = 0, max = 0;
+
+                hts_expand(hfile_part, fp->nparts+1, fp->maxparts, fp->parts);
+                part = &fp->parts[fp->nparts++];
+                part->url = NULL;
+                part->headers = NULL;
+
+                if (t.type != '{') return t.type;
+                while (hts_json_fnext(json, &t, b) != '}') {
+                    if (t.type != 's') return '?';
+
+                    if (strcmp(t.str, "url") == 0) {
+                        if (hts_json_fnext(json, &t, b) != 's') return t.type;
+                        part->url = ks_release(b);
+                    }
+                    else if (strcmp(t.str, "headers") == 0) {
+                        if (hts_json_fnext(json, &t, b) != '{') return t.type;
+
+                        while (hts_json_fnext(json, &t, header) != '}') {
+                            if (t.type != 's') return '?';
+
+                            if (hts_json_fnext(json, &t, b) != 's')
+                                return t.type;
+
+                            kputs(": ", header);
+                            kputs(t.str, header);
+                            n++;
+                            hts_expand(char *, n+1, max, part->headers);
+                            part->headers[n-1] = ks_release(header);
+                            part->headers[n] = NULL;
+                        }
+                    }
+                    else if (hts_json_fskip_value(json, '\0') != 'v')
+                        return '?';
+                }
+
+                if (! part->url) return 'i';
+            }
+        }
+        else if (strcmp(t.str, "format") == 0) {
+            if (hts_json_fnext(json, &t, b) != 's') return t.type;
+
+            hts_log_debug("GA4GH JSON redirection to multipart %s data", t.str);
+        }
+        else if (hts_json_fskip_value(json, '\0') != 'v') return '?';
+    }
+
+    return 'v';
+}
+
+// Returns 'v' (valid value), 'i' (invalid; required GA4GH field missing),
+// or upon encountering an unexpected token, that token's type.
+// Explicit `return '?'` means a JSON parsing error, typically a member key
+// that is not a string.  An unexpected token may be a valid token that was
+// not the type expected for a particular GA4GH field, or it may be '?' or
+// '\0' which should be propagated.
+static char
+parse_ga4gh_redirect_json(hFILE_multipart *fp, hFILE *json,
+                          kstring_t *b, kstring_t *header) {
+    hts_json_token t;
+
+    if (hts_json_fnext(json, &t, b) != '{') return t.type;
+    while (hts_json_fnext(json, &t, b) != '}') {
+        if (t.type != 's') return '?';
+
+        if (strcmp(t.str, "htsget") == 0) {
+            char ret = parse_ga4gh_body_json(fp, json, b, header);
+            if (ret != 'v') return ret;
+        }
+        else return '?';
+    }
+
+    if (hts_json_fnext(json, &t, b) != '\0') return '?';
+
+    return 'v';
+}
+
+hFILE *hopen_htsget_redirect(hFILE *hfile, const char *mode)
+{
+    hFILE_multipart *fp;
+    kstring_t s1 = { 0, 0, NULL }, s2 = { 0, 0, NULL };
+    char ret;
+
+    fp = (hFILE_multipart *) hfile_init(sizeof (hFILE_multipart), mode, 0);
+    if (fp == NULL) return NULL;
+
+    fp->parts = NULL;
+    fp->nparts = fp->maxparts = 0;
+
+    ret = parse_ga4gh_redirect_json(fp, hfile, &s1, &s2);
+    free(s1.s);
+    free(s2.s);
+    if (ret != 'v') {
+        free_all_parts(fp);
+        hfile_destroy((hFILE *) fp);
+        errno = (ret == '?' || ret == '\0')? EPROTO : EINVAL;
+        return NULL;
+    }
+
+    fp->current = 0;
+    fp->currentfp = NULL;
+    fp->base.backend = &multipart_backend;
+    return &fp->base;
+}
diff --git a/ext/htslib/os/lzma_stub.h b/ext/htslib/os/lzma_stub.h
new file mode 100644
index 0000000..5dd9c1a
--- /dev/null
+++ b/ext/htslib/os/lzma_stub.h
@@ -0,0 +1,85 @@
+#ifndef LZMA_STUB_H
+#define LZMA_STUB_H
+
+/* Some platforms, notably macOS, ship a usable liblzma shared library but
+   do not ship any LZMA header files. The <lzma.h> and <lzma/{*}.h> header
+   files that come with the library contain the following statement:
+
+     *
+     * Author: Lasse Collin
+     *
+     * This file has been put into the public domain.
+     * You can do whatever you want with this file.
+     *
+
+   Accordingly the following declarations have been copied and distilled
+   from <lzma/base.h> and <lzma/container.h> (primarily) and are sufficient
+   to compile cram/cram_io.c in the absence of proper LZMA headers.
+
+   This file, lzma_stub.h, remains in the public domain.  */
+
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef enum { LZMA_OK = 0, LZMA_STREAM_END = 1 } lzma_ret;
+typedef enum { LZMA_RUN = 0, LZMA_FINISH = 3 } lzma_action;
+typedef enum { LZMA_CHECK_CRC32 = 1 } lzma_check;
+typedef enum { LZMA_RESERVED_ENUM = 0 } lzma_reserved_enum;
+
+struct lzma_allocator;
+struct lzma_internal;
+
+typedef struct {
+    const uint8_t *next_in;
+    size_t avail_in;
+    uint64_t total_in;
+
+    uint8_t *next_out;
+    size_t avail_out;
+    uint64_t total_out;
+
+    const struct lzma_allocator *allocator;
+    struct lzma_internal *internal;
+
+    void *reserved_ptr1;
+    void *reserved_ptr2;
+    void *reserved_ptr3;
+    void *reserved_ptr4;
+    uint64_t reserved_int1;
+    uint64_t reserved_int2;
+    size_t reserved_int3;
+    size_t reserved_int4;
+    lzma_reserved_enum reserved_enum1;
+    lzma_reserved_enum reserved_enum2;
+} lzma_stream;
+
+#define LZMA_STREAM_INIT \
+    { NULL, 0, 0, NULL, 0, 0, NULL, NULL, \
+    NULL, NULL, NULL, NULL, 0, 0, 0, 0, \
+    LZMA_RESERVED_ENUM, LZMA_RESERVED_ENUM }
+
+extern size_t lzma_stream_buffer_bound(size_t uncompressed_size);
+
+extern lzma_ret lzma_easy_buffer_encode(
+        uint32_t preset, lzma_check check,
+        const struct lzma_allocator *allocator,
+        const uint8_t *in, size_t in_size,
+        uint8_t *out, size_t *out_pos, size_t out_size);
+
+extern lzma_ret lzma_stream_decoder(
+        lzma_stream *strm, uint64_t memlimit, uint32_t flags);
+
+extern uint64_t lzma_easy_decoder_memusage(uint32_t preset);
+
+extern lzma_ret lzma_code(lzma_stream *strm, lzma_action action);
+
+extern void lzma_end(lzma_stream *strm);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/ext/htslib/os/rand.c b/ext/htslib/os/rand.c
new file mode 100644
index 0000000..7ceafa8
--- /dev/null
+++ b/ext/htslib/os/rand.c
@@ -0,0 +1,97 @@
+/*  rand.c -- drand48 implementation from the FreeBSD source tree. */
+
+// This file is an amalgamation of the many small files in FreeBSD to do with
+// drand48 and friends implementations.
+// It comprises _rand48.c, rand48.h, srand48.c, drand48.c, erand48.c, lrand48.c
+
+/*
+ * Copyright (c) 1993 Martin Birgmeier
+ * All rights reserved.
+ *
+ * You may redistribute unmodified or modified versions of this source
+ * code provided that the above copyright notice and this and the
+ * following conditions are retained.
+ *
+ * This software is provided ``as is'', and comes with no warranties
+ * of any kind. I shall in no event be liable for anything that happens
+ * to anyone/anything when using this software.
+ */
+
+//#include <sys/cdefs.h>
+//__FBSDID("$FreeBSD: src/lib/libc/gen/_rand48.c,v 1.2 2002/03/22 21:52:05 obrien Exp $");
+
+#include <math.h>
+
+#define	RAND48_SEED_0	(0x330e)
+#define	RAND48_SEED_1	(0xabcd)
+#define	RAND48_SEED_2	(0x1234)
+#define	RAND48_MULT_0	(0xe66d)
+#define	RAND48_MULT_1	(0xdeec)
+#define	RAND48_MULT_2	(0x0005)
+#define	RAND48_ADD	(0x000b)
+
+static unsigned short _rand48_seed[3] = {
+	RAND48_SEED_0,
+	RAND48_SEED_1,
+	RAND48_SEED_2
+};
+static unsigned short _rand48_mult[3] = {
+	RAND48_MULT_0,
+	RAND48_MULT_1,
+	RAND48_MULT_2
+};
+static unsigned short _rand48_add = RAND48_ADD;
+
+static void
+_dorand48(unsigned short xseed[3])
+{
+	unsigned long accu;
+	unsigned short temp[2];
+
+	accu = (unsigned long) _rand48_mult[0] * (unsigned long) xseed[0] +
+	 (unsigned long) _rand48_add;
+	temp[0] = (unsigned short) accu;	/* lower 16 bits */
+	accu >>= sizeof(unsigned short) * 8;
+	accu += (unsigned long) _rand48_mult[0] * (unsigned long) xseed[1] +
+	 (unsigned long) _rand48_mult[1] * (unsigned long) xseed[0];
+	temp[1] = (unsigned short) accu;	/* middle 16 bits */
+	accu >>= sizeof(unsigned short) * 8;
+	accu += _rand48_mult[0] * xseed[2] + _rand48_mult[1] * xseed[1] + _rand48_mult[2] * xseed[0];
+	xseed[0] = temp[0];
+	xseed[1] = temp[1];
+	xseed[2] = (unsigned short) accu;
+}
+
+HTSLIB_EXPORT
+void hts_srand48(long seed)
+{
+	_rand48_seed[0] = RAND48_SEED_0;
+	_rand48_seed[1] = (unsigned short) seed;
+	_rand48_seed[2] = (unsigned short) (seed >> 16);
+	_rand48_mult[0] = RAND48_MULT_0;
+	_rand48_mult[1] = RAND48_MULT_1;
+	_rand48_mult[2] = RAND48_MULT_2;
+	_rand48_add = RAND48_ADD;
+}
+
+HTSLIB_EXPORT
+double hts_erand48(unsigned short xseed[3])
+{
+	_dorand48(xseed);
+	return ldexp((double) xseed[0], -48) +
+	       ldexp((double) xseed[1], -32) +
+	       ldexp((double) xseed[2], -16);
+}
+
+HTSLIB_EXPORT
+double hts_drand48(void)
+{
+	return hts_erand48(_rand48_seed);
+}
+
+HTSLIB_EXPORT
+long hts_lrand48(void)
+{
+	_dorand48(_rand48_seed);
+	return ((long) _rand48_seed[2] << 15) + ((long) _rand48_seed[1] >> 1);
+}
diff --git a/ext/htslib/plugin.c b/ext/htslib/plugin.c
new file mode 100644
index 0000000..670081f
--- /dev/null
+++ b/ext/htslib/plugin.c
@@ -0,0 +1,220 @@
+/*  plugin.c -- low-level path parsing and plugin functions.
+
+    Copyright (C) 2015-2016, 2020 Genome Research Ltd.
+
+    Author: John Marshall <jm18@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#include <config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+
+#include <dirent.h>
+#include <dlfcn.h>
+
+#include "hts_internal.h"
+#include "htslib/kstring.h"
+
+#ifndef PLUGINPATH
+#define PLUGINPATH ""
+#endif
+
+static DIR *open_nextdir(struct hts_path_itr *itr)
+{
+    DIR *dir;
+
+    while (1) {
+        const char *colon = strchr(itr->pathdir, HTS_PATH_SEPARATOR_CHAR);
+        if (colon == NULL) return NULL;
+
+        itr->entry.l = 0;
+        kputsn(itr->pathdir, colon - itr->pathdir, &itr->entry);
+        itr->pathdir = &colon[1];
+        if (itr->entry.l == 0) continue;
+
+        dir = opendir(itr->entry.s);
+        if (dir) break;
+
+        if (hts_verbose >= 4)
+            fprintf(stderr,
+                    "[W::hts_path_itr] can't scan directory \"%s\": %s\n",
+                    itr->entry.s, strerror(errno));
+    }
+
+    if (itr->entry.s[itr->entry.l-1] != '/') kputc('/', &itr->entry);
+    itr->entry_dir_l = itr->entry.l;
+    return dir;
+}
+
+void hts_path_itr_setup(struct hts_path_itr *itr, const char *path,
+        const char *builtin_path, const char *prefix, size_t prefix_len,
+        const char *suffix, size_t suffix_len)
+{
+    itr->prefix = prefix;
+    itr->prefix_len = prefix_len;
+
+    if (suffix) itr->suffix = suffix, itr->suffix_len = suffix_len;
+    else itr->suffix = PLUGIN_EXT, itr->suffix_len = strlen(PLUGIN_EXT);
+
+    itr->path.l = itr->path.m = 0; itr->path.s = NULL;
+    itr->entry.l = itr->entry.m = 0; itr->entry.s = NULL;
+
+    if (! builtin_path) builtin_path = PLUGINPATH;
+    if (! path) {
+        path = getenv("HTS_PATH");
+        if (! path) path = "";
+    }
+
+    while (1) {
+        size_t len = strcspn(path, HTS_PATH_SEPARATOR_STR);
+        if (len == 0) kputs(builtin_path, &itr->path);
+        else kputsn(path, len, &itr->path);
+        kputc(HTS_PATH_SEPARATOR_CHAR, &itr->path);
+
+        path += len;
+        if (*path == HTS_PATH_SEPARATOR_CHAR) path++;
+        else break;
+    }
+
+    // Note that ':' now terminates entries rather than separates them
+    itr->pathdir = itr->path.s;
+    itr->dirv = open_nextdir(itr);
+}
+
+const char *hts_path_itr_next(struct hts_path_itr *itr)
+{
+    while (itr->dirv) {
+        struct dirent *e;
+        while ((e = readdir((DIR *) itr->dirv)) != NULL) {
+            size_t d_name_len = strlen(e->d_name);
+            if (strncmp(e->d_name, itr->prefix, itr->prefix_len) == 0 &&
+                d_name_len >= itr->suffix_len &&
+                strncmp(e->d_name + d_name_len - itr->suffix_len, itr->suffix,
+                        itr->suffix_len) == 0) {
+                itr->entry.l = itr->entry_dir_l;
+                kputs(e->d_name, &itr->entry);
+                return itr->entry.s;
+            }
+        }
+
+        closedir((DIR *) itr->dirv);
+        itr->dirv = open_nextdir(itr);
+    }
+
+    itr->pathdir = NULL;
+    free(itr->path.s); itr->path.s = NULL;
+    free(itr->entry.s); itr->entry.s = NULL;
+    return NULL;
+}
+
+
+#ifndef RTLD_NOLOAD
+#define RTLD_NOLOAD 0
+#endif
+
+plugin_void_func *load_plugin(void **pluginp, const char *filename, const char *symbol)
+{
+    void *lib = dlopen(filename, RTLD_NOW | RTLD_LOCAL);
+    if (lib == NULL) goto error;
+
+    plugin_void_func *sym;
+    *(void **) &sym = dlsym(lib, symbol);
+    if (sym == NULL) {
+        // Reopen the plugin with RTLD_GLOBAL and check for uniquified symbol
+        void *libg = dlopen(filename, RTLD_NOLOAD | RTLD_NOW | RTLD_GLOBAL);
+        if (libg == NULL) goto error;
+        dlclose(lib);
+        lib = libg;
+
+        kstring_t symbolg = { 0, 0, NULL };
+        kputs(symbol, &symbolg);
+        kputc('_', &symbolg);
+        const char *slash = strrchr(filename, '/');
+        const char *basename = slash? slash+1 : filename;
+        kputsn(basename, strcspn(basename, ".-+"), &symbolg);
+
+        *(void **) &sym = dlsym(lib, symbolg.s);
+        free(symbolg.s);
+        if (sym == NULL) goto error;
+    }
+
+    *pluginp = lib;
+    return sym;
+
+error:
+    if (hts_verbose >= 4)
+        fprintf(stderr, "[W::%s] can't load plugin \"%s\": %s\n",
+                __func__, filename, dlerror());
+    if (lib) dlclose(lib);
+    return NULL;
+}
+
+void *plugin_sym(void *plugin, const char *name, const char **errmsg)
+{
+    void *sym = dlsym(plugin, name);
+    if (sym == NULL) *errmsg = dlerror();
+    return sym;
+}
+
+plugin_void_func *plugin_func(void *plugin, const char *name, const char **errmsg)
+{
+    plugin_void_func *sym;
+    *(void **) &sym = plugin_sym(plugin, name, errmsg);
+    return sym;
+}
+
+void close_plugin(void *plugin)
+{
+    if (dlclose(plugin) != 0) {
+        if (hts_verbose >= 4)
+            fprintf(stderr, "[W::%s] dlclose() failed: %s\n",
+                    __func__, dlerror());
+    }
+}
+
+const char *hts_plugin_path(void) {
+#ifdef ENABLE_PLUGINS
+    char *path = getenv("HTS_PATH");
+    if (!path) path = "";
+
+    kstring_t ks = {0};
+    while(1) {
+        size_t len = strcspn(path, HTS_PATH_SEPARATOR_STR);
+        if (len == 0) kputs(PLUGINPATH, &ks);
+        else kputsn(path, len, &ks);
+        kputc(HTS_PATH_SEPARATOR_CHAR, &ks);
+
+        path += len;
+        if (*path == HTS_PATH_SEPARATOR_CHAR) path++;
+        else break;
+    }
+
+    static char s_path[1024];
+    snprintf(s_path, sizeof(s_path), "%s", ks.s ? ks.s : "");
+    free(ks.s);
+
+    return s_path;
+#else
+    return NULL;
+#endif
+}
diff --git a/ext/htslib/probaln.c b/ext/htslib/probaln.c
new file mode 100644
index 0000000..b42f856
--- /dev/null
+++ b/ext/htslib/probaln.c
@@ -0,0 +1,468 @@
+/* The MIT License
+
+   Copyright (C) 2003-2006, 2008-2010 by Heng Li <lh3lh3@live.co.uk>
+   Copyright (C) 2016-2017, 2020, 2023 Genome Research Ltd.
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be
+   included in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+*/
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdint.h>
+#include <limits.h>
+#include <math.h>
+#include <errno.h>
+#include "htslib/hts.h"
+
+/*****************************************
+ * Probabilistic banded glocal alignment *
+ *****************************************/
+
+#define EI .25
+#define EM .33333333333
+
+static float g_qual2prob[256];
+
+#define set_u(u, b, i, k) { int x=(i)-(b); x=x>0?x:0; (u)=((k)-x+1)*3; }
+
+/*
+  The topology of the profile HMM:
+
+           /\             /\        /\             /\
+           I[1]           I[k-1]    I[k]           I[L]
+            ^   \      \    ^    \   ^   \      \   ^
+            |    \      \   |     \  |    \      \  |
+    M[0]   M[1] -> ... -> M[k-1] -> M[k] -> ... -> M[L]   M[L+1]
+                \      \/        \/      \/      /
+                 \     /\        /\      /\     /
+                       -> D[k-1] -> D[k] ->
+
+   M[0] points to every {M,I}[k] and every {M,I}[k] points to M[L+1].
+
+   On input, ref is the reference sequence and query is the query
+   sequence. Both are sequences of 0/1/2/3/4 where 4 stands for an
+   ambiguous residue. iqual is the base quality. c sets the gap open
+   probability, gap extension probability and band width.
+
+   On output, state and q are arrays of length l_query. The higher 30
+   bits give the reference position the query base is matched to and the
+   lower two bits can be 0 (an alignment match) or 1 (an
+   insertion). q[i] gives the phred scaled posterior probability of
+   state[i] being wrong.
+
+   Returns phred-scaled likelihood score, or INT_MIN on failure.
+ */
+int probaln_glocal(const uint8_t *ref, int l_ref, const uint8_t *query,
+                   int l_query, const uint8_t *iqual, const probaln_par_t *c,
+                   int *state, uint8_t *q)
+{
+    double *f = NULL, *b = NULL, *s = NULL, m[9], sI, sM, bI, bM;
+    float *qual = NULL;
+    int bw, bw2, i, k, is_backward = 1, Pr;
+
+    if ( l_ref<0 || l_query<0 || l_query >= INT_MAX - 2) {
+        errno = EINVAL;
+        return INT_MIN;
+    }
+    if (l_ref==0 || l_query==0)
+        return 0;  // Is this actually invalid??
+
+    /*** initialization ***/
+    is_backward = state && q? 1 : 0;
+    bw = l_ref > l_query? l_ref : l_query;
+    if (bw > c->bw) bw = c->bw;
+    if (bw < abs(l_ref - l_query)) bw = abs(l_ref - l_query);
+    bw2 = bw * 2 + 1;
+    size_t i_dim = bw2 < l_ref ? (size_t) bw2*3+6 : (size_t) l_ref*3+6;
+
+    // allocate the forward and backward matrices f[][] and b[][]
+    // and the scaling array s[]
+    // Ideally these callocs would be mallocs + initialisation of
+    // the few bits needed.
+    if (SIZE_MAX / (l_query+1) / i_dim < sizeof(double)) {
+        errno = ENOMEM; // Allocation would fail
+        return INT_MIN;
+    }
+    f = calloc((l_query+1)*i_dim, sizeof(double));
+    if (!f) goto fail;
+    if (is_backward) {
+        b = calloc((l_query+1)*i_dim, sizeof(double));
+        if (!b) goto fail;
+    }
+
+    // s[] is the scaling factor to avoid underflow
+    s = malloc((l_query+2) * sizeof(double));
+    if (!s) goto fail;
+
+    // initialize qual
+    qual = malloc(l_query * sizeof(float));
+    if (!qual) goto fail;
+    if (g_qual2prob[0] == 0)
+        for (i = 0; i < 256; ++i)
+            g_qual2prob[i] = pow(10, -i/10.);
+    qual[0] = 0.0; // Should be unused
+    for (i = 0; i < l_query; ++i)
+        qual[i] = g_qual2prob[iqual? iqual[i] : 30];
+
+    // initialize transition probability
+    // the value here seems not to affect results; FIXME: need proof
+    sM = sI = 1. / (2 * l_query + 2);
+    m[0*3+0] = (1 - c->d - c->d) * (1 - sM);
+    m[0*3+1] = m[0*3+2] = c->d * (1 - sM);
+    m[1*3+0] = (1 - c->e) * (1 - sI);
+    m[1*3+1] = c->e * (1 - sI);
+    m[1*3+2] = 0.;
+    m[2*3+0] = 1 - c->e;
+    m[2*3+1] = 0.;
+    m[2*3+2] = c->e;
+    bM = (1 - c->d) / l_ref; // (bM+bI)*l_ref==1
+    bI = c->d / l_ref;
+
+    // f[] and b[] are 2-d arrays of three scores, with rows along the
+    // query and columns across the band.  The first query base and
+    // first band position appear at index 1 allowing edge conditions
+    // to be stored in index 0.  Hence the loops below appear to use
+    // 1-based indexing instead of 0-based as you'd normally expect in C,
+    // and the sequences are accessed using query[i - 1] and ref[k - 1].
+
+    /*** forward ***/
+    // f[0]
+    set_u(k, bw, 0, 0);
+    f[0*i_dim+k] = s[0] = 1.;
+    { // f[1]
+        double *fi = &f[1*i_dim], sum;
+        int beg = 1, end = l_ref < bw + 1? l_ref : bw + 1;
+        for (k = beg, sum = 0.; k <= end; ++k) {
+            int u;
+            double e = (ref[k - 1] > 3 || query[0] > 3)? 1. : ref[k - 1] == query[0]? 1. - qual[0] : qual[0] * EM;
+            set_u(u, bw, 1, k);
+            fi[u+0] = e * bM; fi[u+1] = EI * bI;
+            sum += fi[u] + fi[u+1];
+        }
+        s[1] = sum;
+    }
+    // f[2..l_query]
+    for (i = 2; i <= l_query; ++i) {
+        double *fi = &f[i*i_dim], *fi1 = &f[(i-1)*i_dim], sum, qli = qual[i-1];
+        int beg = 1, end = l_ref, x;
+        uint8_t qyi = query[i - 1];
+        x = i - bw; beg = beg > x? beg : x; // band start
+        x = i + bw; end = end < x? end : x; // band end
+
+        // NB end-beg is almost always 14 (99.9% of the time)
+        // Hence not a large volume to parallelise.
+        //
+        // Maybe stripe in diagonal doing 14 lines together?
+        //
+        // Consider rotation? 150x14 vs 14x150 so inner loop
+        // takes longer.
+
+        double E[] = {
+            qli * EM, // 00
+            1. - qli, // 01
+            1.,       // 10
+            1.,       // 11
+        };
+        double M = 1./s[i-1];
+
+        // Note this code has the original version listed here (albeit
+        // with improved formatting), but we do not compile using
+        // -DPROBALN_ORIG.  The purpose of this code is to act as an
+        // easier(?) to understand version of the heavily optimised
+        // version following it and as an easy validation path in case
+        // of any differences in results.
+#ifdef PROBALN_ORIG
+        for (k = beg, sum = 0.; k <= end; ++k) {
+            int u, v11, v01, v10;
+            double e;
+            e = E[(ref[k - 1] > 3 || qyi > 3)*2 + (ref[k - 1] == qyi)];
+            set_u(u, bw, i, k);
+            set_u(v11, bw, i-1, k-1);
+            set_u(v10, bw, i-1, k);
+            set_u(v01, bw, i, k-1);
+            fi[u+0] = e * (m[0] * M*fi1[v11+0] + m[3] * M*fi1[v11+1] + m[6] * M*fi1[v11+2]);
+            fi[u+1] = EI * (m[1] * M*fi1[v10+0] + m[4] * M*fi1[v10+1]);
+            fi[u+2] = m[2] * fi[v01+0] + m[8] * fi[v01+2];
+            sum += fi[u] + fi[u+1] + fi[u+2];
+        }
+#else
+        // We use EI*(M*m[1]*? + M*m[4]*?) a lot.  So factor it out here.
+        double xm[5];
+        xm[0] =    M*m[0];
+        xm[1] =    M*m[3];
+        xm[2] =    M*m[6];
+        xm[3] = EI*M*m[1];
+        xm[4] = EI*M*m[4];
+
+        {
+            int u, v11;
+            set_u(u,   bw, i,   beg);
+            set_u(v11, bw, i-1, beg-1);
+            // Rather than recompute k->{u,v01,v10,v11} each loop
+            // we just increment the pointers.
+            double *xi = &fi[u];
+            double *yi = &fi1[v11];
+            // Derived from xi[0,2] in previous loop iter.
+            double  l_x0 = m[2]*xi[0];
+            double  l_x2 = m[8]*xi[2];
+            for (k = beg, sum = 0.; k <= end; ++k, xi+=3, yi+=3) {
+                int cond = (ref[k-1] > 3 || qyi > 3)*2 + (ref[k-1] == qyi);
+
+                double z0 = xm[0]*yi[0];
+                double z1 = xm[1]*yi[1];
+                double z2 = xm[2]*yi[2];
+                double z3 = xm[3]*yi[3];
+                double z4 = xm[4]*yi[4];
+
+                xi[0] = E[cond] * (z0+z1+z2);
+                xi[1] = z3 + z4;
+                xi[2] = l_x0 + l_x2;
+                sum  += xi[0] + xi[1] + xi[2];
+
+                l_x0 = m[2]*xi[0];
+                l_x2 = m[8]*xi[2];
+            }
+        }
+#endif
+        s[i] = sum;
+    }
+
+    { // f[l_query+1]
+        double sum;
+        double M = 1./s[l_query];
+        // Note that this goes from 1 to l_ref inclusive, but as the
+        // alignment is banded not all of the values will have been
+        // calculated (the rest are taken as 0), so the summation
+        // actually goes over the values set in the last iteration of
+        // the previous loop (when i = l_query).  For some reason lost to
+        // time this is done by looking for valid values of 'u' instead of
+        // working out 'beg' and 'end'.
+
+        // From HTSlib 1.8 to 1.17, the endpoint was incorrectly set
+        // to i_dim - 3.  When l_query <= bandwidth, this caused the last
+        // column to be missed, and if l_ref == l_query then a match at the end
+        // could incorrectly be reported as an insertion.  See #1605.
+
+        for (k = 1, sum = 0.; k <= l_ref; ++k) {
+            int u;
+            set_u(u, bw, l_query, k);
+            if (u < 3 || u >= i_dim) continue;
+            sum += M*f[l_query*i_dim + u+0] * sM + M*f[l_query*i_dim + u+1] * sI;
+        }
+        s[l_query+1] = sum; // the last scaling factor
+    }
+    { // compute likelihood
+        double p = 1., Pr1 = 0.;
+        for (i = 0; i <= l_query + 1; ++i) {
+            p *= s[i];
+            if (p < 1e-100) Pr1 += -4.343 * log(p), p = 1.;
+        }
+        Pr1 += -4.343 * log(p * l_ref * l_query);
+        Pr = (int)(Pr1 + .499);
+        if (!is_backward) { // skip backward and MAP
+            free(f); free(s); free(qual);
+            return Pr;
+        }
+    }
+    /*** backward ***/
+    // b[l_query] (b[l_query+1][0]=1 and thus \tilde{b}[][]=1/s[l_query+1]; this is where s[l_query+1] comes from)
+    for (k = 1; k <= l_ref; ++k) {
+        int u;
+        double *bi = &b[l_query*i_dim];
+        set_u(u, bw, l_query, k);
+        if (u < 3 || u >= i_dim) continue;
+        bi[u+0] = sM / s[l_query] / s[l_query+1]; bi[u+1] = sI / s[l_query] / s[l_query+1];
+    }
+    // b[l_query-1..1]
+    for (i = l_query - 1; i >= 1; --i) {
+        int beg = 1, end = l_ref, x;
+        double *bi = &b[i*i_dim], *bi1 = &b[(i+1)*i_dim], y = (i > 1), qli1 = qual[i];
+        uint8_t qyi1 = query[i];
+        x = i - bw; beg = beg > x? beg : x;
+        x = i + bw; end = end < x? end : x;
+        double E[] = {
+            qli1 * EM, //000
+            1. - qli1, //001
+            1.,        //010
+            1.,        //011
+            //0,0,0,0    //1xx
+        };
+
+#ifdef PROBALN_ORIG
+        for (k = end; k >= beg; --k) {
+            int u, v11, v01, v10;
+            double e;
+            set_u(u, bw, i, k);
+            set_u(v11, bw, i+1, k+1);
+            set_u(v10, bw, i+1, k);
+            set_u(v01, bw, i, k+1);
+            e = (k>=l_ref)?0 :E[(ref[k] > 3 || qyi1 > 3)*2 + (ref[k] == qyi1)] * bi1[v11];
+            bi[u+0] = e * m[0] + EI * m[1] * bi1[v10+1] + m[2] * bi[v01+2]; // bi1[v11] has been foled into e.
+            bi[u+1] = e * m[3] + EI * m[4] * bi1[v10+1];
+            bi[u+2] = (e * m[6] + m[8] * bi[v01+2]) * y;
+//          fprintf(stderr, "B (%d,%d;%d): %lg,%lg,%lg\n", i, k, u, bi[u], bi[u+1], bi[u+2]); // DEBUG
+        }
+        // rescale
+        int _beg, _end;
+        set_u(_beg, bw, i, beg); set_u(_end, bw, i, end); _end += 2;
+        for (k = _beg, y = 1./s[i]; k <= _end; ++k) bi[k] *= y;
+#else
+        {
+            int u, v10;
+            set_u(u,   bw, i,   end);
+            set_u(v10, bw, i+1, end);
+            // Rather than recompute k->{u,v01,v10,v11} each loop
+            // we just increment the pointers.
+            double *xi = &bi[u];
+            double *yi = &bi1[v10];
+            // NB xi[5] is equiv to v01+2.
+            double xi_5 = xi[5];
+            // Manual loop invariant removal
+            double e1 = EI*m[1];
+            double e4 = EI*m[4];
+            // Do renorm too in the same pass.
+            double n = 1./s[i];
+            for (k = end; k >= beg; --k, xi -= 3, yi -= 3) {
+                double e = (k>=l_ref)
+                    ? 0
+                    : E[(ref[k]>3 || qyi1>3)*2 + (ref[k] == qyi1)] * yi[3];
+
+                xi[1] =  e * m[3] +   e4 * yi[1];
+                xi[0] =  e * m[0] +   e1 * yi[1]  + m[2] * xi_5;
+                xi[2] = (e * m[6] + m[8] * xi_5) * y;
+                // bi[u+2] from this iter becomes bi[v01+2] in next iter
+                xi_5 = xi[2];
+
+                // rescale
+                xi[1] *= n;
+                xi[0] *= n;
+                xi[2] *= n;
+            }
+        }
+#endif
+    }
+    { // b[0]
+        int beg = 1, end = l_ref < bw + 1? l_ref : bw + 1;
+        double sum = 0.;
+        for (k = end; k >= beg; --k) {
+            int u;
+            double e = (ref[k - 1] > 3 || query[0] > 3)? 1. : ref[k - 1] == query[0]? 1. - qual[0] : qual[0] * EM;
+            set_u(u, bw, 1, k);
+            if (u < 3 || u >= i_dim) continue;
+            sum += e * b[1*i_dim + u+0] * bM + EI * b[1*i_dim + u+1] * bI;
+        }
+        set_u(k, bw, 0, 0);
+        b[0*i_dim + k] = sum / s[0]; // if everything works as is expected, b[0][k] == 1.0
+    }
+    /*** MAP ***/
+    for (i = 1; i <= l_query; ++i) {
+        double sum = 0., *fi = &f[i*i_dim], *bi = &b[i*i_dim], max = 0.;
+        int beg = 1, end = l_ref, x, max_k = -1;
+        x = i - bw; beg = beg > x? beg : x;
+        x = i + bw; end = end < x? end : x;
+        double M = 1./s[i];
+#ifdef PROBALN_ORIG
+        for (k = beg; k <= end; ++k) {
+            int u;
+            double z;
+            set_u(u, bw, i, k);
+            z = M*fi[u+0] * bi[u+0];
+            if (z > max) max = z, max_k = (k-1)<<2 | 0;
+            sum += z;
+            z = M*fi[u+1] * bi[u+1];
+            if (z > max) max = z, max_k = (k-1)<<2 | 1;
+            sum += z;
+        }
+#else
+        {
+            int u;
+            set_u(u, bw, i, beg);
+            for (k = beg; k <= end; ++k, u+=3) {
+                double z1, z2;
+                z1 = M*fi[u+0] * bi[u+0];
+                z2 = M*fi[u+1] * bi[u+1];
+                int which = z2 > z1; // strictly z2 >= z1 matches old code
+                double zm = which ? z2 : z1;
+                if (zm > max) {
+                    max   = zm;
+                    max_k = (k-1)<<2 | which;
+                }
+                sum += z1 + z2;
+            }
+        }
+#endif
+        max /= sum; sum *= s[i]; // if everything works as is expected, sum == 1.0
+        if (state) state[i-1] = max_k;
+        if (q) k = (int)(-4.343 * log(1. - max) + .499), q[i-1] = k > 100? 99 : k;
+#ifdef PROBALN_MAIN
+        k = 0;
+        set_u(k, bw, 0, 0);
+        fprintf(stderr, "(%.10lg,%.10lg) (%d,%d:%c,%c:%d) %lg\n", b[0][k], sum, i-1, max_k>>2,
+                "ACGT"[query[i - 1]], "ACGT"[ref[(max_k>>2)]], max_k&3, max); // DEBUG
+#endif
+    }
+
+    /*** free ***/
+    free(f); free(b); free(s); free(qual);
+    return Pr;
+
+ fail:
+    free(f); free(b); free(s); free(qual);
+    return INT_MIN;
+}
+
+#ifdef PROBALN_MAIN
+#include <unistd.h>
+int main(int argc, char *argv[])
+{
+    uint8_t conv[256], *iqual, *ref, *query;
+    probaln_par_t par = { 0.001, 0.1, 10 };
+    int c, l_ref, l_query, i, q = 30, b = 10, P;
+    while ((c = getopt(argc, argv, "b:q:")) >= 0) {
+        switch (c) {
+        case 'b': b = atoi(optarg); break;
+        case 'q': q = atoi(optarg); break;
+        }
+    }
+    if (optind + 2 > argc) {
+        fprintf(stderr, "Usage: %s [-q %d] [-b %d] <ref> <query>\n", argv[0], q, b); // example: acttc attc
+        return 1;
+    }
+    memset(conv, 4, 256);
+    conv['a'] = conv['A'] = 0; conv['c'] = conv['C'] = 1;
+    conv['g'] = conv['G'] = 2; conv['t'] = conv['T'] = 3;
+    ref = (uint8_t*)argv[optind]; query = (uint8_t*)argv[optind+1];
+    l_ref = strlen((char*)ref); l_query = strlen((char*)query);
+    for (i = 0; i < l_ref; ++i) ref[i] = conv[ref[i]];
+    for (i = 0; i < l_query; ++i) query[i] = conv[query[i]];
+    iqual = malloc(l_query);
+    memset(iqual, q, l_query);
+    par.bw = b;
+    P = probaln_glocal(ref, l_ref, query, l_query, iqual, &par, 0, 0);
+    fprintf(stderr, "%d\n", P);
+    free(iqual);
+    return 0;
+}
+#endif
diff --git a/ext/htslib/realn.c b/ext/htslib/realn.c
new file mode 100644
index 0000000..d7e8255
--- /dev/null
+++ b/ext/htslib/realn.c
@@ -0,0 +1,331 @@
+/*  realn.c -- BAQ calculation and realignment.
+
+    Copyright (C) 2009-2011, 2014-2016, 2018, 2021, 2023 Genome Research Ltd.
+    Portions copyright (C) 2009-2011 Broad Institute.
+
+    Author: Heng Li <lh3@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <limits.h>
+#include <stdint.h>
+#include <errno.h>
+#include <assert.h>
+#include "htslib/hts.h"
+#include "htslib/sam.h"
+
+int sam_cap_mapq(bam1_t *b, const char *ref, hts_pos_t ref_len, int thres)
+{
+    uint8_t *seq = bam_get_seq(b), *qual = bam_get_qual(b);
+    uint32_t *cigar = bam_get_cigar(b);
+    bam1_core_t *c = &b->core;
+    int i, y, mm, q, len, clip_l, clip_q;
+    hts_pos_t x;
+    double t;
+    if (thres < 0) thres = 40; // set the default
+    mm = q = len = clip_l = clip_q = 0;
+    for (i = y = 0, x = c->pos; i < c->n_cigar; ++i) {
+        int j, l = cigar[i]>>4, op = cigar[i]&0xf;
+        if (op == BAM_CMATCH || op == BAM_CEQUAL || op == BAM_CDIFF) {
+            for (j = 0; j < l; ++j) {
+                int c1, c2, z = y + j;
+                if (x+j >= ref_len || ref[x+j] == '\0') break; // out of bounds
+                c1 = bam_seqi(seq, z), c2 = seq_nt16_table[(unsigned char)ref[x+j]];
+                if (c2 != 15 && c1 != 15 && qual[z] >= 13) { // not ambiguous
+                    ++len;
+                    if (c1 && c1 != c2 && qual[z] >= 13) { // mismatch
+                        ++mm;
+                        q += qual[z] > 33? 33 : qual[z];
+                    }
+                }
+            }
+            if (j < l) break;
+            x += l; y += l; len += l;
+        } else if (op == BAM_CDEL) {
+            for (j = 0; j < l; ++j)
+                if (x+j >= ref_len || ref[x+j] == '\0') break;
+            if (j < l) break;
+            x += l;
+        } else if (op == BAM_CSOFT_CLIP) {
+            for (j = 0; j < l; ++j) clip_q += qual[y+j];
+            clip_l += l;
+            y += l;
+        } else if (op == BAM_CHARD_CLIP) {
+            clip_q += 13 * l;
+            clip_l += l;
+        } else if (op == BAM_CINS) y += l;
+        else if (op == BAM_CREF_SKIP) x += l;
+    }
+    for (i = 0, t = 1; i < mm; ++i)
+        t *= (double)len / (i+1);
+    t = q - 4.343 * log(t) + clip_q / 5.;
+    if (t > thres) return -1;
+    if (t < 0) t = 0;
+    t = sqrt((thres - t) / thres) * thres;
+    //fprintf(stderr, "%s %lf %d\n", bam_get_qname(b), t, q);
+    return (int)(t + .499);
+}
+
+static int realn_check_tag(const uint8_t *tg, enum htsLogLevel severity,
+                           const char *type, const bam1_t *b) {
+    if (*tg != 'Z') {
+        hts_log(severity, __func__, "Incorrect %s tag type (%c) for read %s",
+                type, *tg, bam_get_qname(b));
+        return -1;
+    }
+    if (b->core.l_qseq != strlen((const char *) tg + 1)) {
+        hts_log(severity, __func__, "Read %s %s tag is wrong length",
+                bam_get_qname(b), type);
+        return -1;
+    }
+    return 0;
+}
+
+int sam_prob_realn(bam1_t *b, const char *ref, hts_pos_t ref_len, int flag) {
+    int k, bw, y, yb, ye, xb, xe, fix_bq = 0, apply_baq = flag & BAQ_APPLY,
+        extend_baq = flag & BAQ_EXTEND, redo_baq = flag & BAQ_REDO;
+    enum htsRealnFlags system = flag & (0xff << 3);
+    hts_pos_t i, x;
+    uint32_t *cigar = bam_get_cigar(b);
+    bam1_core_t *c = &b->core;
+
+    // d(I) e(M) band
+    probaln_par_t conf = { 0.001, 0.1, 10 }; // Illumina
+
+    if (b->core.l_qseq > 1000 || system > BAQ_ILLUMINA) {
+        // Params that work well on PacBio CCS 15k.  Unknown if they
+        // help other long-read platforms yet, but likely better than
+        // the short-read tuned ones.
+        //
+        // This function has no access to the SAM header.
+        // Ideally the calling function would check for e.g.
+        // @RG PL = "PACBIO" and DS contains "READTYPE=CCS".
+        //
+        // In the absense of this, we simply auto-detect via a crude
+        // short vs long strategy.
+        conf.d = 1e-7;
+        conf.e = 1e-1;
+    }
+
+    uint8_t *bq = NULL, *zq = NULL, *qual = bam_get_qual(b);
+    int *state = NULL;
+    if ((c->flag & BAM_FUNMAP) || b->core.l_qseq == 0 || qual[0] == (uint8_t)-1)
+        return -1; // do nothing
+
+    // test if BQ or ZQ is present, and make sanity checks
+    if ((bq = bam_aux_get(b, "BQ")) != NULL) {
+        if (!redo_baq) {
+            if (realn_check_tag(bq, HTS_LOG_WARNING, "BQ", b) < 0)
+                fix_bq = 1;
+        }
+        ++bq;
+    }
+    if ((zq = bam_aux_get(b, "ZQ")) != NULL) {
+        if (realn_check_tag(zq, HTS_LOG_ERROR, "ZQ", b) < 0)
+            return -4;
+        ++zq;
+    }
+    if (bq && redo_baq)
+    {
+        bam_aux_del(b, bq-1);
+        bq = 0;
+    }
+    if (bq && zq) { // remove the ZQ tag
+        bam_aux_del(b, zq-1);
+        zq = 0;
+    }
+    if (!zq && fix_bq) { // Need to fix invalid BQ tag (by realigning)
+        assert(bq != NULL);
+        bam_aux_del(b, bq-1);
+        bq = 0;
+    }
+
+    if (bq || zq) {
+        if ((apply_baq && zq) || (!apply_baq && bq)) return -3; // in both cases, do nothing
+        if (bq && apply_baq) { // then convert BQ to ZQ
+            for (i = 0; i < c->l_qseq; ++i)
+                qual[i] = qual[i] + 64 < bq[i]? 0 : qual[i] - ((int)bq[i] - 64);
+            *(bq - 3) = 'Z';
+        } else if (zq && !apply_baq) { // then convert ZQ to BQ
+            for (i = 0; i < c->l_qseq; ++i)
+                qual[i] += (int)zq[i] - 64;
+            *(zq - 3) = 'B';
+        }
+        return 0;
+    }
+    // find the start and end of the alignment
+    x = c->pos, y = 0, yb = ye = xb = xe = -1;
+    for (k = 0; k < c->n_cigar; ++k) {
+        int op, l;
+        op = cigar[k]&0xf; l = cigar[k]>>4;
+        if (op == BAM_CMATCH || op == BAM_CEQUAL || op == BAM_CDIFF) {
+            if (yb < 0) yb = y;
+            if (xb < 0) xb = x;
+            ye = y + l; xe = x + l;
+            x += l; y += l;
+        } else if (op == BAM_CSOFT_CLIP || op == BAM_CINS) y += l;
+        else if (op == BAM_CDEL) x += l;
+        else if (op == BAM_CREF_SKIP) return -1; // do nothing if there is a reference skip
+    }
+    if (xb == -1) // No matches in CIGAR.
+        return -1;
+    // set bandwidth and the start and the end
+    bw = 7;
+    if (abs((xe - xb) - (ye - yb)) > bw)
+        bw = abs((xe - xb) - (ye - yb)) + 3;
+    conf.bw = bw;
+
+    xb -= yb + bw/2; if (xb < 0) xb = 0;
+    xe += c->l_qseq - ye + bw/2;
+    if (xe - xb - c->l_qseq > bw)
+        xb += (xe - xb - c->l_qseq - bw) / 2, xe -= (xe - xb - c->l_qseq - bw) / 2;
+    { // glocal
+        uint8_t *seq = bam_get_seq(b);
+        uint8_t *tseq; // translated seq A=>0,C=>1,G=>2,T=>3,other=>4
+        uint8_t *tref; // translated ref
+        uint8_t *q; // Probability of incorrect alignment from probaln_glocal()
+        size_t lref = xe > xb ? xe - xb : 1;
+        size_t align_lqseq;
+        if (extend_baq && lref < c->l_qseq)
+            lref = c->l_qseq; // So we can recycle tseq,tref for left,rght below
+        // Try to make q,tref,tseq reasonably well aligned
+        align_lqseq = ((c->l_qseq + 1) | 0xf) + 1;
+        // Overflow check - 3 for *bq, sizeof(int) for *state
+        if ((SIZE_MAX - lref) / (3 + sizeof(int)) < align_lqseq) {
+            errno = ENOMEM;
+            goto fail;
+        }
+
+        assert(bq == NULL); // bq was used above, but should now be NULL
+        bq = malloc(align_lqseq * 3 + lref);
+        if (!bq) goto fail;
+        q = bq + align_lqseq;
+        tseq = q + align_lqseq;
+        tref = tseq + align_lqseq;
+
+        memcpy(bq, qual, c->l_qseq); bq[c->l_qseq] = 0;
+        for (i = 0; i < c->l_qseq; ++i)
+            tseq[i] = seq_nt16_int[bam_seqi(seq, i)];
+        for (i = xb; i < xe; ++i) {
+            if (i >= ref_len || ref[i] == '\0') { xe = i; break; }
+            tref[i-xb] = seq_nt16_int[seq_nt16_table[(unsigned char)ref[i]]];
+        }
+
+        state = malloc(c->l_qseq * sizeof(int));
+        if (!state) goto fail;
+        if (probaln_glocal(tref, xe-xb, tseq, c->l_qseq, qual,
+                           &conf, state, q) == INT_MIN) {
+            goto fail;
+        }
+
+        if (!extend_baq) { // in this block, bq[] is capped by base quality qual[]
+            for (k = 0, x = c->pos, y = 0; k < c->n_cigar; ++k) {
+                int op = cigar[k]&0xf, l = cigar[k]>>4;
+                if (l == 0) continue;
+                if (op == BAM_CMATCH || op == BAM_CEQUAL || op == BAM_CDIFF) {
+                    // Sanity check running off the end of the sequence
+                    // Can only happen if the alignment is broken
+                    if (l > c->l_qseq - y)
+                        l = c->l_qseq - y;
+                    for (i = y; i < y + l; ++i) {
+                        if ((state[i]&3) != 0 || state[i]>>2 != x - xb + (i - y)) bq[i] = 0;
+                        else bq[i] = bq[i] < q[i]? bq[i] : q[i];
+                    }
+                    x += l; y += l;
+                } else if (op == BAM_CSOFT_CLIP || op == BAM_CINS) {
+                    // Need sanity check here too.
+                    if (l > c->l_qseq - y)
+                        l = c->l_qseq - y;
+                    y += l;
+                } else if (op == BAM_CDEL) {
+                    x += l;
+                }
+            }
+            for (i = 0; i < c->l_qseq; ++i) bq[i] = qual[i] - bq[i] + 64; // finalize BQ
+        } else { // in this block, bq[] is BAQ that can be larger than qual[] (different from the above!)
+            // tseq,tref are no longer needed, so we can steal them to avoid mallocs
+            uint8_t *left = tseq;
+            uint8_t *rght = tref;
+            int len = 0;
+
+            for (k = 0, x = c->pos, y = 0; k < c->n_cigar; ++k) {
+                int op = cigar[k]&0xf, l = cigar[k]>>4;
+
+                // concatenate alignment matches (including sequence (mis)matches)
+                // otherwise 50M50M gives a different result to 100M
+                if (op == BAM_CMATCH || op == BAM_CEQUAL || op == BAM_CDIFF) {
+                    if ((k + 1) < c->n_cigar) {
+                        int next_op = bam_cigar_op(cigar[k + 1]);
+
+                        if (next_op == BAM_CMATCH || next_op == BAM_CEQUAL || next_op == BAM_CDIFF) {
+                            len += l;
+                            continue;
+                        }
+                    }
+
+                    // last of M/X/= ops
+                    l += len;
+                    len = 0;
+                }
+
+                if (l == 0) continue;
+                if (op == BAM_CMATCH || op == BAM_CEQUAL || op == BAM_CDIFF) {
+                    // Sanity check running off the end of the sequence
+                    // Can only happen if the alignment is broken
+                    if (l > c->l_qseq - y)
+                        l = c->l_qseq - y;
+                    for (i = y; i < y + l; ++i)
+                        bq[i] = ((state[i]&3) != 0 || state[i]>>2 != x - xb + (i - y))? 0 : q[i];
+                    for (left[y] = bq[y], i = y + 1; i < y + l; ++i)
+                        left[i] = bq[i] > left[i-1]? bq[i] : left[i-1];
+                    for (rght[y+l-1] = bq[y+l-1], i = y + l - 2; i >= y; --i)
+                        rght[i] = bq[i] > rght[i+1]? bq[i] : rght[i+1];
+                    for (i = y; i < y + l; ++i)
+                        bq[i] = left[i] < rght[i]? left[i] : rght[i];
+                    x += l; y += l;
+                } else if (op == BAM_CSOFT_CLIP || op == BAM_CINS) {
+                    // Need sanity check here too.
+                    if (l > c->l_qseq - y)
+                        l = c->l_qseq - y;
+                    y += l;
+                } else if (op == BAM_CDEL) {
+                    x += l;
+                }
+            }
+            for (i = 0; i < c->l_qseq; ++i) bq[i] = 64 + (qual[i] <= bq[i]? 0 : qual[i] - bq[i]); // finalize BQ
+        }
+        if (apply_baq) {
+            for (i = 0; i < c->l_qseq; ++i) qual[i] -= bq[i] - 64; // modify qual
+            bam_aux_append(b, "ZQ", 'Z', c->l_qseq + 1, bq);
+        } else bam_aux_append(b, "BQ", 'Z', c->l_qseq + 1, bq);
+        free(bq); free(state);
+    }
+
+    return 0;
+
+ fail:
+    free(bq); free(state);
+    return -4;
+}
diff --git a/ext/htslib/regidx.c b/ext/htslib/regidx.c
new file mode 100644
index 0000000..602edeb
--- /dev/null
+++ b/ext/htslib/regidx.c
@@ -0,0 +1,688 @@
+/*
+    Copyright (C) 2014-2019 Genome Research Ltd.
+
+    Author: Petr Danecek <pd3@sanger.ac.uk>
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in
+    all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+    THE SOFTWARE.
+*/
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+#include <strings.h>
+#include <assert.h>
+#include "htslib/hts.h"
+#include "htslib/kstring.h"
+#include "htslib/kseq.h"
+#include "htslib/khash_str2int.h"
+#include "htslib/regidx.h"
+#include "hts_internal.h"
+
+#define MAX_COOR_0 REGIDX_MAX   // CSI and hts_itr_query limit, 0-based
+
+#define iBIN(x) ((x)>>13)
+
+typedef struct
+{
+    hts_pos_t beg, end;
+}
+reg_t;
+
+typedef struct
+{
+    hts_pos_t pos;  // position
+    uint32_t  ireg; // index to reglist.reg and reglist.dat
+}
+pos_t;
+
+typedef struct reglist_t reglist_t;
+
+typedef struct
+{
+    hts_pos_t beg, end; // query region
+    uint32_t ireg;      // index of active region
+    regidx_t *ridx;
+    reglist_t *list;
+    int active;
+}
+itr_t_;
+
+// List of regions for one chromosome.
+struct reglist_t
+{
+    uint32_t *idx, nidx;    // index to list.reg+1
+    uint32_t nreg, mreg;    // n:used, m:allocated
+    reg_t *reg;             // regions
+    uint8_t *dat;           // payload data
+    char *seq;              // sequence name
+    int unsorted;
+};
+
+// Container of all sequences
+struct regidx_t
+{
+    int nseq, mseq;         // n:used, m:alloced
+    reglist_t *seq;         // regions for each sequence
+    void *seq2regs;         // hash for fast lookup from chr name to regions
+    char **seq_names;
+    regidx_free_f free;     // function to free any data allocated by regidx_parse_f
+    regidx_parse_f parse;   // parse one input line
+    void *usr;              // user data to pass to regidx_parse_f
+    int payload_size;
+    void *payload;          // temporary payload data set by regidx_parse_f (sequence is not known beforehand)
+    kstring_t str;
+};
+
+int regidx_seq_nregs(regidx_t *idx, const char *seq)
+{
+    int iseq;
+    if ( khash_str2int_get(idx->seq2regs, seq, &iseq)!=0 ) return 0; // no such sequence
+    return idx->seq[iseq].nreg;
+}
+
+int regidx_nregs(regidx_t *idx)
+{
+    int i, nreg = 0;
+    for (i=0; i<idx->nseq; i++) nreg += idx->seq[i].nreg;
+    return nreg;
+}
+
+char **regidx_seq_names(regidx_t *idx, int *n)
+{
+    *n = idx->nseq;
+    return idx->seq_names;
+}
+
+int regidx_insert_list(regidx_t *idx, char *line, char delim)
+{
+    kstring_t tmp = KS_INITIALIZE;
+    char *ss = line;
+    while ( *ss )
+    {
+        char *se = ss;
+        while ( *se && *se!=delim ) se++;
+        kputsn(ss, se-ss, ks_clear(&tmp));
+        if ( regidx_insert(idx,tmp.s) < 0 )
+        {
+            ks_free(&tmp);
+            return -1;
+        }
+        if ( !*se ) break;
+        ss = se+1;
+    }
+    ks_free(&tmp);
+    return 0;
+}
+
+static inline int cmp_regs(reg_t *a, reg_t *b)
+{
+    if ( a->beg < b->beg ) return -1;
+    if ( a->beg > b->beg ) return 1;
+    if ( a->end < b->end ) return 1;    // longer intervals come first
+    if ( a->end > b->end ) return -1;
+    if ( a < b ) return -1; // this is are just for qsort reproducibility across platforms
+    if ( a > b ) return 1;
+    return 0;
+}
+static int cmp_reg_ptrs(const void *a, const void *b)
+{
+    return cmp_regs((reg_t*)a,(reg_t*)b);
+}
+static int cmp_reg_ptrs2(const void *a, const void *b)
+{
+    return cmp_regs(*((reg_t**)a),*((reg_t**)b));
+}
+
+int regidx_push(regidx_t *idx, char *chr_beg, char *chr_end, hts_pos_t beg, hts_pos_t end, void *payload)
+{
+    if (beg < 0) beg = 0;
+    if (end < 0) end = 0;
+    if ( beg > MAX_COOR_0 ) beg = MAX_COOR_0;
+    if ( end > MAX_COOR_0 ) end = MAX_COOR_0;
+
+    int rid;
+    if (kputsn(chr_beg, chr_end-chr_beg+1, ks_clear(&idx->str)) < 0) return -1;
+    if ( khash_str2int_get(idx->seq2regs, idx->str.s, &rid)!=0 )
+    {
+        // new chromosome
+        int m_tmp = idx->mseq;
+        if (hts_resize(char*, idx->nseq + 1, &m_tmp,
+                       &idx->seq_names, HTS_RESIZE_CLEAR) < 0) {
+            return -1;
+        }
+        if (hts_resize(reglist_t, idx->nseq + 1, &idx->mseq,
+                       &idx->seq, HTS_RESIZE_CLEAR) < 0) {
+            return -1;
+        }
+        assert(m_tmp == idx->mseq);
+        idx->seq_names[idx->nseq] = strdup(idx->str.s);
+        rid = khash_str2int_inc(idx->seq2regs, idx->seq_names[idx->nseq]);
+        idx->nseq++;
+    }
+
+    reglist_t *list = &idx->seq[rid];
+    list->seq = idx->seq_names[rid];
+    int mreg = list->mreg;
+    if (hts_resize(reg_t, list->nreg + 1, &list->mreg, &list->reg, 0) < 0)
+        return -1;
+    list->reg[list->nreg].beg = beg;
+    list->reg[list->nreg].end = end;
+    if ( idx->payload_size ) {
+        if ( mreg != list->mreg ) {
+            uint8_t *new_dat = realloc(list->dat, idx->payload_size*list->mreg);
+            if (!new_dat) return -1;
+            list->dat = new_dat;
+        }
+        memcpy(list->dat + idx->payload_size*list->nreg, payload, idx->payload_size);
+    }
+    list->nreg++;
+    if ( !list->unsorted && list->nreg>1 && cmp_regs(&list->reg[list->nreg-2],&list->reg[list->nreg-1])>0 ) list->unsorted = 1;
+    return 0;
+}
+
+int regidx_insert(regidx_t *idx, char *line)
+{
+    if ( !line ) return 0;
+    char *chr_from, *chr_to;
+    hts_pos_t beg,end;
+    int ret = idx->parse(line,&chr_from,&chr_to,&beg,&end,idx->payload,idx->usr);
+    if ( ret==-2 ) return -1;   // error
+    if ( ret==-1 ) return 0;    // skip the line
+    return regidx_push(idx, chr_from,chr_to,beg,end,idx->payload);
+}
+
+regidx_t *regidx_init_string(const char *str, regidx_parse_f parser, regidx_free_f free_f, size_t payload_size, void *usr_dat)
+{
+    kstring_t tmp = KS_INITIALIZE;
+    regidx_t *idx = (regidx_t*) calloc(1,sizeof(regidx_t));
+    if ( !idx ) return NULL;
+
+    idx->free  = free_f;
+    idx->parse = parser ? parser : regidx_parse_tab;
+    idx->usr   = usr_dat;
+    idx->seq2regs = khash_str2int_init();
+    if (!idx->seq2regs) goto fail;
+    idx->payload_size = payload_size;
+    if ( payload_size ) {
+        idx->payload = malloc(payload_size);
+        if (!idx->payload) goto fail;
+    }
+
+    const char *ss = str;
+    while ( *ss )
+    {
+        while ( *ss && isspace_c(*ss) ) ss++;
+        const char *se = ss;
+        while ( *se && *se!='\r' && *se!='\n' ) se++;
+        if (kputsn(ss, se-ss, ks_clear(&tmp)) < 0) goto fail;
+        if (regidx_insert(idx, tmp.s) < 0) goto fail;
+        while ( *se && isspace_c(*se) ) se++;
+        ss = se;
+    }
+    ks_free(&tmp);
+    return idx;
+
+ fail:
+    regidx_destroy(idx);
+    ks_free(&tmp);
+    return NULL;
+}
+
+regidx_t *regidx_init(const char *fname, regidx_parse_f parser, regidx_free_f free_f, size_t payload_size, void *usr_dat)
+{
+    if ( !parser )
+    {
+        if ( !fname ) parser = regidx_parse_tab;
+        else
+        {
+            int len = strlen(fname);
+            if ( len>=7 && !strcasecmp(".bed.gz",fname+len-7) )
+                parser = regidx_parse_bed;
+            else if ( len>=8 && !strcasecmp(".bed.bgz",fname+len-8) )
+                parser = regidx_parse_bed;
+            else if ( len>=4 && !strcasecmp(".bed",fname+len-4) )
+                parser = regidx_parse_bed;
+            else if ( len>=4 && !strcasecmp(".vcf",fname+len-4) )
+                parser = regidx_parse_vcf;
+            else if ( len>=7 && !strcasecmp(".vcf.gz",fname+len-7) )
+                parser = regidx_parse_vcf;
+            else
+                parser = regidx_parse_tab;
+        }
+    }
+
+    kstring_t str = KS_INITIALIZE;
+    htsFile *fp = NULL;
+    int ret;
+    regidx_t *idx = (regidx_t*) calloc(1,sizeof(regidx_t));
+    if (!idx) return NULL;
+    idx->free  = free_f;
+    idx->parse = parser;
+    idx->usr   = usr_dat;
+    idx->seq2regs = khash_str2int_init();
+    if (!idx->seq2regs) goto error;
+    idx->payload_size = payload_size;
+    if ( payload_size ) {
+        idx->payload = malloc(payload_size);
+        if (!idx->payload) goto error;
+    }
+
+    if ( !fname ) return idx;
+
+    fp = hts_open(fname,"r");
+    if ( !fp ) goto error;
+
+    while ((ret = hts_getline(fp, KS_SEP_LINE, &str)) > 0 ) {
+        if ( regidx_insert(idx, str.s) ) goto error;
+    }
+    if (ret < -1) goto error;
+
+    ret = hts_close(fp);
+    fp = NULL;
+    if ( ret != 0 ) {
+        hts_log_error("Close failed .. %s", fname);
+        goto error;
+    }
+    ks_free(&str);
+    return idx;
+
+error:
+    ks_free(&str);
+    if ( fp ) hts_close(fp);
+    regidx_destroy(idx);
+    return NULL;
+}
+
+void regidx_destroy(regidx_t *idx)
+{
+    int i, j;
+    if (!idx) return;
+    for (i=0; i<idx->nseq; i++)
+    {
+        reglist_t *list = &idx->seq[i];
+        if ( idx->free )
+        {
+            for (j=0; j<list->nreg; j++)
+                idx->free((char *)list->dat + idx->payload_size*j);
+        }
+        free(list->dat);
+        free(list->reg);
+        free(list->idx);
+    }
+    free(idx->seq_names);
+    free(idx->seq);
+    free(idx->str.s);
+    free(idx->payload);
+    khash_str2int_destroy_free(idx->seq2regs);
+    free(idx);
+}
+
+static int reglist_build_index_(regidx_t *regidx, reglist_t *list)
+{
+    int i;
+    if ( list->unsorted ) {
+        if ( !regidx->payload_size ) {
+            qsort(list->reg,list->nreg,sizeof(reg_t),cmp_reg_ptrs);
+        } else {
+            reg_t **ptr = malloc(sizeof(*ptr)*list->nreg);
+            if (!ptr) return -1;
+            for (i=0; i<list->nreg; i++) ptr[i] = list->reg + i;
+            qsort(ptr,list->nreg,sizeof(*ptr),cmp_reg_ptrs2);
+
+            uint8_t *tmp_dat = malloc(regidx->payload_size*list->nreg);
+            if (!tmp_dat) { free(ptr); return -1; }
+            for (i=0; i<list->nreg; i++) {
+                size_t iori = ptr[i] - list->reg;
+                memcpy(tmp_dat+i*regidx->payload_size,
+                       list->dat+iori*regidx->payload_size,
+                       regidx->payload_size);
+            }
+            free(list->dat);
+            list->dat = tmp_dat;
+
+            reg_t *tmp_reg = (reg_t*) malloc(sizeof(reg_t)*list->nreg);
+            if (!tmp_reg) { free(ptr); return -1; }
+            for (i=0; i<list->nreg; i++) {
+                size_t iori = ptr[i] - list->reg;
+                tmp_reg[i] = list->reg[iori];
+            }
+            free(ptr);
+            free(list->reg);
+            list->reg  = tmp_reg;
+            list->mreg = list->nreg;
+        }
+        list->unsorted = 0;
+    }
+
+    list->nidx = 0;
+    uint32_t j,k, midx = 0;
+    // Find highest index bin.  It's possible that we could just look at
+    // the last region, but go through the list in case some entries overlap.
+    for (j=0; j<list->nreg; j++) {
+        int iend = iBIN(list->reg[j].end);
+        if (midx <= iend) midx = iend;
+    }
+    midx++;
+    uint32_t *new_idx = calloc(midx, sizeof(uint32_t));
+    if (!new_idx) return -1;
+    free(list->idx); // Should be NULL on entry, but just in case...
+    list->idx = new_idx;
+    list->nidx = midx;
+
+    for (j=0; j<list->nreg; j++) {
+        int ibeg = iBIN(list->reg[j].beg);
+        int iend = iBIN(list->reg[j].end);
+        if ( ibeg==iend ) {
+            if ( !list->idx[ibeg] ) list->idx[ibeg] = j + 1;
+        } else {
+            for (k=ibeg; k<=iend; k++)
+                if ( !list->idx[k] ) list->idx[k] = j + 1;
+        }
+    }
+
+    return 0;
+}
+
+int regidx_overlap(regidx_t *regidx, const char *chr, hts_pos_t beg, hts_pos_t end, regitr_t *regitr)
+{
+    if ( regitr ) regitr->seq = NULL;
+
+    int iseq, ireg;
+    if ( khash_str2int_get(regidx->seq2regs, chr, &iseq)!=0 ) return 0;    // no such sequence
+
+    reglist_t *list = &regidx->seq[iseq];
+    if ( !list->nreg ) return 0;
+
+    if ( list->nreg==1 )
+    {
+        if ( beg > list->reg[0].end ) return 0;
+        if ( end < list->reg[0].beg ) return 0;
+        ireg = 0;
+    }
+    else
+    {
+        if ( !list->idx ) {
+            if (reglist_build_index_(regidx,list) < 0) return -1;
+        }
+
+        int ibeg = iBIN(beg);
+        if ( ibeg >= list->nidx ) return 0;     // beg is too big
+
+        // find a matching region
+        uint32_t i = list->idx[ibeg];
+        if ( !i )
+        {
+            int iend = iBIN(end);
+            if ( iend > list->nidx ) iend = list->nidx;
+            for (i=ibeg; i<=iend; i++)
+                if ( list->idx[i] ) break;
+            if ( i>iend ) return 0;
+            i = list->idx[i];
+        }
+        for (ireg=i-1; ireg<list->nreg; ireg++)
+        {
+            if ( list->reg[ireg].beg > end ) return 0;   // no match, past the query region
+            if ( list->reg[ireg].end >= beg && list->reg[ireg].beg <= end ) break; // found
+        }
+
+        if ( ireg >= list->nreg ) return 0;   // no match
+    }
+
+    if ( !regitr ) return 1;    // match, but no more info to save
+
+    // may need to iterate over the matching regions later
+    itr_t_ *itr = (itr_t_*)regitr->itr;
+    itr->ridx = regidx;
+    itr->list = list;
+    itr->beg  = beg;
+    itr->end  = end;
+    itr->ireg = ireg;
+    itr->active = 0;
+
+    regitr->seq = list->seq;
+    regitr->beg = list->reg[ireg].beg;
+    regitr->end = list->reg[ireg].end;
+    if ( regidx->payload_size )
+        regitr->payload = list->dat + regidx->payload_size*ireg;
+
+    return 1;
+}
+
+int regidx_parse_bed(const char *line, char **chr_beg, char **chr_end, hts_pos_t *beg, hts_pos_t *end, void *payload, void *usr)
+{
+    char *ss = (char*) line;
+    while ( *ss && isspace_c(*ss) ) ss++;
+    if ( !*ss ) return -1;      // skip blank lines
+    if ( *ss=='#' ) return -1;  // skip comments
+
+    char *se = ss;
+    while ( *se && !isspace_c(*se) ) se++;
+
+    *chr_beg = ss;
+    *chr_end = se-1;
+
+    if ( !*se )
+    {
+        // just the chromosome name
+        *beg = 0;
+        *end = MAX_COOR_0;
+        return 0;
+    }
+
+    ss = se+1;
+    *beg = hts_parse_decimal(ss, &se, 0);
+    if ( ss==se ) { hts_log_error("Could not parse bed line: %s", line); return -2; }
+
+    ss = se+1;
+    *end = hts_parse_decimal(ss, &se, 0) - 1;
+    if ( ss==se ) { hts_log_error("Could not parse bed line: %s", line); return -2; }
+
+    return 0;
+}
+
+int regidx_parse_tab(const char *line, char **chr_beg, char **chr_end, hts_pos_t *beg, hts_pos_t *end, void *payload, void *usr)
+{
+    char *ss = (char*) line;
+    while ( *ss && isspace_c(*ss) ) ss++;
+    if ( !*ss ) return -1;      // skip blank lines
+    if ( *ss=='#' ) return -1;  // skip comments
+
+    char *se = ss;
+    while ( *se && !isspace_c(*se) ) se++;
+
+    *chr_beg = ss;
+    *chr_end = se-1;
+
+    if ( !*se )
+    {
+        // just the chromosome name
+        *beg = 0;
+        *end = MAX_COOR_0;
+        return 0;
+    }
+
+    ss = se+1;
+    *beg = hts_parse_decimal(ss, &se, 0);
+    if ( ss==se ) { hts_log_error("Could not parse tab line: %s", line); return -2; }
+    if ( *beg==0 ) { hts_log_error("Could not parse tab line, expected 1-based coordinate: %s", line); return -2; }
+    (*beg)--;
+
+    if ( !se[0] || !se[1] )
+        *end = *beg;
+    else
+    {
+        ss = se+1;
+        *end = hts_parse_decimal(ss, &se, 0);
+        if ( ss==se || (*se && !isspace_c(*se)) ) *end = *beg;
+        else if ( *end==0 ) { hts_log_error("Could not parse tab line, expected 1-based coordinate: %s", line); return -2; }
+        else (*end)--;
+    }
+    return 0;
+}
+
+int regidx_parse_vcf(const char *line, char **chr_beg, char **chr_end, hts_pos_t *beg, hts_pos_t *end, void *payload, void *usr)
+{
+    int ret = regidx_parse_tab(line, chr_beg, chr_end, beg, end, payload, usr);
+    if ( !ret ) *end = *beg;
+    return ret;
+}
+
+int regidx_parse_reg(const char *line, char **chr_beg, char **chr_end, hts_pos_t *beg, hts_pos_t *end, void *payload, void *usr)
+{
+    char *ss = (char*) line;
+    while ( *ss && isspace_c(*ss) ) ss++;
+    if ( !*ss ) return -1;      // skip blank lines
+    if ( *ss=='#' ) return -1;  // skip comments
+
+    char *se = ss;
+    while ( *se && *se!=':' ) se++;
+
+    *chr_beg = ss;
+    *chr_end = se-1;
+
+    if ( !*se )
+    {
+        *beg = 0;
+        *end = MAX_COOR_0;
+        return 0;
+    }
+
+    ss = se+1;
+    *beg = hts_parse_decimal(ss, &se, 0);
+    if ( ss==se ) { hts_log_error("Could not parse reg line: %s", line); return -2; }
+    if ( *beg==0 ) { hts_log_error("Could not parse reg line, expected 1-based coordinate: %s", line); return -2; }
+    (*beg)--;
+
+    if ( !se[0] || !se[1] )
+        *end = se[0]=='-' ? MAX_COOR_0 : *beg;
+    else
+    {
+        ss = se+1;
+        *end = hts_parse_decimal(ss, &se, 0);
+        if ( ss==se ) *end = *beg;
+        else if ( *end==0 ) { hts_log_error("Could not parse reg line, expected 1-based coordinate: %s", line); return -2; }
+        else (*end)--;
+    }
+    return 0;
+}
+
+regitr_t *regitr_init(regidx_t *regidx)
+{
+    regitr_t *regitr = (regitr_t*) calloc(1,sizeof(regitr_t));
+    if (!regitr) return NULL;
+    regitr->itr  = (itr_t_*) calloc(1,sizeof(itr_t_));
+    if (!regitr->itr) {
+        free(regitr);
+        return NULL;
+    }
+    itr_t_ *itr = (itr_t_*) regitr->itr;
+    itr->ridx = regidx;
+    itr->list = NULL;
+    return regitr;
+}
+
+void regitr_reset(regidx_t *regidx, regitr_t *regitr)
+{
+    itr_t_ *itr = (itr_t_*) regitr->itr;
+    memset(itr,0,sizeof(itr_t_));
+    itr->ridx = regidx;
+}
+
+void regitr_destroy(regitr_t *regitr)
+{
+    free(regitr->itr);
+    free(regitr);
+}
+
+int regitr_overlap(regitr_t *regitr)
+{
+    if ( !regitr || !regitr->seq || !regitr->itr ) return 0;
+
+    itr_t_ *itr = (itr_t_*) regitr->itr;
+    if ( !itr->active )
+    {
+        // is this the first call after regidx_overlap?
+        itr->active = 1;
+        itr->ireg++;
+        return 1;
+    }
+
+    reglist_t *list = itr->list;
+
+    int i;
+    for (i=itr->ireg; i<list->nreg; i++)
+    {
+        if ( list->reg[i].beg > itr->end ) return 0;   // no match, past the query region
+        if ( list->reg[i].end >= itr->beg && list->reg[i].beg <= itr->end ) break; // found
+    }
+
+    if ( i >= list->nreg ) return 0;   // no match
+
+    itr->ireg = i + 1;
+    regitr->seq = list->seq;
+    regitr->beg = list->reg[i].beg;
+    regitr->end = list->reg[i].end;
+    if ( itr->ridx->payload_size )
+        regitr->payload = (char *)list->dat + itr->ridx->payload_size*i;
+
+    return 1;
+}
+
+int regitr_loop(regitr_t *regitr)
+{
+    if ( !regitr || !regitr->itr ) return 0;
+
+    itr_t_ *itr = (itr_t_*) regitr->itr;
+    regidx_t *regidx = itr->ridx;
+
+    if ( !itr->list )    // first time here
+    {
+        itr->list = regidx->seq;
+        itr->ireg = 0;
+    }
+
+    size_t iseq = itr->list - regidx->seq;
+    if ( iseq >= regidx->nseq ) return 0;
+
+    if ( itr->ireg >= itr->list->nreg )
+    {
+        iseq++;
+        if ( iseq >= regidx->nseq ) return 0; // no more sequences, done
+        itr->ireg = 0;
+        itr->list = &regidx->seq[iseq];
+    }
+
+    regitr->seq = itr->list->seq;
+    regitr->beg = itr->list->reg[itr->ireg].beg;
+    regitr->end = itr->list->reg[itr->ireg].end;
+    if ( regidx->payload_size )
+        regitr->payload = (char *)itr->list->dat + regidx->payload_size*itr->ireg;
+    itr->ireg++;
+
+    return 1;
+}
+
+
+void regitr_copy(regitr_t *dst, regitr_t *src)
+{
+    itr_t_ *dst_itr = (itr_t_*) dst->itr;
+    itr_t_ *src_itr = (itr_t_*) src->itr;
+    *dst_itr = *src_itr;
+    *dst = *src;
+    dst->itr = dst_itr;
+}
diff --git a/ext/htslib/region.c b/ext/htslib/region.c
new file mode 100644
index 0000000..8b570e0
--- /dev/null
+++ b/ext/htslib/region.c
@@ -0,0 +1,276 @@
+/*  region.c -- Functions to create and free region lists
+
+    Copyright (C) 2019 Genome Research Ltd.
+
+    Author: Valeriu Ohan <vo2@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+
+#include "htslib/hts.h"
+#include "htslib/khash.h"
+
+typedef struct reglist
+{
+    uint32_t n, m;
+    hts_pair_pos_t *a;
+    int tid;
+} reglist_t;
+
+KHASH_MAP_INIT_INT(reg, reglist_t)
+typedef kh_reg_t reghash_t;
+
+static int compare_hts_pair_pos_t (const void *av, const void *bv)
+{
+    hts_pair_pos_t *a = (hts_pair_pos_t *) av;
+    hts_pair_pos_t *b = (hts_pair_pos_t *) bv;
+    if (a->beg < b->beg) return -1;
+    if (a->beg > b->beg) return  1;
+    if (a->end < b->end) return -1;
+    if (a->end > b->end) return  1;
+
+    return 0;
+}
+
+#if 0
+/**
+ * Good to have around for debugging
+ */
+static void reg_print(reghash_t *h) {
+    reglist_t *p;
+    khint_t k;
+    uint32_t i;
+    khint32_t key;
+
+    if (!h) {
+        fprintf(stderr, "Hash table is empty!\n");
+        return;
+    }
+    for (k = kh_begin(h); k < kh_end(h); k++) {
+        if (kh_exist(h,k)) {
+            key = kh_key(h,k);
+            fprintf(stderr, "Region: key %u tid %d\n", key, p->tid);
+            if ((p = &kh_val(h,k)) != NULL && p->n > 0) {
+                for (i=0; i<p->n; i++) {
+                    fprintf(stderr, "\tinterval[%d]: %"PRIhts_pos"-%"PRIhts_pos"\n", i,
+                            p->a[i].beg, p->a[i].end);
+                }
+            } else {
+                fprintf(stderr, "Region key %u has no intervals!\n", key);
+            }
+        }
+    }
+}
+#endif
+
+/**
+ * Sort and merge overlapping or adjacent intervals.
+ */
+static int reg_compact(reghash_t *h) {
+    khint_t i;
+    uint32_t j, new_n;
+    reglist_t *p;
+    int count = 0;
+
+    if (!h)
+        return 0;
+
+    for (i = kh_begin(h); i < kh_end(h); i++) {
+        if (!kh_exist(h,i) || !(p = &kh_val(h,i)) || !(p->n))
+            continue;
+
+        qsort(p->a, p->n, sizeof(p->a[0]), compare_hts_pair_pos_t);
+        for (new_n = 0, j = 1; j < p->n; j++) {
+            if (p->a[new_n].end < p->a[j].beg) {
+                p->a[++new_n].beg = p->a[j].beg;
+                p->a[new_n].end = p->a[j].end;
+            } else {
+                if (p->a[new_n].end < p->a[j].end)
+                    p->a[new_n].end = p->a[j].end;
+            }
+        }
+        ++new_n;
+        if (p->n > new_n) {
+            // Shrink array to required size.
+            hts_pair_pos_t *new_a = realloc(p->a, new_n * sizeof(p->a[0]));
+            if (new_a) p->a = new_a;
+        }
+        p->n = new_n;
+        count++;
+    }
+
+    return count;
+}
+
+static int reg_insert(reghash_t *h, int tid, hts_pos_t beg, hts_pos_t end) {
+
+    khint_t k;
+    reglist_t *p;
+
+    if (!h)
+        return -1;
+
+    // Put reg in the hash table if not already there
+    k = kh_get(reg, h, tid);
+    if (k == kh_end(h)) { // absent from the hash table
+        int ret;
+        k = kh_put(reg, h, tid, &ret);
+        if (-1 == ret) {
+            return -1;
+        }
+        memset(&kh_val(h, k), 0, sizeof(reglist_t));
+        kh_val(h, k).tid = tid;
+    }
+    p = &kh_val(h, k);
+
+    // Add beg and end to the list
+    if (p->n == p->m) {
+        uint32_t new_m = p->m ? p->m<<1 : 4;
+        if (new_m == 0) return -1;
+        hts_pair_pos_t *new_a = realloc(p->a, new_m * sizeof(p->a[0]));
+        if (new_a == NULL) return -1;
+        p->m = new_m;
+        p->a = new_a;
+    }
+    p->a[p->n].beg = beg;
+    p->a[p->n++].end = end;
+
+    return 0;
+}
+
+static void reg_destroy(reghash_t *h) {
+
+    khint_t k;
+
+    if (!h)
+        return;
+
+    for (k = 0; k < kh_end(h); ++k) {
+        if (kh_exist(h, k)) {
+            free(kh_val(h, k).a);
+        }
+    }
+    kh_destroy(reg, h);
+}
+
+/**
+ * Take a char array of reg:interval elements and produce a hts_reglis_t with r_count elements.
+ */
+hts_reglist_t *hts_reglist_create(char **argv, int argc, int *r_count, void *hdr,  hts_name2id_f getid) {
+
+    if (!argv || argc < 1)
+        return NULL;
+
+    reghash_t *h = NULL;
+    reglist_t *p;
+    hts_reglist_t *h_reglist = NULL;
+
+    khint_t k;
+    int i, l_count = 0, tid;
+    const char *q;
+    hts_pos_t beg, end;
+
+    /* First, transform the char array into a hash table */
+    h = kh_init(reg);
+    if (!h) {
+        hts_log_error("Error when creating the region hash table");
+        return NULL;
+    }
+
+    for (i=0; i<argc; i++) {
+        if (!strcmp(argv[i], ".")) {
+            q = argv[i] + 1;
+            tid = HTS_IDX_START; beg = 0; end = INT64_MAX;
+        } else if (!strcmp(argv[i], "*")) {
+            q = argv[i] + 1;
+            tid = HTS_IDX_NOCOOR; beg = 0; end = INT64_MAX;
+        } else {
+            q = hts_parse_region(argv[i], &tid, &beg, &end, getid, hdr,
+                                 HTS_PARSE_THOUSANDS_SEP);
+        }
+        if (!q) {
+            if (tid < -1) {
+                hts_log_error("Failed to parse header");
+                goto fail;
+            } else {
+                // not parsable as a region
+                hts_log_warning("Region '%s' specifies an unknown reference name. Continue anyway", argv[i]);
+                continue;
+            }
+        }
+
+        if (reg_insert(h, tid, beg, end) != 0) {
+            hts_log_error("Error when inserting region='%s' in the bed hash table at address=%p", argv[i], (void *) h);
+            goto fail;
+        }
+    }
+
+    *r_count = reg_compact(h);
+    if (!*r_count)
+        goto fail;
+
+    /* Transform the hash table into a list */
+    h_reglist = (hts_reglist_t *)calloc(*r_count, sizeof(hts_reglist_t));
+    if (!h_reglist)
+        goto fail;
+
+    for (k = kh_begin(h); k < kh_end(h) && l_count < *r_count; k++) {
+        if (!kh_exist(h,k) || !(p = &kh_val(h,k)))
+            continue;
+
+        h_reglist[l_count].tid = p->tid;
+        h_reglist[l_count].intervals = p->a;
+        h_reglist[l_count].count = p->n;
+        p->a = NULL; // As we stole it.
+
+        // After reg_compact(), list is ordered and non-overlapping, so...
+        if (p->n > 0) {
+            h_reglist[l_count].min_beg = h_reglist[l_count].intervals[0].beg;
+            h_reglist[l_count].max_end = h_reglist[l_count].intervals[p->n - 1].end;
+        } else {
+            h_reglist[l_count].min_beg = 0;
+            h_reglist[l_count].max_end = 0;
+        }
+
+        l_count++;
+    }
+    reg_destroy(h);
+
+    return h_reglist;
+
+fail:
+    reg_destroy(h);
+    if(h_reglist) hts_reglist_free(h_reglist, l_count);
+
+    return NULL;
+}
+
+void hts_reglist_free(hts_reglist_t *reglist, int count) {
+
+    int i;
+    if(reglist) {
+        for (i = 0; i < count; i++) {
+            if (reglist[i].intervals)
+                free(reglist[i].intervals);
+        }
+        free(reglist);
+    }
+}
diff --git a/ext/htslib/sam.5 b/ext/htslib/sam.5
new file mode 100644
index 0000000..d44719e
--- /dev/null
+++ b/ext/htslib/sam.5
@@ -0,0 +1,68 @@
+'\" t
+.TH sam 5 "August 2013" "htslib" "Bioinformatics formats"
+.SH NAME
+sam \- Sequence Alignment/Map file format
+.\"
+.\" Copyright (C) 2009, 2013-2014 Genome Research Ltd.
+.\"
+.\" Author: Heng Li <lh3@sanger.ac.uk>
+.\"
+.\" Permission is hereby granted, free of charge, to any person obtaining a
+.\" copy of this software and associated documentation files (the "Software"),
+.\" to deal in the Software without restriction, including without limitation
+.\" the rights to use, copy, modify, merge, publish, distribute, sublicense,
+.\" and/or sell copies of the Software, and to permit persons to whom the
+.\" Software is furnished to do so, subject to the following conditions:
+.\"
+.\" The above copyright notice and this permission notice shall be included in
+.\" all copies or substantial portions of the Software.
+.\"
+.\" THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+.\" IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+.\" FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+.\" THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+.\" LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+.\" FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+.\" DEALINGS IN THE SOFTWARE.
+.\"
+.SH DESCRIPTION
+Sequence Alignment/Map (SAM) format is TAB-delimited. Apart from the header lines, which are started
+with the `@' symbol, each alignment line consists of:
+.TS
+nlbl.
+1	QNAME	Query template/pair NAME
+2	FLAG	bitwise FLAG
+3	RNAME	Reference sequence NAME
+4	POS	1-based leftmost POSition/coordinate of clipped sequence
+5	MAPQ	MAPping Quality (Phred-scaled)
+6	CIGAR	extended CIGAR string
+7	MRNM	Mate Reference sequence NaMe (`=' if same as RNAME)
+8	MPOS	1-based Mate POSition
+9	TLEN	inferred Template LENgth (insert size)
+10	SEQ	query SEQuence on the same strand as the reference
+11	QUAL	query QUALity (ASCII-33 gives the Phred base quality)
+12+	OPT	variable OPTional fields in the format TAG:VTYPE:VALUE
+.TE
+.PP
+Each bit in the FLAG field is defined as:
+.TS
+lcbl.
+0x0001	p	the read is paired in sequencing
+0x0002	P	the read is mapped in a proper pair
+0x0004	u	the query sequence itself is unmapped
+0x0008	U	the mate is unmapped
+0x0010	r	strand of the query (1 for reverse)
+0x0020	R	strand of the mate
+0x0040	1	the read is the first read in a pair
+0x0080	2	the read is the second read in a pair
+0x0100	s	the alignment is not primary
+0x0200	f	the read fails platform/vendor quality checks
+0x0400	d	the read is either a PCR or an optical duplicate
+0x0800	S	the alignment is supplementary
+.TE
+.P
+where the second column gives the string representation of the FLAG field.
+.SH SEE ALSO
+.TP
+https://github.com/samtools/hts-specs
+The full SAM/BAM file format specification
diff --git a/ext/htslib/sam.c b/ext/htslib/sam.c
new file mode 100644
index 0000000..7e58da6
--- /dev/null
+++ b/ext/htslib/sam.c
@@ -0,0 +1,6391 @@
+/*  sam.c -- SAM and BAM file I/O and manipulation.
+
+    Copyright (C) 2008-2010, 2012-2024 Genome Research Ltd.
+    Copyright (C) 2010, 2012, 2013 Broad Institute.
+
+    Author: Heng Li <lh3@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+
+#include <strings.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <zlib.h>
+#include <assert.h>
+#include <signal.h>
+#include <inttypes.h>
+#include <unistd.h>
+
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+#include "fuzz_settings.h"
+#endif
+
+// Suppress deprecation message for cigar_tab, which we initialise
+#include "htslib/hts_defs.h"
+#undef HTS_DEPRECATED
+#define HTS_DEPRECATED(message)
+
+#include "htslib/sam.h"
+#include "htslib/bgzf.h"
+#include "cram/cram.h"
+#include "hts_internal.h"
+#include "sam_internal.h"
+#include "htslib/hfile.h"
+#include "htslib/hts_endian.h"
+#include "htslib/hts_expr.h"
+#include "header.h"
+
+#include "htslib/khash.h"
+KHASH_DECLARE(s2i, kh_cstr_t, int64_t)
+KHASH_SET_INIT_INT(tag)
+
+#ifndef EFTYPE
+#define EFTYPE ENOEXEC
+#endif
+#ifndef EOVERFLOW
+#define EOVERFLOW ERANGE
+#endif
+
+/**********************
+ *** BAM header I/O ***
+ **********************/
+
+HTSLIB_EXPORT
+const int8_t bam_cigar_table[256] = {
+    // 0 .. 47
+    -1, -1, -1, -1,  -1, -1, -1, -1,  -1, -1, -1, -1,  -1, -1, -1, -1,
+    -1, -1, -1, -1,  -1, -1, -1, -1,  -1, -1, -1, -1,  -1, -1, -1, -1,
+    -1, -1, -1, -1,  -1, -1, -1, -1,  -1, -1, -1, -1,  -1, -1, -1, -1,
+
+    // 48 .. 63  (including =)
+    -1, -1, -1, -1,  -1, -1, -1, -1,  -1, -1, -1, -1,  -1, BAM_CEQUAL, -1, -1,
+
+    // 64 .. 79  (including MIDNHB)
+    -1, -1, BAM_CBACK, -1,  BAM_CDEL, -1, -1, -1,
+        BAM_CHARD_CLIP, BAM_CINS, -1, -1,  -1, BAM_CMATCH, BAM_CREF_SKIP, -1,
+
+    // 80 .. 95  (including SPX)
+    BAM_CPAD, -1, -1, BAM_CSOFT_CLIP,  -1, -1, -1, -1,
+        BAM_CDIFF, -1, -1, -1,  -1, -1, -1, -1,
+
+    // 96 .. 127
+    -1, -1, -1, -1,  -1, -1, -1, -1,  -1, -1, -1, -1,  -1, -1, -1, -1,
+    -1, -1, -1, -1,  -1, -1, -1, -1,  -1, -1, -1, -1,  -1, -1, -1, -1,
+
+    // 128 .. 255
+    -1, -1, -1, -1,  -1, -1, -1, -1,  -1, -1, -1, -1,  -1, -1, -1, -1,
+    -1, -1, -1, -1,  -1, -1, -1, -1,  -1, -1, -1, -1,  -1, -1, -1, -1,
+    -1, -1, -1, -1,  -1, -1, -1, -1,  -1, -1, -1, -1,  -1, -1, -1, -1,
+    -1, -1, -1, -1,  -1, -1, -1, -1,  -1, -1, -1, -1,  -1, -1, -1, -1,
+    -1, -1, -1, -1,  -1, -1, -1, -1,  -1, -1, -1, -1,  -1, -1, -1, -1,
+    -1, -1, -1, -1,  -1, -1, -1, -1,  -1, -1, -1, -1,  -1, -1, -1, -1,
+    -1, -1, -1, -1,  -1, -1, -1, -1,  -1, -1, -1, -1,  -1, -1, -1, -1,
+    -1, -1, -1, -1,  -1, -1, -1, -1,  -1, -1, -1, -1,  -1, -1, -1, -1
+};
+
+sam_hdr_t *sam_hdr_init(void)
+{
+    sam_hdr_t *bh = (sam_hdr_t*)calloc(1, sizeof(sam_hdr_t));
+    if (bh == NULL) return NULL;
+
+    bh->cigar_tab = bam_cigar_table;
+    return bh;
+}
+
+void sam_hdr_destroy(sam_hdr_t *bh)
+{
+    int32_t i;
+
+    if (bh == NULL) return;
+
+    if (bh->ref_count > 0) {
+        --bh->ref_count;
+        return;
+    }
+
+    if (bh->target_name) {
+        for (i = 0; i < bh->n_targets; ++i)
+            free(bh->target_name[i]);
+        free(bh->target_name);
+        free(bh->target_len);
+    }
+    free(bh->text);
+    if (bh->hrecs)
+        sam_hrecs_free(bh->hrecs);
+    if (bh->sdict)
+        kh_destroy(s2i, (khash_t(s2i) *) bh->sdict);
+    free(bh);
+}
+
+// Copy the sam_hdr_t::sdict hash, used to store the real lengths of long
+// references before sam_hdr_t::hrecs is populated
+int sam_hdr_dup_sdict(const sam_hdr_t *h0, sam_hdr_t *h)
+{
+    const khash_t(s2i) *src_long_refs = (khash_t(s2i) *) h0->sdict;
+    khash_t(s2i) *dest_long_refs = kh_init(s2i);
+    int i;
+    if (!dest_long_refs) return -1;
+
+    for (i = 0; i < h->n_targets; i++) {
+        int ret;
+        khiter_t ksrc, kdest;
+        if (h->target_len[i] < UINT32_MAX) continue;
+        ksrc = kh_get(s2i, src_long_refs, h->target_name[i]);
+        if (ksrc == kh_end(src_long_refs)) continue;
+        kdest = kh_put(s2i, dest_long_refs, h->target_name[i], &ret);
+        if (ret < 0) {
+            kh_destroy(s2i, dest_long_refs);
+            return -1;
+        }
+        kh_val(dest_long_refs, kdest) = kh_val(src_long_refs, ksrc);
+    }
+
+    h->sdict = dest_long_refs;
+    return 0;
+}
+
+sam_hdr_t *sam_hdr_dup(const sam_hdr_t *h0)
+{
+    if (h0 == NULL) return NULL;
+    sam_hdr_t *h;
+    if ((h = sam_hdr_init()) == NULL) return NULL;
+    // copy the simple data
+    h->n_targets = 0;
+    h->ignore_sam_err = h0->ignore_sam_err;
+    h->l_text = 0;
+
+    // Then the pointery stuff
+
+    if (!h0->hrecs) {
+        h->target_len = (uint32_t*)calloc(h0->n_targets, sizeof(uint32_t));
+        if (!h->target_len) goto fail;
+        h->target_name = (char**)calloc(h0->n_targets, sizeof(char*));
+        if (!h->target_name) goto fail;
+
+        int i;
+        for (i = 0; i < h0->n_targets; ++i) {
+            h->target_len[i] = h0->target_len[i];
+            h->target_name[i] = strdup(h0->target_name[i]);
+            if (!h->target_name[i]) break;
+        }
+        h->n_targets = i;
+        if (i < h0->n_targets) goto fail;
+
+        if (h0->sdict) {
+            if (sam_hdr_dup_sdict(h0, h) < 0) goto fail;
+        }
+    }
+
+    if (h0->hrecs) {
+        kstring_t tmp = { 0, 0, NULL };
+        if (sam_hrecs_rebuild_text(h0->hrecs, &tmp) != 0) {
+            free(ks_release(&tmp));
+            goto fail;
+        }
+
+        h->l_text = tmp.l;
+        h->text   = ks_release(&tmp);
+
+        if (sam_hdr_update_target_arrays(h, h0->hrecs, 0) != 0)
+            goto fail;
+    } else {
+        h->l_text = h0->l_text;
+        h->text = malloc(h->l_text + 1);
+        if (!h->text) goto fail;
+        memcpy(h->text, h0->text, h->l_text);
+        h->text[h->l_text] = '\0';
+    }
+
+    return h;
+
+ fail:
+    sam_hdr_destroy(h);
+    return NULL;
+}
+
+sam_hdr_t *bam_hdr_read(BGZF *fp)
+{
+    sam_hdr_t *h;
+    uint8_t buf[4];
+    int magic_len, has_EOF;
+    int32_t i, name_len, num_names = 0;
+    size_t bufsize;
+    ssize_t bytes;
+    // check EOF
+    has_EOF = bgzf_check_EOF(fp);
+    if (has_EOF < 0) {
+        perror("[W::bam_hdr_read] bgzf_check_EOF");
+    } else if (has_EOF == 0) {
+        hts_log_warning("EOF marker is absent. The input is probably truncated");
+    }
+    // read "BAM1"
+    magic_len = bgzf_read(fp, buf, 4);
+    if (magic_len != 4 || memcmp(buf, "BAM\1", 4)) {
+        hts_log_error("Invalid BAM binary header");
+        return 0;
+    }
+    h = sam_hdr_init();
+    if (!h) goto nomem;
+
+    // read plain text and the number of reference sequences
+    bytes = bgzf_read(fp, buf, 4);
+    if (bytes != 4) goto read_err;
+    h->l_text = le_to_u32(buf);
+
+    bufsize = h->l_text + 1;
+    if (bufsize < h->l_text) goto nomem; // so large that adding 1 overflowed
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    if (bufsize > FUZZ_ALLOC_LIMIT) goto nomem;
+#endif
+    h->text = (char*)malloc(bufsize);
+    if (!h->text) goto nomem;
+    h->text[h->l_text] = 0; // make sure it is NULL terminated
+    bytes = bgzf_read(fp, h->text, h->l_text);
+    if (bytes != h->l_text) goto read_err;
+
+    bytes = bgzf_read(fp, &h->n_targets, 4);
+    if (bytes != 4) goto read_err;
+    if (fp->is_be) ed_swap_4p(&h->n_targets);
+
+    if (h->n_targets < 0) goto invalid;
+
+    // read reference sequence names and lengths
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    if (h->n_targets > (FUZZ_ALLOC_LIMIT - bufsize)/(sizeof(char*)+sizeof(uint32_t)))
+        goto nomem;
+#endif
+    if (h->n_targets > 0) {
+        h->target_name = (char**)calloc(h->n_targets, sizeof(char*));
+        if (!h->target_name) goto nomem;
+        h->target_len = (uint32_t*)calloc(h->n_targets, sizeof(uint32_t));
+        if (!h->target_len) goto nomem;
+    }
+    else {
+        h->target_name = NULL;
+        h->target_len = NULL;
+    }
+
+    for (i = 0; i != h->n_targets; ++i) {
+        bytes = bgzf_read(fp, &name_len, 4);
+        if (bytes != 4) goto read_err;
+        if (fp->is_be) ed_swap_4p(&name_len);
+        if (name_len <= 0) goto invalid;
+
+        h->target_name[i] = (char*)malloc(name_len);
+        if (!h->target_name[i]) goto nomem;
+        num_names++;
+
+        bytes = bgzf_read(fp, h->target_name[i], name_len);
+        if (bytes != name_len) goto read_err;
+
+        if (h->target_name[i][name_len - 1] != '\0') {
+            /* Fix missing NUL-termination.  Is this being too nice?
+               We could alternatively bail out with an error. */
+            char *new_name;
+            if (name_len == INT32_MAX) goto invalid;
+            new_name = realloc(h->target_name[i], name_len + 1);
+            if (new_name == NULL) goto nomem;
+            h->target_name[i] = new_name;
+            h->target_name[i][name_len] = '\0';
+        }
+
+        bytes = bgzf_read(fp, &h->target_len[i], 4);
+        if (bytes != 4) goto read_err;
+        if (fp->is_be) ed_swap_4p(&h->target_len[i]);
+    }
+    return h;
+
+ nomem:
+    hts_log_error("Out of memory");
+    goto clean;
+
+ read_err:
+    if (bytes < 0) {
+        hts_log_error("Error reading BGZF stream");
+    } else {
+        hts_log_error("Truncated BAM header");
+    }
+    goto clean;
+
+ invalid:
+    hts_log_error("Invalid BAM binary header");
+
+ clean:
+    if (h != NULL) {
+        h->n_targets = num_names; // ensure we free only allocated target_names
+        sam_hdr_destroy(h);
+    }
+    return NULL;
+}
+
+int bam_hdr_write(BGZF *fp, const sam_hdr_t *h)
+{
+    int32_t i, name_len, x;
+    kstring_t hdr_ks = { 0, 0, NULL };
+    char *text;
+    uint32_t l_text;
+
+    if (!h) return -1;
+
+    if (h->hrecs) {
+        if (sam_hrecs_rebuild_text(h->hrecs, &hdr_ks) != 0) return -1;
+        if (hdr_ks.l > UINT32_MAX) {
+            hts_log_error("Header too long for BAM format");
+            free(hdr_ks.s);
+            return -1;
+        } else if (hdr_ks.l > INT32_MAX) {
+            hts_log_warning("Header too long for BAM specification (>2GB)");
+            hts_log_warning("Output file may not be portable");
+        }
+        text = hdr_ks.s;
+        l_text = hdr_ks.l;
+    } else {
+        if (h->l_text > UINT32_MAX) {
+            hts_log_error("Header too long for BAM format");
+            return -1;
+        } else if (h->l_text > INT32_MAX) {
+            hts_log_warning("Header too long for BAM specification (>2GB)");
+            hts_log_warning("Output file may not be portable");
+        }
+        text = h->text;
+        l_text = h->l_text;
+    }
+    // write "BAM1"
+    if (bgzf_write(fp, "BAM\1", 4) < 0) { free(hdr_ks.s); return -1; }
+    // write plain text and the number of reference sequences
+    if (fp->is_be) {
+        x = ed_swap_4(l_text);
+        if (bgzf_write(fp, &x, 4) < 0) { free(hdr_ks.s); return -1; }
+        if (l_text) {
+            if (bgzf_write(fp, text, l_text) < 0) { free(hdr_ks.s); return -1; }
+        }
+        x = ed_swap_4(h->n_targets);
+        if (bgzf_write(fp, &x, 4) < 0) { free(hdr_ks.s); return -1; }
+    } else {
+        if (bgzf_write(fp, &l_text, 4) < 0) { free(hdr_ks.s); return -1; }
+        if (l_text) {
+            if (bgzf_write(fp, text, l_text) < 0) { free(hdr_ks.s); return -1; }
+        }
+        if (bgzf_write(fp, &h->n_targets, 4) < 0) { free(hdr_ks.s); return -1; }
+    }
+    free(hdr_ks.s);
+    // write sequence names and lengths
+    for (i = 0; i != h->n_targets; ++i) {
+        char *p = h->target_name[i];
+        name_len = strlen(p) + 1;
+        if (fp->is_be) {
+            x = ed_swap_4(name_len);
+            if (bgzf_write(fp, &x, 4) < 0) return -1;
+        } else {
+            if (bgzf_write(fp, &name_len, 4) < 0) return -1;
+        }
+        if (bgzf_write(fp, p, name_len) < 0) return -1;
+        if (fp->is_be) {
+            x = ed_swap_4(h->target_len[i]);
+            if (bgzf_write(fp, &x, 4) < 0) return -1;
+        } else {
+            if (bgzf_write(fp, &h->target_len[i], 4) < 0) return -1;
+        }
+    }
+    if (bgzf_flush(fp) < 0) return -1;
+    return 0;
+}
+
+const char *sam_parse_region(sam_hdr_t *h, const char *s, int *tid,
+                             hts_pos_t *beg, hts_pos_t *end, int flags) {
+    return hts_parse_region(s, tid, beg, end, (hts_name2id_f)bam_name2id, h, flags);
+}
+
+/*************************
+ *** BAM alignment I/O ***
+ *************************/
+
+bam1_t *bam_init1(void)
+{
+    return (bam1_t*)calloc(1, sizeof(bam1_t));
+}
+
+int sam_realloc_bam_data(bam1_t *b, size_t desired)
+{
+    uint32_t new_m_data;
+    uint8_t *new_data;
+    new_m_data = desired;
+    kroundup32(new_m_data); // next power of 2
+    new_m_data += 32; // reduces malloc arena migrations?
+    if (new_m_data < desired) {
+        errno = ENOMEM; // Not strictly true but we can't store the size
+        return -1;
+    }
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    if (new_m_data > FUZZ_ALLOC_LIMIT) {
+        errno = ENOMEM;
+        return -1;
+    }
+#endif
+    if ((bam_get_mempolicy(b) & BAM_USER_OWNS_DATA) == 0) {
+        new_data = realloc(b->data, new_m_data);
+    } else {
+        if ((new_data = malloc(new_m_data)) != NULL) {
+            if (b->l_data > 0)
+                memcpy(new_data, b->data,
+                       b->l_data < b->m_data ? b->l_data : b->m_data);
+            bam_set_mempolicy(b, bam_get_mempolicy(b) & (~BAM_USER_OWNS_DATA));
+        }
+    }
+    if (!new_data) return -1;
+    b->data = new_data;
+    b->m_data = new_m_data;
+    return 0;
+}
+
+void bam_destroy1(bam1_t *b)
+{
+    if (b == 0) return;
+    if ((bam_get_mempolicy(b) & BAM_USER_OWNS_DATA) == 0) {
+        free(b->data);
+        if ((bam_get_mempolicy(b) & BAM_USER_OWNS_STRUCT) != 0) {
+            // In case of reuse
+            b->data = NULL;
+            b->m_data = 0;
+            b->l_data = 0;
+        }
+    }
+
+    if ((bam_get_mempolicy(b) & BAM_USER_OWNS_STRUCT) == 0)
+        free(b);
+}
+
+bam1_t *bam_copy1(bam1_t *bdst, const bam1_t *bsrc)
+{
+    if (realloc_bam_data(bdst, bsrc->l_data) < 0) return NULL;
+    memcpy(bdst->data, bsrc->data, bsrc->l_data); // copy var-len data
+    memcpy(&bdst->core, &bsrc->core, sizeof(bsrc->core)); // copy the rest
+    bdst->l_data = bsrc->l_data;
+    bdst->id = bsrc->id;
+    return bdst;
+}
+
+bam1_t *bam_dup1(const bam1_t *bsrc)
+{
+    if (bsrc == NULL) return NULL;
+    bam1_t *bdst = bam_init1();
+    if (bdst == NULL) return NULL;
+    if (bam_copy1(bdst, bsrc) == NULL) {
+        bam_destroy1(bdst);
+        return NULL;
+    }
+    return bdst;
+}
+
+static void bam_cigar2rqlens(int n_cigar, const uint32_t *cigar,
+                             hts_pos_t *rlen, hts_pos_t *qlen)
+{
+    int k;
+    *rlen = *qlen = 0;
+    for (k = 0; k < n_cigar; ++k) {
+        int type = bam_cigar_type(bam_cigar_op(cigar[k]));
+        int len = bam_cigar_oplen(cigar[k]);
+        if (type & 1) *qlen += len;
+        if (type & 2) *rlen += len;
+    }
+}
+
+static int subtract_check_underflow(size_t length, size_t *limit)
+{
+    if (length <= *limit) {
+        *limit -= length;
+        return 0;
+    }
+
+    return -1;
+}
+
+int bam_set1(bam1_t *bam,
+             size_t l_qname, const char *qname,
+             uint16_t flag, int32_t tid, hts_pos_t pos, uint8_t mapq,
+             size_t n_cigar, const uint32_t *cigar,
+             int32_t mtid, hts_pos_t mpos, hts_pos_t isize,
+             size_t l_seq, const char *seq, const char *qual,
+             size_t l_aux)
+{
+    // use a default qname "*" if none is provided
+    if (l_qname == 0) {
+        l_qname = 1;
+        qname = "*";
+    }
+
+    // note: the qname is stored nul terminated and padded as described in the
+    // documentation for the bam1_t struct.
+    size_t qname_nuls = 4 - l_qname % 4;
+
+    // the aligment length, needed for bam_reg2bin(), is calculated as in bam_endpos().
+    // can't use bam_endpos() directly as some fields not yet set up.
+    hts_pos_t rlen = 0, qlen = 0;
+    if (!(flag & BAM_FUNMAP)) {
+        bam_cigar2rqlens((int)n_cigar, cigar, &rlen, &qlen);
+    }
+    if (rlen == 0) {
+        rlen = 1;
+    }
+
+    // validate parameters
+    if (l_qname > 254) {
+        hts_log_error("Query name too long");
+        errno = EINVAL;
+        return -1;
+    }
+    if (HTS_POS_MAX - rlen <= pos) {
+        hts_log_error("Read ends beyond highest supported position");
+        errno = EINVAL;
+        return -1;
+    }
+    if (!(flag & BAM_FUNMAP) && l_seq > 0 && n_cigar == 0) {
+        hts_log_error("Mapped query must have a CIGAR");
+        errno = EINVAL;
+        return -1;
+    }
+    if (!(flag & BAM_FUNMAP) && l_seq > 0 && l_seq != qlen) {
+        hts_log_error("CIGAR and query sequence are of different length");
+        errno = EINVAL;
+        return -1;
+    }
+
+    size_t limit = INT32_MAX;
+    int u = subtract_check_underflow(l_qname + qname_nuls, &limit);
+    u    += subtract_check_underflow(n_cigar * 4, &limit);
+    u    += subtract_check_underflow((l_seq + 1) / 2, &limit);
+    u    += subtract_check_underflow(l_seq, &limit);
+    u    += subtract_check_underflow(l_aux, &limit);
+    if (u != 0) {
+        hts_log_error("Size overflow");
+        errno = EINVAL;
+        return -1;
+    }
+
+    // re-allocate the data buffer as needed.
+    size_t data_len = l_qname + qname_nuls + n_cigar * 4 + (l_seq + 1) / 2 + l_seq;
+    if (realloc_bam_data(bam, data_len + l_aux) < 0) {
+        return -1;
+    }
+
+    bam->l_data = (int)data_len;
+    bam->core.pos = pos;
+    bam->core.tid = tid;
+    bam->core.bin = bam_reg2bin(pos, pos + rlen);
+    bam->core.qual = mapq;
+    bam->core.l_extranul = (uint8_t)(qname_nuls - 1);
+    bam->core.flag = flag;
+    bam->core.l_qname = (uint16_t)(l_qname + qname_nuls);
+    bam->core.n_cigar = (uint32_t)n_cigar;
+    bam->core.l_qseq = (int32_t)l_seq;
+    bam->core.mtid = mtid;
+    bam->core.mpos = mpos;
+    bam->core.isize = isize;
+
+    uint8_t *cp = bam->data;
+    strncpy((char *)cp, qname, l_qname);
+    int i;
+    for (i = 0; i < qname_nuls; i++) {
+        cp[l_qname + i] = '\0';
+    }
+    cp += l_qname + qname_nuls;
+
+    if (n_cigar > 0) {
+        memcpy(cp, cigar, n_cigar * 4);
+    }
+    cp += n_cigar * 4;
+
+#define NN 16
+    const uint8_t *useq = (uint8_t *)seq;
+    for (i = 0; i + NN < l_seq; i += NN) {
+        int j;
+        const uint8_t *u2 = useq+i;
+        for (j = 0; j < NN/2; j++)
+            cp[j] = (seq_nt16_table[u2[j*2]]<<4) | seq_nt16_table[u2[j*2+1]];
+        cp += NN/2;
+    }
+    for (; i + 1 < l_seq; i += 2) {
+        *cp++ = (seq_nt16_table[useq[i]] << 4) | seq_nt16_table[useq[i + 1]];
+    }
+
+    for (; i < l_seq; i++) {
+        *cp++ = seq_nt16_table[(unsigned char)seq[i]] << 4;
+    }
+
+    if (qual) {
+        memcpy(cp, qual, l_seq);
+    }
+    else {
+        memset(cp, '\xff', l_seq);
+    }
+
+    return (int)data_len;
+}
+
+hts_pos_t bam_cigar2qlen(int n_cigar, const uint32_t *cigar)
+{
+    int k;
+    hts_pos_t l;
+    for (k = l = 0; k < n_cigar; ++k)
+        if (bam_cigar_type(bam_cigar_op(cigar[k]))&1)
+            l += bam_cigar_oplen(cigar[k]);
+    return l;
+}
+
+hts_pos_t bam_cigar2rlen(int n_cigar, const uint32_t *cigar)
+{
+    int k;
+    hts_pos_t l;
+    for (k = l = 0; k < n_cigar; ++k)
+        if (bam_cigar_type(bam_cigar_op(cigar[k]))&2)
+            l += bam_cigar_oplen(cigar[k]);
+    return l;
+}
+
+hts_pos_t bam_endpos(const bam1_t *b)
+{
+    hts_pos_t rlen = (b->core.flag & BAM_FUNMAP)? 0 : bam_cigar2rlen(b->core.n_cigar, bam_get_cigar(b));
+    if (rlen == 0) rlen = 1;
+    return b->core.pos + rlen;
+}
+
+static int bam_tag2cigar(bam1_t *b, int recal_bin, int give_warning) // return 0 if CIGAR is untouched; 1 if CIGAR is updated with CG
+{
+    bam1_core_t *c = &b->core;
+
+    // Bail out as fast as possible for the easy case
+    uint32_t test_CG = BAM_CSOFT_CLIP | (c->l_qseq << BAM_CIGAR_SHIFT);
+    if (c->n_cigar == 0 || test_CG != *bam_get_cigar(b))
+        return 0;
+
+    // The above isn't fool proof - we may have old CIGAR tags that aren't used,
+    // but this is much less likely so do as a secondary check.
+    if (c->tid < 0 || c->pos < 0)
+        return 0;
+
+    // Do we have a CG tag?
+    uint8_t *CG = bam_aux_get(b, "CG");
+    int saved_errno = errno;
+    if (!CG) {
+        if (errno != ENOENT) return -1;  // Bad aux data
+        errno = saved_errno; // restore errno on expected no-CG-tag case
+        return 0;
+    }
+
+    // Now we start with the serious work migrating CG to CIGAR
+    uint32_t cigar_st, n_cigar4, CG_st, CG_en, ori_len = b->l_data,
+        *cigar0, CG_len, fake_bytes;
+    cigar0 = bam_get_cigar(b);
+    fake_bytes = c->n_cigar * 4;
+    if (CG[0] != 'B' || !(CG[1] == 'I' || CG[1] == 'i'))
+        return 0; // not of type B,I
+    CG_len = le_to_u32(CG + 2);
+    // don't move if the real CIGAR length is shorter than the fake cigar length
+    if (CG_len < c->n_cigar || CG_len >= 1U<<29) return 0;
+
+    // move from the CG tag to the right position
+    cigar_st = (uint8_t*)cigar0 - b->data;
+    c->n_cigar = CG_len;
+    n_cigar4 = c->n_cigar * 4;
+    CG_st = CG - b->data - 2;
+    CG_en = CG_st + 8 + n_cigar4;
+    if (possibly_expand_bam_data(b, n_cigar4 - fake_bytes) < 0) return -1;
+    // we need c->n_cigar-fake_bytes bytes to swap CIGAR to the right place
+    b->l_data = b->l_data - fake_bytes + n_cigar4;
+    // insert c->n_cigar-fake_bytes empty space to make room
+    memmove(b->data + cigar_st + n_cigar4, b->data + cigar_st + fake_bytes, ori_len - (cigar_st + fake_bytes));
+    // copy the real CIGAR to the right place; -fake_bytes for the fake CIGAR
+    memcpy(b->data + cigar_st, b->data + (n_cigar4 - fake_bytes) + CG_st + 8, n_cigar4);
+    if (ori_len > CG_en) // move data after the CG tag
+        memmove(b->data + CG_st + n_cigar4 - fake_bytes, b->data + CG_en + n_cigar4 - fake_bytes, ori_len - CG_en);
+    b->l_data -= n_cigar4 + 8; // 8: CGBI (4 bytes) and CGBI length (4)
+    if (recal_bin)
+        b->core.bin = hts_reg2bin(b->core.pos, bam_endpos(b), 14, 5);
+    if (give_warning)
+        hts_log_warning("%s encodes a CIGAR with %d operators at the CG tag", bam_get_qname(b), c->n_cigar);
+    return 1;
+}
+
+static inline int aux_type2size(uint8_t type)
+{
+    switch (type) {
+    case 'A': case 'c': case 'C':
+        return 1;
+    case 's': case 'S':
+        return 2;
+    case 'i': case 'I': case 'f':
+        return 4;
+    case 'd':
+        return 8;
+    case 'Z': case 'H': case 'B':
+        return type;
+    default:
+        return 0;
+    }
+}
+
+static void swap_data(const bam1_core_t *c, int l_data, uint8_t *data, int is_host)
+{
+    uint32_t *cigar = (uint32_t*)(data + c->l_qname);
+    uint32_t i;
+    for (i = 0; i < c->n_cigar; ++i) ed_swap_4p(&cigar[i]);
+}
+
+// Fix bad records where qname is not terminated correctly.
+static int fixup_missing_qname_nul(bam1_t *b) {
+    bam1_core_t *c = &b->core;
+
+    // Note this is called before c->l_extranul is added to c->l_qname
+    if (c->l_extranul > 0) {
+        b->data[c->l_qname++] = '\0';
+        c->l_extranul--;
+    } else {
+        if (b->l_data > INT_MAX - 4) return -1;
+        if (realloc_bam_data(b, b->l_data + 4) < 0) return -1;
+        b->l_data += 4;
+        b->data[c->l_qname++] = '\0';
+        c->l_extranul = 3;
+    }
+    return 0;
+}
+
+/*
+ * Note a second interface that returns a bam pointer instead would avoid bam_copy1
+ * in multi-threaded handling.  This may be worth considering for htslib2.
+ */
+int bam_read1(BGZF *fp, bam1_t *b)
+{
+    bam1_core_t *c = &b->core;
+    int32_t block_len, ret, i;
+    uint32_t new_l_data;
+    uint8_t tmp[32], *x;
+
+    b->l_data = 0;
+
+    if ((ret = bgzf_read_small(fp, &block_len, 4)) != 4) {
+        if (ret == 0) return -1; // normal end-of-file
+        else return -2; // truncated
+    }
+    if (fp->is_be)
+        ed_swap_4p(&block_len);
+    if (block_len < 32) return -4;  // block_len includes core data
+    if (fp->block_length - fp->block_offset > 32) {
+        // Avoid bgzf_read and a temporary copy to a local buffer
+        x = (uint8_t *)fp->uncompressed_block + fp->block_offset;
+        fp->block_offset += 32;
+    } else {
+        x = tmp;
+        if (bgzf_read(fp, x, 32) != 32) return -3;
+    }
+
+    c->tid        = le_to_u32(x);
+    c->pos        = le_to_i32(x+4);
+    uint32_t x2   = le_to_u32(x+8);
+    c->bin        = x2>>16;
+    c->qual       = x2>>8&0xff;
+    c->l_qname    = x2&0xff;
+    c->l_extranul = (c->l_qname%4 != 0)? (4 - c->l_qname%4) : 0;
+    uint32_t x3   = le_to_u32(x+12);
+    c->flag       = x3>>16;
+    c->n_cigar    = x3&0xffff;
+    c->l_qseq     = le_to_u32(x+16);
+    c->mtid       = le_to_u32(x+20);
+    c->mpos       = le_to_i32(x+24);
+    c->isize      = le_to_i32(x+28);
+
+    new_l_data = block_len - 32 + c->l_extranul;
+    if (new_l_data > INT_MAX || c->l_qseq < 0 || c->l_qname < 1) return -4;
+    if (((uint64_t) c->n_cigar << 2) + c->l_qname + c->l_extranul
+        + (((uint64_t) c->l_qseq + 1) >> 1) + c->l_qseq > (uint64_t) new_l_data)
+        return -4;
+    if (realloc_bam_data(b, new_l_data) < 0) return -4;
+    b->l_data = new_l_data;
+
+    if (bgzf_read_small(fp, b->data, c->l_qname) != c->l_qname) return -4;
+    if (b->data[c->l_qname - 1] != '\0') { // try to fix missing nul termination
+        if (fixup_missing_qname_nul(b) < 0) return -4;
+    }
+    for (i = 0; i < c->l_extranul; ++i) b->data[c->l_qname+i] = '\0';
+    c->l_qname += c->l_extranul;
+    if (b->l_data < c->l_qname ||
+        bgzf_read_small(fp, b->data + c->l_qname, b->l_data - c->l_qname) != b->l_data - c->l_qname)
+        return -4;
+    if (fp->is_be) swap_data(c, b->l_data, b->data, 0);
+    if (bam_tag2cigar(b, 0, 0) < 0)
+        return -4;
+
+    // TODO: consider making this conditional
+    if (c->n_cigar > 0) { // recompute "bin" and check CIGAR-qlen consistency
+        hts_pos_t rlen, qlen;
+        bam_cigar2rqlens(c->n_cigar, bam_get_cigar(b), &rlen, &qlen);
+        if ((b->core.flag & BAM_FUNMAP) || rlen == 0) rlen = 1;
+        b->core.bin = hts_reg2bin(b->core.pos, b->core.pos + rlen, 14, 5);
+        // Sanity check for broken CIGAR alignments
+        if (c->l_qseq > 0 && !(c->flag & BAM_FUNMAP) && qlen != c->l_qseq) {
+            hts_log_error("CIGAR and query sequence lengths differ for %s",
+                    bam_get_qname(b));
+            return -4;
+        }
+    }
+
+    return 4 + block_len;
+}
+
+int bam_write1(BGZF *fp, const bam1_t *b)
+{
+    const bam1_core_t *c = &b->core;
+    uint32_t x[8], block_len = b->l_data - c->l_extranul + 32, y;
+    int i, ok;
+    if (c->l_qname - c->l_extranul > 255) {
+        hts_log_error("QNAME \"%s\" is longer than 254 characters", bam_get_qname(b));
+        errno = EOVERFLOW;
+        return -1;
+    }
+    if (c->n_cigar > 0xffff) block_len += 16; // "16" for "CGBI", 4-byte tag length and 8-byte fake CIGAR
+    if (c->pos > INT_MAX ||
+        c->mpos > INT_MAX ||
+        c->isize < INT_MIN || c->isize > INT_MAX) {
+        hts_log_error("Positional data is too large for BAM format");
+        return -1;
+    }
+    x[0] = c->tid;
+    x[1] = c->pos;
+    x[2] = (uint32_t)c->bin<<16 | c->qual<<8 | (c->l_qname - c->l_extranul);
+    if (c->n_cigar > 0xffff) x[3] = (uint32_t)c->flag << 16 | 2;
+    else x[3] = (uint32_t)c->flag << 16 | (c->n_cigar & 0xffff);
+    x[4] = c->l_qseq;
+    x[5] = c->mtid;
+    x[6] = c->mpos;
+    x[7] = c->isize;
+    ok = (bgzf_flush_try(fp, 4 + block_len) >= 0);
+    if (fp->is_be) {
+        for (i = 0; i < 8; ++i) ed_swap_4p(x + i);
+        y = block_len;
+        if (ok) ok = (bgzf_write_small(fp, ed_swap_4p(&y), 4) >= 0);
+        swap_data(c, b->l_data, b->data, 1);
+    } else {
+        if (ok) ok = (bgzf_write_small(fp, &block_len, 4) >= 0);
+    }
+    if (ok) ok = (bgzf_write_small(fp, x, 32) >= 0);
+    if (ok) ok = (bgzf_write_small(fp, b->data, c->l_qname - c->l_extranul) >= 0);
+    if (c->n_cigar <= 0xffff) { // no long CIGAR; write normally
+        if (ok) ok = (bgzf_write_small(fp, b->data + c->l_qname, b->l_data - c->l_qname) >= 0);
+    } else { // with long CIGAR, insert a fake CIGAR record and move the real CIGAR to the CG:B,I tag
+        uint8_t buf[8];
+        uint32_t cigar_st, cigar_en, cigar[2];
+        hts_pos_t cigreflen = bam_cigar2rlen(c->n_cigar, bam_get_cigar(b));
+        if (cigreflen >= (1<<28)) {
+            // Length of reference covered is greater than the biggest
+            // CIGAR operation currently allowed.
+            hts_log_error("Record %s with %d CIGAR ops and ref length %"PRIhts_pos
+                          " cannot be written in BAM.  Try writing SAM or CRAM instead.\n",
+                          bam_get_qname(b), c->n_cigar, cigreflen);
+            return -1;
+        }
+        cigar_st = (uint8_t*)bam_get_cigar(b) - b->data;
+        cigar_en = cigar_st + c->n_cigar * 4;
+        cigar[0] = (uint32_t)c->l_qseq << 4 | BAM_CSOFT_CLIP;
+        cigar[1] = (uint32_t)cigreflen << 4 | BAM_CREF_SKIP;
+        u32_to_le(cigar[0], buf);
+        u32_to_le(cigar[1], buf + 4);
+        if (ok) ok = (bgzf_write_small(fp, buf, 8) >= 0); // write cigar: <read_length>S<ref_length>N
+        if (ok) ok = (bgzf_write_small(fp, &b->data[cigar_en], b->l_data - cigar_en) >= 0); // write data after CIGAR
+        if (ok) ok = (bgzf_write_small(fp, "CGBI", 4) >= 0); // write CG:B,I
+        u32_to_le(c->n_cigar, buf);
+        if (ok) ok = (bgzf_write_small(fp, buf, 4) >= 0); // write the true CIGAR length
+        if (ok) ok = (bgzf_write_small(fp, &b->data[cigar_st], c->n_cigar * 4) >= 0); // write the real CIGAR
+    }
+    if (fp->is_be) swap_data(c, b->l_data, b->data, 0);
+    return ok? 4 + block_len : -1;
+}
+
+/*
+ * Write a BAM file and append to the in-memory index simultaneously.
+ */
+static int bam_write_idx1(htsFile *fp, const sam_hdr_t *h, const bam1_t *b) {
+    BGZF *bfp = fp->fp.bgzf;
+
+    if (!fp->idx)
+        return bam_write1(bfp, b);
+
+    uint32_t block_len = b->l_data - b->core.l_extranul + 32;
+    if (bgzf_flush_try(bfp, 4 + block_len) < 0)
+        return -1;
+    if (!bfp->mt)
+        hts_idx_amend_last(fp->idx, bgzf_tell(bfp));
+
+    int ret = bam_write1(bfp, b);
+    if (ret < 0)
+        return -1;
+
+    if (bgzf_idx_push(bfp, fp->idx, b->core.tid, b->core.pos, bam_endpos(b), bgzf_tell(bfp), !(b->core.flag&BAM_FUNMAP)) < 0) {
+        hts_log_error("Read '%s' with ref_name='%s', ref_length=%"PRIhts_pos", flags=%d, pos=%"PRIhts_pos" cannot be indexed",
+                bam_get_qname(b), sam_hdr_tid2name(h, b->core.tid), sam_hdr_tid2len(h, b->core.tid), b->core.flag, b->core.pos+1);
+        ret = -1;
+    }
+
+    return ret;
+}
+
+/*
+ * Set the qname in a BAM record
+ */
+int bam_set_qname(bam1_t *rec, const char *qname)
+{
+    if (!rec) return -1;
+    if (!qname || !*qname) return -1;
+
+    size_t old_len = rec->core.l_qname;
+    size_t new_len = strlen(qname) + 1;
+    if (new_len < 1 || new_len > 255) return -1;
+
+    int extranul = (new_len%4 != 0) ? (4 - new_len%4) : 0;
+
+    size_t new_data_len = rec->l_data - old_len + new_len + extranul;
+    if (realloc_bam_data(rec, new_data_len) < 0) return -1;
+
+    // Make room
+    if (new_len + extranul != rec->core.l_qname)
+        memmove(rec->data + new_len + extranul, rec->data + rec->core.l_qname, rec->l_data - rec->core.l_qname);
+    // Copy in new name and pad if needed
+    memcpy(rec->data, qname, new_len);
+    int n;
+    for (n = 0; n < extranul; n++) rec->data[new_len + n] = '\0';
+
+    rec->l_data = new_data_len;
+    rec->core.l_qname = new_len + extranul;
+    rec->core.l_extranul = extranul;
+
+    return 0;
+}
+
+/********************
+ *** BAM indexing ***
+ ********************/
+
+static hts_idx_t *sam_index(htsFile *fp, int min_shift)
+{
+    int n_lvls, i, fmt, ret;
+    bam1_t *b;
+    hts_idx_t *idx;
+    sam_hdr_t *h;
+    h = sam_hdr_read(fp);
+    if (h == NULL) return NULL;
+    if (min_shift > 0) {
+        hts_pos_t max_len = 0, s;
+        for (i = 0; i < h->n_targets; ++i) {
+            hts_pos_t len = sam_hdr_tid2len(h, i);
+            if (max_len < len) max_len = len;
+        }
+        max_len += 256;
+        for (n_lvls = 0, s = 1<<min_shift; max_len > s; ++n_lvls, s <<= 3);
+        fmt = HTS_FMT_CSI;
+    } else min_shift = 14, n_lvls = 5, fmt = HTS_FMT_BAI;
+    idx = hts_idx_init(h->n_targets, fmt, bgzf_tell(fp->fp.bgzf), min_shift, n_lvls);
+    b = bam_init1();
+    while ((ret = sam_read1(fp, h, b)) >= 0) {
+        ret = hts_idx_push(idx, b->core.tid, b->core.pos, bam_endpos(b), bgzf_tell(fp->fp.bgzf), !(b->core.flag&BAM_FUNMAP));
+        if (ret < 0) { // unsorted or doesn't fit
+            hts_log_error("Read '%s' with ref_name='%s', ref_length=%"PRIhts_pos", flags=%d, pos=%"PRIhts_pos" cannot be indexed", bam_get_qname(b), sam_hdr_tid2name(h, b->core.tid), sam_hdr_tid2len(h, b->core.tid), b->core.flag, b->core.pos+1);
+            goto err;
+        }
+    }
+    if (ret < -1) goto err; // corrupted BAM file
+
+    hts_idx_finish(idx, bgzf_tell(fp->fp.bgzf));
+    sam_hdr_destroy(h);
+    bam_destroy1(b);
+    return idx;
+
+err:
+    bam_destroy1(b);
+    hts_idx_destroy(idx);
+    return NULL;
+}
+
+int sam_index_build3(const char *fn, const char *fnidx, int min_shift, int nthreads)
+{
+    hts_idx_t *idx;
+    htsFile *fp;
+    int ret = 0;
+
+    if ((fp = hts_open(fn, "r")) == 0) return -2;
+    if (nthreads)
+        hts_set_threads(fp, nthreads);
+
+    switch (fp->format.format) {
+    case cram:
+
+        ret = cram_index_build(fp->fp.cram, fn, fnidx);
+        break;
+
+    case bam:
+    case sam:
+        if (fp->format.compression != bgzf) {
+            hts_log_error("%s file \"%s\" not BGZF compressed",
+                          fp->format.format == bam ? "BAM" : "SAM", fn);
+            ret = -1;
+            break;
+        }
+        idx = sam_index(fp, min_shift);
+        if (idx) {
+            ret = hts_idx_save_as(idx, fn, fnidx, (min_shift > 0)? HTS_FMT_CSI : HTS_FMT_BAI);
+            if (ret < 0) ret = -4;
+            hts_idx_destroy(idx);
+        }
+        else ret = -1;
+        break;
+
+    default:
+        ret = -3;
+        break;
+    }
+    hts_close(fp);
+
+    return ret;
+}
+
+int sam_index_build2(const char *fn, const char *fnidx, int min_shift)
+{
+    return sam_index_build3(fn, fnidx, min_shift, 0);
+}
+
+int sam_index_build(const char *fn, int min_shift)
+{
+    return sam_index_build3(fn, NULL, min_shift, 0);
+}
+
+// Provide bam_index_build() symbol for binary compatibility with earlier HTSlib
+#undef bam_index_build
+int bam_index_build(const char *fn, int min_shift)
+{
+    return sam_index_build2(fn, NULL, min_shift);
+}
+
+// Initialise fp->idx for the current format type.
+// This must be called after the header has been written but no other data.
+int sam_idx_init(htsFile *fp, sam_hdr_t *h, int min_shift, const char *fnidx) {
+    fp->fnidx = fnidx;
+    if (fp->format.format == bam || fp->format.format == bcf ||
+        (fp->format.format == sam && fp->format.compression == bgzf)) {
+        int n_lvls, fmt = HTS_FMT_CSI;
+        if (min_shift > 0) {
+            int64_t max_len = 0, s;
+            int i;
+            for (i = 0; i < h->n_targets; ++i)
+                if (max_len < h->target_len[i]) max_len = h->target_len[i];
+            max_len += 256;
+            for (n_lvls = 0, s = 1<<min_shift; max_len > s; ++n_lvls, s <<= 3);
+
+        } else min_shift = 14, n_lvls = 5, fmt = HTS_FMT_BAI;
+
+        fp->idx = hts_idx_init(h->n_targets, fmt, bgzf_tell(fp->fp.bgzf), min_shift, n_lvls);
+        return fp->idx ? 0 : -1;
+    }
+
+    if (fp->format.format == cram) {
+        fp->fp.cram->idxfp = bgzf_open(fnidx, "wg");
+        return fp->fp.cram->idxfp ? 0 : -1;
+    }
+
+    return -1;
+}
+
+// Finishes an index. Call after the last record has been written.
+// Returns 0 on success, <0 on failure.
+int sam_idx_save(htsFile *fp) {
+    if (fp->format.format == bam || fp->format.format == bcf ||
+        fp->format.format == vcf || fp->format.format == sam) {
+        int ret;
+        if ((ret = sam_state_destroy(fp)) < 0) {
+            errno = -ret;
+            return -1;
+        }
+        if (!fp->is_bgzf || bgzf_flush(fp->fp.bgzf) < 0)
+            return -1;
+        hts_idx_amend_last(fp->idx, bgzf_tell(fp->fp.bgzf));
+
+        if (hts_idx_finish(fp->idx, bgzf_tell(fp->fp.bgzf)) < 0)
+            return -1;
+
+        return hts_idx_save_but_not_close(fp->idx, fp->fnidx, hts_idx_fmt(fp->idx));
+
+    } else if (fp->format.format == cram) {
+        // flushed and closed by cram_close
+    }
+
+    return 0;
+}
+
+static int sam_readrec(BGZF *ignored, void *fpv, void *bv, int *tid, hts_pos_t *beg, hts_pos_t *end)
+{
+    htsFile *fp = (htsFile *)fpv;
+    bam1_t *b = bv;
+    fp->line.l = 0;
+    int ret = sam_read1(fp, fp->bam_header, b);
+    if (ret >= 0) {
+        *tid = b->core.tid;
+        *beg = b->core.pos;
+        *end = bam_endpos(b);
+    }
+    return ret;
+}
+
+// This is used only with read_rest=1 iterators, so need not set tid/beg/end.
+static int sam_readrec_rest(BGZF *ignored, void *fpv, void *bv, int *tid, hts_pos_t *beg, hts_pos_t *end)
+{
+    htsFile *fp = (htsFile *)fpv;
+    bam1_t *b = bv;
+    fp->line.l = 0;
+    int ret = sam_read1(fp, fp->bam_header, b);
+    return ret;
+}
+
+// Internal (for now) func used by bam_sym_lookup.  This is copied from
+// samtools/bam.c.
+static const char *bam_get_library(const bam_hdr_t *h, const bam1_t *b)
+{
+    const char *rg;
+    kstring_t lib = { 0, 0, NULL };
+    rg = (char *)bam_aux_get(b, "RG");
+
+    if (!rg)
+        return NULL;
+    else
+        rg++;
+
+    if (sam_hdr_find_tag_id((bam_hdr_t *)h, "RG", "ID", rg, "LB", &lib)  < 0)
+        return NULL;
+
+    static char LB_text[1024];
+    int len = lib.l < sizeof(LB_text) - 1 ? lib.l : sizeof(LB_text) - 1;
+
+    memcpy(LB_text, lib.s, len);
+    LB_text[len] = 0;
+
+    free(lib.s);
+
+    return LB_text;
+}
+
+
+// Bam record pointer and SAM header combined
+typedef struct {
+    const sam_hdr_t *h;
+    const bam1_t *b;
+} hb_pair;
+
+// Looks up variable names in str and replaces them with their value.
+// Also supports aux tags.
+//
+// Note the expression parser deliberately overallocates str size so it
+// is safe to use memcmp over strcmp.
+static int bam_sym_lookup(void *data, char *str, char **end,
+                          hts_expr_val_t *res) {
+    hb_pair *hb = (hb_pair *)data;
+    const bam1_t *b = hb->b;
+
+    res->is_str = 0;
+    switch(*str) {
+    case 'c':
+        if (memcmp(str, "cigar", 5) == 0) {
+            *end = str+5;
+            res->is_str = 1;
+            ks_clear(&res->s);
+            uint32_t *cigar = bam_get_cigar(b);
+            int i, n = b->core.n_cigar, r = 0;
+            if (n) {
+                for (i = 0; i < n; i++) {
+                    r |= kputw (bam_cigar_oplen(cigar[i]), &res->s) < 0;
+                    r |= kputc_(bam_cigar_opchr(cigar[i]), &res->s) < 0;
+                }
+                r |= kputs("", &res->s) < 0;
+            } else {
+                r |= kputs("*", &res->s) < 0;
+            }
+            return r ? -1 : 0;
+        }
+        break;
+
+    case 'e':
+        if (memcmp(str, "endpos", 6) == 0) {
+            *end = str+6;
+            res->d = bam_endpos(b);
+            return 0;
+        }
+        break;
+
+    case 'f':
+        if (memcmp(str, "flag", 4) == 0) {
+            str = *end = str+4;
+            if (*str != '.') {
+                res->d = b->core.flag;
+                return 0;
+            } else {
+                str++;
+                if (!memcmp(str, "paired", 6)) {
+                    *end = str+6;
+                    res->d = b->core.flag & BAM_FPAIRED;
+                    return 0;
+                } else if (!memcmp(str, "proper_pair", 11)) {
+                    *end = str+11;
+                    res->d = b->core.flag & BAM_FPROPER_PAIR;
+                    return 0;
+                } else if (!memcmp(str, "unmap", 5)) {
+                    *end = str+5;
+                    res->d = b->core.flag & BAM_FUNMAP;
+                    return 0;
+                } else if (!memcmp(str, "munmap", 6)) {
+                    *end = str+6;
+                    res->d = b->core.flag & BAM_FMUNMAP;
+                    return 0;
+                } else if (!memcmp(str, "reverse", 7)) {
+                    *end = str+7;
+                    res->d = b->core.flag & BAM_FREVERSE;
+                    return 0;
+                } else if (!memcmp(str, "mreverse", 8)) {
+                    *end = str+8;
+                    res->d = b->core.flag & BAM_FMREVERSE;
+                    return 0;
+                } else if (!memcmp(str, "read1", 5)) {
+                    *end = str+5;
+                    res->d = b->core.flag & BAM_FREAD1;
+                    return 0;
+                } else if (!memcmp(str, "read2", 5)) {
+                    *end = str+5;
+                    res->d = b->core.flag & BAM_FREAD2;
+                    return 0;
+                } else if (!memcmp(str, "secondary", 9)) {
+                    *end = str+9;
+                    res->d = b->core.flag & BAM_FSECONDARY;
+                    return 0;
+                } else if (!memcmp(str, "qcfail", 6)) {
+                    *end = str+6;
+                    res->d = b->core.flag & BAM_FQCFAIL;
+                    return 0;
+                } else if (!memcmp(str, "dup", 3)) {
+                    *end = str+3;
+                    res->d = b->core.flag & BAM_FDUP;
+                    return 0;
+                } else if (!memcmp(str, "supplementary", 13)) {
+                    *end = str+13;
+                    res->d = b->core.flag & BAM_FSUPPLEMENTARY;
+                    return 0;
+                } else {
+                    hts_log_error("Unrecognised flag string");
+                    return -1;
+                }
+            }
+        }
+        break;
+
+    case 'h':
+        if (memcmp(str, "hclen", 5) == 0) {
+            int hclen = 0;
+            uint32_t *cigar = bam_get_cigar(b);
+            uint32_t ncigar = b->core.n_cigar;
+
+            // left
+            if (ncigar > 0 && bam_cigar_op(cigar[0]) == BAM_CHARD_CLIP)
+                hclen = bam_cigar_oplen(cigar[0]);
+
+            // right
+            if (ncigar > 1 && bam_cigar_op(cigar[ncigar-1]) == BAM_CHARD_CLIP)
+                hclen += bam_cigar_oplen(cigar[ncigar-1]);
+
+            *end = str+5;
+            res->d = hclen;
+            return 0;
+        }
+        break;
+
+    case 'l':
+        if (memcmp(str, "library", 7) == 0) {
+            *end = str+7;
+            res->is_str = 1;
+            const char *lib = bam_get_library(hb->h, b);
+            kputs(lib ? lib : "", ks_clear(&res->s));
+            return 0;
+        }
+        break;
+
+    case 'm':
+        if (memcmp(str, "mapq", 4) == 0) {
+            *end = str+4;
+            res->d = b->core.qual;
+            return 0;
+        } else if (memcmp(str, "mpos", 4) == 0) {
+            *end = str+4;
+            res->d = b->core.mpos+1;
+            return 0;
+        } else if (memcmp(str, "mrname", 6) == 0) {
+            *end = str+6;
+            res->is_str = 1;
+            const char *rn = sam_hdr_tid2name(hb->h, b->core.mtid);
+            kputs(rn ? rn : "*", ks_clear(&res->s));
+            return 0;
+        } else if (memcmp(str, "mrefid", 6) == 0) {
+            *end = str+6;
+            res->d = b->core.mtid;
+            return 0;
+        }
+        break;
+
+    case 'n':
+        if (memcmp(str, "ncigar", 6) == 0) {
+            *end = str+6;
+            res->d = b->core.n_cigar;
+            return 0;
+        }
+        break;
+
+    case 'p':
+        if (memcmp(str, "pos", 3) == 0) {
+            *end = str+3;
+            res->d = b->core.pos+1;
+            return 0;
+        } else if (memcmp(str, "pnext", 5) == 0) {
+            *end = str+5;
+            res->d = b->core.mpos+1;
+            return 0;
+        }
+        break;
+
+    case 'q':
+        if (memcmp(str, "qlen", 4) == 0) {
+            *end = str+4;
+            res->d = bam_cigar2qlen(b->core.n_cigar, bam_get_cigar(b));
+            return 0;
+        } else if (memcmp(str, "qname", 5) == 0) {
+            *end = str+5;
+            res->is_str = 1;
+            kputs(bam_get_qname(b), ks_clear(&res->s));
+            return 0;
+        } else if (memcmp(str, "qual", 4) == 0) {
+            *end = str+4;
+            ks_clear(&res->s);
+            if (ks_resize(&res->s, b->core.l_qseq+1) < 0)
+                return -1;
+            memcpy(res->s.s, bam_get_qual(b), b->core.l_qseq);
+            res->s.l = b->core.l_qseq;
+            res->is_str = 1;
+            return 0;
+        }
+        break;
+
+    case 'r':
+        if (memcmp(str, "rlen", 4) == 0) {
+            *end = str+4;
+            res->d = bam_cigar2rlen(b->core.n_cigar, bam_get_cigar(b));
+            return 0;
+        } else if (memcmp(str, "rname", 5) == 0) {
+            *end = str+5;
+            res->is_str = 1;
+            const char *rn = sam_hdr_tid2name(hb->h, b->core.tid);
+            kputs(rn ? rn : "*", ks_clear(&res->s));
+            return 0;
+        } else if (memcmp(str, "rnext", 5) == 0) {
+            *end = str+5;
+            res->is_str = 1;
+            const char *rn = sam_hdr_tid2name(hb->h, b->core.mtid);
+            kputs(rn ? rn : "*", ks_clear(&res->s));
+            return 0;
+        } else if (memcmp(str, "refid", 5) == 0) {
+            *end = str+5;
+            res->d = b->core.tid;
+            return 0;
+        }
+        break;
+
+    case 's':
+        if (memcmp(str, "seq", 3) == 0) {
+            *end = str+3;
+            ks_clear(&res->s);
+            if (ks_resize(&res->s, b->core.l_qseq+1) < 0)
+                return -1;
+            nibble2base(bam_get_seq(b), res->s.s, b->core.l_qseq);
+            res->s.s[b->core.l_qseq] = 0;
+            res->s.l = b->core.l_qseq;
+            res->is_str = 1;
+            return 0;
+        } else if (memcmp(str, "sclen", 5) == 0) {
+            int sclen = 0;
+            uint32_t *cigar = bam_get_cigar(b);
+            int ncigar = b->core.n_cigar;
+            int left = 0;
+
+            // left
+            if (ncigar > 0
+                && bam_cigar_op(cigar[0]) == BAM_CSOFT_CLIP)
+                left = 0, sclen += bam_cigar_oplen(cigar[0]);
+            else if (ncigar > 1
+                     && bam_cigar_op(cigar[0]) == BAM_CHARD_CLIP
+                     && bam_cigar_op(cigar[1]) == BAM_CSOFT_CLIP)
+                left = 1, sclen += bam_cigar_oplen(cigar[1]);
+
+            // right
+            if (ncigar-1 > left
+                && bam_cigar_op(cigar[ncigar-1]) == BAM_CSOFT_CLIP)
+                sclen += bam_cigar_oplen(cigar[ncigar-1]);
+            else if (ncigar-2 > left
+                     && bam_cigar_op(cigar[ncigar-1]) == BAM_CHARD_CLIP
+                     && bam_cigar_op(cigar[ncigar-2]) == BAM_CSOFT_CLIP)
+                sclen += bam_cigar_oplen(cigar[ncigar-2]);
+
+            *end = str+5;
+            res->d = sclen;
+            return 0;
+        }
+        break;
+
+    case 't':
+        if (memcmp(str, "tlen", 4) == 0) {
+            *end = str+4;
+            res->d = b->core.isize;
+            return 0;
+        }
+        break;
+
+    case '[':
+        if (*str == '[' && str[1] && str[2] && str[3] == ']') {
+            /* aux tags */
+            *end = str+4;
+
+            uint8_t *aux = bam_aux_get(b, str+1);
+            if (aux) {
+                // we define the truth of a tag to be its presence, even if 0.
+                res->is_true = 1;
+                switch (*aux) {
+                case 'Z':
+                case 'H':
+                    res->is_str = 1;
+                    kputs((char *)aux+1, ks_clear(&res->s));
+                    break;
+
+                case 'A':
+                    res->is_str = 1;
+                    kputsn((char *)aux+1, 1, ks_clear(&res->s));
+                    break;
+
+                case 'i': case 'I':
+                case 's': case 'S':
+                case 'c': case 'C':
+                    res->is_str = 0;
+                    res->d = bam_aux2i(aux);
+                    break;
+
+                case 'f':
+                case 'd':
+                    res->is_str = 0;
+                    res->d = bam_aux2f(aux);
+                    break;
+
+                default:
+                    hts_log_error("Aux type '%c not yet supported by filters",
+                                  *aux);
+                    return -1;
+                }
+                return 0;
+
+            } else {
+                // hence absent tags are always false (and strings)
+                res->is_str = 1;
+                res->s.l = 0;
+                res->d = 0;
+                res->is_true = 0;
+                return 0;
+            }
+        }
+        break;
+    }
+
+    // All successful matches in switch should return 0.
+    // So if we didn't match, it's a parse error.
+    return -1;
+}
+
+// Returns 1 when accepted by the filter, 0 if not, -1 on error.
+int sam_passes_filter(const sam_hdr_t *h, const bam1_t *b, hts_filter_t *filt)
+{
+    hb_pair hb = {h, b};
+    hts_expr_val_t res = HTS_EXPR_VAL_INIT;
+    if (hts_filter_eval2(filt, &hb, bam_sym_lookup, &res)) {
+        hts_log_error("Couldn't process filter expression");
+        hts_expr_val_free(&res);
+        return -1;
+    }
+
+    int t = res.is_true;
+    hts_expr_val_free(&res);
+
+    return t;
+}
+
+static int cram_readrec(BGZF *ignored, void *fpv, void *bv, int *tid, hts_pos_t *beg, hts_pos_t *end)
+{
+    htsFile *fp = fpv;
+    bam1_t *b = bv;
+    int pass_filter, ret;
+
+    do {
+        ret = cram_get_bam_seq(fp->fp.cram, &b);
+        if (ret < 0)
+            return cram_eof(fp->fp.cram) ? -1 : -2;
+
+        if (bam_tag2cigar(b, 1, 1) < 0)
+            return -2;
+
+        *tid = b->core.tid;
+        *beg = b->core.pos;
+        *end = bam_endpos(b);
+
+        if (fp->filter) {
+            pass_filter = sam_passes_filter(fp->bam_header, b, fp->filter);
+            if (pass_filter < 0)
+                return -2;
+        } else {
+            pass_filter = 1;
+        }
+    } while (pass_filter == 0);
+
+    return ret;
+}
+
+static int cram_pseek(void *fp, int64_t offset, int whence)
+{
+    cram_fd *fd =  (cram_fd *)fp;
+
+    if ((0 != cram_seek(fd, offset, SEEK_SET))
+     && (0 != cram_seek(fd, offset - fd->first_container, SEEK_CUR)))
+        return -1;
+
+    fd->curr_position = offset;
+
+    if (fd->ctr) {
+        cram_free_container(fd->ctr);
+        if (fd->ctr_mt && fd->ctr_mt != fd->ctr)
+            cram_free_container(fd->ctr_mt);
+
+        fd->ctr = NULL;
+        fd->ctr_mt = NULL;
+        fd->ooc = 0;
+    }
+
+    return 0;
+}
+
+/*
+ * cram_ptell is a pseudo-tell function, because it matches the position of the disk cursor only
+ *   after a fresh seek call. Otherwise it indicates that the read takes place inside the buffered
+ *   container previously fetched. It was designed like this to integrate with the functionality
+ *   of the iterator stepping logic.
+ */
+
+static int64_t cram_ptell(void *fp)
+{
+    cram_fd *fd = (cram_fd *)fp;
+    cram_container *c;
+    cram_slice *s;
+    int64_t ret = -1L;
+
+    if (fd) {
+        if ((c = fd->ctr) != NULL) {
+            if ((s = c->slice) != NULL && s->max_rec) {
+                if ((c->curr_slice + s->curr_rec/s->max_rec) >= (c->max_slice + 1))
+                    fd->curr_position += c->offset + c->length;
+            }
+        }
+        ret = fd->curr_position;
+    }
+
+    return ret;
+}
+
+static int bam_pseek(void *fp, int64_t offset, int whence)
+{
+    BGZF *fd = (BGZF *)fp;
+
+    return bgzf_seek(fd, offset, whence);
+}
+
+static int64_t bam_ptell(void *fp)
+{
+    BGZF *fd = (BGZF *)fp;
+    if (!fd)
+        return -1L;
+
+    return bgzf_tell(fd);
+}
+
+
+
+static hts_idx_t *index_load(htsFile *fp, const char *fn, const char *fnidx, int flags)
+{
+    switch (fp->format.format) {
+    case bam:
+    case sam:
+        return hts_idx_load3(fn, fnidx, HTS_FMT_BAI, flags);
+
+    case cram: {
+        if (cram_index_load(fp->fp.cram, fn, fnidx) < 0) return NULL;
+
+        // Cons up a fake "index" just pointing at the associated cram_fd:
+        hts_cram_idx_t *idx = malloc(sizeof (hts_cram_idx_t));
+        if (idx == NULL) return NULL;
+        idx->fmt = HTS_FMT_CRAI;
+        idx->cram = fp->fp.cram;
+        return (hts_idx_t *) idx;
+        }
+
+    default:
+        return NULL; // TODO Would use tbx_index_load if it returned hts_idx_t
+    }
+}
+
+hts_idx_t *sam_index_load3(htsFile *fp, const char *fn, const char *fnidx, int flags)
+{
+    return index_load(fp, fn, fnidx, flags);
+}
+
+hts_idx_t *sam_index_load2(htsFile *fp, const char *fn, const char *fnidx) {
+    return index_load(fp, fn, fnidx, HTS_IDX_SAVE_REMOTE);
+}
+
+hts_idx_t *sam_index_load(htsFile *fp, const char *fn)
+{
+    return index_load(fp, fn, NULL, HTS_IDX_SAVE_REMOTE);
+}
+
+static hts_itr_t *cram_itr_query(const hts_idx_t *idx, int tid, hts_pos_t beg, hts_pos_t end, hts_readrec_func *readrec)
+{
+    const hts_cram_idx_t *cidx = (const hts_cram_idx_t *) idx;
+    hts_itr_t *iter = (hts_itr_t *) calloc(1, sizeof(hts_itr_t));
+    if (iter == NULL) return NULL;
+
+    // Cons up a dummy iterator for which hts_itr_next() will simply invoke
+    // the readrec function:
+    iter->is_cram = 1;
+    iter->read_rest = 1;
+    iter->off = NULL;
+    iter->bins.a = NULL;
+    iter->readrec = readrec;
+
+    if (tid >= 0 || tid == HTS_IDX_NOCOOR || tid == HTS_IDX_START) {
+        cram_range r = { tid, beg+1, end };
+        int ret = cram_set_option(cidx->cram, CRAM_OPT_RANGE, &r);
+
+        iter->curr_off = 0;
+        // The following fields are not required by hts_itr_next(), but are
+        // filled in in case user code wants to look at them.
+        iter->tid = tid;
+        iter->beg = beg;
+        iter->end = end;
+
+        switch (ret) {
+        case 0:
+            break;
+
+        case -2:
+            // No data vs this ref, so mark iterator as completed.
+            // Same as HTS_IDX_NONE.
+            iter->finished = 1;
+            break;
+
+        default:
+            free(iter);
+            return NULL;
+        }
+    }
+    else switch (tid) {
+    case HTS_IDX_REST:
+        iter->curr_off = 0;
+        break;
+    case HTS_IDX_NONE:
+        iter->curr_off = 0;
+        iter->finished = 1;
+        break;
+    default:
+        hts_log_error("Query with tid=%d not implemented for CRAM files", tid);
+        abort();
+        break;
+    }
+
+    return iter;
+}
+
+hts_itr_t *sam_itr_queryi(const hts_idx_t *idx, int tid, hts_pos_t beg, hts_pos_t end)
+{
+    const hts_cram_idx_t *cidx = (const hts_cram_idx_t *) idx;
+    if (idx == NULL)
+        return hts_itr_query(NULL, tid, beg, end, sam_readrec_rest);
+    else if (cidx->fmt == HTS_FMT_CRAI)
+        return cram_itr_query(idx, tid, beg, end, sam_readrec);
+    else
+        return hts_itr_query(idx, tid, beg, end, sam_readrec);
+}
+
+static int cram_name2id(void *fdv, const char *ref)
+{
+    cram_fd *fd = (cram_fd *) fdv;
+    return sam_hdr_name2tid(fd->header, ref);
+}
+
+hts_itr_t *sam_itr_querys(const hts_idx_t *idx, sam_hdr_t *hdr, const char *region)
+{
+    const hts_cram_idx_t *cidx = (const hts_cram_idx_t *) idx;
+    return hts_itr_querys(idx, region, (hts_name2id_f)(bam_name2id), hdr,
+                          cidx->fmt == HTS_FMT_CRAI ? cram_itr_query : hts_itr_query,
+                          sam_readrec);
+}
+
+hts_itr_t *sam_itr_regarray(const hts_idx_t *idx, sam_hdr_t *hdr, char **regarray, unsigned int regcount)
+{
+    const hts_cram_idx_t *cidx = (const hts_cram_idx_t *) idx;
+    hts_reglist_t *r_list = NULL;
+    int r_count = 0;
+
+    if (!cidx || !hdr)
+        return NULL;
+
+    hts_itr_t *itr = NULL;
+    if (cidx->fmt == HTS_FMT_CRAI) {
+        r_list = hts_reglist_create(regarray, regcount, &r_count, cidx->cram, cram_name2id);
+        if (!r_list)
+            return NULL;
+        itr = hts_itr_regions(idx, r_list, r_count, cram_name2id, cidx->cram,
+                   hts_itr_multi_cram, cram_readrec, cram_pseek, cram_ptell);
+    } else {
+        r_list = hts_reglist_create(regarray, regcount, &r_count, hdr, (hts_name2id_f)(bam_name2id));
+        if (!r_list)
+            return NULL;
+        itr = hts_itr_regions(idx, r_list, r_count, (hts_name2id_f)(bam_name2id), hdr,
+                   hts_itr_multi_bam, sam_readrec, bam_pseek, bam_ptell);
+    }
+
+    if (!itr)
+        hts_reglist_free(r_list, r_count);
+
+    return itr;
+}
+
+hts_itr_t *sam_itr_regions(const hts_idx_t *idx, sam_hdr_t *hdr, hts_reglist_t *reglist, unsigned int regcount)
+{
+    const hts_cram_idx_t *cidx = (const hts_cram_idx_t *) idx;
+
+    if(!cidx || !hdr || !reglist)
+        return NULL;
+
+    if (cidx->fmt == HTS_FMT_CRAI)
+        return hts_itr_regions(idx, reglist, regcount, cram_name2id, cidx->cram,
+                   hts_itr_multi_cram, cram_readrec, cram_pseek, cram_ptell);
+    else
+        return hts_itr_regions(idx, reglist, regcount, (hts_name2id_f)(bam_name2id), hdr,
+                   hts_itr_multi_bam, sam_readrec, bam_pseek, bam_ptell);
+}
+
+/**********************
+ *** SAM header I/O ***
+ **********************/
+
+#include "htslib/kseq.h"
+#include "htslib/kstring.h"
+
+sam_hdr_t *sam_hdr_parse(size_t l_text, const char *text)
+{
+    sam_hdr_t *bh = sam_hdr_init();
+    if (!bh) return NULL;
+
+    if (sam_hdr_add_lines(bh, text, l_text) != 0) {
+        sam_hdr_destroy(bh);
+        return NULL;
+    }
+
+    return bh;
+}
+
+static int valid_sam_header_type(const char *s) {
+    if (s[0] != '@') return 0;
+    switch (s[1]) {
+    case 'H':
+        return s[2] == 'D' && s[3] == '\t';
+    case 'S':
+        return s[2] == 'Q' && s[3] == '\t';
+    case 'R':
+    case 'P':
+        return s[2] == 'G' && s[3] == '\t';
+    case 'C':
+        return s[2] == 'O';
+    }
+    return 0;
+}
+
+// Minimal sanitisation of a header to ensure.
+// - null terminated string.
+// - all lines start with @ (also implies no blank lines).
+//
+// Much more could be done, but currently is not, including:
+// - checking header types are known (HD, SQ, etc).
+// - syntax (eg checking tab separated fields).
+// - validating n_targets matches @SQ records.
+// - validating target lengths against @SQ records.
+static sam_hdr_t *sam_hdr_sanitise(sam_hdr_t *h) {
+    if (!h)
+        return NULL;
+
+    // Special case for empty headers.
+    if (h->l_text == 0)
+        return h;
+
+    size_t i;
+    unsigned int lnum = 0;
+    char *cp = h->text, last = '\n';
+    for (i = 0; i < h->l_text; i++) {
+        // NB: l_text excludes terminating nul.  This finds early ones.
+        if (cp[i] == 0)
+            break;
+
+        // Error on \n[^@], including duplicate newlines
+        if (last == '\n') {
+            lnum++;
+            if (cp[i] != '@') {
+                hts_log_error("Malformed SAM header at line %u", lnum);
+                sam_hdr_destroy(h);
+                return NULL;
+            }
+        }
+
+        last = cp[i];
+    }
+
+    if (i < h->l_text) { // Early nul found.  Complain if not just padding.
+        size_t j = i;
+        while (j < h->l_text && cp[j] == '\0') j++;
+        if (j < h->l_text)
+            hts_log_warning("Unexpected NUL character in header. Possibly truncated");
+    }
+
+    // Add trailing newline and/or trailing nul if required.
+    if (last != '\n') {
+        hts_log_warning("Missing trailing newline on SAM header. Possibly truncated");
+
+        if (h->l_text < 2 || i >= h->l_text - 2) {
+            if (h->l_text >= SIZE_MAX - 2) {
+                hts_log_error("No room for extra newline");
+                sam_hdr_destroy(h);
+                return NULL;
+            }
+
+            cp = realloc(h->text, (size_t) h->l_text+2);
+            if (!cp) {
+                sam_hdr_destroy(h);
+                return NULL;
+            }
+            h->text = cp;
+        }
+        cp[i++] = '\n';
+
+        // l_text may be larger already due to multiple nul padding
+        if (h->l_text < i)
+            h->l_text = i;
+        cp[h->l_text] = '\0';
+    }
+
+    return h;
+}
+
+static void known_stderr(const char *tool, const char *advice) {
+    hts_log_warning("SAM file corrupted by embedded %s error/log message", tool);
+    hts_log_warning("%s", advice);
+}
+
+static void warn_if_known_stderr(const char *line) {
+    if (strstr(line, "M::bwa_idx_load_from_disk") != NULL)
+        known_stderr("bwa", "Use `bwa mem -o file.sam ...` or `bwa sampe -f file.sam ...` instead of `bwa ... > file.sam`");
+    else if (strstr(line, "M::mem_pestat") != NULL)
+        known_stderr("bwa", "Use `bwa mem -o file.sam ...` instead of `bwa mem ... > file.sam`");
+    else if (strstr(line, "loaded/built the index") != NULL)
+        known_stderr("minimap2", "Use `minimap2 -o file.sam ...` instead of `minimap2 ... > file.sam`");
+}
+
+static sam_hdr_t *sam_hdr_create(htsFile* fp) {
+    kstring_t str = { 0, 0, NULL };
+    khint_t k;
+    sam_hdr_t* h = sam_hdr_init();
+    const char *q, *r;
+    char* sn = NULL;
+    khash_t(s2i) *d = kh_init(s2i);
+    khash_t(s2i) *long_refs = NULL;
+    if (!h || !d)
+        goto error;
+
+    int ret, has_SQ = 0;
+    int next_c = '@';
+    while (next_c == '@' && (ret = hts_getline(fp, KS_SEP_LINE, &fp->line)) >= 0) {
+        if (fp->line.s[0] != '@')
+            break;
+
+        if (fp->line.l > 3 && strncmp(fp->line.s, "@SQ", 3) == 0) {
+            has_SQ = 1;
+            hts_pos_t ln = -1;
+            for (q = fp->line.s + 4;; ++q) {
+                if (strncmp(q, "SN:", 3) == 0) {
+                    q += 3;
+                    for (r = q;*r != '\t' && *r != '\n' && *r != '\0';++r);
+
+                    if (sn) {
+                        hts_log_warning("SQ header line has more than one SN: tag");
+                        free(sn);
+                    }
+                    sn = (char*)calloc(r - q + 1, 1);
+                    if (!sn)
+                        goto error;
+
+                    strncpy(sn, q, r - q);
+                    q = r;
+                } else {
+                    if (strncmp(q, "LN:", 3) == 0)
+                        ln = strtoll(q + 3, (char**)&q, 10);
+                }
+
+                while (*q != '\t' && *q != '\n' && *q != '\0')
+                    ++q;
+                if (*q == '\0' || *q == '\n')
+                    break;
+            }
+            if (sn) {
+                if (ln >= 0) {
+                    int absent;
+                    k = kh_put(s2i, d, sn, &absent);
+                    if (absent < 0)
+                        goto error;
+
+                    if (!absent) {
+                        hts_log_warning("Duplicated sequence \"%s\" in file \"%s\"", sn, fp->fn);
+                        free(sn);
+                    } else {
+                        sn = NULL;
+                        if (ln >= UINT32_MAX) {
+                            // Stash away ref length that
+                            // doesn't fit in target_len array
+                            int k2;
+                            if (!long_refs) {
+                                long_refs = kh_init(s2i);
+                                if (!long_refs)
+                                    goto error;
+                            }
+                            k2 = kh_put(s2i, long_refs, kh_key(d, k), &absent);
+                            if (absent < 0)
+                                goto error;
+                            kh_val(long_refs, k2) = ln;
+                            kh_val(d, k) = ((int64_t) (kh_size(d) - 1) << 32
+                                            | UINT32_MAX);
+                        } else {
+                            kh_val(d, k) = (int64_t) (kh_size(d) - 1) << 32 | ln;
+                        }
+                    }
+                } else {
+                    hts_log_warning("Ignored @SQ SN:%s : bad or missing LN tag", sn);
+                    warn_if_known_stderr(fp->line.s);
+                    free(sn);
+                }
+            } else {
+                hts_log_warning("Ignored @SQ line with missing SN: tag");
+                warn_if_known_stderr(fp->line.s);
+            }
+            sn = NULL;
+        }
+        else if (!valid_sam_header_type(fp->line.s)) {
+            hts_log_error("Invalid header line: must start with @HD/@SQ/@RG/@PG/@CO");
+            warn_if_known_stderr(fp->line.s);
+            goto error;
+        }
+
+        if (kputsn(fp->line.s, fp->line.l, &str) < 0)
+            goto error;
+
+        if (kputc('\n', &str) < 0)
+            goto error;
+
+        if (fp->is_bgzf) {
+            next_c = bgzf_peek(fp->fp.bgzf);
+        } else {
+            unsigned char nc;
+            ssize_t pret = hpeek(fp->fp.hfile, &nc, 1);
+            next_c = pret > 0 ? nc : pret - 1;
+        }
+        if (next_c < -1)
+            goto error;
+    }
+    if (next_c != '@')
+        fp->line.l = 0;
+
+    if (ret < -1)
+        goto error;
+
+    if (!has_SQ && fp->fn_aux) {
+        kstring_t line = { 0, 0, NULL };
+
+        /* The reference index (.fai) is actually needed here */
+        char *fai_fn = fp->fn_aux;
+        char *fn_delim = strstr(fp->fn_aux, HTS_IDX_DELIM);
+        if (fn_delim)
+            fai_fn = fn_delim + strlen(HTS_IDX_DELIM);
+
+        hFILE* f = hopen(fai_fn, "r");
+        int e = 0, absent;
+        if (f == NULL)
+            goto error;
+
+        while (line.l = 0, kgetline(&line, (kgets_func*) hgets, f) >= 0) {
+            char* tab = strchr(line.s, '\t');
+            hts_pos_t ln;
+
+            if (tab == NULL)
+                continue;
+
+            sn = (char*)calloc(tab-line.s+1, 1);
+            if (!sn) {
+                e = 1;
+                break;
+            }
+            memcpy(sn, line.s, tab-line.s);
+            k = kh_put(s2i, d, sn, &absent);
+            if (absent < 0) {
+                e = 1;
+                break;
+            }
+
+            ln = strtoll(tab, NULL, 10);
+
+            if (!absent) {
+                hts_log_warning("Duplicated sequence \"%s\" in the file \"%s\"", sn, fai_fn);
+                free(sn);
+                sn = NULL;
+            } else {
+                sn = NULL;
+                if (ln >= UINT32_MAX) {
+                    // Stash away ref length that
+                    // doesn't fit in target_len array
+                    khint_t k2;
+                    int absent = -1;
+                    if (!long_refs) {
+                        long_refs = kh_init(s2i);
+                        if (!long_refs) {
+                            e = 1;
+                            break;
+                        }
+                    }
+                    k2 = kh_put(s2i, long_refs, kh_key(d, k), &absent);
+                    if (absent < 0) {
+                         e = 1;
+                         break;
+                    }
+                    kh_val(long_refs, k2) = ln;
+                    kh_val(d, k) = ((int64_t) (kh_size(d) - 1) << 32
+                                    | UINT32_MAX);
+                } else {
+                    kh_val(d, k) = (int64_t) (kh_size(d) - 1) << 32 | ln;
+                }
+                has_SQ = 1;
+            }
+
+            e |= kputs("@SQ\tSN:", &str) < 0;
+            e |= kputsn(line.s, tab - line.s, &str) < 0;
+            e |= kputs("\tLN:", &str) < 0;
+            e |= kputll(ln, &str) < 0;
+            e |= kputc('\n', &str) < 0;
+            if (e)
+                break;
+        }
+
+        ks_free(&line);
+        if (hclose(f) != 0) {
+            hts_log_error("Error on closing %s", fai_fn);
+            e = 1;
+        }
+        if (e)
+            goto error;
+    }
+
+    if (has_SQ) {
+        // Populate the targets array
+        h->n_targets = kh_size(d);
+
+        h->target_name = (char**) malloc(sizeof(char*) * h->n_targets);
+        if (!h->target_name) {
+            h->n_targets = 0;
+            goto error;
+        }
+
+        h->target_len = (uint32_t*) malloc(sizeof(uint32_t) * h->n_targets);
+        if (!h->target_len) {
+            h->n_targets = 0;
+            goto error;
+        }
+
+        for (k = kh_begin(d); k != kh_end(d); ++k) {
+            if (!kh_exist(d, k))
+                continue;
+
+            h->target_name[kh_val(d, k) >> 32] = (char*) kh_key(d, k);
+            h->target_len[kh_val(d, k) >> 32] = kh_val(d, k) & 0xffffffffUL;
+            kh_val(d, k) >>= 32;
+        }
+    }
+
+    // Repurpose sdict to hold any references longer than UINT32_MAX
+    h->sdict = long_refs;
+
+    kh_destroy(s2i, d);
+
+    if (str.l == 0)
+        kputsn("", 0, &str);
+    h->l_text = str.l;
+    h->text = ks_release(&str);
+    fp->bam_header = sam_hdr_sanitise(h);
+    fp->bam_header->ref_count = 1;
+
+    return fp->bam_header;
+
+ error:
+    if (h && d && (!h->target_name || !h->target_len)) {
+        for (k = kh_begin(d); k != kh_end(d); ++k)
+            if (kh_exist(d, k)) free((void *)kh_key(d, k));
+    }
+    sam_hdr_destroy(h);
+    ks_free(&str);
+    kh_destroy(s2i, d);
+    kh_destroy(s2i, long_refs);
+    if (sn) free(sn);
+    return NULL;
+}
+
+sam_hdr_t *sam_hdr_read(htsFile *fp)
+{
+    if (!fp) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    switch (fp->format.format) {
+    case bam:
+        return sam_hdr_sanitise(bam_hdr_read(fp->fp.bgzf));
+
+    case cram:
+        return sam_hdr_sanitise(sam_hdr_dup(fp->fp.cram->header));
+
+    case sam:
+        return sam_hdr_create(fp);
+
+    case fastq_format:
+    case fasta_format:
+        return sam_hdr_init();
+
+    case empty_format:
+        errno = EPIPE;
+        return NULL;
+
+    default:
+        errno = EFTYPE;
+        return NULL;
+    }
+}
+
+int sam_hdr_write(htsFile *fp, const sam_hdr_t *h)
+{
+    if (!fp || !h) {
+        errno = EINVAL;
+        return -1;
+    }
+
+    switch (fp->format.format) {
+    case binary_format:
+        fp->format.category = sequence_data;
+        fp->format.format = bam;
+        /* fall-through */
+    case bam:
+        if (bam_hdr_write(fp->fp.bgzf, h) < 0) return -1;
+        break;
+
+    case cram: {
+        cram_fd *fd = fp->fp.cram;
+        if (cram_set_header2(fd, h) < 0) return -1;
+        if (fp->fn_aux)
+            cram_load_reference(fd, fp->fn_aux);
+        if (cram_write_SAM_hdr(fd, fd->header) < 0) return -1;
+        }
+        break;
+
+    case text_format:
+        fp->format.category = sequence_data;
+        fp->format.format = sam;
+        /* fall-through */
+    case sam: {
+        if (!h->hrecs && !h->text)
+            return 0;
+        char *text;
+        kstring_t hdr_ks = { 0, 0, NULL };
+        size_t l_text;
+        ssize_t bytes;
+        int r = 0, no_sq = 0;
+
+        if (h->hrecs) {
+            if (sam_hrecs_rebuild_text(h->hrecs, &hdr_ks) != 0)
+                return -1;
+            text = hdr_ks.s;
+            l_text = hdr_ks.l;
+        } else {
+            const char *p = NULL;
+            do {
+                const char *q = p == NULL ? h->text : p + 4;
+                p = strstr(q, "@SQ\t");
+            } while (!(p == NULL || p == h->text || *(p - 1) == '\n'));
+            no_sq = p == NULL;
+            text = h->text;
+            l_text = h->l_text;
+        }
+
+        if (fp->is_bgzf) {
+            bytes = bgzf_write(fp->fp.bgzf, text, l_text);
+        } else {
+            bytes = hwrite(fp->fp.hfile, text, l_text);
+        }
+        free(hdr_ks.s);
+        if (bytes != l_text)
+            return -1;
+
+        if (no_sq) {
+            int i;
+            for (i = 0; i < h->n_targets; ++i) {
+                fp->line.l = 0;
+                r |= kputsn("@SQ\tSN:", 7, &fp->line) < 0;
+                r |= kputs(h->target_name[i], &fp->line) < 0;
+                r |= kputsn("\tLN:", 4, &fp->line) < 0;
+                r |= kputw(h->target_len[i], &fp->line) < 0;
+                r |= kputc('\n', &fp->line) < 0;
+                if (r != 0)
+                    return -1;
+
+                if (fp->is_bgzf) {
+                    bytes = bgzf_write(fp->fp.bgzf, fp->line.s, fp->line.l);
+                } else {
+                    bytes = hwrite(fp->fp.hfile, fp->line.s, fp->line.l);
+                }
+                if (bytes != fp->line.l)
+                    return -1;
+            }
+        }
+        if (fp->is_bgzf) {
+            if (bgzf_flush(fp->fp.bgzf) != 0) return -1;
+        } else {
+            if (hflush(fp->fp.hfile) != 0) return -1;
+        }
+        }
+        break;
+
+    case fastq_format:
+    case fasta_format:
+        // Nothing to output; FASTQ has no file headers.
+        break;
+
+    default:
+        errno = EBADF;
+        return -1;
+    }
+    return 0;
+}
+
+static int old_sam_hdr_change_HD(sam_hdr_t *h, const char *key, const char *val)
+{
+    char *p, *q, *beg = NULL, *end = NULL, *newtext;
+    size_t new_l_text;
+    if (!h || !key)
+        return -1;
+
+    if (h->l_text > 3) {
+        if (strncmp(h->text, "@HD", 3) == 0) { //@HD line exists
+            if ((p = strchr(h->text, '\n')) == 0) return -1;
+            *p = '\0'; // for strstr call
+
+            char tmp[5] = { '\t', key[0], key[0] ? key[1] : '\0', ':', '\0' };
+
+            if ((q = strstr(h->text, tmp)) != 0) { // key exists
+                *p = '\n'; // change back
+
+                // mark the key:val
+                beg = q;
+                for (q += 4; *q != '\n' && *q != '\t'; ++q);
+                end = q;
+
+                if (val && (strncmp(beg + 4, val, end - beg - 4) == 0)
+                    && strlen(val) == end - beg - 4)
+                     return 0; // val is the same, no need to change
+
+            } else {
+                beg = end = p;
+                *p = '\n';
+            }
+        }
+    }
+    if (beg == NULL) { // no @HD
+        new_l_text = h->l_text;
+        if (new_l_text > SIZE_MAX - strlen(SAM_FORMAT_VERSION) - 9)
+            return -1;
+        new_l_text += strlen(SAM_FORMAT_VERSION) + 8;
+        if (val) {
+            if (new_l_text > SIZE_MAX - strlen(val) - 5)
+                return -1;
+            new_l_text += strlen(val) + 4;
+        }
+        newtext = (char*)malloc(new_l_text + 1);
+        if (!newtext) return -1;
+
+        if (val)
+            snprintf(newtext, new_l_text + 1,
+                    "@HD\tVN:%s\t%s:%s\n%s", SAM_FORMAT_VERSION, key, val, h->text);
+        else
+            snprintf(newtext, new_l_text + 1,
+                    "@HD\tVN:%s\n%s", SAM_FORMAT_VERSION, h->text);
+    } else { // has @HD but different or no key
+        new_l_text = (beg - h->text) + (h->text + h->l_text - end);
+        if (val) {
+            if (new_l_text > SIZE_MAX - strlen(val) - 5)
+                return -1;
+            new_l_text += strlen(val) + 4;
+        }
+        newtext = (char*)malloc(new_l_text + 1);
+        if (!newtext) return -1;
+
+        if (val) {
+            snprintf(newtext, new_l_text + 1, "%.*s\t%s:%s%s",
+                    (int) (beg - h->text), h->text, key, val, end);
+        } else { //delete key
+            snprintf(newtext, new_l_text + 1, "%.*s%s",
+                    (int) (beg - h->text), h->text, end);
+        }
+    }
+    free(h->text);
+    h->text = newtext;
+    h->l_text = new_l_text;
+    return 0;
+}
+
+
+int sam_hdr_change_HD(sam_hdr_t *h, const char *key, const char *val)
+{
+    if (!h || !key)
+        return -1;
+
+    if (!h->hrecs)
+        return old_sam_hdr_change_HD(h, key, val);
+
+    if (val) {
+        if (sam_hdr_update_line(h, "HD", NULL, NULL, key, val, NULL) != 0)
+            return -1;
+    } else {
+        if (sam_hdr_remove_tag_id(h, "HD", NULL, NULL, key) != 0)
+            return -1;
+    }
+    return sam_hdr_rebuild(h);
+}
+/**********************
+ *** SAM record I/O ***
+ **********************/
+
+// The speed of this code can vary considerably depending on minor code
+// changes elsewhere as some of the tight loops are particularly prone to
+// speed changes when the instruction blocks are split over a 32-byte
+// boundary.  To protect against this, we explicitly specify an alignment
+// for this function.  If this is insufficient, we may also wish to
+// consider alignment of blocks within this function via
+// __attribute__((optimize("align-loops=5"))) (gcc) or clang equivalents.
+// However it's not very portable.
+// Instead we break into separate functions so we can explicitly specify
+// use __attribute__((aligned(32))) instead and force consistent loop
+// alignment.
+static inline int64_t grow_B_array(bam1_t *b, uint32_t *n, size_t size) {
+    // Avoid overflow on 32-bit platforms, but it breaks BAM anyway
+    if (*n > INT32_MAX*0.666) {
+        errno = ENOMEM;
+        return -1;
+    }
+
+    size_t bytes = (size_t)size * (size_t)(*n>>1);
+    if (possibly_expand_bam_data(b, bytes) < 0) {
+        hts_log_error("Out of memory");
+        return -1;
+    }
+
+    (*n)+=*n>>1;
+    return 0;
+}
+
+
+// This ensures that q always ends up at the next comma after
+// reading a number even if it's followed by junk.  It
+// prevents the possibility of trying to read more than n items.
+#define skip_to_comma_(q) do { while (*(q) > '\t' && *(q) != ',') (q)++; } while (0)
+
+HTS_ALIGN32
+static char *sam_parse_Bc_vals(bam1_t *b, char *q, uint32_t *nused,
+                               uint32_t *nalloc, int *overflow) {
+    while (*q == ',') {
+        if ((*nused)++ >= (*nalloc)) {
+            if (grow_B_array(b, nalloc, 1) < 0)
+                return NULL;
+        }
+        *(b->data + b->l_data) = hts_str2int(q + 1, &q, 8, overflow);
+        b->l_data++;
+    }
+    return q;
+}
+
+HTS_ALIGN32
+static char *sam_parse_BC_vals(bam1_t *b, char *q, uint32_t *nused,
+                               uint32_t *nalloc, int *overflow) {
+    while (*q == ',') {
+        if ((*nused)++ >= (*nalloc)) {
+            if (grow_B_array(b, nalloc, 1) < 0)
+                return NULL;
+        }
+        if (q[1] != '-') {
+            *(b->data + b->l_data) = hts_str2uint(q + 1, &q, 8, overflow);
+            b->l_data++;
+        } else {
+            *overflow = 1;
+            q++;
+            skip_to_comma_(q);
+        }
+    }
+    return q;
+}
+
+HTS_ALIGN32
+static char *sam_parse_Bs_vals(bam1_t *b, char *q, uint32_t *nused,
+                               uint32_t *nalloc, int *overflow) {
+    while (*q == ',') {
+        if ((*nused)++ >= (*nalloc)) {
+            if (grow_B_array(b, nalloc, 2) < 0)
+                return NULL;
+        }
+        i16_to_le(hts_str2int(q + 1, &q, 16, overflow),
+                  b->data + b->l_data);
+        b->l_data += 2;
+    }
+    return q;
+}
+
+HTS_ALIGN32
+static char *sam_parse_BS_vals(bam1_t *b, char *q, uint32_t *nused,
+                               uint32_t *nalloc, int *overflow) {
+    while (*q == ',') {
+        if ((*nused)++ >= (*nalloc)) {
+            if (grow_B_array(b, nalloc, 2) < 0)
+                return NULL;
+        }
+        if (q[1] != '-') {
+            u16_to_le(hts_str2uint(q + 1, &q, 16, overflow),
+                      b->data + b->l_data);
+            b->l_data += 2;
+        } else {
+            *overflow = 1;
+            q++;
+            skip_to_comma_(q);
+        }
+    }
+    return q;
+}
+
+HTS_ALIGN32
+static char *sam_parse_Bi_vals(bam1_t *b, char *q, uint32_t *nused,
+                               uint32_t *nalloc, int *overflow) {
+    while (*q == ',') {
+        if ((*nused)++ >= (*nalloc)) {
+            if (grow_B_array(b, nalloc, 4) < 0)
+                return NULL;
+        }
+        i32_to_le(hts_str2int(q + 1, &q, 32, overflow),
+                  b->data + b->l_data);
+        b->l_data += 4;
+    }
+    return q;
+}
+
+HTS_ALIGN32
+static char *sam_parse_BI_vals(bam1_t *b, char *q, uint32_t *nused,
+                               uint32_t *nalloc, int *overflow) {
+    while (*q == ',') {
+        if ((*nused)++ >= (*nalloc)) {
+            if (grow_B_array(b, nalloc, 4) < 0)
+                return NULL;
+        }
+        if (q[1] != '-') {
+            u32_to_le(hts_str2uint(q + 1, &q, 32, overflow),
+                      b->data + b->l_data);
+            b->l_data += 4;
+        } else {
+            *overflow = 1;
+            q++;
+            skip_to_comma_(q);
+        }
+    }
+    return q;
+}
+
+HTS_ALIGN32
+static char *sam_parse_Bf_vals(bam1_t *b, char *q, uint32_t *nused,
+                               uint32_t *nalloc, int *overflow) {
+    while (*q == ',') {
+        if ((*nused)++ >= (*nalloc)) {
+            if (grow_B_array(b, nalloc, 4) < 0)
+                return NULL;
+        }
+        float_to_le(strtod(q + 1, &q), b->data + b->l_data);
+        b->l_data += 4;
+    }
+    return q;
+}
+
+HTS_ALIGN32
+static int sam_parse_B_vals_r(char type, uint32_t nalloc, char *in,
+                              char **end, bam1_t *b,
+                              int *ctr) {
+    // Protect against infinite recursion when dealing with invalid input.
+    // An example string is "XX:B:C,-".  The lack of a number means min=0,
+    // but it overflowed due to "-" and so we repeat ad-infinitum.
+    //
+    // Loop detection is the safest solution incase there are other
+    // strange corner cases with malformed inputs.
+    if (++(*ctr) > 2) {
+        hts_log_error("Malformed data in B:%c array", type);
+        return -1;
+    }
+
+    int orig_l = b->l_data;
+    char *q = in;
+    int32_t size;
+    size_t bytes;
+    int overflow = 0;
+
+    size = aux_type2size(type);
+    if (size <= 0 || size > 4) {
+        hts_log_error("Unrecognized type B:%c", type);
+        return -1;
+    }
+
+    // Ensure space for type + values.
+    // The first pass through here we don't know the number of entries and
+    // nalloc == 0.  We start with a small working set and then parse the
+    // data, growing as needed.
+    //
+    // If we have a second pass through we do know the number of entries
+    // and nalloc is already known.  We have no need to expand the bam data.
+    if (!nalloc)
+         nalloc=7;
+
+    // Ensure allocated memory is big enough (for current nalloc estimate)
+    bytes = (size_t) nalloc * (size_t) size;
+    if (bytes / size != nalloc
+        || possibly_expand_bam_data(b, bytes + 2 + sizeof(uint32_t))) {
+        hts_log_error("Out of memory");
+        return -1;
+    }
+
+    uint32_t nused = 0;
+
+    b->data[b->l_data++] = 'B';
+    b->data[b->l_data++] = type;
+    // 32-bit B-array length is inserted later once we know it.
+    int b_len_idx = b->l_data;
+    b->l_data += sizeof(uint32_t);
+
+    if (type == 'c') {
+        if (!(q = sam_parse_Bc_vals(b, q, &nused, &nalloc, &overflow)))
+            return -1;
+    } else if (type == 'C') {
+        if (!(q = sam_parse_BC_vals(b, q, &nused, &nalloc, &overflow)))
+            return -1;
+    } else if (type == 's') {
+        if (!(q = sam_parse_Bs_vals(b, q, &nused, &nalloc, &overflow)))
+            return -1;
+    } else if (type == 'S') {
+        if (!(q = sam_parse_BS_vals(b, q, &nused, &nalloc, &overflow)))
+            return -1;
+    } else if (type == 'i') {
+        if (!(q = sam_parse_Bi_vals(b, q, &nused, &nalloc, &overflow)))
+            return -1;
+    } else if (type == 'I') {
+        if (!(q = sam_parse_BI_vals(b, q, &nused, &nalloc, &overflow)))
+            return -1;
+    } else if (type == 'f') {
+        if (!(q = sam_parse_Bf_vals(b, q, &nused, &nalloc, &overflow)))
+            return -1;
+    }
+    if (*q != '\t' && *q != '\0') {
+        // Unknown B array type or junk in the numbers
+        hts_log_error("Malformed B:%c", type);
+        return -1;
+    }
+    i32_to_le(nused, b->data + b_len_idx);
+
+    if (!overflow) {
+        *end = q;
+        return 0;
+    } else {
+        int64_t max = 0, min = 0, val;
+        // Given type was incorrect.  Try to rescue the situation.
+        char *r = q;
+        q = in;
+        overflow = 0;
+        b->l_data = orig_l;
+        // Find out what range of values is present
+        while (q < r) {
+            val = hts_str2int(q + 1, &q, 64, &overflow);
+            if (max < val) max = val;
+            if (min > val) min = val;
+            skip_to_comma_(q);
+        }
+        // Retry with appropriate type
+        if (!overflow) {
+            if (min < 0) {
+                if (min >= INT8_MIN && max <= INT8_MAX) {
+                    return sam_parse_B_vals_r('c', nalloc, in, end, b, ctr);
+                } else if (min >= INT16_MIN && max <= INT16_MAX) {
+                    return sam_parse_B_vals_r('s', nalloc, in, end, b, ctr);
+                } else if (min >= INT32_MIN && max <= INT32_MAX) {
+                    return sam_parse_B_vals_r('i', nalloc, in, end, b, ctr);
+                }
+            } else {
+                if (max < UINT8_MAX) {
+                    return sam_parse_B_vals_r('C', nalloc, in, end, b, ctr);
+                } else if (max <= UINT16_MAX) {
+                    return sam_parse_B_vals_r('S', nalloc, in, end, b, ctr);
+                } else if (max <= UINT32_MAX) {
+                    return sam_parse_B_vals_r('I', nalloc, in, end, b, ctr);
+                }
+            }
+        }
+        // If here then at least one of the values is too big to store
+        hts_log_error("Numeric value in B array out of allowed range");
+        return -1;
+    }
+#undef skip_to_comma_
+}
+
+HTS_ALIGN32
+static int sam_parse_B_vals(char type, char *in, char **end, bam1_t *b)
+{
+    int ctr = 0;
+    uint32_t nalloc = 0;
+    return sam_parse_B_vals_r(type, nalloc, in, end, b, &ctr);
+}
+
+static inline unsigned int parse_sam_flag(char *v, char **rv, int *overflow) {
+    if (*v >= '1' && *v <= '9') {
+        return hts_str2uint(v, rv, 16, overflow);
+    }
+    else if (*v == '0') {
+        // handle single-digit "0" directly; otherwise it's hex or octal
+        if (v[1] == '\t') { *rv = v+1; return 0; }
+        else {
+            unsigned long val = strtoul(v, rv, 0);
+            if (val > 65535) { *overflow = 1; return 65535; }
+            return val;
+        }
+    }
+    else {
+        // TODO implement symbolic flag letters
+        *rv = v;
+        return 0;
+    }
+}
+
+// Parse tag line and append to bam object b.
+// Shared by both SAM and FASTQ parsers.
+//
+// The difference between the two is how lenient we are to recognising
+// non-compliant strings.  The FASTQ parser glosses over arbitrary
+// non-SAM looking strings.
+static inline int aux_parse(char *start, char *end, bam1_t *b, int lenient,
+                            khash_t(tag) *tag_whitelist) {
+    int overflow = 0;
+    int checkpoint;
+    char logbuf[40];
+    char *q = start, *p = end;
+
+#define _parse_err(cond, ...)                   \
+    do {                                        \
+        if (cond) {                             \
+            if (lenient) {                      \
+                while (q < p && !isspace_c(*q))   \
+                    q++;                        \
+                while (q < p && isspace_c(*q))    \
+                    q++;                        \
+                b->l_data = checkpoint;         \
+                goto loop;                      \
+            } else {                            \
+                hts_log_error(__VA_ARGS__);     \
+                goto err_ret;                   \
+            }                                   \
+        }                                       \
+    } while (0)
+
+    while (q < p) loop: {
+        char type;
+        checkpoint = b->l_data;
+        if (p - q < 5) {
+            if (lenient) {
+                break;
+            } else {
+                hts_log_error("Incomplete aux field");
+                goto err_ret;
+            }
+        }
+        _parse_err(q[0] < '!' || q[1] < '!', "invalid aux tag id");
+
+        if (lenient && (q[2] | q[4]) != ':') {
+            while (q < p && !isspace_c(*q))
+                q++;
+            while (q < p && isspace_c(*q))
+                q++;
+            continue;
+        }
+
+        if (tag_whitelist) {
+            int tt = q[0]*256 + q[1];
+            if (kh_get(tag, tag_whitelist, tt) == kh_end(tag_whitelist)) {
+                while (q < p && *q != '\t')
+                    q++;
+                continue;
+            }
+        }
+
+        // Copy over id
+        if (possibly_expand_bam_data(b, 2) < 0) goto err_ret;
+        memcpy(b->data + b->l_data, q, 2); b->l_data += 2;
+        q += 3; type = *q++; ++q; // q points to value
+        if (type != 'Z' && type != 'H') // the only zero length acceptable fields
+            _parse_err(*q <= '\t', "incomplete aux field");
+
+        // Ensure enough space for a double + type allocated.
+        if (possibly_expand_bam_data(b, 16) < 0) goto err_ret;
+
+        if (type == 'A' || type == 'a' || type == 'c' || type == 'C') {
+            b->data[b->l_data++] = 'A';
+            b->data[b->l_data++] = *q++;
+        } else if (type == 'i' || type == 'I') {
+            if (*q == '-') {
+                int32_t x = hts_str2int(q, &q, 32, &overflow);
+                if (x >= INT8_MIN) {
+                    b->data[b->l_data++] = 'c';
+                    b->data[b->l_data++] = x;
+                } else if (x >= INT16_MIN) {
+                    b->data[b->l_data++] = 's';
+                    i16_to_le(x, b->data + b->l_data);
+                    b->l_data += 2;
+                } else {
+                    b->data[b->l_data++] = 'i';
+                    i32_to_le(x, b->data + b->l_data);
+                    b->l_data += 4;
+                }
+            } else {
+                uint32_t x = hts_str2uint(q, &q, 32, &overflow);
+                if (x <= UINT8_MAX) {
+                    b->data[b->l_data++] = 'C';
+                    b->data[b->l_data++] = x;
+                } else if (x <= UINT16_MAX) {
+                    b->data[b->l_data++] = 'S';
+                    u16_to_le(x, b->data + b->l_data);
+                    b->l_data += 2;
+                } else {
+                    b->data[b->l_data++] = 'I';
+                    u32_to_le(x, b->data + b->l_data);
+                    b->l_data += 4;
+                }
+            }
+        } else if (type == 'f') {
+            b->data[b->l_data++] = 'f';
+            float_to_le(strtod(q, &q), b->data + b->l_data);
+            b->l_data += sizeof(float);
+        } else if (type == 'd') {
+            b->data[b->l_data++] = 'd';
+            double_to_le(strtod(q, &q), b->data + b->l_data);
+            b->l_data += sizeof(double);
+        } else if (type == 'Z' || type == 'H') {
+            char *end = strchr(q, '\t');
+            if (!end) end = q + strlen(q);
+            _parse_err(type == 'H' && ((end-q)&1) != 0,
+                       "hex field does not have an even number of digits");
+            b->data[b->l_data++] = type;
+            if (possibly_expand_bam_data(b, end - q + 1) < 0) goto err_ret;
+            memcpy(b->data + b->l_data, q, end - q);
+            b->l_data += end - q;
+            b->data[b->l_data++] = '\0';
+            q = end;
+        } else if (type == 'B') {
+            type = *q++; // q points to the first ',' following the typing byte
+            _parse_err(*q && *q != ',' && *q != '\t',
+                       "B aux field type not followed by ','");
+
+            if (sam_parse_B_vals(type, q, &q, b) < 0)
+                goto err_ret;
+        } else _parse_err(1, "unrecognized type %s", hts_strprint(logbuf, sizeof logbuf, '\'', &type, 1));
+
+        while (*q > '\t') { q++; } // Skip any junk to next tab
+        q++;
+    }
+
+    _parse_err(!lenient && overflow != 0, "numeric value out of allowed range");
+#undef _parse_err
+
+    return 0;
+
+err_ret:
+    return -2;
+}
+
+int sam_parse1(kstring_t *s, sam_hdr_t *h, bam1_t *b)
+{
+#define _read_token(_p) (_p); do { char *tab = strchr((_p), '\t'); if (!tab) goto err_ret; *tab = '\0'; (_p) = tab + 1; } while (0)
+
+#if HTS_ALLOW_UNALIGNED != 0 && ULONG_MAX == 0xffffffffffffffff
+
+// Macro that operates on 64-bits at a time.
+#define COPY_MINUS_N(to,from,n,l,failed)                        \
+    do {                                                        \
+        uint64_u *from8 = (uint64_u *)(from);                   \
+        uint64_u *to8 = (uint64_u *)(to);                       \
+        uint64_t uflow = 0;                                     \
+        size_t l8 = (l)>>3, i;                                  \
+        for (i = 0; i < l8; i++) {                              \
+            to8[i] = from8[i] - (n)*0x0101010101010101UL;       \
+            uflow |= to8[i];                                    \
+        }                                                       \
+        for (i<<=3; i < (l); ++i) {                             \
+            to[i] = from[i] - (n);                              \
+            uflow |= to[i];                                     \
+        }                                                       \
+        failed = (uflow & 0x8080808080808080UL) > 0;            \
+    } while (0)
+
+#else
+
+// Basic version which operates a byte at a time
+#define COPY_MINUS_N(to,from,n,l,failed) do {                \
+        uint8_t uflow = 0;                                   \
+        for (i = 0; i < (l); ++i) {                          \
+            (to)[i] = (from)[i] - (n);                       \
+            uflow |= (uint8_t) (to)[i];                      \
+        }                                                    \
+        failed = (uflow & 0x80) > 0;                         \
+    } while (0)
+
+#endif
+
+#define _get_mem(type_t, x, b, l) if (possibly_expand_bam_data((b), (l)) < 0) goto err_ret; *(x) = (type_t*)((b)->data + (b)->l_data); (b)->l_data += (l)
+#define _parse_err(cond, ...) do { if (cond) { hts_log_error(__VA_ARGS__); goto err_ret; } } while (0)
+#define _parse_warn(cond, ...) do { if (cond) { hts_log_warning(__VA_ARGS__); } } while (0)
+
+    uint8_t *t;
+
+    char *p = s->s, *q;
+    int i, overflow = 0;
+    char logbuf[40];
+    hts_pos_t cigreflen;
+    bam1_core_t *c = &b->core;
+
+    b->l_data = 0;
+    memset(c, 0, 32);
+
+    // qname
+    q = _read_token(p);
+
+    _parse_warn(p - q <= 1, "empty query name");
+    _parse_err(p - q > 255, "query name too long");
+    // resize large enough for name + extranul
+    if (possibly_expand_bam_data(b, (p - q) + 4) < 0) goto err_ret;
+    memcpy(b->data + b->l_data, q, p-q); b->l_data += p-q;
+
+    c->l_extranul = (4 - (b->l_data & 3)) & 3;
+    memcpy(b->data + b->l_data, "\0\0\0\0", c->l_extranul);
+    b->l_data += c->l_extranul;
+
+    c->l_qname = p - q + c->l_extranul;
+
+    // flag
+    c->flag = parse_sam_flag(p, &p, &overflow);
+    if (*p++ != '\t') goto err_ret; // malformated flag
+
+    // chr
+    q = _read_token(p);
+    if (strcmp(q, "*")) {
+        _parse_err(h->n_targets == 0, "no SQ lines present in the header");
+        c->tid = bam_name2id(h, q);
+        _parse_err(c->tid < -1, "failed to parse header");
+        _parse_warn(c->tid < 0, "unrecognized reference name %s; treated as unmapped", hts_strprint(logbuf, sizeof logbuf, '"', q, SIZE_MAX));
+    } else c->tid = -1;
+
+    // pos
+    c->pos = hts_str2uint(p, &p, 62, &overflow) - 1;
+    if (*p++ != '\t') goto err_ret;
+    if (c->pos < 0 && c->tid >= 0) {
+        _parse_warn(1, "mapped query cannot have zero coordinate; treated as unmapped");
+        c->tid = -1;
+    }
+    if (c->tid < 0) c->flag |= BAM_FUNMAP;
+
+    // mapq
+    c->qual = hts_str2uint(p, &p, 8, &overflow);
+    if (*p++ != '\t') goto err_ret;
+    // cigar
+    if (*p != '*') {
+        uint32_t *cigar = NULL;
+        int old_l_data = b->l_data;
+        int n_cigar = bam_parse_cigar(p, &p, b);
+        if (n_cigar < 1 || *p++ != '\t') goto err_ret;
+        cigar = (uint32_t *)(b->data + old_l_data);
+
+        // can't use bam_endpos() directly as some fields not yet set up
+        cigreflen = (!(c->flag&BAM_FUNMAP))? bam_cigar2rlen(c->n_cigar, cigar) : 1;
+        if (cigreflen == 0) cigreflen = 1;
+    } else {
+        _parse_warn(!(c->flag&BAM_FUNMAP), "mapped query must have a CIGAR; treated as unmapped");
+        c->flag |= BAM_FUNMAP;
+        q = _read_token(p);
+        cigreflen = 1;
+    }
+    _parse_err(HTS_POS_MAX - cigreflen <= c->pos,
+               "read ends beyond highest supported position");
+    c->bin = hts_reg2bin(c->pos, c->pos + cigreflen, 14, 5);
+    // mate chr
+    q = _read_token(p);
+    if (strcmp(q, "=") == 0) {
+        c->mtid = c->tid;
+    } else if (strcmp(q, "*") == 0) {
+        c->mtid = -1;
+    } else {
+        c->mtid = bam_name2id(h, q);
+        _parse_err(c->mtid < -1, "failed to parse header");
+        _parse_warn(c->mtid < 0, "unrecognized mate reference name %s; treated as unmapped", hts_strprint(logbuf, sizeof logbuf, '"', q, SIZE_MAX));
+    }
+    // mpos
+    c->mpos = hts_str2uint(p, &p, 62, &overflow) - 1;
+    if (*p++ != '\t') goto err_ret;
+    if (c->mpos < 0 && c->mtid >= 0) {
+        _parse_warn(1, "mapped mate cannot have zero coordinate; treated as unmapped");
+        c->mtid = -1;
+    }
+    // tlen
+    c->isize = hts_str2int(p, &p, 63, &overflow);
+    if (*p++ != '\t') goto err_ret;
+    _parse_err(overflow, "number outside allowed range");
+    // seq
+    q = _read_token(p);
+    if (strcmp(q, "*")) {
+        _parse_err(p - q - 1 > INT32_MAX, "read sequence is too long");
+        c->l_qseq = p - q - 1;
+        hts_pos_t ql = bam_cigar2qlen(c->n_cigar, (uint32_t*)(b->data + c->l_qname));
+        _parse_err(c->n_cigar && ql != c->l_qseq, "CIGAR and query sequence are of different length");
+        i = (c->l_qseq + 1) >> 1;
+        _get_mem(uint8_t, &t, b, i);
+
+        unsigned int lqs2 = c->l_qseq&~1, i;
+        for (i = 0; i < lqs2; i+=2)
+            t[i>>1] = (seq_nt16_table[(unsigned char)q[i]] << 4) | seq_nt16_table[(unsigned char)q[i+1]];
+        for (; i < c->l_qseq; ++i)
+            t[i>>1] = seq_nt16_table[(unsigned char)q[i]] << ((~i&1)<<2);
+    } else c->l_qseq = 0;
+    // qual
+    _get_mem(uint8_t, &t, b, c->l_qseq);
+    if (p[0] == '*' && (p[1] == '\t' || p[1] == '\0')) {
+        memset(t, 0xff, c->l_qseq);
+        p += 2;
+    } else {
+        int failed = 0;
+        _parse_err(s->l - (p - s->s) < c->l_qseq
+                   || (p[c->l_qseq] != '\t' && p[c->l_qseq] != '\0'),
+                   "SEQ and QUAL are of different length");
+        COPY_MINUS_N(t, p, 33, c->l_qseq, failed);
+        _parse_err(failed, "invalid QUAL character");
+        p += c->l_qseq + 1;
+    }
+
+    // aux
+    if (aux_parse(p, s->s + s->l, b, 0, NULL) < 0)
+        goto err_ret;
+
+    if (bam_tag2cigar(b, 1, 1) < 0)
+        return -2;
+    return 0;
+
+#undef _parse_warn
+#undef _parse_err
+#undef _get_mem
+#undef _read_token
+err_ret:
+    return -2;
+}
+
+static uint32_t read_ncigar(const char *q) {
+    uint32_t n_cigar = 0;
+    for (; *q && *q != '\t'; ++q)
+        if (!isdigit_c(*q)) ++n_cigar;
+    if (!n_cigar) {
+        hts_log_error("No CIGAR operations");
+        return 0;
+    }
+    if (n_cigar >= 2147483647) {
+        hts_log_error("Too many CIGAR operations");
+        return 0;
+    }
+
+    return n_cigar;
+}
+
+/*! @function
+ @abstract  Parse a CIGAR string into preallocated a uint32_t array
+ @param  in      [in]  pointer to the source string
+ @param  a_cigar [out]  address of the destination uint32_t buffer
+ @return         number of processed input characters; 0 on error
+ */
+static int parse_cigar(const char *in, uint32_t *a_cigar, uint32_t n_cigar) {
+    int i, overflow = 0;
+    const char *p = in;
+    for (i = 0; i < n_cigar; i++) {
+        uint32_t len;
+        int op;
+        char *q;
+        len = hts_str2uint(p, &q, 28, &overflow)<<BAM_CIGAR_SHIFT;
+        if (q == p) {
+            hts_log_error("CIGAR length invalid at position %d (%s)", (int)(i+1), p);
+            return 0;
+        }
+        if (overflow) {
+            hts_log_error("CIGAR length too long at position %d (%.*s)", (int)(i+1), (int)(q-p+1), p);
+            return 0;
+        }
+        p = q;
+        op = bam_cigar_table[(unsigned char)*p++];
+        if (op < 0) {
+            hts_log_error("Unrecognized CIGAR operator");
+            return 0;
+        }
+        a_cigar[i] = len;
+        a_cigar[i] |= op;
+    }
+
+    return p-in;
+}
+
+ssize_t sam_parse_cigar(const char *in, char **end, uint32_t **a_cigar, size_t *a_mem) {
+    size_t n_cigar = 0;
+    int diff;
+
+    if (!in || !a_cigar || !a_mem) {
+        hts_log_error("NULL pointer arguments");
+        return -1;
+    }
+    if (end) *end = (char *)in;
+
+    if (*in == '*') {
+        if (end) (*end)++;
+        return 0;
+    }
+    n_cigar = read_ncigar(in);
+    if (!n_cigar) return 0;
+    if (n_cigar > *a_mem) {
+        uint32_t *a_tmp = realloc(*a_cigar, n_cigar*sizeof(**a_cigar));
+        if (a_tmp) {
+            *a_cigar = a_tmp;
+            *a_mem = n_cigar;
+        } else {
+            hts_log_error("Memory allocation error");
+            return -1;
+        }
+    }
+
+    if (!(diff = parse_cigar(in, *a_cigar, n_cigar))) return -1;
+    if (end) *end = (char *)in+diff;
+
+    return n_cigar;
+}
+
+ssize_t bam_parse_cigar(const char *in, char **end, bam1_t *b) {
+    size_t n_cigar = 0;
+    int diff;
+
+    if (!in || !b) {
+        hts_log_error("NULL pointer arguments");
+        return -1;
+    }
+    if (end) *end = (char *)in;
+
+    n_cigar = (*in == '*') ? 0 : read_ncigar(in);
+    if (!n_cigar && b->core.n_cigar == 0) {
+        if (end) *end = (char *)in+1;
+        return 0;
+    }
+
+    ssize_t cig_diff = n_cigar - b->core.n_cigar;
+    if (cig_diff > 0 &&
+        possibly_expand_bam_data(b, cig_diff * sizeof(uint32_t)) < 0) {
+        hts_log_error("Memory allocation error");
+        return -1;
+    }
+
+    uint32_t *cig = bam_get_cigar(b);
+    if ((uint8_t *)cig != b->data + b->l_data) {
+        // Modifying an BAM existing BAM record
+        uint8_t  *seq = bam_get_seq(b);
+        memmove(cig + n_cigar, seq, (b->data + b->l_data) - seq);
+    }
+
+    if (n_cigar) {
+        if (!(diff = parse_cigar(in, cig, n_cigar)))
+            return -1;
+    } else {
+        diff = 1; // handle "*"
+    }
+
+    b->l_data += cig_diff * sizeof(uint32_t);
+    b->core.n_cigar = n_cigar;
+    if (end) *end = (char *)in + diff;
+
+    return n_cigar;
+}
+
+/*
+ * -----------------------------------------------------------------------------
+ * SAM threading
+ */
+// Size of SAM text block (reading)
+#define SAM_NBYTES 240000
+
+// Number of BAM records (writing, up to NB_mem in size)
+#define SAM_NBAM 1000
+
+struct SAM_state;
+
+// Output job - a block of BAM records
+typedef struct sp_bams {
+    struct sp_bams *next;
+    int serial;
+
+    bam1_t *bams;
+    int nbams, abams; // used and alloc for bams[] array
+    size_t bam_mem;   // very approximate total size
+
+    struct SAM_state *fd;
+} sp_bams;
+
+// Input job - a block of SAM text
+typedef struct sp_lines {
+    struct sp_lines *next;
+    int serial;
+
+    char *data;
+    int data_size;
+    int alloc;
+
+    struct SAM_state *fd;
+    sp_bams *bams;
+} sp_lines;
+
+enum sam_cmd {
+    SAM_NONE = 0,
+    SAM_CLOSE,
+    SAM_CLOSE_DONE,
+};
+
+typedef struct SAM_state {
+    sam_hdr_t *h;
+
+    hts_tpool *p;
+    int own_pool;
+    pthread_mutex_t lines_m;
+    hts_tpool_process *q;
+    pthread_t dispatcher;
+    int dispatcher_set;
+
+    sp_lines *lines;
+    sp_bams *bams;
+
+    sp_bams *curr_bam;
+    int curr_idx;
+    int serial;
+
+    // Be warned: moving these mutexes around in this struct can reduce
+    // threading performance by up to 70%!
+    pthread_mutex_t command_m;
+    pthread_cond_t command_c;
+    enum sam_cmd command;
+
+    // One of the E* errno codes
+    int errcode;
+
+    htsFile *fp;
+} SAM_state;
+
+// Returns a SAM_state struct from a generic hFILE.
+//
+// Returns NULL on failure.
+static SAM_state *sam_state_create(htsFile *fp) {
+    // Ideally sam_open wouldn't be a #define to hts_open but instead would
+    // be a redirect call with an additional 'S' mode.  This in turn would
+    // correctly set the designed format to sam instead of a generic
+    // text_format.
+    if (fp->format.format != sam && fp->format.format != text_format)
+        return NULL;
+
+    SAM_state *fd = calloc(1, sizeof(*fd));
+    if (!fd)
+        return NULL;
+
+    fp->state = fd;
+    fd->fp = fp;
+
+    return fd;
+}
+
+static int sam_format1_append(const bam_hdr_t *h, const bam1_t *b, kstring_t *str);
+static void *sam_format_worker(void *arg);
+
+static void sam_state_err(SAM_state *fd, int errcode) {
+    pthread_mutex_lock(&fd->command_m);
+    if (!fd->errcode)
+        fd->errcode = errcode;
+    pthread_mutex_unlock(&fd->command_m);
+}
+
+static void sam_free_sp_bams(sp_bams *b) {
+    if (!b)
+        return;
+
+    if (b->bams) {
+        int i;
+        for (i = 0; i < b->abams; i++) {
+            if (b->bams[i].data)
+                free(b->bams[i].data);
+        }
+        free(b->bams);
+    }
+    free(b);
+}
+
+// Destroys the state produce by sam_state_create.
+int sam_state_destroy(htsFile *fp) {
+    int ret = 0;
+
+    if (!fp->state)
+        return 0;
+
+    SAM_state *fd = fp->state;
+    if (fd->p) {
+        if (fd->h) {
+            // Notify sam_dispatcher we're closing
+            pthread_mutex_lock(&fd->command_m);
+            if (fd->command != SAM_CLOSE_DONE)
+                fd->command = SAM_CLOSE;
+            pthread_cond_signal(&fd->command_c);
+            ret = -fd->errcode;
+            if (fd->q)
+                hts_tpool_wake_dispatch(fd->q); // unstick the reader
+
+            if (!fp->is_write && fd->q && fd->dispatcher_set) {
+                for (;;) {
+                    // Avoid deadlocks with dispatcher
+                    if (fd->command == SAM_CLOSE_DONE)
+                        break;
+                    hts_tpool_wake_dispatch(fd->q);
+                    pthread_mutex_unlock(&fd->command_m);
+                    usleep(10000);
+                    pthread_mutex_lock(&fd->command_m);
+                }
+            }
+            pthread_mutex_unlock(&fd->command_m);
+
+            if (fp->is_write) {
+                // Dispatch the last partial block.
+                sp_bams *gb = fd->curr_bam;
+                if (!ret && gb && gb->nbams > 0 && fd->q)
+                    ret = hts_tpool_dispatch(fd->p, fd->q, sam_format_worker, gb);
+
+                // Flush and drain output
+                if (fd->q)
+                    hts_tpool_process_flush(fd->q);
+                pthread_mutex_lock(&fd->command_m);
+                if (!ret) ret = -fd->errcode;
+                pthread_mutex_unlock(&fd->command_m);
+
+                while (!ret && fd->q && !hts_tpool_process_empty(fd->q)) {
+                    usleep(10000);
+                    pthread_mutex_lock(&fd->command_m);
+                    ret = -fd->errcode;
+                    // not empty but shutdown implies error
+                    if (hts_tpool_process_is_shutdown(fd->q) && !ret)
+                        ret = EIO;
+                    pthread_mutex_unlock(&fd->command_m);
+                }
+                if (fd->q)
+                    hts_tpool_process_shutdown(fd->q);
+            }
+
+            // Wait for it to acknowledge
+            if (fd->dispatcher_set)
+                pthread_join(fd->dispatcher, NULL);
+            if (!ret) ret = -fd->errcode;
+        }
+
+        // Tidy up memory
+        if (fd->q)
+            hts_tpool_process_destroy(fd->q);
+
+        if (fd->own_pool && fp->format.compression == no_compression) {
+            hts_tpool_destroy(fd->p);
+            fd->p = NULL;
+        }
+        pthread_mutex_destroy(&fd->lines_m);
+        pthread_mutex_destroy(&fd->command_m);
+        pthread_cond_destroy(&fd->command_c);
+
+        sp_lines *l = fd->lines;
+        while (l) {
+            sp_lines *n = l->next;
+            free(l->data);
+            free(l);
+            l = n;
+        }
+
+        sp_bams *b = fd->bams;
+        while (b) {
+            if (fd->curr_bam == b)
+                fd->curr_bam = NULL;
+            sp_bams *n = b->next;
+            sam_free_sp_bams(b);
+            b = n;
+        }
+
+        if (fd->curr_bam)
+            sam_free_sp_bams(fd->curr_bam);
+
+        // Decrement counter by one, maybe destroying too.
+        // This is to permit the caller using bam_hdr_destroy
+        // before sam_close without triggering decode errors
+        // in the background threads.
+        bam_hdr_destroy(fd->h);
+    }
+
+    free(fp->state);
+    fp->state = NULL;
+    return ret;
+}
+
+// Cleanup function - job for sam_parse_worker; result for sam_format_worker
+static void cleanup_sp_lines(void *arg) {
+    sp_lines *gl = (sp_lines *)arg;
+    if (!gl) return;
+
+    // Should always be true for lines passed to / from thread workers.
+    assert(gl->next == NULL);
+
+    free(gl->data);
+    sam_free_sp_bams(gl->bams);
+    free(gl);
+}
+
+// Run from one of the worker threads.
+// Convert a passed in array of lines to array of BAMs, returning
+// the result back to the thread queue.
+static void *sam_parse_worker(void *arg) {
+    sp_lines *gl = (sp_lines *)arg;
+    sp_bams *gb = NULL;
+    char *lines = gl->data;
+    int i;
+    bam1_t *b;
+    SAM_state *fd = gl->fd;
+
+    // Use a block of BAM structs we had earlier if available.
+    pthread_mutex_lock(&fd->lines_m);
+    if (fd->bams) {
+        gb = fd->bams;
+        fd->bams = gb->next;
+    }
+    pthread_mutex_unlock(&fd->lines_m);
+
+    if (gb == NULL) {
+        gb = calloc(1, sizeof(*gb));
+        if (!gb) {
+            return NULL;
+        }
+        gb->abams = 100;
+        gb->bams = b = calloc(gb->abams, sizeof(*b));
+        if (!gb->bams) {
+            sam_state_err(fd, ENOMEM);
+            goto err;
+        }
+        gb->nbams = 0;
+        gb->bam_mem = 0;
+    }
+    gb->serial = gl->serial;
+    gb->next = NULL;
+
+    b = (bam1_t *)gb->bams;
+    if (!b) {
+        sam_state_err(fd, ENOMEM);
+        goto err;
+    }
+
+    i = 0;
+    char *cp = lines, *cp_end = lines + gl->data_size;
+    while (cp < cp_end) {
+        if (i >= gb->abams) {
+            int old_abams = gb->abams;
+            gb->abams *= 2;
+            b = (bam1_t *)realloc(gb->bams, gb->abams*sizeof(bam1_t));
+            if (!b) {
+                gb->abams /= 2;
+                sam_state_err(fd, ENOMEM);
+                goto err;
+            }
+            memset(&b[old_abams], 0, (gb->abams - old_abams)*sizeof(*b));
+            gb->bams = b;
+        }
+
+        // Ideally we'd get sam_parse1 to return the number of
+        // bytes decoded and to be able to stop on newline as
+        // well as \0.
+        //
+        // We can then avoid the additional strchr loop.
+        // It's around 6% of our CPU cost, albeit threadable.
+        //
+        // However this is an API change so for now we copy.
+
+        char *nl = strchr(cp, '\n');
+        char *line_end;
+        if (nl) {
+            line_end = nl;
+            if (line_end > cp && *(line_end - 1) == '\r')
+                line_end--;
+            nl++;
+        } else {
+            nl = line_end = cp_end;
+        }
+        *line_end = '\0';
+        kstring_t ks = { line_end - cp, gl->alloc, cp };
+        if (sam_parse1(&ks, fd->h, &b[i]) < 0) {
+            sam_state_err(fd, errno ? errno : EIO);
+            cleanup_sp_lines(gl);
+            goto err;
+        }
+
+        cp = nl;
+        i++;
+    }
+    gb->nbams = i;
+
+    pthread_mutex_lock(&fd->lines_m);
+    gl->next = fd->lines;
+    fd->lines = gl;
+    pthread_mutex_unlock(&fd->lines_m);
+    return gb;
+
+ err:
+    sam_free_sp_bams(gb);
+    return NULL;
+}
+
+static void *sam_parse_eof(void *arg) {
+    return NULL;
+}
+
+// Cleanup function - result for sam_parse_worker; job for sam_format_worker
+static void cleanup_sp_bams(void *arg) {
+    sam_free_sp_bams((sp_bams *) arg);
+}
+
+// Runs in its own thread.
+// Reads a block of text (SAM) and sends a new job to the thread queue to
+// translate this to BAM.
+static void *sam_dispatcher_read(void *vp) {
+    htsFile *fp = vp;
+    kstring_t line = {0};
+    int line_frag = 0;
+    SAM_state *fd = fp->state;
+    sp_lines *l = NULL;
+
+    // Pre-allocate buffer for left-over bits of line (exact size doesn't
+    // matter as it will grow if necessary).
+    if (ks_resize(&line, 1000) < 0)
+        goto err;
+
+    for (;;) {
+        // Check for command
+        pthread_mutex_lock(&fd->command_m);
+        switch (fd->command) {
+
+        case SAM_CLOSE:
+            pthread_cond_signal(&fd->command_c);
+            pthread_mutex_unlock(&fd->command_m);
+            hts_tpool_process_shutdown(fd->q);
+            goto tidyup;
+
+        default:
+            break;
+        }
+        pthread_mutex_unlock(&fd->command_m);
+
+        pthread_mutex_lock(&fd->lines_m);
+        if (fd->lines) {
+            // reuse existing line buffer
+            l = fd->lines;
+            fd->lines = l->next;
+        }
+        pthread_mutex_unlock(&fd->lines_m);
+
+        if (l == NULL) {
+            // none to reuse, to create a new one
+            l = calloc(1, sizeof(*l));
+            if (!l)
+                goto err;
+            l->alloc = SAM_NBYTES;
+            l->data = malloc(l->alloc+8); // +8 for optimisation in sam_parse1
+            if (!l->data) {
+                free(l);
+                l = NULL;
+                goto err;
+            }
+            l->fd = fd;
+        }
+        l->next = NULL;
+
+        if (l->alloc < line_frag+SAM_NBYTES/2) {
+            char *rp = realloc(l->data, line_frag+SAM_NBYTES/2 +8);
+            if (!rp)
+                goto err;
+            l->alloc = line_frag+SAM_NBYTES/2;
+            l->data = rp;
+        }
+        memcpy(l->data, line.s, line_frag);
+
+        l->data_size = line_frag;
+        ssize_t nbytes;
+    longer_line:
+        if (fp->is_bgzf)
+            nbytes = bgzf_read(fp->fp.bgzf, l->data + line_frag, l->alloc - line_frag);
+        else
+            nbytes = hread(fp->fp.hfile, l->data + line_frag, l->alloc - line_frag);
+        if (nbytes < 0) {
+            sam_state_err(fd, errno ? errno : EIO);
+            goto err;
+        } else if (nbytes == 0)
+            break; // EOF
+        l->data_size += nbytes;
+
+        // trim to last \n. Maybe \r\n, but that's still fine
+        if (nbytes == l->alloc - line_frag) {
+            char *cp_end = l->data + l->data_size;
+            char *cp = cp_end-1;
+
+            while (cp > (char *)l->data && *cp != '\n')
+                cp--;
+
+            // entire buffer is part of a single line
+            if (cp == l->data) {
+                line_frag = l->data_size;
+                char *rp = realloc(l->data, l->alloc * 2 + 8);
+                if (!rp)
+                    goto err;
+                l->alloc *= 2;
+                l->data = rp;
+                assert(l->alloc >= l->data_size);
+                assert(l->alloc >= line_frag);
+                assert(l->alloc >= l->alloc - line_frag);
+                goto longer_line;
+            }
+            cp++;
+
+            // line holds the remainder of our line.
+            if (ks_resize(&line, cp_end - cp) < 0)
+                goto err;
+            memcpy(line.s, cp, cp_end - cp);
+            line_frag = cp_end - cp;
+            l->data_size = l->alloc - line_frag;
+        } else {
+            // out of buffer
+            line_frag = 0;
+        }
+
+        l->serial = fd->serial++;
+        //fprintf(stderr, "Dispatching %p, %d bytes, serial %d\n", l, l->data_size, l->serial);
+        if (hts_tpool_dispatch3(fd->p, fd->q, sam_parse_worker, l,
+                                cleanup_sp_lines, cleanup_sp_bams, 0) < 0)
+            goto err;
+        pthread_mutex_lock(&fd->command_m);
+        if (fd->command == SAM_CLOSE) {
+            pthread_mutex_unlock(&fd->command_m);
+            l = NULL;
+            goto tidyup;
+        }
+        l = NULL;  // Now "owned" by sam_parse_worker()
+        pthread_mutex_unlock(&fd->command_m);
+    }
+
+    if (hts_tpool_dispatch(fd->p, fd->q, sam_parse_eof, NULL) < 0)
+        goto err;
+
+    // At EOF, wait for close request.
+    // (In future if we add support for seek, this is where we need to catch it.)
+    for (;;) {
+        pthread_mutex_lock(&fd->command_m);
+        if (fd->command == SAM_NONE)
+            pthread_cond_wait(&fd->command_c, &fd->command_m);
+        switch (fd->command) {
+        case SAM_CLOSE:
+            pthread_cond_signal(&fd->command_c);
+            pthread_mutex_unlock(&fd->command_m);
+            hts_tpool_process_shutdown(fd->q);
+            goto tidyup;
+
+        default:
+            pthread_mutex_unlock(&fd->command_m);
+            break;
+        }
+    }
+
+ tidyup:
+    pthread_mutex_lock(&fd->command_m);
+    fd->command = SAM_CLOSE_DONE;
+    pthread_cond_signal(&fd->command_c);
+    pthread_mutex_unlock(&fd->command_m);
+
+    if (l) {
+        pthread_mutex_lock(&fd->lines_m);
+        l->next = fd->lines;
+        fd->lines = l;
+        pthread_mutex_unlock(&fd->lines_m);
+    }
+    free(line.s);
+
+    return NULL;
+
+ err:
+    sam_state_err(fd, errno ? errno : ENOMEM);
+    hts_tpool_process_shutdown(fd->q);
+    goto tidyup;
+}
+
+// Runs in its own thread.
+// Takes encoded blocks of SAM off the thread results queue and writes them
+// to our output stream.
+static void *sam_dispatcher_write(void *vp) {
+    htsFile *fp = vp;
+    SAM_state *fd = fp->state;
+    hts_tpool_result *r;
+
+    // Iterates until result queue is shutdown, where it returns NULL.
+    while ((r = hts_tpool_next_result_wait(fd->q))) {
+        sp_lines *gl = (sp_lines *)hts_tpool_result_data(r);
+        if (!gl) {
+            sam_state_err(fd, ENOMEM);
+            goto err;
+        }
+
+        if (fp->idx) {
+            sp_bams *gb = gl->bams;
+            int i = 0, count = 0;
+            while (i < gl->data_size) {
+                int j = i;
+                while (i < gl->data_size && gl->data[i] != '\n')
+                    i++;
+                if (i < gl->data_size)
+                    i++;
+
+                if (fp->is_bgzf) {
+                    if (bgzf_flush_try(fp->fp.bgzf, i-j) < 0)
+                        goto err;
+                    if (bgzf_write(fp->fp.bgzf, &gl->data[j], i-j) != i-j)
+                        goto err;
+                } else {
+                    if (hwrite(fp->fp.hfile, &gl->data[j], i-j) != i-j)
+                        goto err;
+                }
+
+                bam1_t *b = &gb->bams[count++];
+                if (fp->format.compression == bgzf) {
+                    if (bgzf_idx_push(fp->fp.bgzf, fp->idx,
+                                      b->core.tid, b->core.pos, bam_endpos(b),
+                                      bgzf_tell(fp->fp.bgzf),
+                                      !(b->core.flag&BAM_FUNMAP)) < 0) {
+                        sam_state_err(fd, errno ? errno : ENOMEM);
+                        hts_log_error("Read '%s' with ref_name='%s', ref_length=%"PRIhts_pos", flags=%d, pos=%"PRIhts_pos" cannot be indexed",
+                                bam_get_qname(b), sam_hdr_tid2name(fd->h, b->core.tid), sam_hdr_tid2len(fd->h, b->core.tid), b->core.flag, b->core.pos+1);
+                        goto err;
+                    }
+                } else {
+                    if (hts_idx_push(fp->idx, b->core.tid, b->core.pos, bam_endpos(b),
+                                     bgzf_tell(fp->fp.bgzf), !(b->core.flag&BAM_FUNMAP)) < 0) {
+                        sam_state_err(fd, errno ? errno : ENOMEM);
+                        hts_log_error("Read '%s' with ref_name='%s', ref_length=%"PRIhts_pos", flags=%d, pos=%"PRIhts_pos" cannot be indexed",
+                                bam_get_qname(b), sam_hdr_tid2name(fd->h, b->core.tid), sam_hdr_tid2len(fd->h, b->core.tid), b->core.flag, b->core.pos+1);
+                        goto err;
+                    }
+                }
+            }
+
+            assert(count == gb->nbams);
+
+            // Add bam array to free-list
+            pthread_mutex_lock(&fd->lines_m);
+            gb->next = fd->bams;
+            fd->bams = gl->bams;
+            gl->bams = NULL;
+            pthread_mutex_unlock(&fd->lines_m);
+        } else {
+            if (fp->is_bgzf) {
+                // We keep track of how much in the current block we have
+                // remaining => R.  We look for the last newline in input
+                // [i] to [i+R], backwards => position N.
+                //
+                // If we find a newline, we write out bytes i to N.
+                // We know we cannot fit the next record in this bgzf block,
+                // so we flush what we have and copy input N to i+R into
+                // the start of a new block, and recompute a new R for that.
+                //
+                // If we don't find a newline (i==N) then we cannot extend
+                // the current block at all, so flush whatever is in it now
+                // if it ends on a newline.
+                // We still copy i(==N) to i+R to the next block and
+                // continue as before with a new R.
+                //
+                // The only exception on the flush is when we run out of
+                // data in the input.  In that case we skip it as we don't
+                // yet know if the next record will fit.
+                //
+                // Both conditions share the same code here:
+                // - Look for newline (pos N)
+                // - Write i to N (which maybe 0)
+                // - Flush if block ends on newline and not end of input
+                // - write N to i+R
+
+                int i = 0;
+                BGZF *fb = fp->fp.bgzf;
+                while (i < gl->data_size) {
+                    // remaining space in block
+                    int R = BGZF_BLOCK_SIZE - fb->block_offset;
+                    int eod = 0;
+                    if (R > gl->data_size-i)
+                        R = gl->data_size-i, eod = 1;
+
+                    // Find last newline in input data
+                    int N = i + R;
+                    while (--N > i) {
+                        if (gl->data[N] == '\n')
+                            break;
+                    }
+
+                    if (N != i) {
+                        // Found a newline
+                        N++;
+                        if (bgzf_write(fb, &gl->data[i], N-i) != N-i)
+                            goto err;
+                    }
+
+                    // Flush bgzf block
+                    int b_off = fb->block_offset;
+                    if (!eod && b_off &&
+                        ((char *)fb->uncompressed_block)[b_off-1] == '\n')
+                        if (bgzf_flush_try(fb, BGZF_BLOCK_SIZE) < 0)
+                            goto err;
+
+                    // Copy from N onwards into next block
+                    if (i+R > N)
+                        if (bgzf_write(fb, &gl->data[N], i+R - N)
+                            != i+R - N)
+                            goto err;
+
+                    i = i+R;
+                }
+            } else {
+                if (hwrite(fp->fp.hfile, gl->data, gl->data_size) != gl->data_size)
+                    goto err;
+            }
+        }
+
+        hts_tpool_delete_result(r, 0);
+
+        // Also updated by main thread
+        pthread_mutex_lock(&fd->lines_m);
+        gl->next = fd->lines;
+        fd->lines = gl;
+        pthread_mutex_unlock(&fd->lines_m);
+    }
+
+    sam_state_err(fd, 0); // success
+    hts_tpool_process_shutdown(fd->q);
+    return NULL;
+
+ err:
+    sam_state_err(fd, errno ? errno : EIO);
+    return (void *)-1;
+}
+
+// Run from one of the worker threads.
+// Convert a passed in array of BAMs (sp_bams) and converts to a block
+// of text SAM records (sp_lines).
+static void *sam_format_worker(void *arg) {
+    sp_bams *gb = (sp_bams *)arg;
+    sp_lines *gl = NULL;
+    int i;
+    SAM_state *fd = gb->fd;
+    htsFile *fp = fd->fp;
+
+    // Use a block of SAM strings we had earlier if available.
+    pthread_mutex_lock(&fd->lines_m);
+    if (fd->lines) {
+        gl = fd->lines;
+        fd->lines = gl->next;
+    }
+    pthread_mutex_unlock(&fd->lines_m);
+
+    if (gl == NULL) {
+        gl = calloc(1, sizeof(*gl));
+        if (!gl) {
+            sam_state_err(fd, ENOMEM);
+            return NULL;
+        }
+        gl->alloc = gl->data_size = 0;
+        gl->data = NULL;
+    }
+    gl->serial = gb->serial;
+    gl->next = NULL;
+
+    kstring_t ks = {0, gl->alloc, gl->data};
+
+    for (i = 0; i < gb->nbams; i++) {
+        if (sam_format1_append(fd->h, &gb->bams[i], &ks) < 0) {
+            sam_state_err(fd, errno ? errno : EIO);
+            goto err;
+        }
+        kputc('\n', &ks);
+    }
+
+    pthread_mutex_lock(&fd->lines_m);
+    gl->data_size = ks.l;
+    gl->alloc = ks.m;
+    gl->data = ks.s;
+
+    if (fp->idx) {
+        // Keep hold of the bam array a little longer as
+        // sam_dispatcher_write needs to use them for building the index.
+        gl->bams = gb;
+    } else {
+        // Add bam array to free-list
+        gb->next = fd->bams;
+        fd->bams = gb;
+    }
+    pthread_mutex_unlock(&fd->lines_m);
+
+    return gl;
+
+ err:
+    // Possible race between this and fd->curr_bam.
+    // Easier to not free and leave it on the input list so it
+    // gets freed there instead?
+    // sam_free_sp_bams(gb);
+    if (gl) {
+        free(gl->data);
+        free(gl);
+    }
+    return NULL;
+}
+
+int sam_set_thread_pool(htsFile *fp, htsThreadPool *p) {
+    if (fp->state)
+        return 0;
+
+    if (!(fp->state = sam_state_create(fp)))
+        return -1;
+    SAM_state *fd = (SAM_state *)fp->state;
+
+    pthread_mutex_init(&fd->lines_m, NULL);
+    pthread_mutex_init(&fd->command_m, NULL);
+    pthread_cond_init(&fd->command_c, NULL);
+    fd->p = p->pool;
+    int qsize = p->qsize;
+    if (!qsize)
+        qsize = 2*hts_tpool_size(fd->p);
+    fd->q = hts_tpool_process_init(fd->p, qsize, 0);
+    if (!fd->q) {
+        sam_state_destroy(fp);
+        return -1;
+    }
+
+    if (fp->format.compression == bgzf)
+        return bgzf_thread_pool(fp->fp.bgzf, p->pool, p->qsize);
+
+    return 0;
+}
+
+int sam_set_threads(htsFile *fp, int nthreads) {
+    if (nthreads <= 0)
+        return 0;
+
+    htsThreadPool p;
+    p.pool = hts_tpool_init(nthreads);
+    p.qsize = nthreads*2;
+
+    int ret = sam_set_thread_pool(fp, &p);
+    if (ret < 0)
+        return ret;
+
+    SAM_state *fd = (SAM_state *)fp->state;
+    fd->own_pool = 1;
+
+    return 0;
+}
+
+typedef struct {
+    kstring_t name;
+    kstring_t comment; // NB: pointer into name, do not free
+    kstring_t seq;
+    kstring_t qual;
+    int casava;
+    int aux;
+    int rnum;
+    char BC[3];         // aux tag ID for barcode
+    khash_t(tag) *tags; // which aux tags to use (if empty, use all).
+    char nprefix;
+    int sra_names;
+} fastq_state;
+
+// Initialise fastq state.
+// Name char of '@' or '>' distinguishes fastq vs fasta variant
+static fastq_state *fastq_state_init(int name_char) {
+    fastq_state *x = (fastq_state *)calloc(1, sizeof(*x));
+    if (!x)
+        return NULL;
+    strcpy(x->BC, "BC");
+    x->nprefix = name_char;
+
+    return x;
+}
+
+void fastq_state_destroy(htsFile *fp) {
+    if (fp->state) {
+        fastq_state *x = (fastq_state *)fp->state;
+        if (x->tags)
+            kh_destroy(tag, x->tags);
+        ks_free(&x->name);
+        ks_free(&x->seq);
+        ks_free(&x->qual);
+        free(fp->state);
+    }
+}
+
+int fastq_state_set(samFile *fp, enum hts_fmt_option opt, ...) {
+    va_list args;
+
+    if (!fp)
+        return -1;
+    if (!fp->state)
+        if (!(fp->state = fastq_state_init(fp->format.format == fastq_format
+                                           ? '@' : '>')))
+            return -1;
+
+    fastq_state *x = (fastq_state *)fp->state;
+
+    switch (opt) {
+    case FASTQ_OPT_CASAVA:
+        x->casava = 1;
+        break;
+
+    case FASTQ_OPT_NAME2:
+        x->sra_names = 1;
+        break;
+
+    case FASTQ_OPT_AUX: {
+        va_start(args, opt);
+        x->aux = 1;
+        char *tag = va_arg(args, char *);
+        va_end(args);
+        if (tag && strcmp(tag, "1") != 0) {
+            if (!x->tags)
+                if (!(x->tags = kh_init(tag)))
+                    return -1;
+
+            size_t i, tlen = strlen(tag);
+            for (i = 0; i+3 <= tlen+1; i += 3) {
+                if (tag[i+0] == ',' || tag[i+1] == ',' ||
+                    !(tag[i+2] == ',' || tag[i+2] == '\0')) {
+                    hts_log_warning("Bad tag format '%.3s'; skipping option", tag+i);
+                    break;
+                }
+                int ret, tcode = tag[i+0]*256 + tag[i+1];
+                kh_put(tag, x->tags, tcode, &ret);
+                if (ret < 0)
+                    return -1;
+            }
+        }
+        break;
+    }
+
+    case FASTQ_OPT_BARCODE: {
+        va_start(args, opt);
+        char *bc = va_arg(args, char *);
+        va_end(args);
+        strncpy(x->BC, bc, 2);
+        x->BC[2] = 0;
+        break;
+    }
+
+    case FASTQ_OPT_RNUM:
+        x->rnum = 1;
+        break;
+
+    default:
+        break;
+    }
+    return 0;
+}
+
+static int fastq_parse1(htsFile *fp, bam1_t *b) {
+    fastq_state *x = (fastq_state *)fp->state;
+    size_t i, l;
+    int ret = 0;
+
+    if (fp->format.format == fasta_format && fp->line.s) {
+        // For FASTA we've already read the >name line; steal it
+        // Not the most efficient, but we don't optimise for fasta reading.
+        if (fp->line.l == 0)
+            return -1; // EOF
+
+        free(x->name.s);
+        x->name = fp->line;
+        fp->line.l = fp->line.m = 0;
+        fp->line.s = NULL;
+    } else {
+        // Read a FASTQ format entry.
+        ret = hts_getline(fp, KS_SEP_LINE, &x->name);
+        if (ret == -1)
+            return -1;  // EOF
+        else if (ret < -1)
+            return ret; // ERR
+    }
+
+    // Name
+    if (*x->name.s != x->nprefix)
+        return -2;
+
+    // Reverse the SRA strangeness of putting the run_name.number before
+    // the read name.
+    i = 0;
+    char *name = x->name.s+1;
+    if (x->sra_names) {
+        char *cp = strpbrk(x->name.s, " \t");
+        if (cp) {
+            while (*cp == ' ' || *cp == '\t')
+                cp++;
+            *--cp = '@';
+            i = cp - x->name.s;
+            name = cp+1;
+        }
+    }
+
+    l = x->name.l;
+    char *s = x->name.s;
+    while (i < l && !isspace_c(s[i]))
+        i++;
+    if (i < l) {
+        s[i] = 0;
+        x->name.l = i++;
+    }
+
+    // Comment; a kstring struct, but pointer into name line.  (Do not free)
+    while (i < l && isspace_c(s[i]))
+        i++;
+    x->comment.s = s+i;
+    x->comment.l = l - i;
+
+    // Seq
+    x->seq.l = 0;
+    for (;;) {
+        if ((ret = hts_getline(fp, KS_SEP_LINE, &fp->line)) < 0)
+            if (fp->format.format == fastq_format || ret < -1)
+                return -2;
+        if (ret == -1 ||
+            *fp->line.s == (fp->format.format == fastq_format ? '+' : '>'))
+            break;
+        if (kputsn(fp->line.s, fp->line.l, &x->seq) < 0)
+            return -2;
+    }
+
+    // Qual
+    if (fp->format.format == fastq_format) {
+        size_t remainder = x->seq.l;
+        x->qual.l = 0;
+        do {
+            if (hts_getline(fp, KS_SEP_LINE, &fp->line) < 0)
+                return -2;
+            if (fp->line.l > remainder)
+                return -2;
+            if (kputsn(fp->line.s, fp->line.l, &x->qual) < 0)
+                return -2;
+            remainder -= fp->line.l;
+        } while (remainder > 0);
+
+        // Decr qual
+        for (i = 0; i < x->qual.l; i++)
+            x->qual.s[i] -= '!';
+    }
+
+    int flag = BAM_FUNMAP; int pflag = BAM_FMUNMAP | BAM_FPAIRED;
+    if (x->name.l > 2 &&
+        x->name.s[x->name.l-2] == '/' &&
+        isdigit_c(x->name.s[x->name.l-1])) {
+        switch(x->name.s[x->name.l-1]) {
+        case '1': flag |= BAM_FREAD1 | pflag; break;
+        case '2': flag |= BAM_FREAD2 | pflag; break;
+        default : flag |= BAM_FREAD1 | BAM_FREAD2 | pflag; break;
+        }
+        x->name.s[x->name.l-=2] = 0;
+    }
+
+    // Convert to BAM
+    ret = bam_set1(b,
+                   x->name.s + x->name.l - name, name,
+                   flag,
+                   -1, -1, 0, // ref '*', pos, mapq,
+                   0, NULL,     // no cigar,
+                   -1, -1, 0,    // mate
+                   x->seq.l, x->seq.s, x->qual.s,
+                   0);
+
+    // Identify Illumina CASAVA strings.
+    // <read>:<is_filtered>:<control_bits>:<barcode_sequence>
+    char *barcode = NULL;
+    int barcode_len = 0;
+    kstring_t *kc = &x->comment;
+    char *endptr;
+    if (x->casava &&
+        // \d:[YN]:\d+:[ACGTN]+
+        kc->l > 6 && (kc->s[1] | kc->s[3]) == ':' && isdigit_c(kc->s[0]) &&
+        strtol(kc->s+4, &endptr, 10) >= 0 && endptr != kc->s+4
+        && *endptr == ':') {
+
+        // read num
+        switch(kc->s[0]) {
+        case '1': b->core.flag |= BAM_FREAD1 | pflag; break;
+        case '2': b->core.flag |= BAM_FREAD2 | pflag; break;
+        default : b->core.flag |= BAM_FREAD1 | BAM_FREAD2 | pflag; break;
+        }
+
+        if (kc->s[2] == 'Y')
+            b->core.flag |= BAM_FQCFAIL;
+
+        // Barcode, maybe numeric in which case we skip it
+        if (!isdigit_c(endptr[1])) {
+            barcode = endptr+1;
+            for (i = barcode - kc->s; i < kc->l; i++)
+                if (isspace_c(kc->s[i]))
+                    break;
+
+            kc->s[i] = 0;
+            barcode_len = i+1-(barcode - kc->s);
+        }
+    }
+
+    if (ret >= 0 && barcode_len)
+        if (bam_aux_append(b, x->BC, 'Z', barcode_len, (uint8_t *)barcode) < 0)
+            ret = -2;
+
+    if (!x->aux)
+        return ret;
+
+    // Identify any SAM style aux tags in comments too.
+    if (aux_parse(&kc->s[barcode_len], kc->s + kc->l, b, 1, x->tags) < 0)
+        ret = -2;
+
+    return ret;
+}
+
+// Internal component of sam_read1 below
+static inline int sam_read1_bam(htsFile *fp, sam_hdr_t *h, bam1_t *b) {
+    int ret = bam_read1(fp->fp.bgzf, b);
+    if (h && ret >= 0) {
+        if (b->core.tid  >= h->n_targets || b->core.tid  < -1 ||
+            b->core.mtid >= h->n_targets || b->core.mtid < -1) {
+            errno = ERANGE;
+            return -3;
+        }
+    }
+    return ret;
+}
+
+// Internal component of sam_read1 below
+static inline int sam_read1_cram(htsFile *fp, sam_hdr_t *h, bam1_t **b) {
+    int ret = cram_get_bam_seq(fp->fp.cram, b);
+    if (ret < 0)
+        return cram_eof(fp->fp.cram) ? -1 : -2;
+
+    if (bam_tag2cigar(*b, 1, 1) < 0)
+        return -2;
+
+    return ret;
+}
+
+// Internal component of sam_read1 below
+static inline int sam_read1_sam(htsFile *fp, sam_hdr_t *h, bam1_t *b) {
+    int ret;
+
+    // Consume 1st line after header parsing as it wasn't using peek
+    if (fp->line.l != 0) {
+        ret = sam_parse1(&fp->line, h, b);
+        fp->line.l = 0;
+        return ret;
+    }
+
+    if (fp->state) {
+        SAM_state *fd = (SAM_state *)fp->state;
+
+        if (fp->format.compression == bgzf && fp->fp.bgzf->seeked) {
+            // We don't support multi-threaded SAM parsing with seeks yet.
+            int ret;
+            if ((ret = sam_state_destroy(fp)) < 0) {
+                errno = -ret;
+                return -2;
+            }
+            if (bgzf_seek(fp->fp.bgzf, fp->fp.bgzf->seeked, SEEK_SET) < 0)
+                return -1;
+            fp->fp.bgzf->seeked = 0;
+            goto err_recover;
+        }
+
+        if (!fd->h) {
+            fd->h = h;
+            fd->h->ref_count++;
+            // Ensure hrecs is initialised now as we don't want multiple
+            // threads trying to do this simultaneously.
+            if (!fd->h->hrecs && sam_hdr_fill_hrecs(fd->h) < 0)
+                return -2;
+
+            // We can only do this once we've got a header
+            if (pthread_create(&fd->dispatcher, NULL, sam_dispatcher_read,
+                               fp) != 0)
+                return -2;
+            fd->dispatcher_set = 1;
+        }
+
+        if (fd->h != h) {
+            hts_log_error("SAM multi-threaded decoding does not support changing header");
+            return -1;
+        }
+
+        sp_bams *gb = fd->curr_bam;
+        if (!gb) {
+            if (fd->errcode) {
+                // In case reader failed
+                errno = fd->errcode;
+                return -2;
+            }
+            hts_tpool_result *r = hts_tpool_next_result_wait(fd->q);
+            if (!r)
+                return -2;
+            fd->curr_bam = gb = (sp_bams *)hts_tpool_result_data(r);
+            hts_tpool_delete_result(r, 0);
+        }
+        if (!gb)
+            return fd->errcode ? -2 : -1;
+        bam1_t *b_array = (bam1_t *)gb->bams;
+        if (fd->curr_idx < gb->nbams)
+            if (!bam_copy1(b, &b_array[fd->curr_idx++]))
+                return -2;
+        if (fd->curr_idx == gb->nbams) {
+            pthread_mutex_lock(&fd->lines_m);
+            gb->next = fd->bams;
+            fd->bams = gb;
+            pthread_mutex_unlock(&fd->lines_m);
+
+            fd->curr_bam = NULL;
+            fd->curr_idx = 0;
+        // Consider prefetching next record?  I.e.
+        // } else {
+        //     __builtin_prefetch(&b_array[fd->curr_idx], 0, 3);
+        }
+
+        ret = 0;
+
+    } else  {
+    err_recover:
+        ret = hts_getline(fp, KS_SEP_LINE, &fp->line);
+        if (ret < 0) return ret;
+
+        ret = sam_parse1(&fp->line, h, b);
+        fp->line.l = 0;
+        if (ret < 0) {
+            hts_log_warning("Parse error at line %lld", (long long)fp->lineno);
+            if (h && h->ignore_sam_err) goto err_recover;
+        }
+    }
+
+    return ret;
+}
+
+// Returns 0 on success,
+//        -1 on EOF,
+//       <-1 on error
+int sam_read1(htsFile *fp, sam_hdr_t *h, bam1_t *b)
+{
+    int ret, pass_filter;
+
+    do {
+        switch (fp->format.format) {
+        case bam:
+            ret = sam_read1_bam(fp, h, b);
+            break;
+
+        case cram:
+            ret = sam_read1_cram(fp, h, &b);
+            break;
+
+        case sam:
+            ret = sam_read1_sam(fp, h, b);
+            break;
+
+        case fasta_format:
+        case fastq_format: {
+            fastq_state *x = (fastq_state *)fp->state;
+            if (!x) {
+                if (!(fp->state = fastq_state_init(fp->format.format
+                                                   == fastq_format ? '@' : '>')))
+                    return -2;
+            }
+
+            return fastq_parse1(fp, b);
+        }
+
+        case empty_format:
+            errno = EPIPE;
+            return -3;
+
+        default:
+            errno = EFTYPE;
+            return -3;
+        }
+
+        pass_filter = (ret >= 0 && fp->filter)
+            ? sam_passes_filter(h, b, fp->filter)
+            : 1;
+    } while (pass_filter == 0);
+
+    return pass_filter < 0 ? -2 : ret;
+}
+
+// With gcc, -O3 or -ftree-loop-vectorize is really key here as otherwise
+// this code isn't vectorised and runs far slower than is necessary (even
+// with the restrict keyword being used).
+static inline void HTS_OPT3
+add33(uint8_t *a, const uint8_t * b, int32_t len) {
+    uint32_t i;
+    for (i = 0; i < len; i++)
+        a[i] = b[i]+33;
+}
+
+static int sam_format1_append(const bam_hdr_t *h, const bam1_t *b, kstring_t *str)
+{
+    int i, r = 0;
+    uint8_t *s, *end;
+    const bam1_core_t *c = &b->core;
+
+    if (c->l_qname == 0)
+        return -1;
+    r |= kputsn_(bam_get_qname(b), c->l_qname-1-c->l_extranul, str);
+    r |= kputc_('\t', str); // query name
+    r |= kputw(c->flag, str); r |= kputc_('\t', str); // flag
+    if (c->tid >= 0) { // chr
+        r |= kputs(h->target_name[c->tid] , str);
+        r |= kputc_('\t', str);
+    } else r |= kputsn_("*\t", 2, str);
+    r |= kputll(c->pos + 1, str); r |= kputc_('\t', str); // pos
+    r |= kputw(c->qual, str); r |= kputc_('\t', str); // qual
+    if (c->n_cigar) { // cigar
+        uint32_t *cigar = bam_get_cigar(b);
+        for (i = 0; i < c->n_cigar; ++i) {
+            r |= kputw(bam_cigar_oplen(cigar[i]), str);
+            r |= kputc_(bam_cigar_opchr(cigar[i]), str);
+        }
+    } else r |= kputc_('*', str);
+    r |= kputc_('\t', str);
+    if (c->mtid < 0) r |= kputsn_("*\t", 2, str); // mate chr
+    else if (c->mtid == c->tid) r |= kputsn_("=\t", 2, str);
+    else {
+        r |= kputs(h->target_name[c->mtid], str);
+        r |= kputc_('\t', str);
+    }
+    r |= kputll(c->mpos + 1, str); r |= kputc_('\t', str); // mate pos
+    r |= kputll(c->isize, str); r |= kputc_('\t', str); // template len
+    if (c->l_qseq) { // seq and qual
+        uint8_t *s = bam_get_seq(b);
+        if (ks_resize(str, str->l+2+2*c->l_qseq) < 0) goto mem_err;
+        char *cp = str->s + str->l;
+
+        // Sequence, 2 bases at a time
+        nibble2base(s, cp, c->l_qseq);
+        cp[c->l_qseq] = '\t';
+        cp += c->l_qseq+1;
+
+        // Quality
+        s = bam_get_qual(b);
+        i = 0;
+        if (s[0] == 0xff) {
+            cp[i++] = '*';
+        } else {
+            add33((uint8_t *)cp, s, c->l_qseq); // cp[i] = s[i]+33;
+            i = c->l_qseq;
+        }
+        cp[i] = 0;
+        cp += i;
+        str->l = cp - str->s;
+    } else r |= kputsn_("*\t*", 3, str);
+
+    s = bam_get_aux(b); // aux
+    end = b->data + b->l_data;
+
+    while (end - s >= 4) {
+        r |= kputc_('\t', str);
+        if ((s = (uint8_t *)sam_format_aux1(s, s[2], s+3, end, str)) == NULL)
+            goto bad_aux;
+    }
+    r |= kputsn("", 0, str); // nul terminate
+    if (r < 0) goto mem_err;
+
+    return str->l;
+
+ bad_aux:
+    hts_log_error("Corrupted aux data for read %.*s flag %d",
+                  b->core.l_qname, bam_get_qname(b), b->core.flag);
+    errno = EINVAL;
+    return -1;
+
+ mem_err:
+    hts_log_error("Out of memory");
+    errno = ENOMEM;
+    return -1;
+}
+
+int sam_format1(const bam_hdr_t *h, const bam1_t *b, kstring_t *str)
+{
+    str->l = 0;
+    return sam_format1_append(h, b, str);
+}
+
+static inline uint8_t *skip_aux(uint8_t *s, uint8_t *end);
+int fastq_format1(fastq_state *x, const bam1_t *b, kstring_t *str)
+{
+    unsigned flag = b->core.flag;
+    int i, e = 0, len = b->core.l_qseq;
+    uint8_t *seq, *qual;
+
+    str->l = 0;
+
+    // Name
+    if (kputc(x->nprefix, str) == EOF || kputs(bam_get_qname(b), str) == EOF)
+        return -1;
+
+    // /1 or /2 suffix
+    if (x && x->rnum && (flag & BAM_FPAIRED)) {
+        int r12 = flag & (BAM_FREAD1 | BAM_FREAD2);
+        if (r12 == BAM_FREAD1) {
+            if (kputs("/1", str) == EOF)
+                return -1;
+        } else if (r12 == BAM_FREAD2) {
+            if (kputs("/2", str) == EOF)
+                return -1;
+        }
+    }
+
+    // Illumina CASAVA tag.
+    // This is <rnum>:<Y/N qcfail>:<control-bits>:<barcode-or-zero>
+    if (x && x->casava) {
+        int rnum = (flag & BAM_FREAD1)? 1 : (flag & BAM_FREAD2)? 2 : 0;
+        char filtered = (flag & BAM_FQCFAIL)? 'Y' : 'N';
+        uint8_t *bc = bam_aux_get(b, x->BC);
+        if (ksprintf(str, " %d:%c:0:%s", rnum, filtered,
+                     bc ? (char *)bc+1 : "0") < 0)
+            return -1;
+
+        if (bc && (*bc != 'Z' || (!isupper_c(bc[1]) && !islower_c(bc[1])))) {
+            hts_log_warning("BC tag starts with non-sequence base; using '0'");
+            str->l -= strlen((char *)bc)-2; // limit to 1 char
+            str->s[str->l-1] = '0';
+            str->s[str->l] = 0;
+            bc = NULL;
+        }
+
+        // Replace any non-alpha with '+'.  Ie seq-seq to seq+seq
+        if (bc) {
+            int l = strlen((char *)bc+1);
+            char *c = (char *)str->s + str->l - l;
+            for (i = 0; i < l; i++) {
+                if (!isalpha_c(c[i]))
+                    c[i] = '+';
+                else if (islower_c(c[i]))
+                    c[i] = toupper_c(c[i]);
+            }
+        }
+    }
+
+    // Aux tags
+    if (x && x->aux) {
+        uint8_t *s = bam_get_aux(b), *end = b->data + b->l_data;
+        while (s && end - s >= 4) {
+            int tt = s[0]*256 + s[1];
+            if (x->tags == NULL ||
+                kh_get(tag, x->tags, tt) != kh_end(x->tags)) {
+                e |= kputc_('\t', str) < 0;
+                if (!(s = (uint8_t *)sam_format_aux1(s, s[2], s+3, end, str)))
+                    return -1;
+            } else {
+                s = skip_aux(s+2, end);
+            }
+        }
+        e |= kputsn("", 0, str) < 0; // nul terminate
+    }
+
+    if (ks_resize(str, str->l + 1 + len+1 + 2 + len+1 + 1) < 0) return -1;
+    e |= kputc_('\n', str) < 0;
+
+    // Seq line
+    seq = bam_get_seq(b);
+    if (flag & BAM_FREVERSE)
+        for (i = len-1; i >= 0; i--)
+            e |= kputc_("!TGKCYSBAWRDMHVN"[bam_seqi(seq, i)], str) < 0;
+    else
+        for (i = 0; i < len; i++)
+            e |= kputc_(seq_nt16_str[bam_seqi(seq, i)], str) < 0;
+
+
+    // Qual line
+    if (x->nprefix == '@') {
+        kputsn("\n+\n", 3, str);
+        qual = bam_get_qual(b);
+        if (qual[0] == 0xff)
+            for (i = 0; i < len; i++)
+                e |= kputc_('B', str) < 0;
+        else if (flag & BAM_FREVERSE)
+            for (i = len-1; i >= 0; i--)
+                e |= kputc_(33 + qual[i], str) < 0;
+        else
+            for (i = 0; i < len; i++)
+                e |= kputc_(33 + qual[i], str) < 0;
+
+    }
+    e |= kputc('\n', str) < 0;
+
+    return e ? -1 : str->l;
+}
+
+// Sadly we need to be able to modify the bam_hdr here so we can
+// reference count the structure.
+int sam_write1(htsFile *fp, const sam_hdr_t *h, const bam1_t *b)
+{
+    switch (fp->format.format) {
+    case binary_format:
+        fp->format.category = sequence_data;
+        fp->format.format = bam;
+        /* fall-through */
+    case bam:
+        return bam_write_idx1(fp, h, b);
+
+    case cram:
+        return cram_put_bam_seq(fp->fp.cram, (bam1_t *)b);
+
+    case text_format:
+        fp->format.category = sequence_data;
+        fp->format.format = sam;
+        /* fall-through */
+    case sam:
+        if (fp->state) {
+            SAM_state *fd = (SAM_state *)fp->state;
+
+            // Threaded output
+            if (!fd->h) {
+                // NB: discard const.  We don't actually modify sam_hdr_t here,
+                // just data pointed to by it (which is a bit weasely still),
+                // but out cached pointer must be non-const as we want to
+                // destroy it later on and sam_hdr_destroy takes non-const.
+                //
+                // We do this because some tools do sam_hdr_destroy; sam_close
+                // while others do sam_close; sam_hdr_destroy.  The former is
+                // an issue as we need the header still when flushing.
+                fd->h = (sam_hdr_t *)h;
+                fd->h->ref_count++;
+
+                if (pthread_create(&fd->dispatcher, NULL, sam_dispatcher_write,
+                                   fp) != 0)
+                    return -2;
+                fd->dispatcher_set = 1;
+            }
+
+            if (fd->h != h) {
+                hts_log_error("SAM multi-threaded decoding does not support changing header");
+                return -2;
+            }
+
+            // Find a suitable BAM array to copy to
+            sp_bams *gb = fd->curr_bam;
+            if (!gb) {
+                pthread_mutex_lock(&fd->lines_m);
+                if (fd->bams) {
+                    fd->curr_bam = gb = fd->bams;
+                    fd->bams = gb->next;
+                    gb->next = NULL;
+                    gb->nbams = 0;
+                    gb->bam_mem = 0;
+                    pthread_mutex_unlock(&fd->lines_m);
+                } else {
+                    pthread_mutex_unlock(&fd->lines_m);
+                    if (!(gb = calloc(1, sizeof(*gb)))) return -1;
+                    if (!(gb->bams = calloc(SAM_NBAM, sizeof(*gb->bams)))) {
+                        free(gb);
+                        return -1;
+                    }
+                    gb->nbams = 0;
+                    gb->abams = SAM_NBAM;
+                    gb->bam_mem = 0;
+                    gb->fd = fd;
+                    fd->curr_idx = 0;
+                    fd->curr_bam = gb;
+                }
+            }
+
+            if (!bam_copy1(&gb->bams[gb->nbams++], b))
+                return -2;
+            gb->bam_mem += b->l_data + sizeof(*b);
+
+            // Dispatch if full
+            if (gb->nbams == SAM_NBAM || gb->bam_mem > SAM_NBYTES*0.8) {
+                gb->serial = fd->serial++;
+                pthread_mutex_lock(&fd->command_m);
+                if (fd->errcode != 0) {
+                    pthread_mutex_unlock(&fd->command_m);
+                    return -fd->errcode;
+                }
+                if (hts_tpool_dispatch3(fd->p, fd->q, sam_format_worker, gb,
+                                        cleanup_sp_bams,
+                                        cleanup_sp_lines, 0) < 0) {
+                    pthread_mutex_unlock(&fd->command_m);
+                    return -1;
+                }
+                pthread_mutex_unlock(&fd->command_m);
+                fd->curr_bam = NULL;
+            }
+
+            // Dummy value as we don't know how long it really is.
+            // We could track file sizes via a SAM_state field, but I don't think
+            // it is necessary.
+            return 1;
+        } else {
+            if (sam_format1(h, b, &fp->line) < 0) return -1;
+            kputc('\n', &fp->line);
+            if (fp->is_bgzf) {
+                if (bgzf_flush_try(fp->fp.bgzf, fp->line.l) < 0)
+                    return -1;
+                if ( bgzf_write(fp->fp.bgzf, fp->line.s, fp->line.l) != fp->line.l ) return -1;
+            } else {
+                if ( hwrite(fp->fp.hfile, fp->line.s, fp->line.l) != fp->line.l ) return -1;
+            }
+
+            if (fp->idx) {
+                if (fp->format.compression == bgzf) {
+                    if (bgzf_idx_push(fp->fp.bgzf, fp->idx, b->core.tid, b->core.pos, bam_endpos(b),
+                                      bgzf_tell(fp->fp.bgzf), !(b->core.flag&BAM_FUNMAP)) < 0) {
+                        hts_log_error("Read '%s' with ref_name='%s', ref_length=%"PRIhts_pos", flags=%d, pos=%"PRIhts_pos" cannot be indexed",
+                                bam_get_qname(b), sam_hdr_tid2name(h, b->core.tid), sam_hdr_tid2len(h, b->core.tid), b->core.flag, b->core.pos+1);
+                        return -1;
+                    }
+                } else {
+                    if (hts_idx_push(fp->idx, b->core.tid, b->core.pos, bam_endpos(b),
+                                     bgzf_tell(fp->fp.bgzf), !(b->core.flag&BAM_FUNMAP)) < 0) {
+                        hts_log_error("Read '%s' with ref_name='%s', ref_length=%"PRIhts_pos", flags=%d, pos=%"PRIhts_pos" cannot be indexed",
+                                bam_get_qname(b), sam_hdr_tid2name(h, b->core.tid), sam_hdr_tid2len(h, b->core.tid), b->core.flag, b->core.pos+1);
+                        return -1;
+                    }
+                }
+            }
+
+            return fp->line.l;
+        }
+
+
+    case fasta_format:
+    case fastq_format: {
+        fastq_state *x = (fastq_state *)fp->state;
+        if (!x) {
+            if (!(fp->state = fastq_state_init(fp->format.format
+                                               == fastq_format ? '@' : '>')))
+                return -2;
+        }
+
+        if (fastq_format1(fp->state, b, &fp->line) < 0)
+            return -1;
+        if (fp->is_bgzf) {
+            if (bgzf_flush_try(fp->fp.bgzf, fp->line.l) < 0)
+                return -1;
+            if (bgzf_write(fp->fp.bgzf, fp->line.s, fp->line.l) != fp->line.l)
+                return -1;
+        } else {
+            if (hwrite(fp->fp.hfile, fp->line.s, fp->line.l) != fp->line.l)
+                return -1;
+        }
+        return fp->line.l;
+    }
+
+    default:
+        errno = EBADF;
+        return -1;
+    }
+}
+
+/************************
+ *** Auxiliary fields ***
+ ************************/
+#ifndef HTS_LITTLE_ENDIAN
+static int aux_to_le(char type, uint8_t *out, const uint8_t *in, size_t len) {
+    int tsz = aux_type2size(type);
+
+    if (tsz >= 2 && tsz <= 8 && (len & (tsz - 1)) != 0) return -1;
+
+    switch (tsz) {
+        case 'H': case 'Z': case 1:  // Trivial
+            memcpy(out, in, len);
+            break;
+
+#define aux_val_to_le(type_t, store_le) do {                            \
+        type_t v;                                                       \
+        size_t i;                                                       \
+        for (i = 0; i < len; i += sizeof(type_t), out += sizeof(type_t)) { \
+            memcpy(&v, in + i, sizeof(type_t));                         \
+            store_le(v, out);                                           \
+        }                                                               \
+    } while (0)
+
+        case 2: aux_val_to_le(uint16_t, u16_to_le); break;
+        case 4: aux_val_to_le(uint32_t, u32_to_le); break;
+        case 8: aux_val_to_le(uint64_t, u64_to_le); break;
+
+#undef aux_val_to_le
+
+        case 'B': { // Recurse!
+            uint32_t n;
+            if (len < 5) return -1;
+            memcpy(&n, in + 1, 4);
+            out[0] = in[0];
+            u32_to_le(n, out + 1);
+            return aux_to_le(in[0], out + 5, in + 5, len - 5);
+        }
+
+        default: // Unknown type code
+            return -1;
+    }
+
+
+
+    return 0;
+}
+#endif
+
+int bam_aux_append(bam1_t *b, const char tag[2], char type, int len, const uint8_t *data)
+{
+    uint32_t new_len;
+
+    assert(b->l_data >= 0);
+    new_len = b->l_data + 3 + len;
+    if (new_len > INT32_MAX || new_len < b->l_data) goto nomem;
+
+    if (realloc_bam_data(b, new_len) < 0) return -1;
+
+    b->data[b->l_data] = tag[0];
+    b->data[b->l_data + 1] = tag[1];
+    b->data[b->l_data + 2] = type;
+
+#ifdef HTS_LITTLE_ENDIAN
+    memcpy(b->data + b->l_data + 3, data, len);
+#else
+    if (aux_to_le(type, b->data + b->l_data + 3, data, len) != 0) {
+        errno = EINVAL;
+        return -1;
+    }
+#endif
+
+    b->l_data = new_len;
+
+    return 0;
+
+ nomem:
+    errno = ENOMEM;
+    return -1;
+}
+
+static inline uint8_t *skip_aux(uint8_t *s, uint8_t *end)
+{
+    int size;
+    uint32_t n;
+    if (s >= end) return end;
+    size = aux_type2size(*s); ++s; // skip type
+    switch (size) {
+    case 'Z':
+    case 'H':
+        while (s < end && *s) ++s;
+        return s < end ? s + 1 : end;
+    case 'B':
+        if (end - s < 5) return NULL;
+        size = aux_type2size(*s); ++s;
+        n = le_to_u32(s);
+        s += 4;
+        if (size == 0 || end - s < size * n) return NULL;
+        return s + size * n;
+    case 0:
+        return NULL;
+    default:
+        if (end - s < size) return NULL;
+        return s + size;
+    }
+}
+
+uint8_t *bam_aux_first(const bam1_t *b)
+{
+    uint8_t *s = bam_get_aux(b);
+    uint8_t *end = b->data + b->l_data;
+    if (end - s <= 2) { errno = ENOENT; return NULL; }
+    return s+2;
+}
+
+uint8_t *bam_aux_next(const bam1_t *b, const uint8_t *s)
+{
+    uint8_t *end = b->data + b->l_data;
+    uint8_t *next = s? skip_aux((uint8_t *) s, end) : end;
+    if (next == NULL) goto bad_aux;
+    if (end - next <= 2) { errno = ENOENT; return NULL; }
+    return next+2;
+
+ bad_aux:
+    hts_log_error("Corrupted aux data for read %s flag %d",
+                  bam_get_qname(b), b->core.flag);
+    errno = EINVAL;
+    return NULL;
+}
+
+uint8_t *bam_aux_get(const bam1_t *b, const char tag[2])
+{
+    uint8_t *s;
+    for (s = bam_aux_first(b); s; s = bam_aux_next(b, s))
+        if (s[-2] == tag[0] && s[-1] == tag[1]) {
+            // Check the tag value is valid and complete
+            uint8_t *e = skip_aux(s, b->data + b->l_data);
+            if (e == NULL) goto bad_aux;
+            if ((*s == 'Z' || *s == 'H') && *(e - 1) != '\0') goto bad_aux;
+
+            return s;
+        }
+
+    // errno now as set by bam_aux_first()/bam_aux_next()
+    return NULL;
+
+ bad_aux:
+    hts_log_error("Corrupted aux data for read %s flag %d",
+                  bam_get_qname(b), b->core.flag);
+    errno = EINVAL;
+    return NULL;
+}
+
+int bam_aux_del(bam1_t *b, uint8_t *s)
+{
+    s = bam_aux_remove(b, s);
+    return (s || errno == ENOENT)? 0 : -1;
+}
+
+uint8_t *bam_aux_remove(bam1_t *b, uint8_t *s)
+{
+    uint8_t *end = b->data + b->l_data;
+    uint8_t *next = skip_aux(s, end);
+    if (next == NULL) goto bad_aux;
+
+    b->l_data -= next - (s-2);
+    if (next >= end) { errno = ENOENT; return NULL; }
+
+    memmove(s-2, next, end - next);
+    return s;
+
+ bad_aux:
+    hts_log_error("Corrupted aux data for read %s flag %d",
+                  bam_get_qname(b), b->core.flag);
+    errno = EINVAL;
+    return NULL;
+}
+
+int bam_aux_update_str(bam1_t *b, const char tag[2], int len, const char *data)
+{
+    // FIXME: This is not at all efficient!
+    size_t ln = len >= 0 ? len : strlen(data) + 1;
+    size_t old_ln = 0;
+    int need_nul = ln == 0 || data[ln - 1] != '\0';
+    int save_errno = errno;
+    int new_tag = 0;
+    uint8_t *s = bam_aux_get(b,tag), *e;
+
+    if (s) {  // Replacing existing tag
+        char type = *s;
+        if (type != 'Z') {
+            hts_log_error("Called bam_aux_update_str for type '%c' instead of 'Z'", type);
+            errno = EINVAL;
+            return -1;
+        }
+        s++;
+        e = memchr(s, '\0', b->data + b->l_data - s);
+        old_ln = (e ? e - s : b->data + b->l_data - s) + 1;
+        s -= 3;
+    } else {
+        if (errno != ENOENT) { // Invalid aux data, give up
+            return -1;
+        } else { // Tag doesn't exist - put it on the end
+            errno = save_errno;
+            s = b->data + b->l_data;
+            new_tag = 3;
+        }
+    }
+
+    if (old_ln < ln + need_nul + new_tag) {
+        ptrdiff_t s_offset = s - b->data;
+        if (possibly_expand_bam_data(b, ln + need_nul + new_tag - old_ln) < 0)
+            return -1;
+        s = b->data + s_offset;
+    }
+    if (!new_tag) {
+        memmove(s + 3 + ln + need_nul,
+                s + 3 + old_ln,
+                b->l_data - (s + 3 - b->data) - old_ln);
+    }
+    b->l_data += new_tag + ln + need_nul - old_ln;
+
+    s[0] = tag[0];
+    s[1] = tag[1];
+    s[2] = 'Z';
+    memmove(s+3,data,ln);
+    if (need_nul) s[3 + ln] = '\0';
+    return 0;
+}
+
+int bam_aux_update_int(bam1_t *b, const char tag[2], int64_t val)
+{
+    uint32_t sz, old_sz = 0, new = 0;
+    uint8_t *s, type;
+
+    if (val < INT32_MIN || val > UINT32_MAX) {
+        errno = EOVERFLOW;
+        return -1;
+    }
+    if (val < INT16_MIN)       { type = 'i'; sz = 4; }
+    else if (val < INT8_MIN)   { type = 's'; sz = 2; }
+    else if (val < 0)          { type = 'c'; sz = 1; }
+    else if (val < UINT8_MAX)  { type = 'C'; sz = 1; }
+    else if (val < UINT16_MAX) { type = 'S'; sz = 2; }
+    else                       { type = 'I'; sz = 4; }
+
+    s = bam_aux_get(b, tag);
+    if (s) {  // Tag present - how big was the old one?
+        switch (*s) {
+            case 'c': case 'C': old_sz = 1; break;
+            case 's': case 'S': old_sz = 2; break;
+            case 'i': case 'I': old_sz = 4; break;
+            default: errno = EINVAL; return -1;  // Not an integer
+        }
+    } else {
+        if (errno == ENOENT) {  // Tag doesn't exist - add a new one
+            s = b->data + b->l_data;
+            new = 1;
+        }  else { // Invalid aux data, give up.
+            return -1;
+        }
+    }
+
+    if (new || old_sz < sz) {
+        // Make room for new tag
+        ptrdiff_t s_offset = s - b->data;
+        if (possibly_expand_bam_data(b, (new ? 3 : 0) + sz - old_sz) < 0)
+            return -1;
+        s =  b->data + s_offset;
+        if (new) { // Add tag id
+            *s++ = tag[0];
+            *s++ = tag[1];
+        } else {   // Shift following data so we have space
+            memmove(s + sz, s + old_sz, b->l_data - s_offset - old_sz);
+        }
+    } else {
+        // Reuse old space.  Data value may be bigger than necessary but
+        // we avoid having to move everything else
+        sz = old_sz;
+        type = (val < 0 ? "\0cs\0i" : "\0CS\0I")[old_sz];
+        assert(type > 0);
+    }
+    *s++ = type;
+#ifdef HTS_LITTLE_ENDIAN
+    memcpy(s, &val, sz);
+#else
+    switch (sz) {
+        case 4:  u32_to_le(val, s); break;
+        case 2:  u16_to_le(val, s); break;
+        default: *s = val; break;
+    }
+#endif
+    b->l_data += (new ? 3 : 0) + sz - old_sz;
+    return 0;
+}
+
+int bam_aux_update_float(bam1_t *b, const char tag[2], float val)
+{
+    uint8_t *s = bam_aux_get(b, tag);
+    int shrink = 0, new = 0;
+
+    if (s) { // Tag present - what was it?
+        switch (*s) {
+            case 'f': break;
+            case 'd': shrink = 1; break;
+            default: errno = EINVAL; return -1;  // Not a float
+        }
+    } else {
+        if (errno == ENOENT) {  // Tag doesn't exist - add a new one
+            new = 1;
+        }  else { // Invalid aux data, give up.
+            return -1;
+        }
+    }
+
+    if (new) { // Ensure there's room
+        if (possibly_expand_bam_data(b, 3 + 4) < 0)
+            return -1;
+        s = b->data + b->l_data;
+        *s++ = tag[0];
+        *s++ = tag[1];
+    } else if (shrink) { // Convert non-standard double tag to float
+        memmove(s + 5, s + 9, b->l_data - ((s + 9) - b->data));
+        b->l_data -= 4;
+    }
+    *s++ = 'f';
+    float_to_le(val, s);
+    if (new) b->l_data += 7;
+
+    return 0;
+}
+
+int bam_aux_update_array(bam1_t *b, const char tag[2],
+                         uint8_t type, uint32_t items, void *data)
+{
+    uint8_t *s = bam_aux_get(b, tag);
+    size_t old_sz = 0, new_sz;
+    int new = 0;
+
+    if (s) { // Tag present
+        if (*s != 'B') { errno = EINVAL; return -1; }
+        old_sz = aux_type2size(s[1]);
+        if (old_sz < 1 || old_sz > 4) { errno = EINVAL; return -1; }
+        old_sz *= le_to_u32(s + 2);
+    } else {
+        if (errno == ENOENT) {  // Tag doesn't exist - add a new one
+            s = b->data + b->l_data;
+            new = 1;
+        }  else { // Invalid aux data, give up.
+            return -1;
+        }
+    }
+
+    new_sz = aux_type2size(type);
+    if (new_sz < 1 || new_sz > 4) { errno = EINVAL; return -1; }
+    if (items > INT32_MAX / new_sz) { errno = ENOMEM; return -1; }
+    new_sz *= items;
+
+    if (new || old_sz < new_sz) {
+        // Make room for new tag
+        ptrdiff_t s_offset = s - b->data;
+        if (possibly_expand_bam_data(b, (new ? 8 : 0) + new_sz - old_sz) < 0)
+            return -1;
+        s =  b->data + s_offset;
+    }
+    if (new) { // Add tag id and type
+        *s++ = tag[0];
+        *s++ = tag[1];
+        *s = 'B';
+        b->l_data += 8 + new_sz;
+    } else if (old_sz != new_sz) { // shift following data if necessary
+        memmove(s + 6 + new_sz, s + 6 + old_sz,
+                b->l_data - ((s + 6 + old_sz) - b->data));
+        b->l_data -= old_sz;
+        b->l_data += new_sz;
+    }
+
+    s[1] = type;
+    u32_to_le(items, s + 2);
+#ifdef HTS_LITTLE_ENDIAN
+    memcpy(s + 6, data, new_sz);
+    return 0;
+#else
+    return aux_to_le(type, s + 6, data, new_sz);
+#endif
+}
+
+static inline int64_t get_int_aux_val(uint8_t type, const uint8_t *s,
+                                      uint32_t idx)
+{
+    switch (type) {
+        case 'c': return le_to_i8(s + idx);
+        case 'C': return s[idx];
+        case 's': return le_to_i16(s + 2 * idx);
+        case 'S': return le_to_u16(s + 2 * idx);
+        case 'i': return le_to_i32(s + 4 * idx);
+        case 'I': return le_to_u32(s + 4 * idx);
+        default:
+            errno = EINVAL;
+            return 0;
+    }
+}
+
+int64_t bam_aux2i(const uint8_t *s)
+{
+    int type;
+    type = *s++;
+    return get_int_aux_val(type, s, 0);
+}
+
+double bam_aux2f(const uint8_t *s)
+{
+    int type;
+    type = *s++;
+    if (type == 'd') return le_to_double(s);
+    else if (type == 'f') return le_to_float(s);
+    else return get_int_aux_val(type, s, 0);
+}
+
+char bam_aux2A(const uint8_t *s)
+{
+    int type;
+    type = *s++;
+    if (type == 'A') return *(char*)s;
+    errno = EINVAL;
+    return 0;
+}
+
+char *bam_aux2Z(const uint8_t *s)
+{
+    int type;
+    type = *s++;
+    if (type == 'Z' || type == 'H') return (char*)s;
+    errno = EINVAL;
+    return 0;
+}
+
+uint32_t bam_auxB_len(const uint8_t *s)
+{
+    if (s[0] != 'B') {
+        errno = EINVAL;
+        return 0;
+    }
+    return le_to_u32(s + 2);
+}
+
+int64_t bam_auxB2i(const uint8_t *s, uint32_t idx)
+{
+    uint32_t len = bam_auxB_len(s);
+    if (idx >= len) {
+        errno = ERANGE;
+        return 0;
+    }
+    return get_int_aux_val(s[1], s + 6, idx);
+}
+
+double bam_auxB2f(const uint8_t *s, uint32_t idx)
+{
+    uint32_t len = bam_auxB_len(s);
+    if (idx >= len) {
+        errno = ERANGE;
+        return 0.0;
+    }
+    if (s[1] == 'f') return le_to_float(s + 6 + 4 * idx);
+    else return get_int_aux_val(s[1], s + 6, idx);
+}
+
+int sam_open_mode(char *mode, const char *fn, const char *format)
+{
+    // TODO Parse "bam5" etc for compression level
+    if (format == NULL) {
+        // Try to pick a format based on the filename extension
+        char extension[HTS_MAX_EXT_LEN];
+        if (find_file_extension(fn, extension) < 0) return -1;
+        return sam_open_mode(mode, fn, extension);
+    }
+    else if (strcasecmp(format, "bam") == 0) strcpy(mode, "b");
+    else if (strcasecmp(format, "cram") == 0) strcpy(mode, "c");
+    else if (strcasecmp(format, "sam") == 0) strcpy(mode, "");
+    else if (strcasecmp(format, "sam.gz") == 0) strcpy(mode, "z");
+    else if (strcasecmp(format, "fastq") == 0 ||
+             strcasecmp(format, "fq") == 0) strcpy(mode, "f");
+    else if (strcasecmp(format, "fastq.gz") == 0 ||
+             strcasecmp(format, "fq.gz") == 0) strcpy(mode, "fz");
+    else if (strcasecmp(format, "fasta") == 0 ||
+             strcasecmp(format, "fa") == 0) strcpy(mode, "F");
+    else if (strcasecmp(format, "fasta.gz") == 0 ||
+             strcasecmp(format, "fa.gz") == 0) strcpy(mode, "Fz");
+    else return -1;
+
+    return 0;
+}
+
+// A version of sam_open_mode that can handle ,key=value options.
+// The format string is allocated and returned, to be freed by the caller.
+// Prefix should be "r" or "w",
+char *sam_open_mode_opts(const char *fn,
+                         const char *mode,
+                         const char *format)
+{
+    char *mode_opts = malloc((format ? strlen(format) : 1) +
+                             (mode   ? strlen(mode)   : 1) + 12);
+    char *opts, *cp;
+    int format_len;
+
+    if (!mode_opts)
+        return NULL;
+
+    strcpy(mode_opts, mode ? mode : "r");
+    cp = mode_opts + strlen(mode_opts);
+
+    if (format == NULL) {
+        // Try to pick a format based on the filename extension
+        char extension[HTS_MAX_EXT_LEN];
+        if (find_file_extension(fn, extension) < 0) {
+            free(mode_opts);
+            return NULL;
+        }
+        if (sam_open_mode(cp, fn, extension) == 0) {
+            return mode_opts;
+        } else {
+            free(mode_opts);
+            return NULL;
+        }
+    }
+
+    if ((opts = strchr(format, ','))) {
+        format_len = opts-format;
+    } else {
+        opts="";
+        format_len = strlen(format);
+    }
+
+    if (strncmp(format, "bam", format_len) == 0) {
+        *cp++ = 'b';
+    } else if (strncmp(format, "cram", format_len) == 0) {
+        *cp++ = 'c';
+    } else if (strncmp(format, "cram2", format_len) == 0) {
+        *cp++ = 'c';
+        strcpy(cp, ",VERSION=2.1");
+        cp += 12;
+    } else if (strncmp(format, "cram3", format_len) == 0) {
+        *cp++ = 'c';
+        strcpy(cp, ",VERSION=3.0");
+        cp += 12;
+    } else if (strncmp(format, "sam", format_len) == 0) {
+        ; // format mode=""
+    } else if (strncmp(format, "sam.gz", format_len) == 0) {
+        *cp++ = 'z';
+    } else if (strncmp(format, "fastq", format_len) == 0 ||
+               strncmp(format, "fq", format_len) == 0) {
+        *cp++ = 'f';
+    } else if (strncmp(format, "fastq.gz", format_len) == 0 ||
+               strncmp(format, "fq.gz", format_len) == 0) {
+        *cp++ = 'f';
+        *cp++ = 'z';
+    } else if (strncmp(format, "fasta", format_len) == 0 ||
+               strncmp(format, "fa", format_len) == 0) {
+        *cp++ = 'F';
+    } else if (strncmp(format, "fasta.gz", format_len) == 0 ||
+               strncmp(format, "fa", format_len) == 0) {
+        *cp++ = 'F';
+        *cp++ = 'z';
+    } else {
+        free(mode_opts);
+        return NULL;
+    }
+
+    strcpy(cp, opts);
+
+    return mode_opts;
+}
+
+#define STRNCMP(a,b,n) (strncasecmp((a),(b),(n)) || strlen(a)!=(n))
+int bam_str2flag(const char *str)
+{
+    char *end, *beg = (char*) str;
+    long int flag = strtol(str, &end, 0);
+    if ( end!=str ) return flag;    // the conversion was successful
+    flag = 0;
+    while ( *str )
+    {
+        end = beg;
+        while ( *end && *end!=',' ) end++;
+        if ( !STRNCMP("PAIRED",beg,end-beg) ) flag |= BAM_FPAIRED;
+        else if ( !STRNCMP("PROPER_PAIR",beg,end-beg) ) flag |= BAM_FPROPER_PAIR;
+        else if ( !STRNCMP("UNMAP",beg,end-beg) ) flag |= BAM_FUNMAP;
+        else if ( !STRNCMP("MUNMAP",beg,end-beg) ) flag |= BAM_FMUNMAP;
+        else if ( !STRNCMP("REVERSE",beg,end-beg) ) flag |= BAM_FREVERSE;
+        else if ( !STRNCMP("MREVERSE",beg,end-beg) ) flag |= BAM_FMREVERSE;
+        else if ( !STRNCMP("READ1",beg,end-beg) ) flag |= BAM_FREAD1;
+        else if ( !STRNCMP("READ2",beg,end-beg) ) flag |= BAM_FREAD2;
+        else if ( !STRNCMP("SECONDARY",beg,end-beg) ) flag |= BAM_FSECONDARY;
+        else if ( !STRNCMP("QCFAIL",beg,end-beg) ) flag |= BAM_FQCFAIL;
+        else if ( !STRNCMP("DUP",beg,end-beg) ) flag |= BAM_FDUP;
+        else if ( !STRNCMP("SUPPLEMENTARY",beg,end-beg) ) flag |= BAM_FSUPPLEMENTARY;
+        else return -1;
+        if ( !*end ) break;
+        beg = end + 1;
+    }
+    return flag;
+}
+
+char *bam_flag2str(int flag)
+{
+    kstring_t str = {0,0,0};
+    if ( flag&BAM_FPAIRED ) ksprintf(&str,"%s%s", str.l?",":"","PAIRED");
+    if ( flag&BAM_FPROPER_PAIR ) ksprintf(&str,"%s%s", str.l?",":"","PROPER_PAIR");
+    if ( flag&BAM_FUNMAP ) ksprintf(&str,"%s%s", str.l?",":"","UNMAP");
+    if ( flag&BAM_FMUNMAP ) ksprintf(&str,"%s%s", str.l?",":"","MUNMAP");
+    if ( flag&BAM_FREVERSE ) ksprintf(&str,"%s%s", str.l?",":"","REVERSE");
+    if ( flag&BAM_FMREVERSE ) ksprintf(&str,"%s%s", str.l?",":"","MREVERSE");
+    if ( flag&BAM_FREAD1 ) ksprintf(&str,"%s%s", str.l?",":"","READ1");
+    if ( flag&BAM_FREAD2 ) ksprintf(&str,"%s%s", str.l?",":"","READ2");
+    if ( flag&BAM_FSECONDARY ) ksprintf(&str,"%s%s", str.l?",":"","SECONDARY");
+    if ( flag&BAM_FQCFAIL ) ksprintf(&str,"%s%s", str.l?",":"","QCFAIL");
+    if ( flag&BAM_FDUP ) ksprintf(&str,"%s%s", str.l?",":"","DUP");
+    if ( flag&BAM_FSUPPLEMENTARY ) ksprintf(&str,"%s%s", str.l?",":"","SUPPLEMENTARY");
+    if ( str.l == 0 ) kputsn("", 0, &str);
+    return str.s;
+}
+
+
+/**************************
+ *** Pileup and Mpileup ***
+ **************************/
+
+#if !defined(BAM_NO_PILEUP)
+
+#include <assert.h>
+
+/*******************
+ *** Memory pool ***
+ *******************/
+
+typedef struct {
+    int k, y;
+    hts_pos_t x, end;
+} cstate_t;
+
+static cstate_t g_cstate_null = { -1, 0, 0, 0 };
+
+typedef struct __linkbuf_t {
+    bam1_t b;
+    hts_pos_t beg, end;
+    cstate_t s;
+    struct __linkbuf_t *next;
+    bam_pileup_cd cd;
+} lbnode_t;
+
+typedef struct {
+    int cnt, n, max;
+    lbnode_t **buf;
+} mempool_t;
+
+static mempool_t *mp_init(void)
+{
+    mempool_t *mp;
+    mp = (mempool_t*)calloc(1, sizeof(mempool_t));
+    return mp;
+}
+static void mp_destroy(mempool_t *mp)
+{
+    int k;
+    for (k = 0; k < mp->n; ++k) {
+        free(mp->buf[k]->b.data);
+        free(mp->buf[k]);
+    }
+    free(mp->buf);
+    free(mp);
+}
+static inline lbnode_t *mp_alloc(mempool_t *mp)
+{
+    ++mp->cnt;
+    if (mp->n == 0) return (lbnode_t*)calloc(1, sizeof(lbnode_t));
+    else return mp->buf[--mp->n];
+}
+static inline void mp_free(mempool_t *mp, lbnode_t *p)
+{
+    --mp->cnt; p->next = 0; // clear lbnode_t::next here
+    if (mp->n == mp->max) {
+        mp->max = mp->max? mp->max<<1 : 256;
+        mp->buf = (lbnode_t**)realloc(mp->buf, sizeof(lbnode_t*) * mp->max);
+    }
+    mp->buf[mp->n++] = p;
+}
+
+/**********************
+ *** CIGAR resolver ***
+ **********************/
+
+/* s->k: the index of the CIGAR operator that has just been processed.
+   s->x: the reference coordinate of the start of s->k
+   s->y: the query coordinate of the start of s->k
+ */
+static inline int resolve_cigar2(bam_pileup1_t *p, hts_pos_t pos, cstate_t *s)
+{
+#define _cop(c) ((c)&BAM_CIGAR_MASK)
+#define _cln(c) ((c)>>BAM_CIGAR_SHIFT)
+
+    bam1_t *b = p->b;
+    bam1_core_t *c = &b->core;
+    uint32_t *cigar = bam_get_cigar(b);
+    int k;
+    // determine the current CIGAR operation
+    //fprintf(stderr, "%s\tpos=%ld\tend=%ld\t(%d,%ld,%d)\n", bam_get_qname(b), pos, s->end, s->k, s->x, s->y);
+    if (s->k == -1) { // never processed
+        p->qpos = 0;
+        if (c->n_cigar == 1) { // just one operation, save a loop
+          if (_cop(cigar[0]) == BAM_CMATCH || _cop(cigar[0]) == BAM_CEQUAL || _cop(cigar[0]) == BAM_CDIFF) s->k = 0, s->x = c->pos, s->y = 0;
+        } else { // find the first match or deletion
+            for (k = 0, s->x = c->pos, s->y = 0; k < c->n_cigar; ++k) {
+                int op = _cop(cigar[k]);
+                int l = _cln(cigar[k]);
+                if (op == BAM_CMATCH || op == BAM_CDEL || op == BAM_CREF_SKIP ||
+                    op == BAM_CEQUAL || op == BAM_CDIFF) break;
+                else if (op == BAM_CINS || op == BAM_CSOFT_CLIP) s->y += l;
+            }
+            assert(k < c->n_cigar);
+            s->k = k;
+        }
+    } else { // the read has been processed before
+        int op, l = _cln(cigar[s->k]);
+        if (pos - s->x >= l) { // jump to the next operation
+            assert(s->k < c->n_cigar); // otherwise a bug: this function should not be called in this case
+            op = _cop(cigar[s->k+1]);
+            if (op == BAM_CMATCH || op == BAM_CDEL || op == BAM_CREF_SKIP || op == BAM_CEQUAL || op == BAM_CDIFF) { // jump to the next without a loop
+              if (_cop(cigar[s->k]) == BAM_CMATCH|| _cop(cigar[s->k]) == BAM_CEQUAL || _cop(cigar[s->k]) == BAM_CDIFF) s->y += l;
+                s->x += l;
+                ++s->k;
+            } else { // find the next M/D/N/=/X
+              if (_cop(cigar[s->k]) == BAM_CMATCH|| _cop(cigar[s->k]) == BAM_CEQUAL || _cop(cigar[s->k]) == BAM_CDIFF) s->y += l;
+                s->x += l;
+                for (k = s->k + 1; k < c->n_cigar; ++k) {
+                    op = _cop(cigar[k]), l = _cln(cigar[k]);
+                    if (op == BAM_CMATCH || op == BAM_CDEL || op == BAM_CREF_SKIP || op == BAM_CEQUAL || op == BAM_CDIFF) break;
+                    else if (op == BAM_CINS || op == BAM_CSOFT_CLIP) s->y += l;
+                }
+                s->k = k;
+            }
+            assert(s->k < c->n_cigar); // otherwise a bug
+        } // else, do nothing
+    }
+    { // collect pileup information
+        int op, l;
+        op = _cop(cigar[s->k]); l = _cln(cigar[s->k]);
+        p->is_del = p->indel = p->is_refskip = 0;
+        if (s->x + l - 1 == pos && s->k + 1 < c->n_cigar) { // peek the next operation
+            int op2 = _cop(cigar[s->k+1]);
+            int l2 = _cln(cigar[s->k+1]);
+            if (op2 == BAM_CDEL && op != BAM_CDEL) {
+                // At start of a new deletion, merge e.g. 1D2D to 3D.
+                // Within a deletion (the 2D in 1D2D) we keep p->indel=0
+                // and rely on is_del=1 as we would for 3D.
+                p->indel = -(int)l2;
+                for (k = s->k+2; k < c->n_cigar; ++k) {
+                    op2 = _cop(cigar[k]); l2 = _cln(cigar[k]);
+                    if (op2 == BAM_CDEL) p->indel -= l2;
+                    else break;
+                }
+            } else if (op2 == BAM_CINS) {
+                p->indel = l2;
+                for (k = s->k+2; k < c->n_cigar; ++k) {
+                    op2 = _cop(cigar[k]); l2 = _cln(cigar[k]);
+                    if (op2 == BAM_CINS) p->indel += l2;
+                    else if (op2 != BAM_CPAD) break;
+                }
+            } else if (op2 == BAM_CPAD && s->k + 2 < c->n_cigar) {
+                int l3 = 0;
+                for (k = s->k + 2; k < c->n_cigar; ++k) {
+                    op2 = _cop(cigar[k]); l2 = _cln(cigar[k]);
+                    if (op2 == BAM_CINS) l3 += l2;
+                    else if (op2 == BAM_CDEL || op2 == BAM_CMATCH || op2 == BAM_CREF_SKIP || op2 == BAM_CEQUAL || op2 == BAM_CDIFF) break;
+                }
+                if (l3 > 0) p->indel = l3;
+            }
+        }
+        if (op == BAM_CMATCH || op == BAM_CEQUAL || op == BAM_CDIFF) {
+            p->qpos = s->y + (pos - s->x);
+        } else if (op == BAM_CDEL || op == BAM_CREF_SKIP) {
+            p->is_del = 1; p->qpos = s->y; // FIXME: distinguish D and N!!!!!
+            p->is_refskip = (op == BAM_CREF_SKIP);
+        } // cannot be other operations; otherwise a bug
+        p->is_head = (pos == c->pos); p->is_tail = (pos == s->end);
+    }
+    p->cigar_ind = s->k;
+    return 1;
+}
+
+/*******************************
+ *** Expansion of insertions ***
+ *******************************/
+
+/*
+ * Fills out the kstring with the padded insertion sequence for the current
+ * location in 'p'.  If this is not an insertion site, the string is blank.
+ *
+ * This variant handles base modifications, but only when "m" is non-NULL.
+ *
+ * Returns the number of inserted base on success, with string length being
+ *        accessable via ins->l;
+ *        -1 on failure.
+ */
+int bam_plp_insertion_mod(const bam_pileup1_t *p,
+                          hts_base_mod_state *m,
+                          kstring_t *ins, int *del_len) {
+    int j, k, indel, nb = 0;
+    uint32_t *cigar;
+
+    if (p->indel <= 0) {
+        if (ks_resize(ins, 1) < 0)
+            return -1;
+        ins->l = 0;
+        ins->s[0] = '\0';
+        return 0;
+    }
+
+    if (del_len)
+        *del_len = 0;
+
+    // Measure indel length including pads
+    indel = 0;
+    k = p->cigar_ind+1;
+    cigar = bam_get_cigar(p->b);
+    while (k < p->b->core.n_cigar) {
+        switch (cigar[k] & BAM_CIGAR_MASK) {
+        case BAM_CPAD:
+        case BAM_CINS:
+            indel += (cigar[k] >> BAM_CIGAR_SHIFT);
+            break;
+        default:
+            k = p->b->core.n_cigar;
+            break;
+        }
+        k++;
+    }
+    nb = ins->l = indel;
+
+    // Produce sequence
+    if (ks_resize(ins, indel+1) < 0)
+        return -1;
+    indel = 0;
+    k = p->cigar_ind+1;
+    j = 1;
+    while (k < p->b->core.n_cigar) {
+        int l, c;
+        switch (cigar[k] & BAM_CIGAR_MASK) {
+        case BAM_CPAD:
+            for (l = 0; l < (cigar[k]>>BAM_CIGAR_SHIFT); l++)
+                ins->s[indel++] = '*';
+            break;
+        case BAM_CINS:
+            for (l = 0; l < (cigar[k]>>BAM_CIGAR_SHIFT); l++, j++) {
+                c = p->qpos + j - p->is_del < p->b->core.l_qseq
+                    ? seq_nt16_str[bam_seqi(bam_get_seq(p->b),
+                                            p->qpos + j - p->is_del)]
+                    : 'N';
+                ins->s[indel++] = c;
+                int nm;
+                hts_base_mod mod[256];
+                if (m && (nm = bam_mods_at_qpos(p->b, p->qpos + j - p->is_del,
+                                                m, mod, 256)) > 0) {
+                    int o_indel = indel;
+                    if (ks_resize(ins, ins->l + nm*16+3) < 0)
+                        return -1;
+                    ins->s[indel++] = '[';
+                    int j;
+                    for (j = 0; j < nm; j++) {
+                        char qual[20];
+                        if (mod[j].qual >= 0)
+                            snprintf(qual, sizeof(qual), "%d", mod[j].qual);
+                        else
+                            *qual=0;
+                        if (mod[j].modified_base < 0)
+                            // ChEBI
+                            indel += snprintf(&ins->s[indel], ins->m - indel,
+                                              "%c(%d)%s",
+                                              "+-"[mod[j].strand],
+                                              -mod[j].modified_base,
+                                              qual);
+                        else
+                            indel += snprintf(&ins->s[indel], ins->m - indel,
+                                              "%c%c%s",
+                                              "+-"[mod[j].strand],
+                                              mod[j].modified_base,
+                                              qual);
+                    }
+                    ins->s[indel++] = ']';
+                    ins->l += indel - o_indel; // grow by amount we used
+                }
+            }
+            break;
+        case BAM_CDEL:
+            // eg cigar 1M2I1D gives mpileup output in T+2AA-1C style
+            if (del_len)
+                *del_len = cigar[k]>>BAM_CIGAR_SHIFT;
+            // fall through
+        default:
+            k = p->b->core.n_cigar;
+            break;
+        }
+        k++;
+    }
+    ins->s[indel] = '\0';
+    ins->l = indel; // string length
+
+    return nb;      // base length
+}
+
+/*
+ * Fills out the kstring with the padded insertion sequence for the current
+ * location in 'p'.  If this is not an insertion site, the string is blank.
+ *
+ * This is the original interface with no capability for reporting base
+ * modifications.
+ *
+ * Returns the length of insertion string on success;
+ *        -1 on failure.
+ */
+int bam_plp_insertion(const bam_pileup1_t *p, kstring_t *ins, int *del_len) {
+    return bam_plp_insertion_mod(p, NULL, ins, del_len);
+}
+
+/***********************
+ *** Pileup iterator ***
+ ***********************/
+
+// Dictionary of overlapping reads
+KHASH_MAP_INIT_STR(olap_hash, lbnode_t *)
+typedef khash_t(olap_hash) olap_hash_t;
+
+struct bam_plp_s {
+    mempool_t *mp;
+    lbnode_t *head, *tail;
+    int32_t tid, max_tid;
+    hts_pos_t pos, max_pos;
+    int is_eof, max_plp, error, maxcnt;
+    uint64_t id;
+    bam_pileup1_t *plp;
+    // for the "auto" interface only
+    bam1_t *b;
+    bam_plp_auto_f func;
+    void *data;
+    olap_hash_t *overlaps;
+
+    // For notification of creation and destruction events
+    // and associated client-owned pointer.
+    int (*plp_construct)(void *data, const bam1_t *b, bam_pileup_cd *cd);
+    int (*plp_destruct )(void *data, const bam1_t *b, bam_pileup_cd *cd);
+};
+
+bam_plp_t bam_plp_init(bam_plp_auto_f func, void *data)
+{
+    bam_plp_t iter;
+    iter = (bam_plp_t)calloc(1, sizeof(struct bam_plp_s));
+    iter->mp = mp_init();
+    iter->head = iter->tail = mp_alloc(iter->mp);
+    iter->max_tid = iter->max_pos = -1;
+    iter->maxcnt = 8000;
+    if (func) {
+        iter->func = func;
+        iter->data = data;
+        iter->b = bam_init1();
+    }
+    return iter;
+}
+
+int bam_plp_init_overlaps(bam_plp_t iter)
+{
+    iter->overlaps = kh_init(olap_hash);  // hash for tweaking quality of bases in overlapping reads
+    return iter->overlaps ? 0 : -1;
+}
+
+void bam_plp_destroy(bam_plp_t iter)
+{
+    lbnode_t *p, *pnext;
+    if ( iter->overlaps ) kh_destroy(olap_hash, iter->overlaps);
+    for (p = iter->head; p != NULL; p = pnext) {
+        if (iter->plp_destruct && p != iter->tail)
+            iter->plp_destruct(iter->data, &p->b, &p->cd);
+        pnext = p->next;
+        mp_free(iter->mp, p);
+    }
+    mp_destroy(iter->mp);
+    if (iter->b) bam_destroy1(iter->b);
+    free(iter->plp);
+    free(iter);
+}
+
+void bam_plp_constructor(bam_plp_t plp,
+                         int (*func)(void *data, const bam1_t *b, bam_pileup_cd *cd)) {
+    plp->plp_construct = func;
+}
+
+void bam_plp_destructor(bam_plp_t plp,
+                        int (*func)(void *data, const bam1_t *b, bam_pileup_cd *cd)) {
+    plp->plp_destruct = func;
+}
+
+//---------------------------------
+//---  Tweak overlapping reads
+//---------------------------------
+
+/**
+ *  cigar_iref2iseq_set()  - find the first CMATCH setting the ref and the read index
+ *  cigar_iref2iseq_next() - get the next CMATCH base
+ *  @cigar:       pointer to current cigar block (rw)
+ *  @cigar_max:   pointer just beyond the last cigar block
+ *  @icig:        position within the current cigar block (rw)
+ *  @iseq:        position in the sequence (rw)
+ *  @iref:        position with respect to the beginning of the read (iref_pos - b->core.pos) (rw)
+ *
+ *  Returns BAM_CMATCH, -1 when there is no more cigar to process or the requested position is not covered,
+ *  or -2 on error.
+ */
+static inline int cigar_iref2iseq_set(const uint32_t **cigar,
+                                      const uint32_t *cigar_max,
+                                      hts_pos_t *icig,
+                                      hts_pos_t *iseq,
+                                      hts_pos_t *iref)
+{
+    hts_pos_t pos = *iref;
+    if ( pos < 0 ) return -1;
+    *icig = 0;
+    *iseq = 0;
+    *iref = 0;
+    while ( *cigar<cigar_max )
+    {
+        int cig  = (**cigar) & BAM_CIGAR_MASK;
+        int ncig = (**cigar) >> BAM_CIGAR_SHIFT;
+
+        if ( cig==BAM_CSOFT_CLIP ) { (*cigar)++; *iseq += ncig; *icig = 0; continue; }
+        if ( cig==BAM_CHARD_CLIP || cig==BAM_CPAD ) { (*cigar)++; *icig = 0; continue; }
+        if ( cig==BAM_CMATCH || cig==BAM_CEQUAL || cig==BAM_CDIFF )
+        {
+            pos -= ncig;
+            if ( pos < 0 ) { *icig = ncig + pos; *iseq += *icig; *iref += *icig; return BAM_CMATCH; }
+            (*cigar)++; *iseq += ncig; *icig = 0; *iref += ncig;
+            continue;
+        }
+        if ( cig==BAM_CINS ) { (*cigar)++; *iseq += ncig; *icig = 0; continue; }
+        if ( cig==BAM_CDEL || cig==BAM_CREF_SKIP )
+        {
+            pos -= ncig;
+            if ( pos<0 ) pos = 0;
+            (*cigar)++; *icig = 0; *iref += ncig;
+            continue;
+        }
+        hts_log_error("Unexpected cigar %d", cig);
+        return -2;
+    }
+    *iseq = -1;
+    return -1;
+}
+static inline int cigar_iref2iseq_next(const uint32_t **cigar,
+                                       const uint32_t *cigar_max,
+                                       hts_pos_t *icig,
+                                       hts_pos_t *iseq,
+                                       hts_pos_t *iref)
+{
+    while ( *cigar < cigar_max )
+    {
+        int cig  = (**cigar) & BAM_CIGAR_MASK;
+        int ncig = (**cigar) >> BAM_CIGAR_SHIFT;
+
+        if ( cig==BAM_CMATCH || cig==BAM_CEQUAL || cig==BAM_CDIFF )
+        {
+            if ( *icig >= ncig - 1 ) { *icig = -1;  (*cigar)++; continue; }
+            (*iseq)++; (*icig)++; (*iref)++;
+            return BAM_CMATCH;
+        }
+        if ( cig==BAM_CDEL || cig==BAM_CREF_SKIP ) { (*cigar)++; (*iref) += ncig; *icig = -1; continue; }
+        if ( cig==BAM_CINS ) { (*cigar)++; *iseq += ncig; *icig = -1; continue; }
+        if ( cig==BAM_CSOFT_CLIP ) { (*cigar)++; *iseq += ncig; *icig = -1; continue; }
+        if ( cig==BAM_CHARD_CLIP || cig==BAM_CPAD ) { (*cigar)++; *icig = -1; continue; }
+        hts_log_error("Unexpected cigar %d", cig);
+        return -2;
+    }
+    *iseq = -1;
+    *iref = -1;
+    return -1;
+}
+
+// Given overlapping read 'a' (left) and 'b' (right) on the same
+// template, adjust quality values to zero for either a or b.
+// Note versions 1.12 and earlier always removed quality from 'b' for
+// matching bases.  Now we select a or b semi-randomly based on name hash.
+// Returns 0 on success,
+//        -1 on failure
+static int tweak_overlap_quality(bam1_t *a, bam1_t *b)
+{
+    const uint32_t *a_cigar = bam_get_cigar(a),
+        *a_cigar_max = a_cigar + a->core.n_cigar;
+    const uint32_t *b_cigar = bam_get_cigar(b),
+        *b_cigar_max = b_cigar + b->core.n_cigar;
+    hts_pos_t a_icig = 0, a_iseq = 0;
+    hts_pos_t b_icig = 0, b_iseq = 0;
+    uint8_t *a_qual = bam_get_qual(a), *b_qual = bam_get_qual(b);
+    uint8_t *a_seq  = bam_get_seq(a), *b_seq = bam_get_seq(b);
+
+    hts_pos_t iref   = b->core.pos;
+    hts_pos_t a_iref = iref - a->core.pos;
+    hts_pos_t b_iref = iref - b->core.pos;
+
+    int a_ret = cigar_iref2iseq_set(&a_cigar, a_cigar_max,
+                                    &a_icig, &a_iseq, &a_iref);
+    if ( a_ret<0 )
+        // no overlap or error
+        return a_ret<-1 ? -1:0;
+
+    int b_ret = cigar_iref2iseq_set(&b_cigar, b_cigar_max,
+                                    &b_icig, &b_iseq, &b_iref);
+    if ( b_ret<0 )
+        // no overlap or error
+        return b_ret<-1 ? -1:0;
+
+    // Determine which seq is the one getting modified qualities.
+    uint8_t amul, bmul;
+    if (__ac_Wang_hash(__ac_X31_hash_string(bam_get_qname(a))) & 1) {
+        amul = 1;
+        bmul = 0;
+    } else {
+        amul = 0;
+        bmul = 1;
+    }
+
+    // Loop over the overlapping region nulling qualities in either
+    // seq a or b.
+    int err = 0;
+    while ( 1 ) {
+        // Step to next matching reference position in a and b
+        while ( a_ret >= 0 && a_iref>=0 && a_iref < iref - a->core.pos )
+            a_ret = cigar_iref2iseq_next(&a_cigar, a_cigar_max,
+                                         &a_icig, &a_iseq, &a_iref);
+        if ( a_ret<0 ) { // done
+            err = a_ret<-1?-1:0;
+            break;
+        }
+
+        while ( b_ret >= 0 && b_iref>=0 && b_iref < iref - b->core.pos )
+            b_ret = cigar_iref2iseq_next(&b_cigar, b_cigar_max, &b_icig,
+                                         &b_iseq, &b_iref);
+        if ( b_ret<0 ) { // done
+            err = b_ret<-1?-1:0;
+            break;
+        }
+
+        if ( iref < a_iref + a->core.pos )
+            iref = a_iref + a->core.pos;
+
+        if ( iref < b_iref + b->core.pos )
+            iref = b_iref + b->core.pos;
+
+        iref++;
+
+        // If A or B has a deletion then we catch up the other to this point.
+        // We also amend quality values using the same rules for mismatch.
+        if (a_iref+a->core.pos != b_iref+b->core.pos) {
+            if (a_iref+a->core.pos < b_iref+b->core.pos
+                && b_cigar > bam_get_cigar(b)
+                && bam_cigar_op(b_cigar[-1]) == BAM_CDEL) {
+                // Del in B means it's moved on further than A
+                do {
+                    a_qual[a_iseq] = amul
+                        ? a_qual[a_iseq]*0.8
+                        : 0;
+                    a_ret = cigar_iref2iseq_next(&a_cigar, a_cigar_max,
+                                                 &a_icig, &a_iseq, &a_iref);
+                    if (a_ret < 0)
+                        return -(a_ret<-1); // 0 or -1
+                } while (a_iref + a->core.pos < b_iref+b->core.pos);
+            } else if (a_cigar > bam_get_cigar(a)
+                       && bam_cigar_op(a_cigar[-1]) == BAM_CDEL) {
+                // Del in A means it's moved on further than B
+                do {
+                    b_qual[b_iseq] = bmul
+                        ? b_qual[b_iseq]*0.8
+                        : 0;
+                    b_ret = cigar_iref2iseq_next(&b_cigar, b_cigar_max,
+                                                 &b_icig, &b_iseq, &b_iref);
+                    if (b_ret < 0)
+                        return -(b_ret<-1); // 0 or -1
+                } while (b_iref + b->core.pos < a_iref+a->core.pos);
+            } else {
+                // Anything else, eg ref-skip, we don't support here
+                continue;
+            }
+        }
+
+        // fprintf(stderr, "a_cig=%ld,%ld b_cig=%ld,%ld iref=%ld "
+        //         "a_iref=%ld b_iref=%ld a_iseq=%ld b_iseq=%ld\n",
+        //         a_cigar-bam_get_cigar(a), a_icig,
+        //         b_cigar-bam_get_cigar(b), b_icig,
+        //         iref, a_iref+a->core.pos+1, b_iref+b->core.pos+1,
+        //         a_iseq, b_iseq);
+
+        if (a_iseq > a->core.l_qseq || b_iseq > b->core.l_qseq)
+            // Fell off end of sequence, bad CIGAR?
+            return -1;
+
+        // We're finally at the same ref base in both a and b.
+        // Check if the bases match (confident) or mismatch
+        // (not so confident).
+        if ( bam_seqi(a_seq,a_iseq) == bam_seqi(b_seq,b_iseq) ) {
+            // We are very confident about this base.  Use sum of quals
+            int qual = a_qual[a_iseq] + b_qual[b_iseq];
+            a_qual[a_iseq] = amul * (qual>200 ? 200 : qual);
+            b_qual[b_iseq] = bmul * (qual>200 ? 200 : qual);;
+        } else {
+            // Not so confident about anymore given the mismatch.
+            // Reduce qual for lowest quality base.
+            if ( a_qual[a_iseq] > b_qual[b_iseq] ) {
+                // A highest qual base; keep
+                a_qual[a_iseq] = 0.8 * a_qual[a_iseq];
+                b_qual[b_iseq] = 0;
+            } else if (a_qual[a_iseq] < b_qual[b_iseq] ) {
+                // B highest qual base; keep
+                b_qual[b_iseq] = 0.8 * b_qual[b_iseq];
+                a_qual[a_iseq] = 0;
+            } else {
+                // Both equal, so pick randomly
+                a_qual[a_iseq] = amul * 0.8 * a_qual[a_iseq];
+                b_qual[b_iseq] = bmul * 0.8 * b_qual[b_iseq];
+            }
+        }
+    }
+
+    return err;
+}
+
+// Fix overlapping reads. Simple soft-clipping did not give good results.
+// Lowering qualities of unwanted bases is more selective and works better.
+//
+// Returns 0 on success, -1 on failure
+static int overlap_push(bam_plp_t iter, lbnode_t *node)
+{
+    if ( !iter->overlaps ) return 0;
+
+    // mapped mates and paired reads only
+    if ( node->b.core.flag&BAM_FMUNMAP || !(node->b.core.flag&BAM_FPROPER_PAIR) ) return 0;
+
+    // no overlap possible, unless some wild cigar
+    if ( (node->b.core.mtid >= 0 && node->b.core.tid != node->b.core.mtid)
+         || (llabs(node->b.core.isize) >= 2*node->b.core.l_qseq
+         && node->b.core.mpos >= node->end) // for those wild cigars
+       ) return 0;
+
+    khiter_t kitr = kh_get(olap_hash, iter->overlaps, bam_get_qname(&node->b));
+    if ( kitr==kh_end(iter->overlaps) )
+    {
+        // Only add reads where the mate is still to arrive
+        if (node->b.core.mpos >= node->b.core.pos ||
+            ((node->b.core.flag & BAM_FPAIRED) && node->b.core.mpos == -1)) {
+            int ret;
+            kitr = kh_put(olap_hash, iter->overlaps, bam_get_qname(&node->b), &ret);
+            if (ret < 0) return -1;
+            kh_value(iter->overlaps, kitr) = node;
+        }
+    }
+    else
+    {
+        lbnode_t *a = kh_value(iter->overlaps, kitr);
+        int err = tweak_overlap_quality(&a->b, &node->b);
+        kh_del(olap_hash, iter->overlaps, kitr);
+        assert(a->end-1 == a->s.end);
+        return err;
+    }
+    return 0;
+}
+
+static void overlap_remove(bam_plp_t iter, const bam1_t *b)
+{
+    if ( !iter->overlaps ) return;
+
+    khiter_t kitr;
+    if ( b )
+    {
+        kitr = kh_get(olap_hash, iter->overlaps, bam_get_qname(b));
+        if ( kitr!=kh_end(iter->overlaps) )
+            kh_del(olap_hash, iter->overlaps, kitr);
+    }
+    else
+    {
+        // remove all
+        for (kitr = kh_begin(iter->overlaps); kitr<kh_end(iter->overlaps); kitr++)
+            if ( kh_exist(iter->overlaps, kitr) ) kh_del(olap_hash, iter->overlaps, kitr);
+    }
+}
+
+
+
+// Prepares next pileup position in bam records collected by bam_plp_auto -> user func -> bam_plp_push. Returns
+// pointer to the piled records if next position is ready or NULL if there is not enough records in the
+// buffer yet (the current position is still the maximum position across all buffered reads).
+const bam_pileup1_t *bam_plp64_next(bam_plp_t iter, int *_tid, hts_pos_t *_pos, int *_n_plp)
+{
+    if (iter->error) { *_n_plp = -1; return NULL; }
+    *_n_plp = 0;
+    if (iter->is_eof && iter->head == iter->tail) return NULL;
+    while (iter->is_eof || iter->max_tid > iter->tid || (iter->max_tid == iter->tid && iter->max_pos > iter->pos)) {
+        int n_plp = 0;
+        // write iter->plp at iter->pos
+        lbnode_t **pptr = &iter->head;
+        while (*pptr != iter->tail) {
+            lbnode_t *p = *pptr;
+            if (p->b.core.tid < iter->tid || (p->b.core.tid == iter->tid && p->end <= iter->pos)) { // then remove
+                overlap_remove(iter, &p->b);
+                if (iter->plp_destruct)
+                    iter->plp_destruct(iter->data, &p->b, &p->cd);
+                *pptr = p->next; mp_free(iter->mp, p);
+            }
+            else {
+                if (p->b.core.tid == iter->tid && p->beg <= iter->pos) { // here: p->end > pos; then add to pileup
+                    if (n_plp == iter->max_plp) { // then double the capacity
+                        iter->max_plp = iter->max_plp? iter->max_plp<<1 : 256;
+                        iter->plp = (bam_pileup1_t*)realloc(iter->plp, sizeof(bam_pileup1_t) * iter->max_plp);
+                    }
+                    iter->plp[n_plp].b = &p->b;
+                    iter->plp[n_plp].cd = p->cd;
+                    if (resolve_cigar2(iter->plp + n_plp, iter->pos, &p->s)) ++n_plp; // actually always true...
+                }
+                pptr = &(*pptr)->next;
+            }
+        }
+        *_n_plp = n_plp; *_tid = iter->tid; *_pos = iter->pos;
+        // update iter->tid and iter->pos
+        if (iter->head != iter->tail) {
+            if (iter->tid > iter->head->b.core.tid) {
+                hts_log_error("Unsorted input. Pileup aborts");
+                iter->error = 1;
+                *_n_plp = -1;
+                return NULL;
+            }
+        }
+        if (iter->tid < iter->head->b.core.tid) { // come to a new reference sequence
+            iter->tid = iter->head->b.core.tid; iter->pos = iter->head->beg; // jump to the next reference
+        } else if (iter->pos < iter->head->beg) { // here: tid == head->b.core.tid
+            iter->pos = iter->head->beg; // jump to the next position
+        } else ++iter->pos; // scan contiguously
+        // return
+        if (n_plp) return iter->plp;
+        if (iter->is_eof && iter->head == iter->tail) break;
+    }
+    return NULL;
+}
+
+const bam_pileup1_t *bam_plp_next(bam_plp_t iter, int *_tid, int *_pos, int *_n_plp)
+{
+    hts_pos_t pos64 = 0;
+    const bam_pileup1_t *p = bam_plp64_next(iter, _tid, &pos64, _n_plp);
+    if (pos64 < INT_MAX) {
+        *_pos = pos64;
+    } else {
+        hts_log_error("Position %"PRId64" too large", pos64);
+        *_pos = INT_MAX;
+        iter->error = 1;
+        *_n_plp = -1;
+        return NULL;
+    }
+    return p;
+}
+
+int bam_plp_push(bam_plp_t iter, const bam1_t *b)
+{
+    if (iter->error) return -1;
+    if (b) {
+        if (b->core.tid < 0) { overlap_remove(iter, b); return 0; }
+        // Skip only unmapped reads here, any additional filtering must be done in iter->func
+        if (b->core.flag & BAM_FUNMAP) { overlap_remove(iter, b); return 0; }
+        if (iter->tid == b->core.tid && iter->pos == b->core.pos && iter->mp->cnt > iter->maxcnt)
+        {
+            overlap_remove(iter, b);
+            return 0;
+        }
+        if (bam_copy1(&iter->tail->b, b) == NULL)
+            return -1;
+        iter->tail->b.id = iter->id++;
+        iter->tail->beg = b->core.pos;
+        // Use raw rlen rather than bam_endpos() which adjusts rlen=0 to rlen=1
+        iter->tail->end = b->core.pos + bam_cigar2rlen(b->core.n_cigar, bam_get_cigar(b));
+        iter->tail->s = g_cstate_null; iter->tail->s.end = iter->tail->end - 1; // initialize cstate_t
+        if (b->core.tid < iter->max_tid) {
+            hts_log_error("The input is not sorted (chromosomes out of order)");
+            iter->error = 1;
+            return -1;
+        }
+        if ((b->core.tid == iter->max_tid) && (iter->tail->beg < iter->max_pos)) {
+            hts_log_error("The input is not sorted (reads out of order)");
+            iter->error = 1;
+            return -1;
+        }
+        iter->max_tid = b->core.tid; iter->max_pos = iter->tail->beg;
+        if (iter->tail->end > iter->pos || iter->tail->b.core.tid > iter->tid) {
+            lbnode_t *next = mp_alloc(iter->mp);
+            if (!next) {
+                iter->error = 1;
+                return -1;
+            }
+            if (iter->plp_construct) {
+                if (iter->plp_construct(iter->data, &iter->tail->b,
+                                        &iter->tail->cd) < 0) {
+                    mp_free(iter->mp, next);
+                    iter->error = 1;
+                    return -1;
+                }
+            }
+            if (overlap_push(iter, iter->tail) < 0) {
+                mp_free(iter->mp, next);
+                iter->error = 1;
+                return -1;
+            }
+            iter->tail->next = next;
+            iter->tail = iter->tail->next;
+        }
+    } else iter->is_eof = 1;
+    return 0;
+}
+
+const bam_pileup1_t *bam_plp64_auto(bam_plp_t iter, int *_tid, hts_pos_t *_pos, int *_n_plp)
+{
+    const bam_pileup1_t *plp;
+    if (iter->func == 0 || iter->error) { *_n_plp = -1; return 0; }
+    if ((plp = bam_plp64_next(iter, _tid, _pos, _n_plp)) != 0) return plp;
+    else { // no pileup line can be obtained; read alignments
+        *_n_plp = 0;
+        if (iter->is_eof) return 0;
+        int ret;
+        while ( (ret=iter->func(iter->data, iter->b)) >= 0) {
+            if (bam_plp_push(iter, iter->b) < 0) {
+                *_n_plp = -1;
+                return 0;
+            }
+            if ((plp = bam_plp64_next(iter, _tid, _pos, _n_plp)) != 0) return plp;
+            // otherwise no pileup line can be returned; read the next alignment.
+        }
+        if ( ret < -1 ) { iter->error = ret; *_n_plp = -1; return 0; }
+        if (bam_plp_push(iter, 0) < 0) {
+            *_n_plp = -1;
+            return 0;
+        }
+        if ((plp = bam_plp64_next(iter, _tid, _pos, _n_plp)) != 0) return plp;
+        return 0;
+    }
+}
+
+const bam_pileup1_t *bam_plp_auto(bam_plp_t iter, int *_tid, int *_pos, int *_n_plp)
+{
+    hts_pos_t pos64 = 0;
+    const bam_pileup1_t *p = bam_plp64_auto(iter, _tid, &pos64, _n_plp);
+    if (pos64 < INT_MAX) {
+        *_pos = pos64;
+    } else {
+        hts_log_error("Position %"PRId64" too large", pos64);
+        *_pos = INT_MAX;
+        iter->error = 1;
+        *_n_plp = -1;
+        return NULL;
+    }
+    return p;
+}
+
+void bam_plp_reset(bam_plp_t iter)
+{
+    overlap_remove(iter, NULL);
+    iter->max_tid = iter->max_pos = -1;
+    iter->tid = iter->pos = 0;
+    iter->is_eof = 0;
+    while (iter->head != iter->tail) {
+        lbnode_t *p = iter->head;
+        iter->head = p->next;
+        mp_free(iter->mp, p);
+    }
+}
+
+void bam_plp_set_maxcnt(bam_plp_t iter, int maxcnt)
+{
+    iter->maxcnt = maxcnt;
+}
+
+/************************
+ *** Mpileup iterator ***
+ ************************/
+
+struct bam_mplp_s {
+    int n;
+    int32_t min_tid, *tid;
+    hts_pos_t min_pos, *pos;
+    bam_plp_t *iter;
+    int *n_plp;
+    const bam_pileup1_t **plp;
+};
+
+bam_mplp_t bam_mplp_init(int n, bam_plp_auto_f func, void **data)
+{
+    int i;
+    bam_mplp_t iter;
+    iter = (bam_mplp_t)calloc(1, sizeof(struct bam_mplp_s));
+    iter->pos = (hts_pos_t*)calloc(n, sizeof(hts_pos_t));
+    iter->tid = (int32_t*)calloc(n, sizeof(int32_t));
+    iter->n_plp = (int*)calloc(n, sizeof(int));
+    iter->plp = (const bam_pileup1_t**)calloc(n, sizeof(bam_pileup1_t*));
+    iter->iter = (bam_plp_t*)calloc(n, sizeof(bam_plp_t));
+    iter->n = n;
+    iter->min_pos = HTS_POS_MAX;
+    iter->min_tid = (uint32_t)-1;
+    for (i = 0; i < n; ++i) {
+        iter->iter[i] = bam_plp_init(func, data[i]);
+        iter->pos[i] = iter->min_pos;
+        iter->tid[i] = iter->min_tid;
+    }
+    return iter;
+}
+
+int bam_mplp_init_overlaps(bam_mplp_t iter)
+{
+    int i, r = 0;
+    for (i = 0; i < iter->n; ++i)
+        r |= bam_plp_init_overlaps(iter->iter[i]);
+    return r == 0 ? 0 : -1;
+}
+
+void bam_mplp_set_maxcnt(bam_mplp_t iter, int maxcnt)
+{
+    int i;
+    for (i = 0; i < iter->n; ++i)
+        iter->iter[i]->maxcnt = maxcnt;
+}
+
+void bam_mplp_destroy(bam_mplp_t iter)
+{
+    int i;
+    for (i = 0; i < iter->n; ++i) bam_plp_destroy(iter->iter[i]);
+    free(iter->iter); free(iter->pos); free(iter->tid);
+    free(iter->n_plp); free(iter->plp);
+    free(iter);
+}
+
+int bam_mplp64_auto(bam_mplp_t iter, int *_tid, hts_pos_t *_pos, int *n_plp, const bam_pileup1_t **plp)
+{
+    int i, ret = 0;
+    hts_pos_t new_min_pos = HTS_POS_MAX;
+    uint32_t new_min_tid = (uint32_t)-1;
+    for (i = 0; i < iter->n; ++i) {
+        if (iter->pos[i] == iter->min_pos && iter->tid[i] == iter->min_tid) {
+            int tid;
+            hts_pos_t pos;
+            iter->plp[i] = bam_plp64_auto(iter->iter[i], &tid, &pos, &iter->n_plp[i]);
+            if ( iter->iter[i]->error ) return -1;
+            if (iter->plp[i]) {
+                iter->tid[i] = tid;
+                iter->pos[i] = pos;
+            } else {
+                iter->tid[i] = 0;
+                iter->pos[i] = 0;
+            }
+        }
+        if (iter->plp[i]) {
+            if (iter->tid[i] < new_min_tid) {
+                new_min_tid = iter->tid[i];
+                new_min_pos = iter->pos[i];
+            } else if (iter->tid[i] == new_min_tid && iter->pos[i] < new_min_pos) {
+                new_min_pos = iter->pos[i];
+            }
+        }
+    }
+    iter->min_pos = new_min_pos;
+    iter->min_tid = new_min_tid;
+    if (new_min_pos == HTS_POS_MAX) return 0;
+    *_tid = new_min_tid; *_pos = new_min_pos;
+    for (i = 0; i < iter->n; ++i) {
+        if (iter->pos[i] == iter->min_pos && iter->tid[i] == iter->min_tid) {
+            n_plp[i] = iter->n_plp[i], plp[i] = iter->plp[i];
+            ++ret;
+        } else n_plp[i] = 0, plp[i] = 0;
+    }
+    return ret;
+}
+
+int bam_mplp_auto(bam_mplp_t iter, int *_tid, int *_pos, int *n_plp, const bam_pileup1_t **plp)
+{
+    hts_pos_t pos64 = 0;
+    int ret = bam_mplp64_auto(iter, _tid, &pos64, n_plp, plp);
+    if (ret >= 0) {
+        if (pos64 < INT_MAX) {
+            *_pos = pos64;
+        } else {
+            hts_log_error("Position %"PRId64" too large", pos64);
+            *_pos = INT_MAX;
+            return -1;
+        }
+    }
+    return ret;
+}
+
+void bam_mplp_reset(bam_mplp_t iter)
+{
+    int i;
+    iter->min_pos = HTS_POS_MAX;
+    iter->min_tid = (uint32_t)-1;
+    for (i = 0; i < iter->n; ++i) {
+        bam_plp_reset(iter->iter[i]);
+        iter->pos[i] = HTS_POS_MAX;
+        iter->tid[i] = (uint32_t)-1;
+        iter->n_plp[i] = 0;
+        iter->plp[i] = NULL;
+    }
+}
+
+void bam_mplp_constructor(bam_mplp_t iter,
+                          int (*func)(void *arg, const bam1_t *b, bam_pileup_cd *cd)) {
+    int i;
+    for (i = 0; i < iter->n; ++i)
+        bam_plp_constructor(iter->iter[i], func);
+}
+
+void bam_mplp_destructor(bam_mplp_t iter,
+                         int (*func)(void *arg, const bam1_t *b, bam_pileup_cd *cd)) {
+    int i;
+    for (i = 0; i < iter->n; ++i)
+        bam_plp_destructor(iter->iter[i], func);
+}
+
+#endif // ~!defined(BAM_NO_PILEUP)
diff --git a/ext/htslib/sam_internal.h b/ext/htslib/sam_internal.h
new file mode 100644
index 0000000..750c597
--- /dev/null
+++ b/ext/htslib/sam_internal.h
@@ -0,0 +1,121 @@
+/*  sam_internal.h -- internal functions; not part of the public API.
+
+    Copyright (C) 2019-2020, 2023-2024 Genome Research Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#ifndef HTSLIB_SAM_INTERNAL_H
+#define HTSLIB_SAM_INTERNAL_H
+
+#include <errno.h>
+#include <stdint.h>
+
+#include "htslib/sam.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Used internally in the SAM format multi-threading.
+int sam_state_destroy(samFile *fp);
+int sam_set_thread_pool(htsFile *fp, htsThreadPool *p);
+int sam_set_threads(htsFile *fp, int nthreads);
+
+// Fastq state
+int fastq_state_set(samFile *fp, enum hts_fmt_option opt, ...);
+void fastq_state_destroy(samFile *fp);
+
+// bam1_t data (re)allocation
+int sam_realloc_bam_data(bam1_t *b, size_t desired);
+
+static inline int realloc_bam_data(bam1_t *b, size_t desired)
+{
+    if (desired <= b->m_data) return 0;
+    return sam_realloc_bam_data(b, desired);
+}
+
+static inline int possibly_expand_bam_data(bam1_t *b, size_t bytes) {
+    size_t new_len = (size_t) b->l_data + bytes;
+
+    if (new_len > INT32_MAX || new_len < bytes) { // Too big or overflow
+        errno = ENOMEM;
+        return -1;
+    }
+    if (new_len <= b->m_data) return 0;
+    return sam_realloc_bam_data(b, new_len);
+}
+
+/*
+ * Convert a nibble encoded BAM sequence to a string of bases.
+ *
+ * We do this 2 bp at a time for speed. Equiv to:
+ *
+ * for (i = 0; i < len; i++)
+ *    seq[i] = seq_nt16_str[bam_seqi(nib, i)];
+ */
+static inline void nibble2base_default(uint8_t *nib, char *seq, int len) {
+    static const char code2base[512] =
+        "===A=C=M=G=R=S=V=T=W=Y=H=K=D=B=N"
+        "A=AAACAMAGARASAVATAWAYAHAKADABAN"
+        "C=CACCCMCGCRCSCVCTCWCYCHCKCDCBCN"
+        "M=MAMCMMMGMRMSMVMTMWMYMHMKMDMBMN"
+        "G=GAGCGMGGGRGSGVGTGWGYGHGKGDGBGN"
+        "R=RARCRMRGRRRSRVRTRWRYRHRKRDRBRN"
+        "S=SASCSMSGSRSSSVSTSWSYSHSKSDSBSN"
+        "V=VAVCVMVGVRVSVVVTVWVYVHVKVDVBVN"
+        "T=TATCTMTGTRTSTVTTTWTYTHTKTDTBTN"
+        "W=WAWCWMWGWRWSWVWTWWWYWHWKWDWBWN"
+        "Y=YAYCYMYGYRYSYVYTYWYYYHYKYDYBYN"
+        "H=HAHCHMHGHRHSHVHTHWHYHHHKHDHBHN"
+        "K=KAKCKMKGKRKSKVKTKWKYKHKKKDKBKN"
+        "D=DADCDMDGDRDSDVDTDWDYDHDKDDDBDN"
+        "B=BABCBMBGBRBSBVBTBWBYBHBKBDBBBN"
+        "N=NANCNMNGNRNSNVNTNWNYNHNKNDNBNN";
+
+    int i, len2 = len/2;
+    seq[0] = 0;
+
+    for (i = 0; i < len2; i++)
+        // Note size_t cast helps gcc optimiser.
+        memcpy(&seq[i*2], &code2base[(size_t)nib[i]*2], 2);
+
+    if ((i *= 2) < len)
+        seq[i] = seq_nt16_str[bam_seqi(nib, i)];
+}
+
+#if defined HAVE_ATTRIBUTE_CONSTRUCTOR && \
+    ((defined __x86_64__ && defined HAVE_ATTRIBUTE_TARGET && defined HAVE_BUILTIN_CPU_SUPPORT_SSSE3) || \
+     (defined __ARM_NEON))
+#define BUILDING_SIMD_NIBBLE2BASE
+#endif
+
+static inline void nibble2base(uint8_t *nib, char *seq, int len) {
+#ifdef BUILDING_SIMD_NIBBLE2BASE
+    extern void (*htslib_nibble2base)(uint8_t *nib, char *seq, int len);
+    htslib_nibble2base(nib, seq, len);
+#else
+    nibble2base_default(nib, seq, len);
+#endif
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/ext/htslib/sam_mods.c b/ext/htslib/sam_mods.c
new file mode 100644
index 0000000..e45f26d
--- /dev/null
+++ b/ext/htslib/sam_mods.c
@@ -0,0 +1,695 @@
+/*  sam_mods.c -- Base modification handling in SAM and BAM.
+
+    Copyright (C) 2020-2024 Genome Research Ltd.
+
+    Author: James Bonfield <jkb@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+#include <assert.h>
+
+#include "htslib/sam.h"
+#include "textutils_internal.h"
+
+// ---------------------------
+// Base Modification retrieval
+//
+// These operate by recording state in an opaque type, allocated and freed
+// via the functions below.
+//
+// Initially we call bam_parse_basemod to process the tags and record the
+// modifications in the state structure, and then functions such as
+// bam_next_basemod can iterate over this cached state.
+
+/* Overview of API.
+
+We start by allocating an hts_base_mod_state and parsing the MM, ML and MN
+tags into it.  This has optional flags controlling how we report base
+modifications in "explicit" coordinates.  See below
+
+    hts_base_mod_state *m = hts_base_mod_state_alloc();
+    bam_parse_basemod2(b, m, HTS_MOD_REPORT_UNCHECKED);
+    // Or: bam_parse_basemod(b, m), which is equiv to flags==0
+    //... do something ...
+    hts_base_mod_state_free(m);
+
+In the default implicit MM coordinate system, any location not
+reported is implicitly assumed to contain no modification.  We only
+report the places we think are likely modified.
+
+Some tools however only look for base modifications in particular
+contexts, eg CpG islands.  Here we need to distinguish between
+not-looked-for and looked-for-but-didn't-find.  These calls have an
+explicit coordinate system, where we only know information about the
+coordinates explicitly listed and everything else is considered to be
+unverified.
+
+By default we don't get reports on the other coordinates in an
+explicit MM tag, but the HTS_MOD_REPORT_UNCHECKED flag will report
+them (with quality HTS_MOD_UNCHECKED) meaning we can do consensus
+modification analysis with accurate counting when dealing with a
+mixture of explicit and implicit records.
+
+
+We have different ways of processing the base modifications.  We can
+iterate either mod-by-mod or position-by-position, or we can simply
+query a specific coordinate as may be done when processing a pileup.
+
+To check for base modifications as a specific location within a
+sequence we can use bam_mods_at_qpos.  This provides complete random
+access within the MM string.  However currently this is inefficiently
+implemented so should only be used for occasional analysis or as a way
+to start iterating at a specific location.  It modifies the state
+position, so after the first use we can then switch to
+bam_mods_at_next_pos to iterate position by position from then on.
+
+    hts_base_mod mods[10];
+    int n = bam_mods_at_qpos(b, pos, m, mods, 10);
+
+For base by base, we have bam_mods_at_next_pos.  This strictly starts
+at the first base and reports entries one at a time.  It's more
+efficient than a loop repeatedly calling ...at-pos.
+
+    hts_base_mod mods[10];
+    int n = bam_mods_at_next_pos(b, m, mods, 10);
+    for (int i = 0; i < n; i++) {
+        // report mod i of n
+    }
+
+Iterating over modifications instead of coordinates is simpler and
+more efficient as it skips reporting of unmodified bases.  This is
+done with bam_next_basemod.
+
+    hts_base_mod mods[10];
+    while ((n=bam_next_basemod(b, m, mods, 10, &pos)) > 0) {
+        for (j = 0; j < n; j++) {
+            // Report 'n'th mod at sequence position 'pos'
+        }
+    }
+
+There are also functions that query meta-data about the MM line rather
+than per-site information.
+
+bam_mods_recorded returns an array of ints holding the +ve code ('m')
+or -ve CHEBI numeric values.
+
+    int ntypes, *types = bam_mods_recorded(m, &ntype);
+
+We can then query a specific modification type to get further
+information on the strand it is operating on, whether it has implicit
+or explicit coordinates, and what it's corresponding canonical base it
+is (The "C" in "C+m").  bam_mods_query_type does this by code name,
+while bam_mods_queryi does this by numeric i^{th} type (from 0 to ntype-1).
+
+    bam_mods_query_type(m, 'c', &strand, &implicit, &canonical);
+    bam_mods_queryi(m, 2, &strand, &implicit, &canonical);
+
+*/
+
+/*
+ * Base modification are stored in MM/Mm tags as <mod_list> defined as
+ *
+ * <mod_list>        ::= <mod_chain><mod_list> | ""
+ * <mod_chain>       ::= <canonical_base><strand><mod-list><delta-list>
+ *
+ * <canonical_base>  ::= "A" | "C" | "G" | "T" | "N".
+ *
+ * <strand>          ::= "+" | "-".
+ *
+ * <mod-list>        ::= <simple-mod-list> | <ChEBI-code>
+ * <simple-mod-list> ::= <simple-mod><simple-mod-list> | <simple-mod>
+ * <ChEBI-code>      ::= <integer>
+ * <simple-mod>      ::= <letter>
+ *
+ * <delta-list>      ::= "," <integer> <delta-list> | ";"
+ *
+ * We do not allocate additional memory other than the fixed size
+ * state, thus we track up to 256 pointers to different locations
+ * within the MM and ML tags.  Each pointer is for a distinct
+ * modification code (simple or ChEBI), meaning some may point to the
+ * same delta-list when multiple codes are combined together
+ * (e.g. "C+mh,1,5,18,3;").  This is the MM[] array.
+ *
+ * Each numeric in the delta-list is tracked in MMcount[], counted
+ * down until it hits zero in which case the next delta is fetched.
+ *
+ * ML array similarly holds the locations in the quality (ML) tag per
+ * type, but these are interleaved so C+mhfc,10,15 will have 4 types
+ * all pointing to the same delta position, but in ML we store
+ * Q(m0)Q(h0)Q(f0)Q(c0) followed by Q(m1)Q(h1)Q(f1)Q(c1).  This ML
+ * also has MLstride indicating how many positions along ML to jump
+ * each time we consume a base. (4 in our above example, but usually 1
+ * for the simple case).
+ *
+ * One complexity of the base modification system is that mods are
+ * always stored in the original DNA orientation.  This is so that
+ * tools that may reverse-complement a sequence (eg "samtools fastq -T
+ * MM,ML") can pass through these modification tags irrespective of
+ * whether they have any knowledge of their internal workings.
+ *
+ * Because we don't wish to allocate extra memory, we cannot simply
+ * reverse the MM and ML tags.  Sadly this means we have to manage the
+ * reverse complementing ourselves on-the-fly.
+ * For reversed reads we start at the right end of MM and no longer
+ * stop at the semicolon.  Instead we use MMend[] array to mark the
+ * termination point.
+ */
+#define MAX_BASE_MOD 256
+struct hts_base_mod_state {
+    int type[MAX_BASE_MOD];     // char or minus-CHEBI
+    int canonical[MAX_BASE_MOD];// canonical base, as seqi (1,2,4,8,15)
+    char strand[MAX_BASE_MOD];  // strand of modification; + or -
+    int MMcount[MAX_BASE_MOD];  // no. canonical bases left until next mod
+    char *MM[MAX_BASE_MOD];     // next pos delta (string)
+    char *MMend[MAX_BASE_MOD];  // end of pos-delta string
+    uint8_t *ML[MAX_BASE_MOD];  // next qual
+    int MLstride[MAX_BASE_MOD]; // bytes between quals for this type
+    int implicit[MAX_BASE_MOD]; // treat unlisted positions as non-modified?
+    int seq_pos;                // current position along sequence
+    int nmods;                  // used array size (0 to MAX_BASE_MOD-1).
+    uint32_t flags;             // Bit-field: see HTS_MOD_REPORT_UNCHECKED
+};
+
+hts_base_mod_state *hts_base_mod_state_alloc(void) {
+    return calloc(1, sizeof(hts_base_mod_state));
+}
+
+void hts_base_mod_state_free(hts_base_mod_state *state) {
+    free(state);
+}
+
+/*
+ * Count frequency of A, C, G, T and N canonical bases in the sequence
+ */
+static void seq_freq(const bam1_t *b, int freq[16]) {
+    int i;
+
+    memset(freq, 0, 16*sizeof(*freq));
+    uint8_t *seq = bam_get_seq(b);
+    for (i = 0; i < b->core.l_qseq; i++)
+        freq[bam_seqi(seq, i)]++;
+    freq[15] = b->core.l_qseq; // all bases count as N for base mods
+}
+
+//0123456789ABCDEF
+//=ACMGRSVTWYHKDBN  aka seq_nt16_str[]
+//=TGKCYSBAWRDMHVN  comp1ement of seq_nt16_str
+//084C2A6E195D3B7F
+static int seqi_rc[] = { 0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15 };
+
+/*
+ * Parse the MM and ML tags to populate the base mod state.
+ * This structure will have been previously allocated via
+ * hts_base_mod_state_alloc, but it does not need to be repeatedly
+ * freed and allocated for each new bam record. (Although obviously
+ * it requires a new call to this function.)
+ *
+ * Flags are copied into the state and used to control reporting functions.
+ * Currently the only flag is HTS_MOD_REPORT_UNCHECKED, to control whether
+ * explicit "C+m?" mods report quality HTS_MOD_UNCHECKED for the bases
+ * outside the explicitly reported region.
+ */
+int bam_parse_basemod2(const bam1_t *b, hts_base_mod_state *state,
+                       uint32_t flags) {
+    // Reset position, else upcoming calls may fail on
+    // seq pos - length comparison
+    state->seq_pos = 0;
+    state->nmods = 0;
+    state->flags = flags;
+
+    // Read MM and ML tags
+    uint8_t *mm = bam_aux_get(b, "MM");
+    if (!mm) mm = bam_aux_get(b, "Mm");
+    if (!mm)
+        return 0;
+    if (mm[0] != 'Z') {
+        hts_log_error("%s: MM tag is not of type Z", bam_get_qname(b));
+        return -1;
+    }
+
+    uint8_t *mi = bam_aux_get(b, "MN");
+    if (mi && bam_aux2i(mi) != b->core.l_qseq && b->core.l_qseq) {
+        // bam_aux2i with set errno = EINVAL and return 0 if the tag
+        // isn't integer, but 0 will be a seq-length mismatch anyway so
+        // triggers an error here too.
+        hts_log_error("%s: MM/MN data length is incompatible with"
+                      " SEQ length", bam_get_qname(b));
+        return -1;
+    }
+
+    uint8_t *ml = bam_aux_get(b, "ML");
+    if (!ml) ml = bam_aux_get(b, "Ml");
+    if (ml && (ml[0] != 'B' || ml[1] != 'C')) {
+        hts_log_error("%s: ML tag is not of type B,C", bam_get_qname(b));
+        return -1;
+    }
+    uint8_t *ml_end = ml ? ml+6 + le_to_u32(ml+2) : NULL;
+    if (ml) ml += 6;
+
+    // Aggregate freqs of ACGTN if reversed, to get final-delta (later)
+    int freq[16];
+    if (b->core.flag & BAM_FREVERSE)
+        seq_freq(b, freq);
+
+    char *cp = (char *)mm+1;
+    int mod_num = 0;
+    int implicit = 1;
+    while (*cp) {
+        for (; *cp; cp++) {
+            // cp should be [ACGTNU][+-]([a-zA-Z]+|[0-9]+)[.?]?(,\d+)*;
+            unsigned char btype = *cp++;
+
+            if (btype != 'A' && btype != 'C' &&
+                btype != 'G' && btype != 'T' &&
+                btype != 'U' && btype != 'N')
+                return -1;
+            if (btype == 'U') btype = 'T';
+
+            btype = seq_nt16_table[btype];
+
+            // Strand
+            if (*cp != '+' && *cp != '-')
+                return -1; // malformed
+            char strand = *cp++;
+
+            // List of modification types
+            char *ms = cp, *me; // mod code start and end
+            char *cp_end = NULL;
+            int chebi = 0;
+            if (isdigit_c(*cp)) {
+                chebi = strtol(cp, &cp_end, 10);
+                cp = cp_end;
+                ms = cp-1;
+            } else {
+                while (*cp && isalpha_c(*cp))
+                    cp++;
+                if (*cp == '\0')
+                    return -1;
+            }
+
+            me = cp;
+
+            // Optional explicit vs implicit marker
+            implicit = 1;
+            if (*cp == '.') {
+                // default is implicit = 1;
+                cp++;
+            } else if (*cp == '?') {
+                implicit = 0;
+                cp++;
+            } else if (*cp != ',' && *cp != ';') {
+                // parse error
+                return -1;
+            }
+
+            long delta;
+            int n = 0; // nth symbol in a multi-mod string
+            int stride = me-ms;
+            int ndelta = 0;
+
+            if (b->core.flag & BAM_FREVERSE) {
+                // We process the sequence in left to right order,
+                // but delta is successive count of bases to skip
+                // counting right to left.  This also means the number
+                // of bases to skip at left edge is unrecorded (as it's
+                // the remainder).
+                //
+                // To output mods in left to right, we step through the
+                // MM list in reverse and need to identify the left-end
+                // "remainder" delta.
+                int total_seq = 0;
+                for (;;) {
+                    cp += (*cp == ',');
+                    if (*cp == 0 || *cp == ';')
+                        break;
+
+                    delta = strtol(cp, &cp_end, 10);
+                    if (cp_end == cp) {
+                        hts_log_error("%s: Hit end of MM tag. Missing "
+                                      "semicolon?", bam_get_qname(b));
+                        return -1;
+                    }
+
+                    cp = cp_end;
+                    total_seq += delta+1;
+                    ndelta++;
+                }
+                delta = freq[seqi_rc[btype]] - total_seq; // remainder
+            } else {
+                delta = *cp == ','
+                    ? strtol(cp+1, &cp_end, 10)
+                    : 0;
+                if (!cp_end) {
+                    // empty list
+                    delta = INT_MAX;
+                    cp_end = cp;
+                }
+            }
+            // Now delta is first in list or computed remainder,
+            // and cp_end is either start or end of the MM list.
+            while (ms < me) {
+                state->type     [mod_num] = chebi ? -chebi : *ms;
+                state->strand   [mod_num] = (strand == '-');
+                state->canonical[mod_num] = btype;
+                state->MLstride [mod_num] = stride;
+                state->implicit [mod_num] = implicit;
+
+                if (delta < 0) {
+                    hts_log_error("%s: MM tag refers to bases beyond sequence "
+                                  "length", bam_get_qname(b));
+                    return -1;
+                }
+                state->MMcount  [mod_num] = delta;
+                if (b->core.flag & BAM_FREVERSE) {
+                    state->MM   [mod_num] = me+1;
+                    state->MMend[mod_num] = cp_end;
+                    state->ML   [mod_num] = ml ? ml+n +(ndelta-1)*stride: NULL;
+                } else {
+                    state->MM   [mod_num] = cp_end;
+                    state->MMend[mod_num] = NULL;
+                    state->ML   [mod_num] = ml ? ml+n : NULL;
+                }
+
+                if (++mod_num >= MAX_BASE_MOD) {
+                    hts_log_error("%s: Too many base modification types",
+                                  bam_get_qname(b));
+                    return -1;
+                }
+                ms++; n++;
+            }
+
+            // Skip modification deltas
+            if (ml) {
+                if (b->core.flag & BAM_FREVERSE) {
+                    ml += ndelta*stride;
+                } else {
+                    while (*cp && *cp != ';') {
+                        if (*cp == ',')
+                            ml+=stride;
+                        cp++;
+                    }
+                }
+                if (ml > ml_end) {
+                    hts_log_error("%s: Insufficient number of entries in ML "
+                                  "tag", bam_get_qname(b));
+                    return -1;
+                }
+            } else {
+                // cp_end already known if FREVERSE
+                if (cp_end && (b->core.flag & BAM_FREVERSE))
+                    cp = cp_end;
+                else
+                    while (*cp && *cp != ';')
+                        cp++;
+            }
+            if (!*cp) {
+                hts_log_error("%s: Hit end of MM tag. Missing semicolon?",
+                              bam_get_qname(b));
+                return -1;
+            }
+        }
+    }
+    if (ml && ml != ml_end) {
+        hts_log_error("%s: Too many entries in ML tag", bam_get_qname(b));
+        return -1;
+    }
+
+    state->nmods = mod_num;
+
+    return 0;
+}
+
+int bam_parse_basemod(const bam1_t *b, hts_base_mod_state *state) {
+    return bam_parse_basemod2(b, state, 0);
+}
+
+/*
+ * Fills out mods[] with the base modifications found.
+ * Returns the number found (0 if none), which may be more than
+ * the size of n_mods if more were found than reported.
+ * Returns <= -1 on error.
+ *
+ * This always marches left to right along sequence, irrespective of
+ * reverse flag or modification strand.
+ */
+int bam_mods_at_next_pos(const bam1_t *b, hts_base_mod_state *state,
+                         hts_base_mod *mods, int n_mods) {
+    if (b->core.flag & BAM_FREVERSE) {
+        if (state->seq_pos < 0)
+            return -1;
+    } else {
+        if (state->seq_pos >= b->core.l_qseq)
+            return -1;
+    }
+
+    int i, j, n = 0;
+    unsigned char base = bam_seqi(bam_get_seq(b), state->seq_pos);
+    state->seq_pos++;
+    if (b->core.flag & BAM_FREVERSE)
+        base = seqi_rc[base];
+
+    for (i = 0; i < state->nmods; i++) {
+        int unchecked = 0;
+        if (state->canonical[i] != base && state->canonical[i] != 15/*N*/)
+            continue;
+
+        if (state->MMcount[i]-- > 0) {
+            if (!state->implicit[i] &&
+                (state->flags & HTS_MOD_REPORT_UNCHECKED))
+                unchecked = 1;
+            else
+                continue;
+        }
+
+        char *MMptr = state->MM[i];
+        if (n < n_mods) {
+            mods[n].modified_base = state->type[i];
+            mods[n].canonical_base = seq_nt16_str[state->canonical[i]];
+            mods[n].strand = state->strand[i];
+            mods[n].qual = unchecked
+                ? HTS_MOD_UNCHECKED
+                : (state->ML[i] ? *state->ML[i] : HTS_MOD_UNKNOWN);
+        }
+        n++;
+
+        if (unchecked)
+            continue;
+
+        if (state->ML[i])
+            state->ML[i] += (b->core.flag & BAM_FREVERSE)
+                ? -state->MLstride[i]
+                : +state->MLstride[i];
+
+        if (b->core.flag & BAM_FREVERSE) {
+            // process MM list backwards
+            char *cp;
+            if (state->MMend[i]-1 < state->MM[i]) {
+                // Should be impossible to hit if coding is correct
+                hts_log_error("Assert failed while processing base modification states");
+                return -1;
+            }
+            for (cp = state->MMend[i]-1; cp != state->MM[i]; cp--)
+                if (*cp == ',')
+                    break;
+            state->MMend[i] = cp;
+            if (cp != state->MM[i])
+                state->MMcount[i] = strtol(cp+1, NULL, 10);
+            else
+                state->MMcount[i] = INT_MAX;
+        } else {
+            if (*state->MM[i] == ',')
+                state->MMcount[i] = strtol(state->MM[i]+1, &state->MM[i], 10);
+            else
+                state->MMcount[i] = INT_MAX;
+        }
+
+        // Multiple mods at the same coords.
+        for (j=i+1; j < state->nmods && state->MM[j] == MMptr; j++) {
+            if (n < n_mods) {
+                mods[n].modified_base = state->type[j];
+                mods[n].canonical_base = seq_nt16_str[state->canonical[j]];
+                mods[n].strand = state->strand[j];
+                mods[n].qual = state->ML[j] ? *state->ML[j] : -1;
+            }
+            n++;
+            state->MMcount[j] = state->MMcount[i];
+            state->MM[j]      = state->MM[i];
+            if (state->ML[j])
+                state->ML[j] += (b->core.flag & BAM_FREVERSE)
+                    ? -state->MLstride[j]
+                    : +state->MLstride[j];
+        }
+        i = j-1;
+    }
+
+    return n;
+}
+
+/*
+ * Return data at the next modified location.
+ *
+ * bam_mods_at_next_pos does quite a bit of work, so we don't want to
+ * repeatedly call it for every location until we find a mod.  Instead
+ * we check how many base types we can consume before the next mod,
+ * and scan through the sequence looking for them.  Once we're at that
+ * site, we defer back to bam_mods_at_next_pos for the return values.
+ */
+int bam_next_basemod(const bam1_t *b, hts_base_mod_state *state,
+                     hts_base_mod *mods, int n_mods, int *pos) {
+    // Look through state->MMcount arrays to see when the next lowest is
+    // per base type;
+    int next[16], freq[16] = {0}, i;
+    memset(next, 0x7f, 16*sizeof(*next));
+    const int unchecked = state->flags & HTS_MOD_REPORT_UNCHECKED;
+    if (b->core.flag & BAM_FREVERSE) {
+        for (i = 0; i < state->nmods; i++) {
+            if (unchecked && !state->implicit[i])
+                next[seqi_rc[state->canonical[i]]] = 1;
+            else if (next[seqi_rc[state->canonical[i]]] > state->MMcount[i])
+                next[seqi_rc[state->canonical[i]]] = state->MMcount[i];
+        }
+    } else {
+        for (i = 0; i < state->nmods; i++) {
+            if (unchecked && !state->implicit[i])
+                next[state->canonical[i]] = 0;
+            else if (next[state->canonical[i]] > state->MMcount[i])
+                next[state->canonical[i]] = state->MMcount[i];
+       }
+    }
+
+    // Now step through the sequence counting off base types.
+    for (i = state->seq_pos; i < b->core.l_qseq; i++) {
+        unsigned char bc = bam_seqi(bam_get_seq(b), i);
+        if (next[bc] <= freq[bc] || next[15] <= freq[15])
+            break;
+        freq[bc]++;
+        if (bc != 15) // N
+            freq[15]++;
+    }
+    *pos = state->seq_pos = i;
+
+    if (b->core.flag & BAM_FREVERSE) {
+        for (i = 0; i < state->nmods; i++)
+            state->MMcount[i] -= freq[seqi_rc[state->canonical[i]]];
+    } else {
+        for (i = 0; i < state->nmods; i++)
+            state->MMcount[i] -= freq[state->canonical[i]];
+    }
+
+    if (b->core.l_qseq && state->seq_pos >= b->core.l_qseq &&
+        !(b->core.flag & BAM_FREVERSE)) {
+        // Spots +ve orientation run-overs.
+        // The -ve orientation is spotted in bam_parse_basemod2
+        int i;
+        for (i = 0; i < state->nmods; i++) {
+            // Check if any remaining items in MM after hitting the end
+            // of the sequence.
+            if (state->MMcount[i] < 0x7f000000 ||
+                (*state->MM[i]!=0 && *state->MM[i]!=';')) {
+                hts_log_warning("MM tag refers to bases beyond sequence length");
+                return -1;
+            }
+        }
+        return 0;
+    }
+
+    int r = bam_mods_at_next_pos(b, state, mods, n_mods);
+    return r > 0 ? r : 0;
+}
+
+/*
+ * As per bam_mods_at_next_pos, but at a specific qpos >= the previous qpos.
+ * This can only march forwards along the read, but can do so by more than
+ * one base-pair.
+ *
+ * This makes it useful for calling from pileup iterators where qpos may
+ * start part way through a read for the first occurrence of that record.
+ */
+int bam_mods_at_qpos(const bam1_t *b, int qpos, hts_base_mod_state *state,
+                    hts_base_mod *mods, int n_mods) {
+    // FIXME: for now this is inefficient in implementation.
+    int r = 0;
+    while (state->seq_pos <= qpos)
+        if ((r = bam_mods_at_next_pos(b, state, mods, n_mods)) < 0)
+            break;
+
+    return r;
+}
+
+/*
+ * Returns the list of base modification codes provided for this
+ * alignment record as an array of character codes (+ve) or ChEBI numbers
+ * (negative).
+ *
+ * Returns the array, with *ntype filled out with the size.
+ *         The array returned should not be freed.
+ *         It is a valid pointer until the state is freed using
+ *         hts_base_mod_free().
+ */
+int *bam_mods_recorded(hts_base_mod_state *state, int *ntype) {
+    *ntype = state->nmods;
+    return state->type;
+}
+
+/*
+ * Returns data about a specific modification type for the alignment record.
+ * Code is either positive (eg 'm') or negative for ChEBI numbers.
+ *
+ * Return 0 on success or -1 if not found.  The strand, implicit and canonical
+ * fields are filled out if passed in as non-NULL pointers.
+ */
+int bam_mods_query_type(hts_base_mod_state *state, int code,
+                        int *strand, int *implicit, char *canonical) {
+    // Find code entry
+    int i;
+    for (i = 0; i < state->nmods; i++) {
+        if (state->type[i] == code)
+            break;
+    }
+    if (i == state->nmods)
+        return -1;
+
+    // Return data
+    if (strand)    *strand    = state->strand[i];
+    if (implicit)  *implicit  = state->implicit[i];
+    if (canonical) *canonical = "?AC?G???T??????N"[state->canonical[i]];
+
+    return 0;
+}
+
+/*
+ * Returns data about the ith modification type for the alignment record.
+ *
+ * Return 0 on success or -1 if not found.  The strand, implicit and canonical
+ * fields are filled out if passed in as non-NULL pointers.
+ */
+int bam_mods_queryi(hts_base_mod_state *state, int i,
+                    int *strand, int *implicit, char *canonical) {
+    if (i < 0 || i >= state->nmods)
+        return -1;
+
+    // Return data
+    if (strand)    *strand    = state->strand[i];
+    if (implicit)  *implicit  = state->implicit[i];
+    if (canonical) *canonical = "?AC?G???T??????N"[state->canonical[i]];
+
+    return 0;
+}
diff --git a/ext/htslib/samples/DEMO.md b/ext/htslib/samples/DEMO.md
new file mode 100644
index 0000000..98c9981
--- /dev/null
+++ b/ext/htslib/samples/DEMO.md
@@ -0,0 +1,1740 @@
+# HTS API
+
+## HTSLib APIs and samtools
+
+HTSLib is a C library implementation used to access and process the genome
+sequence data. HTSLib implements multiple API interfaces, HTS API, VCF API and
+SAM API.  HTS API provides a framework for use by other APIs and applications,
+implements bgzf compression, htscodecs and provides CRAM format support.  VCF
+APIs work with variant data in VCF and BCF format.
+
+SAM API works with sequence data of different formats, SAM / BAM / CRAM /
+FASTA / FASTQ, and provides methods to do operations on the data. It uses
+methods from HTS API.
+
+'samtools' is the utility used to read and modify sequence data. It uses SAM
+APIs from HTSLib to work on the sequence data.
+
+
+## About this document
+
+There are a number of demonstration utilities and their source code in
+'samples' directory of HTSLib and this document gives the description of them
+and the usage of API of HTSLib.  The samples are for demonstration
+purposes only and proper error handling is required for actual usage. This
+document is based on HTSLib version 1.17.
+
+Updates to this document may be made along with later releases when required.
+
+
+## The sample apps
+
+Flags - This application showcases the basic read of alignment files and flag
+access. It reads and shows the count of read1 and read2 alignments.
+
+Split - This application showcases the basic read and write of alignment data.
+It saves the read1 and read2 as separate files in given directory, one as sam
+and other as bam.
+
+Split2 - This application showcases the output file format selection. It saves
+the read1 and read2 as separate files in given directory, both as compressed
+sam though the extensions are different.
+
+Cram - This application showcases the different way in which cram reference
+data is used for cram output creation.
+
+Read_fast - This application showcases the fasta/fastq data read.
+
+Read_header - This application showcases the read and access of header data.
+It can show all header line of given type, data of a given tag on a specific
+header line or for all lines of given type.
+
+Read_ref - This application showcases the read and access of header data.
+It shows all reference names which has length equal or greater to given input.
+
+Read_bam - This application showcases read of different alignment data fields.
+It shows contents of each alignment.
+
+Read_aux - This application showcases read of specific auxiliary tag data in
+alignment. It shows the data retrieved using 2 APIs, one as a string with tag
+data and other as raw data alternatively.
+
+Dump_aux - This application showcases read of all auxiliary tag data one by one
+in an alignment. It shows the data retrieved.
+
+Add_header - This application showcases the write of header lines to a file.
+It adds header line of types, SQ, RG, PG and CO and writes to standard output.
+
+Remove_header - This application showcases removal of header line from a file.
+It removes either all header lines of given type or one specific line of given
+type with given unique identifier. Modified header is written on standard
+output.
+
+Update_header - This application shows the update of header line fields, where
+update is allowed. It takes the header line type, unique identifier for the
+line, tag to be modified and the new value. Updated data is written on standard
+output.
+
+Mod_bam - This application showcases the update of alignment data. It takes
+alignment name, position of field to be modified and new value of it.
+Modified data is written on standard output.
+
+Mod_aux - This application showcases the update of auxiliary data in alignment.
+It takes alignment name, tag to be modified, its type and new value. Modified
+data is written on standard output.
+
+Mod_aux_ba - This application showcases the update of auxiliary array data in
+alignment. It adds count of ATCGN base as an array in auxiliary data, BA:I.
+Modified data is written on standard output.
+
+Write_fast - This application showcases the fasta/fastq data write. It appends
+data to given file.
+
+Index_write - This application showcases the creation of index along with
+output creation. Based on file type and shift, it creates bai, csi or crai
+files.
+
+Index_fast - This application showcases the index creation on fasta/fastq
+reference files.
+
+Read_reg - This application showcases the usage of region specification in
+alignment read.
+
+Read_multireg - This application showcases the usage of multiple region
+specification in alignment read.
+
+Read_fast_index - This application showcases the fasta/fastq data read using
+index.
+
+Pileup - This application showcases the pileup api, where all alignments
+covering a reference position are accessed together. It displays the bases
+covering each position on standard output.
+
+Mpileup - This application showcases the mpileup api, which supports multiple
+input files for pileup and gives a side by side view of them in pileup format.
+It displays the bases covering each position on standard output.
+
+Modstate - This application showcases the access of base modifications in
+alignment. It shows the modifications present in an alignment and accesses them
+using available APIs. There are 2 APIs and which one to be used can be selected
+through input.
+
+Pileup_mod - This application showcases the base modification access in pileup
+mode. It shows the pileup display with base modifications.
+
+Flags_field - This application showcases the read of selected fields alone,
+reducing the overhead / increasing the performance. It reads the flag field
+alone and shows the count of read1 and read2. This has impact only on CRAM
+files.
+
+Split_thread1 - This application showcases the use of threads in file handling.
+It saves the read1 and read2 as separate files in given directory, one as sam
+and other as bam. 2 threads are used for read and 1 each dedicated for each
+output file.
+
+Split_thread2 - This application showcases the use of thread pool in file
+handling. It saves the read1 and read2 as separate files in given directory,
+one as sam and other as bam. A pool of 4 threads is created and shared for both
+read and write.
+
+Qtask_ordered - This application showcases the use of queues and threads for
+custom processing. Alignments in input file are updated with their GC ratio
+on a custom aux tag. The processing may occur in any order but the result is
+retrieved in same order as it was queued and saved to disk.
+
+Qtask_unordered - This application showcases the use of queues and threads
+for custom processing. The count of bases and GC ratio are calculated and
+displayed.  The order of counting is irrelevant and hence ordered retrieval is
+not used.
+
+## Building the sample apps
+
+The samples expect the HTSLib is installed, libraries and header file path are
+part of the PATH environment variable. If not, these paths need to be explicitly
+passed during the build time.
+
+Gcc and compatible compilers can be used to build the samples.
+
+These applications can be linked statically or dynamically to HTSLib.
+For static linking, along with htslib other libraries and/or headers required
+to build are, math, pthread, curl, lzma, z and bz2 libraries.
+
+A makefile is available along with source files which links statically to
+htslib. To use dynamic linking, update the makefile's 'LDFLAGS' and 'rpath'
+path. The 'rpath' path to be set as the path to lib directory of htslib
+installation.
+
+
+## Usage of HTS APIs
+### Sequence data file access for read
+
+The sequence data file for read may be opened using the sam_open method. It
+opens the file and returns samFile (htsFile) pointer on success or NULL on
+failure.  The input can be path to a file in disk, network, cloud or '-'
+designating the standard input.
+
+SAM, BAM and CRAM file formats are supported and the input file format is
+detected from the file content.
+
+Once done with the file, it needs to be closed with sam_close.
+
+Many times, header details would be required and can be read using
+sam_hdr_read api.  It returns sam_hdr_t pointer or NULL. The returned header
+needs to be destroyed using sam_hdr_destroy when no longer required.
+
+The sequence data may be compressed or uncompressed on disk and on memory it
+is read and kept as uncompressed BAM format. It can be read from a file using
+sam_read1 api. samFile pointer, header and bam storage are to be passed as
+argument and it returns 0 on success, -1 on end of file and < -1 in case of
+errors.
+
+The bam storage has to be initialized using bam_init1 api before the call and
+can be reused for successive reads. Once done, it needs to be destroyed using
+bam_destroy1.  The member field named core - bam1_core_t - in bam storage,
+bam1_t, has the sequence data in an easily accessible way. Using the fields
+and macros, data can easily be read from it.
+
+    #include <htslib/sam.h>
+
+    int main(int argc, char *argv[])
+    {
+        ...
+        //initialize
+        if (!(bamdata = bam_init1()))
+           ... // error
+        //open input files - r reading
+        if (!(infile = sam_open(inname, "r")))
+           ... // error
+        //read header
+        if (!(in_samhdr = sam_hdr_read(infile)))
+           ... // error
+
+        //read data, check flags and update count
+        while ((c = sam_read1(infile, in_samhdr, bamdata)) >= 0) {
+            if (bamdata->core.flag & BAM_FREAD1)
+                cntread1++;
+            ...
+
+        //clean up
+        if (in_samhdr)
+            sam_hdr_destroy(in_samhdr);
+
+        if (infile)
+            sam_close(infile);
+
+        if (bamdata)
+            bam_destroy1(bamdata);
+
+        return ret;
+    }
+Refer: flags_demo.c
+
+This shows the count of read1 and read2 alignments.
+
+    ./flags /tmp/sample.sam.gz
+
+To read CRAM files, reference data is required and if it is not available, based
+on configuration, library may try to download it from external repositories.
+
+
+### Sequence data file access for write
+
+File access for write is similar to read with a few additional optional steps.
+
+The output file can be opened using sam_open api as in read, with "w" instead
+of "r" as mode. This opens the file for writing and uses mode to select the
+output file type. "w" alone denotes SAM, "wb" denotes BAM and "wc" denotes CRAM.
+
+Another way is to use sam_open_mode method, which sets the output file type and
+compression based on the file name and explicit textual format specification.
+This method expects a buffer to append type and compression flags. Usually a
+buffer with standard file open flag is used, the buffer past the flag is passed
+to the method to ensure existing flags and updates from this method are present
+in the same buffer without being overwritten. This method will add more flags
+indicating file type and compression based on name. If explicit format detail
+given, then extension is ignored and the explicit specification is used. This
+updated buffer can be used with sam_open to select the file format.
+
+sam_open_format method may also be used to open the file for output as more
+information on the output file can be specified using this. Can use
+mode buffer from sam_open_mode api or explicit format structure for this.
+
+The header data can be written using the sam_hdr_write api. When the header
+data is copied to another variable and has different lifetime, it is good to
+increase the reference count of the header using sam_hdr_incr_ref and
+sam_hdr_destroy called as many times as required.
+
+The alignment data can be written using the sam_write1 api. It takes a samFile
+pointer, header pointer and the alignment data. The header data is required to
+set the reference name in the alignment. It returns -ve value on error.
+
+    int main(int argc, char *argv[])
+    {
+        ...
+        if (!(infile = sam_open(inname, "r")))
+           ... // error
+        outfile1 = sam_open(file1, "w");            //as SAM
+        outfile2 = sam_open(file2, "wb");           //as BAM
+        ...
+        if (!(in_samhdr = sam_hdr_read(infile)))
+           ... // error
+
+        //write header
+        if ((sam_hdr_write(outfile1, in_samhdr) == -1) ||
+         (sam_hdr_write(outfile2, in_samhdr) == -1))
+           ... // error
+
+        while ((c = sam_read1(infile, in_samhdr, bamdata)) >= 0) {
+            if (bamdata->core.flag & BAM_FREAD1) {
+                if (sam_write1(outfile1, in_samhdr, bamdata) < 0) {
+                    ... // error
+    }
+Refer: split.c
+
+This creates 1.sam and 2.bam in /tmp/ containing read1 and read2 respectively.
+
+    ./split /tmp/sample.sam.gz /tmp/
+
+Below code excerpt shows sam_open_mode api usage.
+
+    int main(int argc, char *argv[])
+    {
+        ...
+        //set file open mode based on file name for 1st and as explicit for 2nd
+        if ((sam_open_mode(mode1+1, file1, NULL) == -1) ||
+         (sam_open_mode(mode2+1, file2, "sam.gz") == -1))
+           ... // error
+        if (!(infile = sam_open(inname, "r")))
+           ... // error
+
+        //open output files
+        outfile1 = sam_open(file1, mode1);                          //as compressed SAM through sam_open
+        outfile2 = sam_open_format(file2, mode2, NULL);             //as compressed SAM through sam_open_format
+        ...
+    }
+Refer: split2.c
+
+This creates 1.sam.gz and 2.sam in /tmp/ both having compressed data.
+
+    ./split2 /tmp/sample.sam.gz /tmp/
+
+An htsFormat structure filled appropriately can also be used to specify output
+file format while using sam_open_format api.
+
+
+### CRAM writing
+
+CRAM files uses reference data and compresses alignment data. A CRAM file may
+be created with external reference data file - most appropriate, with embedded
+reference in it or with no reference data at all. It can also be created using
+an autogenerated reference, based on consensus with-in the alignment data.
+The reference detail can be set to an htsFormat structure using hts_parse_format
+api and used with sam_open_format api to create appropriate CRAM file.
+
+    ...
+    snprintf(reffmt1, size1, "cram,reference=%s", reffile);
+    snprintf(reffmt2, size2, "cram,embed_ref=1,reference=%s", reffile);
+    ...
+    if (hts_parse_format(&fmt1, reffmt1) == -1 ||                   //using external reference - uses the M5/UR tags to get
+     reference data during read
+            hts_parse_format(&fmt2, reffmt2) == -1 ||               //embed the reference internally
+            hts_parse_format(&fmt3, "cram,embed_ref=2") == -1 ||    //embed autogenerated reference
+            hts_parse_format(&fmt4, "cram,no_ref=1") == -1) {       //no reference data encoding at all
+       ... // error
+    outfile1 = sam_open_format(file1, "wc", &fmt1); outfile2 = sam_open_format(file2, "wc", &fmt2);
+    ...
+Refer: cram.c
+
+
+### FASTA/FASTQ data access
+
+FASTA/FASTQ files have the raw sequence data and the data can be read one by
+one using sam_read1 or a selected range using a region. The data can be written
+similar to alignment data using sam_write1 api. To write the file, format
+can be set by updating mode buffer using sam_open_mode with file name
+or explicit format text. This mode buffer can be used with sam_open or can be
+used with sam_open_format with explicit format information in htsFormat
+structure.
+
+It is the FASTA format which is mainly in use to store the reference data.
+
+    ...
+    if (!(bamdata = bam_init1()))
+      ... // error
+    if (!(infile = sam_open(inname, "r")))
+       ... // error
+    if (infile->format.format != fasta_format && infile->format.format != fastq_format)
+       ... // error
+    if (!(in_samhdr = sam_hdr_read(infile)))
+       ... // error
+
+    while ((c = sam_read1(infile, in_samhdr, bamdata)) >= 0)
+       ... // error
+        printf("\nsequence: ");
+        for (c = 0; c < bamdata->core.l_qseq; ++c) {
+            printf("%c", seq_nt16_str[bam_seqi(bam_get_seq(bamdata), c)]);
+        }
+        if (infile->format.format == fastq_format) {
+            printf("\nquality: ");
+            for (c = 0; c < bamdata->core.l_qseq; ++c) {
+                printf("%c", bam_get_qual(bamdata)[c] + 33);
+    ...
+Refer: read_fast.c
+
+    ...
+    char mode[4] = "a";
+    ...
+    if (sam_open_mode(mode + 1, outname, NULL) < 0)
+       ... // error
+    if (!(outfile = sam_open(outname, mode)))
+       ... // error
+    if (bam_set1(bamdata, strlen(name), name, BAM_FUNMAP, -1, -1, 0, 0, NULL, -1, -1, 0, strlen(data), data, qual, 0) < 0)
+       ... // error
+    if (sam_write1(outfile, out_samhdr, bamdata) < 0) {
+        printf("Failed to write data\n");
+        ...
+Refer: write_fast.c
+
+
+### Header data read
+
+The header gives the version, reference details, read group, change history
+and comments. These data are stored inside the sam_hdr_t. Each of these
+entries, except comments, have their unique identifier and it is required to
+access different fields of them.  The api sam_hdr_count_lines gives the count
+of the specified type of header line.  The value of a unique identifier to a
+specific type of header line can be retrieved with sam_hdr_line_name api.  The
+api sam_hdr_find_tag_id and sam_hdr_find_tag_pos can get the field data from a
+header line using unique identifier values or using position. The full header
+line can be retrieved using sam_hdr_find_line_pos or sam_hdr_line_id with
+position and unique identifier values respectively.
+
+    ...
+    if (!(in_samhdr = sam_hdr_read(infile)))
+        ... // error
+    ...
+      if (tag)
+          ret = sam_hdr_find_tag_id(in_samhdr, header, id, idval, tag, &data);
+      else
+          ret = sam_hdr_find_line_id(in_samhdr, header, id, idval, &data);
+    ...
+        linecnt = sam_hdr_count_lines(in_samhdr, header);
+        ...
+            if (tag)
+                ret = sam_hdr_find_tag_pos(in_samhdr, header, c, tag, &data);
+            else
+                ret = sam_hdr_find_line_pos(in_samhdr, header, c, &data);
+        ...
+Refer: read_header.c
+
+This will show the VN tag's value from HD header.
+
+    ./read_header /tmp/sample.sam.gz HD VN
+
+Shows the 2nd SQ line's LN field value.
+
+    ./read_header /tmp/sample.sam.gz SQ SN T2 LN
+
+Below code excerpt shows the reference names which has length above given value.
+
+    ...
+    linecnt = sam_hdr_count_lines(in_samhdr, "SQ"); //get reference count
+    ...
+    //iterate and check each reference's length
+    for (pos = 1, c = 0; c < linecnt; ++c) {
+        if ((ret = sam_hdr_find_tag_pos(in_samhdr, "SQ", c, "LN", &data) == -2))
+            ... // error
+
+        size = atoll(data.s);
+        if (size < minsize) {
+            //not required
+            continue;
+        }
+
+        //sam_hdr_find_tag_pos(in_samhdr, "SQ", c, "SN", &data) can also do the same!
+        if (!(id = sam_hdr_line_name(in_samhdr, "SQ", c)))
+            ... // error
+
+        printf("%d,%s,%s\n", pos, id, data.s);
+    ...
+Refer: read_refname.c
+
+
+### Alignment data read
+
+The alignment / sequence data contains many fields. Mainly the read/query
+name, flags indicating the properties of the read, reference sequence name,
+position in reference to which it matches, quality of the read, CIGAR string
+indicating the match status, position of mate / reverse strand, name of
+reference sequence to which mate matches, the insert length, base sequence,
+quality value of each base and auxiliary fields.
+
+Header data would be required to retrieve the reference names as alignment
+contains the position of the reference in the header.
+
+A few of the data are directly visible in bam1_t and the rest are hidden
+inside data member of bam1_t and can easily be retrieved using macros.
+bam_get_qname gives the name of the read, sam_hdr_tid2name gives the reference
+name. bam_get_cigar retrieves the cigar operation array, which can be decoded
+using bam_cigar_oplen to get count of bases to which that operation applicable
+and bam_cigar_opchr to get the cigar operation.  bam_seqi retrieves the base
+data at a given position in alignment and it can be converted to character by
+indexing the seq_nt16_str array.
+
+    ...
+    while ((ret_r = sam_read1(infile, in_samhdr, bamdata)) >= 0)
+    {
+        //QNAME FLAG RNAME POS MAPQ CIGAR RNEXT PNEXT TLEN SEQ QUAL [TAG:TYPE:VALUE]
+        printf("NAME: %s\n", bam_get_qname(bamdata));                           //get the query name using the macro
+        flags = bam_flag2str(bamdata->core.flag);                               //flags as string
+        ...
+        tidname = sam_hdr_tid2name(in_samhdr, bamdata->core.tid);
+        ...
+        printf("MQUAL: %d\n", bamdata->core.qual);                              //map quality value
+        cigar = bam_get_cigar(bamdata);                                         //retrieves the cigar data
+        for (i = 0; i < bamdata->core.n_cigar; ++i) {                           //no. of cigar data entries
+            printf("%d%c", bam_cigar_oplen(cigar[i]), bam_cigar_opchr(cigar[i]));
+            //the macros gives the count of operation and the symbol of operation for given cigar entry
+        }
+        printf("\nTLEN/ISIZE: %"PRIhts_pos"\n", bamdata->core.isize);
+        data = bam_get_seq(bamdata);
+                 //get the sequence data
+        if (bamdata->core.l_qseq != bam_cigar2qlen(bamdata->core.n_cigar, cigar)) { //checks the length with CIGAR and query
+        ...
+        for (i = 0; i < bamdata->core.l_qseq ; ++i) {       //sequence length
+            printf("%c", seq_nt16_str[bam_seqi(data, i)]);  //retrieves the base from (internal compressed) sequence data
+            ...
+            printf("%c", bam_get_qual(bamdata)[i]+33);      //retrieves the quality value
+        ...
+Refer: read_bam.c
+
+Shows the data from alignments.
+
+    ./read_bam /tmp/sample.sam.gz
+
+
+### Aux data read
+
+Auxiliary data gives extra information about the alignment. There can be a
+number of such data and can be accessed by specifying required tag or by
+iterating one by one through them once the alignment is read as bam1_t. The
+auxiliary data are stored along with the variable length data in the data
+field of bam1_t. There are macros defined to retrieve information about
+auxiliary data from the data field of bam1_t.
+
+Data for a specific tag can be retrieved as a string or can be retrieved as raw
+data. bam_aux_get_str retrieves as a string, with tag name, tag type and data.
+bam_aux_get can get raw data and with bam_aux_type and bam_aux2A, bam_aux2f etc.
+the raw data can be extracted.
+
+To iterate through all data, the start of aux data is retrieved using macro
+bam_aux_first and successive ones using bam_aux_next. Macro bam_aux_tag gives
+the tag of the aux field and bam_aux_type gives the information about type of
+the aux field.
+
+Bam_aux2i, bam_aux2f, bam_aux2Z macros retrieve the aux data's value as
+integer, float and string respectively. The integer value may be of different
+precision / size and the bam_aux_type character indicates how to use the
+value. The string/hex data are NULL terminated.
+
+For array data, bam_aux_type will return 'B' and bam_auxB_len gives the length
+of the array. bam_aux_type with the next byte will give the type of data in
+the array. bam_auxB2i, bam_auxB2f will give integer and float data from a
+given position of the array.
+
+    ...
+    while ((ret_r = sam_read1(infile, in_samhdr, bamdata)) >= 0) {
+        //option 1 - get data as string with tag and type
+        if ((c = bam_aux_get_str(bamdata, tag, &sdata)) == 1) {
+            printf("%s\n",sdata.s);
+        ...
+        //option 2 - get raw data
+        if ((data = bam_aux_get(bamdata, tag)) != NULL) {
+            printauxdata(stdout, bam_aux_type(data), -1, data);
+    ...
+Refer: read_aux.c
+
+Shows the MD aux tag from alignments.
+
+    ./read_aux ../../samtools/test/mpileup/mpileup.1.bam MD
+
+    ...
+    while ((ret_r = sam_read1(infile, in_samhdr, bamdata)) >= 0) {
+        data = bam_aux_first(bamdata);                                              //get the first aux data
+        while (data) {
+            printf("%.2s:%c:", bam_aux_tag(data), NULL != strchr("cCsSiI", bam_aux_type(data)) ? 'i' : bam_aux_type(data));
+              //macros gets the tag and type of aux data
+            //dump the data
+            printauxdata(stdout, bam_aux_type(data), -1, data);
+            ...
+            data = bam_aux_next(bamdata, data);                                     //get the next aux data
+    ...
+Refer: dump_aux.c
+
+Shows all the tags from all alignments.
+
+    ./dump_aux ../../samtools/test/mpileup/mpileup.1.bam
+
+
+### Add/Remove/Update header
+
+There are specific types of data that can be part of header data. They have
+a tag from HD, SQ, RG, PG and CO. Fully formatted header lines, separated by new
+line, can be added with sam_hdr_add_lines api. A single header line can be added
+using sam_hdr_add_line api where the header type, tag and value pair are passed
+as arguments, terminated with a NULL argument. The PG header lines are special
+that they have a kind of linkage to previous PG lines. This linkage can be auto
+generated by using sam_hdr_add_pg api which sets the 'PP' field used in linkage.
+sam_hdr_write api does the write of the header data to file.
+
+    ...
+    //add SQ line with SN as TR1 and TR2
+    if (sam_hdr_add_lines(in_samhdr, &sq[0], 0))                                        //length as 0 for NULL terminated data
+        ... // error
+
+    //add RG line with ID as RG1
+    if (sam_hdr_add_line(in_samhdr, "RG", "ID", "RG1", "LB", "Test", "SM", "S1", NULL))
+        ... // error
+
+    //add PG/CO lines
+    if (sam_hdr_add_pg(in_samhdr, "add_header", "VN", "Test", "CL", data.s, NULL))      //NULL is to indicate end of args
+        ... // error
+    if (sam_hdr_add_line(in_samhdr, "CO", "Test data", NULL))                           //NULL is to indicate end of args
+        ... // error
+
+    //write output
+    if (sam_hdr_write(outfile, in_samhdr) < 0)
+        ... // error
+Refer: add_header.c
+
+Not all type of header data can be removed but where it is possible, either a
+specific header line can be removed or all of a header type can be removed. To
+remove a specific line, header type, unique identifier field tag and its value
+to be used. To remove all lines of a type, header type and unique identifier
+field tag are to be used.
+
+    ...
+
+    //remove specific line
+    if (sam_hdr_remove_line_id(in_samhdr, header, id, idval) < 0)
+        ... // error
+
+    //remove multiple lines of a header type
+    if (sam_hdr_remove_lines(in_samhdr, header, id, NULL) < 0)
+        ... // error
+Refer: rem_header.c
+
+Shows the file content after removing SQ line with SN 2.
+    ./rem_header ../../samtools/test/mpileup/mpileup.1.bam SQ 2
+
+The unique identifier for the line needs to be found to update a field, though
+not all types in the header may be modifiable.  The api sam_hdr_update_line
+takes the unique identifier for the header line type, its value, the field
+which needs to be modified and the new value with which to modify it, followed
+by a NULL.
+e.g. To change LN field from 2000 to 2250 in SQ line with unique identifier SN
+as 'chr1', sam_hdr_update_line( header, "SQ", "SN", "chr1", "LN", "2250",
+NULL).  To change PP field from ABC to DEF in PG line with ID APP.10,
+sam_hdr_update_line( header, "PG", "ID", "APP.10", "PP", "DEF", NULL).
+
+    ...
+    //update with new data
+    if (sam_hdr_update_line(in_samhdr, header, id, idval, tag, val, NULL) < 0) {
+        printf("Failed to update data\n");
+        goto end;
+    }
+    ...
+Refer: update_header.c
+
+Shows new sam file with 2nd SQ line having length as 38.
+
+    ./update_header /tmp/sample.sam.gz SQ T1 LN 38
+
+
+### Update alignment data
+
+Many of the bam data fields may be updated by setting new value to appropriate
+field in bam1_core_t structure and for a few, creating a new bam1_t record would
+be easier than update of existing record.
+
+    ...
+    while ((ret_r = sam_read1(infile, in_samhdr, bamdata)) >= 0)
+    {
+    ...
+            case 1:// QNAME
+                ret = bam_set_qname(bamdata, val);
+            break;
+            case 2:// FLAG
+                bamdata->core.flag = atol(val) & 0xFFFF;
+            break;
+            case 3:// RNAME
+            case 7:// RNEXT
+                if ((ret = sam_hdr_name2tid(in_samhdr, val)) < 0)
+                    ... // error
+                if (field == 3) {
+                    //reference
+                    bamdata->core.tid = ret;
+                } else {
+                    //mate reference
+                    bamdata->core.mtid = ret;
+                }
+            break;
+            case 4:// POS
+                bamdata->core.pos = atoll(val);
+            break;
+            case 5:// MAPQ
+                bamdata->core.qual = atoi(val) & 0x0FF;
+            break;
+            case 6:// CIGAR
+            {
+                ...
+                //get cigar array and set all data in new bam record
+                if ((ncigar = sam_parse_cigar(val, NULL, &cigar, &size)) < 0)
+                    ... // error
+                if (bam_set1(newbam, bamdata->core.l_qname, bam_get_qname(bamdata), bamdata->core.flag, bamdata->core.tid,
+                 bamdata->core.pos, bamdata->core.qual, ncigar, cigar, bamdata->core.mtid, bamdata->core.mpos,
+                  bamdata->core.isize, bamdata->core.l_qseq, (const char*)bam_get_seq(bamdata),
+                   (const char*)bam_get_qual(bamdata), bam_get_l_aux(bamdata)) < 0)
+                    ... // error
+
+                //correct sequence data as input is expected in ascii format and not as compressed inside bam!
+                memcpy(bam_get_seq(newbam), bam_get_seq(bamdata), (bamdata->core.l_qseq + 1) / 2);
+                //copy the aux data
+                memcpy(bam_get_aux(newbam), bam_get_aux(bamdata), bam_get_l_aux(bamdata));
+            ...
+            break;
+            case 8:// PNEXT
+                bamdata->core.mpos = atoll(val);
+            break;
+            case 9:// TLEN
+                bamdata->core.isize = atoll(val);
+            break;
+            case 10:// SEQ
+                ...
+                for( c = 0; c < i; ++c) {
+                    bam_set_seqi(bam_get_seq(bamdata), c, seq_nt16_table[(unsigned char)val[c]]);
+                }
+            break;
+            case 11:// QUAL
+                ...
+                for (c = 0; c < i; ++c)
+                    val[c] -= 33;               //phred score from ascii value
+                memcpy(bam_get_qual(bamdata), val, i);
+Refer: mod_bam.c
+
+Shows data with RNAME modified to T2.
+
+    ./mod_bam /tmp/sample.sam ITR1 3 T2
+
+The auxiliary data in bam1_t structure can be modified using
+bam_aux_update_float, bam_aux_update_int etc. apis. If the aux field is not
+present at all, it can be appended using bam_aux_append.
+
+    ...
+    //matched to qname, update aux
+    if (!(data = bam_aux_get(bamdata, tag))) {
+        //tag not present append
+        ... // cut: computed length and val based on tag type
+        if (bam_aux_append(bamdata, tag, type, length, (const uint8_t*)val))
+            ... // error
+    } else {
+        //update the tag with newer value
+        char auxtype = bam_aux_type(data);
+        switch (type) {
+            case 'f':
+            case 'd':
+                ...
+                if (bam_aux_update_float(bamdata, tag, atof(val)))
+                    ... // error
+            case 'C':
+            case 'S':
+            case 'I':
+                ...
+                if (bam_aux_update_int(bamdata, tag, atoll(val)))
+                    ... // error
+            case 'Z':
+                ...
+                if (bam_aux_update_str(bamdata, tag, length, val))
+                    ... // error
+            case 'A':
+                ...
+                //update the char data directly on buffer
+                *(data+1) = val[0];
+Refer: mod_aux.c
+
+Shows the given record's MD tag set to Test.
+
+    ./mod_aux samtools/test/mpileup/mpileup.1.bam ERR013140.6157908 MD Z Test
+
+The array aux fields can be updated using bam_aux_update_array api.
+
+    ...
+    if (bam_aux_update_array(bamdata, "BA", 'I', sizeof(cnt)/sizeof(cnt[0]), cnt))
+        ... // error
+Refer: mod_aux_ba.c
+
+Shows the records updated with an array of integers, containing count of ACGT
+and N in that order. The bases are decoded before count for the sake of
+simplicity. Refer qtask_ordered.c for a better counting where decoding is made
+outside the loop.
+
+    ./mod_aux_ba samtools/test/mpileup/mpileup.1.bam
+
+
+### Create an index
+
+Indexes help to read data faster without iterating sequentially through the
+file. Indexes contain the position information about alignments and that they
+can be read easily. There are different type of indices, BAI, CSI, CRAI, TBI,
+FAI etc. and are usually used with iterators.
+
+Indexing of plain/textual files are not supported, compressed SAM&FASTA/Q, BAM,
+and CRAM files can be indexed. CRAM files are indexed as .crai and the others
+as .bai, .csi, .fai etc. Each of these types have different internal
+representations of the index information. Bai uses a fixed configuration values
+where as csi has them dynamically updated based on the alignment data.
+
+Indexes can be created either with save of alignment data or explicitly by
+read of existing alignment file for alignment data (SAM/BAM/CRAM). For reference
+data it has to be explicitly created (FASTA).
+
+To create index along with alignment write, the sam_idx_init api need to be
+invoked before the start of alignment data write. This api takes the output
+samFile pointer, header pointer, minimum shift and index file path. For BAI
+index, the min shift has to be 0.
+
+At the end of write, sam_idx_save api need to be invoked to save the index.
+
+    ...
+    //write header
+    if (sam_hdr_write(outfile, in_samhdr))
+        ... // error
+    // initialize indexing, before start of write
+    if (sam_idx_init(outfile, in_samhdr, size, fileidx))
+        ... // error
+        if (sam_write1(outfile, in_samhdr, bamdata) < 0)
+            ... // error
+    if (sam_idx_save(outfile))
+        ... // error
+Refer:index_write.c
+
+Creates mpileup.1.bam and mpileup.1.bam.bai in /tmp/.
+
+    ./idx_on_write  ../../samtools/test/mpileup/mpileup.1.bam 0 /tmp/
+
+To create index explicitly on an existing alignment data file, the
+sam_index_build api or its alike can be used. sam_index_build takes the
+alignment file path, min shift for the index and creates the index file in
+same path. The output name will be based on the alignment file format and min
+shift passed.
+
+The sam_index_build2 api takes the index file path as well and gives more
+control than the previous one.  The sam_index_build3 api provides an option to
+configure the number of threads in index creation.
+
+Index for reference data can be created using fai_build3 api. This creates
+index file with .fai extension. If the file is bgzip-ped, a .gzi file is
+created as well. It takes the path to input file and that of fai and gzi files.
+When fai/gzi path are NULL, they are created along with input file.
+These index files will be useful for reference data access.
+
+    ...
+    if (fai_build3(filename, NULL, NULL) == -1)
+        ... // error
+Refer: index_fast.c
+
+A tabix index can be created for compressed vcf/sam/bed and other data using
+tbx_index_build. It is mainly used with vcf and non-sam type files.
+
+
+### Read with iterators
+
+Index file helps to read required data without sequentially accessing the file
+and are required to use iterators. The interested reference, start and end
+position etc. are required to read data with iterators. With index and these
+information, an iterator is created and relevant alignments can be accessed by
+iterating it.
+
+The api sam_index_load and the like does the index loading. It takes input
+samFile pointer and file path. It loads the index file based on the input file
+name, from the same path and with implicit index file extension - cram file
+with .crai and others with .bai. The sam_index_load2 api accepts explicit path
+to index file, which allows loading it from a different location and explicit
+extensions. The sam_index_load3 api supports download/save of the index
+locally from a remote location. These apis returns NULL on failure and index
+pointer on success.
+
+The index file path can be appended to alignment file path and used as well.
+In this case the paths are expected to be separated by '##idx##'.
+
+The sam_iter_queryi or sam_iter_querys apis may be used to create an iterator
+and sam_itr_next api does the alignment data retrieval. Along with retrieval
+of current data, it advances the iterator to next relevant data. The
+sam_iter_queryi takes the interested positions as numeric values and
+sam_iter_querys takes the interested position as a string.
+
+With sam_iter_queryi, the reference id can be the 0 based index of reference
+data, -2 for unmapped alignments, -3 to start read from beginning of file, -4
+to continue from current position, -5 to return nothing. Based on the
+reference id given, alignment covering the given start and end positions will
+be read with sam_iter_next api.
+
+With sam_iter_querys, the reference sequence is identified with the name and
+interested positions can be described with start and end separated by '-' as
+string. When sequence is identified as '.', it begins from the start of file
+and when it is '*', unmapped alignments are read. Reference with <name>[:],
+<name>:S, <name>:S-E, <name>:-E retrieves all data, all data covering position
+S onwards, all data covering position S to E, all data covering upto position
+E of reference with ID <name> respectively on read using sam_iter_next.
+
+The index and iterator created are to be destroyed once the need is over.
+sam_itr_destroy and hts_idx_destroy apis does this.
+
+    ...
+    //load index file
+    if (!(idx = sam_index_load2(infile, inname, idxfile)))
+        ... // error
+    //create iterator
+    if (!(iter = sam_itr_querys(idx, in_samhdr, region)))
+        ... // error
+
+    //read using iterator
+    while ((c = sam_itr_next(infile, iter, bamdata)) >= 0)
+        ... // error
+
+    if (iter)
+        sam_itr_destroy(iter);
+    if (idx)
+        hts_idx_destroy(idx);
+    ...
+Refer:index_reg_read.c
+
+With sample.sam, region as \* will show alignments with name UNMAP2 and UNMAP3
+
+    ./read_reg /tmp/sample.sam.gz \*
+
+With region as \., it shows all alignments
+
+    ./read_reg /tmp/sample.sam.gz \.
+
+With region as T1:1-4, start 1 and end 4 it shows nothing and with T1:1-5 it
+shows alignment with name ITR1.
+
+    ./read_reg /tmp/sample.sam.gz T1:1-5
+
+With region as T2:30-100, it shows alignment with name ITR2M which refers the
+reference data T2.
+
+    ./read_reg /tmp/sample.sam.gz T2:30-100
+
+
+Multiple interested regions can be specified for read using sam_itr_regarray.
+It takes index path, header, count of regions and region descriptions as array
+of char array / string. This array passed need to be released by the user
+itself.
+
+    ...
+    //load index file, assume it to be present in same location
+    if (!(idx = sam_index_load(infile, inname)))
+        ... // error
+    //create iterator
+    if (!(iter = sam_itr_regarray(idx, in_samhdr, regions, regcnt)))
+        ... // error
+    if (regions) {
+        //can be freed as it is no longer required
+        free(regions);
+        regions = NULL;
+    }
+
+    //get required area
+    while ((c = sam_itr_multi_next(infile, iter, bamdata) >= 0))
+        ... // process bamdata
+Refer:index_multireg_read.c
+
+With compressed sample.sam and 2 regions from reference T1 (30 to 32) and 1
+region from T2 (34 onwards), alignments with name A1, B1, A2 and ITR2M would
+be shown.
+
+    ./read_multireg /tmp/sample.sam.gz 2 T1:30-32,T2:34
+
+To use numeric indices instead of textual regions, sam_itr_regions can be used.
+It takes index file path, header, count of regions and an array of region
+description (hts_reglist_t*), which has the start end positions as numerals.
+
+The index and iterators are to be destroyed using the sam_itr_destroy and
+hts_idx_destroy. The hts_reglist_t* array passed is destroyed by the library
+on iterator destroy. The regions array (array of char array/string) needs to be
+destroyed by the user itself.
+
+For fasta/fastq files, the index has to be loaded using fai_load3_format which
+takes the file, index file names and format. With single region specification
+fai_fetch64 can be used to get bases, and fai_fetchqual64 for quality in case
+of fastq data. With multiple region specification, with comma separation,
+faidx_fetch_seq64 and faidx_fetch_qual64 does the job. Regions has to be parsed
+using fai_parse_region in case of multiregion specifications. fai_adjust_region
+is used to adjust the start-end points based on available data.
+
+Below excerpt shows fasta/q access with single and multiregions,
+
+    ...
+    //load index
+    if (!(idx = fai_load3_format(inname, NULL, NULL, FAI_CREATE, fmt)))
+        ... // error
+
+    ...
+    if (!usemulti) {
+        //get data from single given region
+        if (!(data = fai_fetch64(idx, region, &len)))
+            ... // region not found
+
+        printf("Data: %"PRId64" %s\n", len, data);
+        free((void*)data);
+        //get quality for fastq type
+        if (fmt == FAI_FASTQ) {
+            if (!(data = fai_fetchqual64(idx, region, &len)))
+                ... // region not found
+        ...
+
+    } else { // usemulti
+        //parse, get each region and get data for each
+        while ((remaining = fai_parse_region(idx, region, &tid, &beg, &end, HTS_PARSE_LIST))) {     //here expects regions as csv
+            //parsed the region, correct end points based on actual data
+            if (fai_adjust_region(idx, tid, &beg, &end) == -1)
+                ... // error
+            //get data for given region
+            if (!(data = faidx_fetch_seq64(idx, faidx_iseq(idx, tid), beg, end, &len)))
+                ... // region not found
+
+            printf("Data: %"PRIhts_pos" %s\n", len, data);
+            free((void*)data);
+            data = NULL;
+            //get quality data for fastq
+            if (fmt == FAI_FASTQ) {
+                if (!(data = faidx_fetch_qual64(idx, faidx_iseq(idx, tid), beg, end, &len)))
+                    ... // error
+                printf("Qual: %"PRIhts_pos" %s\n", len, data);
+                free((void*)data);
+            ...
+            region = remaining;                                     //parse remaining region defs
+
+    ...
+    if (idx) {
+        fai_destroy(idx);
+    ...
+Refer: read_fast_index.c
+
+
+### Pileup and MPileup
+
+Pileup shows the transposed view of the SAM alignment data, i.e. it shows the
+reference positions and bases which cover that position through different reads
+side by side. MPileup facilitates the piling up of multiple sam files against
+each other and same reference at the same time.
+
+Mpileup has replaced the pileup. The input expects the data to be sorted by
+position.
+
+Pileup needs to be initialized with bam_pileup_init method which takes pointer
+to a method, which will be called by pileup to read data from required files,
+and pointer to data which might be required for this read method to do the
+read operation. It returns a pointer to the pileup iterator.
+
+User can specify methods which need to be invoked during the load and unload
+of an alignment, like constructor and destructor of objects.
+Bam_plp_constructor and bam_plp_destructor methods does the setup of
+these methods in the pileup iterator. During invocation of these methods, the
+pointer to data passed in the initialization is passed as well. If user want
+to do any custom status handling or actions during load or unload, it can be
+done in these methods. Alignment specific data can be created and stored in
+an argument passed to the constructor and the same will be accessible during
+pileup status return. The same will be accessible during destructor as well
+where any deallocation can be made.
+
+User is expected to invoke bam_plp_auto api to get the pileup status. It
+returns the pileup status or NULL on end. During this all alignments are read
+one by one, using the method given in initialization for data read, until one
+for a new reference is found or all alignment covering a position is read. On
+such condition, the pileup status is returned and the same continuous on next
+bam_plp_auto call.  The pileup status returned is an array for all positions
+for which the processing is completed. Along with the result, the reference
+index, position in reference data and number of alignments which covers this
+position are passed. User can iterate the result array and get bases from each
+alignment which covers the given reference position. The alignment specific
+custom data which were created in constructor function will also be available
+in the result.
+
+The bam_plp_auto api invokes the data read method to load an alignment and the
+constructor method is invoked during the load. Once the end of alignment is
+passed, it is removed from the processing and destructor method is invoked,
+that user could do deallocations and custom actions as in load during this
+time. The custom data passed during the initialization is passed to the
+constructor and destructor methods during invocation.
+
+Once the forward and reverse strands are identified, the better of the quality
+is identified and used. Both reads are required for this and hence reads are
+cached until its mate is read. The maximum number of reads that can be cached
+is controlled by bam_plp_set_maxcnt. Reads covering a position are cached and
+as soon as mate is found, quality is adjusted and is removed from cache. Reads
+above the cache limit are discarded.
+
+Once done, the pileup iterator to be discarded by sam_plp_destroy api.
+
+    ...
+    if (!(plpiter = bam_plp_init(readdata, &conf)))
+        ... // error
+    //set constructor destructor callbacks
+    bam_plp_constructor(plpiter, plpconstructor);
+    bam_plp_destructor(plpiter, plpdestructor);
+
+    while ((plp = bam_plp_auto(plpiter, &tid, &refpos, &n))) {
+        printf("%d\t%d\t", tid+1, refpos+1);
+        for (j = 0; j < n; ++j) {
+            //doesnt detect succeeding insertion and deletion together here, only insertion is identified
+            //deletion is detected in plp->is_del as and when pos reaches the position
+            //if detection ahead is required, use bam_plp_insertion here which gives deletion length along with insertion
+            if (plp[j].is_del || plp[j].is_refskip) {
+                printf("*");
+                continue;
+            }
+            //start and end are displayed in UPPER and rest on LOWER
+            printf("%c", plp[j].is_head ? toupper(seq_nt16_str[bam_seqi(bam_get_seq(plp[j].b), plp[j].qpos)]) :
+                            (plp[j].is_tail ? toupper(seq_nt16_str[bam_seqi(bam_get_seq(plp[j].b), plp[j].qpos)]) :
+                             tolower(seq_nt16_str[bam_seqi(bam_get_seq(plp[j].b), plp[j].qpos)])));
+            if (plp[j].indel > 0) {
+                //insertions, anyway not start or end
+                printf("+%d", plp[j].indel);
+                for (k = 0; k < plp[j].indel; ++k) {
+                    printf("%c", tolower(seq_nt16_str[bam_seqi(bam_get_seq(plp[j].b), plp[j].qpos + k + 1)]));
+                }
+            }
+            else if (plp[j].indel < 0) {
+                printf("%d", plp[j].indel);
+                for (k = 0; k < -plp[j].indel; ++k) {
+                    printf("?");
+                }
+    ...
+    if (plpiter)
+        bam_plp_destroy(plpiter);
+    ...
+Refer:pileup.c
+
+The read method may use a simple read or it could be an advanced read using
+indices, iterators and region specifications based on the need. The constructor
+method may create any custom data and store it in the pointer passed to it. The
+same need to be released by use on destructor method.
+
+MPileup works same as the pileup and supports multiple inputs against the same
+reference, giving side by side view of reference and alignments from different
+inputs.
+
+MPileup needs to be initialized with bam_mpileup_init method which takes
+pointer to a method, which will be called by pileup to read data from required
+files, and an array of pointer to data which might be required for this read
+method to do the read operation. It returns a pointer to the mpileup iterator.
+
+User can specify methods which need to be invoked during the load and unload
+of an alignment, like constructor and destructor of objects.
+bam_mplp_constructor and bam_mplp_destructor methods does the setup
+of these methods in the pileup iterator. During invocation of these methods,
+the pointer to data passed in the initialization is passed as well. If user
+want to do any custom status handling or actions during load or unload, it can
+be done on these methods. Alignment specific data can be created and
+stored in the custom data pointer and the same will be accessible during
+return of pileup status. The same will be accessible during destructor as well
+where any deallocation can be made.
+
+User is expected to invoke bam_mplp_auto api to get the pileup status. It
+returns the pileup status. During this all alignments are read one by one,
+using the method given in initialization for data read, until one for a new
+reference is found or all alignment covering a position is read. On such
+condition, the pileup status is returned and the same continuous on next
+bam_mplp_auto call.
+
+The pileup status is returned through a parameter in the method itself, is an
+array for all inputs, each containing array for positions on which the
+processing is completed. Along with the result, the reference index, position
+in reference data and number of alignments which covers this position are
+passed. User can iterate the result array and get bases from each alignment
+which covers the given reference position. The alignment specific custom data
+which were created in constructor function will also be available in the
+result.
+
+Once the forward and reverse strands are identified, the better of the quality
+is identified and used. Both reads are required for this and hence reads are
+cached until its mate is read. The maximum number of reads that can be cached
+is controlled by bam_mplp_set_maxcnt. Reads covering a position are cached and
+as soon as mate is found, quality is adjusted and is removed from cache. Reads
+above the cache limit are discarded.
+
+Once done, the pileup iterator to be discarded by sam_mplp_destroy api.
+
+    ...
+    if (!(mplpiter = bam_mplp_init(argc - 1, readdata, (void**) conf)))
+        ... // error
+    //set constructor destructor callbacks
+    bam_mplp_constructor(mplpiter, plpconstructor);
+    bam_mplp_destructor(mplpiter, plpdestructor);
+
+    while (bam_mplp64_auto(mplpiter, &tid, &refpos, depth, plp) > 0) {
+        printf("%d\t%"PRIhts_pos"\t", tid+1, refpos+1);
+
+        for (input = 0; input < argc - 1; ++input) {
+            for (dpt = 0; dpt  < depth[input]; ++dpt) {
+                if (plp[input][dpt].is_del || plp[input][dpt].is_refskip) {
+                    printf("*");
+                    continue;
+                }
+                //start and end are displayed in UPPER and rest on LOWER
+                printf("%c", plp[input][dpt].is_head ? toupper(seq_nt16_str[bam_seqi(bam_get_seq(plp[input][dpt].b),
+                 plp[input][dpt].qpos)]) : (plp[input]->is_tail ? toupper(seq_nt16_str[bam_seqi(bam_get_seq(plp[input][dpt].b),
+                  plp[input][dpt].qpos)]) : tolower(seq_nt16_str[bam_seqi(bam_get_seq(plp[input][dpt].b),
+                   plp[input][dpt].qpos)])));
+                if (plp[input][dpt].indel > 0) {
+                    //insertions, anyway not start or end
+                    printf("+%d", plp[input][dpt].indel);
+                    for (k = 0; k < plp[input][dpt].indel; ++k) {
+                        printf("%c", tolower(seq_nt16_str[bam_seqi(bam_get_seq(plp[input][dpt].b),
+                         plp[input][dpt].qpos + k + 1)]));
+                    }
+                }
+                else if (plp[input][dpt].indel < 0) {
+                    printf("%d", plp[input][dpt].indel);
+                    for (k = 0; k < -plp[input][dpt].indel; ++k) {
+                        printf("?");
+    ...
+    if (mplpiter) {
+        bam_mplp_destroy(mplpiter);
+    }
+    ...
+    if (plp) {
+        free(plp);
+    ...
+Refer:mpileup.c
+
+This sample takes multiple sam files and shows the pileup of data side by side.
+
+    ./mpileup /tmp/mp.bam /tmp/mp.sam
+
+
+### Base modifications
+
+The alignment data may contain base modification information as well. This
+gives the base, modifications found, orientation in which it was found and the
+quality for the modification. The base modification can be identified using
+hts_parse_basemod api. It stores the modification details on hts_base_mod_state
+and this has to be initialized using hts_base_mod_state_alloc api.
+
+Once the modifications are identified, they can be accessed through different
+ways. bam_mods_recorded api gives the modifications identified for an alignment.
+Modifications can be queried for each base position iteratively using
+bam_mods_at_next_pos api. Check the returned value with buffer size to see
+whether the buffer is big enough to retrieve all modifications.
+Instead of querying for each position, the next modified position can be
+directly retrieved directly using bam_next_basemod api. An alignment can be
+queried to have a specific modification using bam_mods_query_type api. At the
+end of processing, the state need to be released using hts_base_mod_state_free
+api.
+
+    ...
+    if (!(ms = hts_base_mod_state_alloc()))
+        ... // error
+    while ((ret_r = sam_read1(infile, in_samhdr, bamdata)) >= 0)
+    {
+        ...
+        if (bam_parse_basemod(bamdata, ms))
+            ... // error
+        bm = bam_mods_recorded(ms, &cnt);
+        for (k = 0; k < cnt; ++k) {
+            printf("%c", bm[k]);
+        }
+        printf("\n");
+        hts_base_mod mod[5] = {0};  //for ATCGN
+        if (opt) {
+            //option 1
+            for (; i < bamdata->core.l_qseq; ++i) {
+                if ((r = bam_mods_at_next_pos(bamdata, ms, mod, sizeof(mod)/sizeof(mod[0]))) <= -1) {
+                    printf("Failed to get modifications\n");
+                    goto end;
+                }
+                else if (r > (sizeof(mod) / sizeof(mod[0]))) {
+                    printf("More modifications than this app can handle, update the app\n");
+                    goto end;
+                }
+                else if (!r) {
+                    //no modification at this pos
+                    printf("%c", seq_nt16_str[bam_seqi(data, i)]);
+                }
+                //modifications
+                for (j = 0; j < r; ++j) {
+                    printf("%c%c%c", mod[j].canonical_base, mod[j].strand ? '-' : '+', mod[j].modified_base);
+    ...
+        else {
+            //option 2
+            while ((r = bam_next_basemod(bamdata, ms, mod, sizeof(mod)/sizeof(mod[0]), &pos)) >= 0) {
+                for (; i < bamdata->core.l_qseq && i < pos; ++i) {
+                    printf("%c", seq_nt16_str[bam_seqi(data, i)]);
+                }
+                //modifications
+                for (j = 0; j < r; ++j) {
+                    printf("%c%c%c", mod[j].canonical_base, mod[j].strand ? '-' : '+', mod[j].modified_base);
+                }
+    ...
+        //check last alignment's base modification
+        int strand = 0, impl = 0;
+        char canonical = 0, modification[] = "mhfcgebaon";      //possible modifications
+        printf("\n\nLast alignment has \n");
+        for (k = 0; k < sizeof(modification) - 1; ++k) {        //avoiding NUL termination
+            if (bam_mods_query_type(ms, modification[k], &strand, &impl, &canonical)) {
+                printf ("No modification of %c type\n", modification[k]);
+            }
+            else {
+                printf("%s strand has %c modified with %c, can %sassume unlisted as unmodified\n", strand ? "-/bottom/reverse" :
+                "+/top/forward", canonical, modification[k], impl?"" : "not " );
+            }
+        }
+    ...
+    if (ms)
+        hts_base_mod_state_free(ms);
+    ...
+Refer:modstate.c
+
+The modification can be accessed in pileup mode as well. bam_mods_at_qpos gives
+the modification at given pileup position. Insertion and deletion to the given
+position with possible modification can be retrieved using bam_plp_insertion_mod
+api.
+
+    ...
+    int plpconstructor(void *data, const bam1_t *b, bam_pileup_cd *cd) {
+        //when using cd, initialize and use as it will be reused after destructor
+        cd->p = hts_base_mod_state_alloc();
+        //parse the bam data and gather modification data from MM tags
+        return (-1 == bam_parse_basemod(b, (hts_base_mod_state*)cd->p)) ? 1 : 0;
+    }
+
+    int plpdestructor(void *data, const bam1_t *b, bam_pileup_cd *cd) {
+        if (cd->p) {
+            hts_base_mod_state_free((hts_base_mod_state *)cd->p);
+            cd->p = NULL;
+        }
+        return 0;
+    }
+
+    int main(int argc, char *argv[])
+    {
+    ...
+    if (!(plpiter = bam_plp_init(readdata, &conf))) {
+        ... // error
+    //set constructor destructor callbacks
+    bam_plp_constructor(plpiter, plpconstructor);
+    bam_plp_destructor(plpiter, plpdestructor);
+
+    while ((plp = bam_plp_auto(plpiter, &tid, &refpos, &depth))) {
+        memset(&mods, 0, sizeof(mods));
+        printf("%d\t%d\t", tid+1, refpos+1);
+
+        for (j = 0; j < depth; ++j) {
+            dellen = 0;
+            if (plp[j].is_del || plp[j].is_refskip) {
+                printf("*");
+                continue;
+            }
+            /*invoke bam mods_mods_at_qpos before bam_plp_insertion_mod that the base modification
+            is retrieved before change in pileup pos thr' plp_insertion_mod call*/
+            if ((modlen = bam_mods_at_qpos(plp[j].b, plp[j].qpos, plp[j].cd.p, mods, NMODS)) == -1)
+                ... // error
+            //use plp_insertion/_mod to get insertion and del at the same position
+            if ((inslen = bam_plp_insertion_mod(&plp[j], (hts_base_mod_state*)plp[j].cd.p, &insdata, &dellen)) == -1)
+                ... // error
+            //start and end are displayed in UPPER and rest on LOWER, only 1st modification considered
+            //base and modification
+            printf("%c%c%c", plp[j].is_head ? toupper(seq_nt16_str[bam_seqi(bam_get_seq(plp[j].b), plp[j].qpos)]) :
+                (plp[j].is_tail ? toupper(seq_nt16_str[bam_seqi(bam_get_seq(plp[j].b), plp[j].qpos)]) :
+                    tolower(seq_nt16_str[bam_seqi(bam_get_seq(plp[j].b), plp[j].qpos)])),
+                modlen > 0 ? mods[0].strand ? '-' : '+' : '\0', modlen > 0 ? mods[0].modified_base : '\0');
+            //insertion and deletions
+            if (plp[j].indel > 0) {
+                //insertion
+                /*insertion data from plp_insertion_mod, note this shows the quality value as well
+                which is different from base and modification above;the lower case display is not attempted either*/
+                printf("+%d%s", plp[j].indel, insdata.s);
+                //handle deletion if any
+                if (dellen) {
+                    printf("-%d", dellen);
+                    for (k = 0; k < dellen; ++k) {
+                        printf("?");
+                ...
+            else if (plp[j].indel < 0) {
+                //deletion
+                printf("%d", plp[j].indel);
+                for (k = 0; k < -plp[j].indel; ++k) {
+                    printf("?");
+                }
+            }
+    ...
+Refer:pileup_mod.c
+
+
+### Read selected fields
+
+At times the whole alignment data may not be of interest and it would be
+better to read required fields alone from the alignment data. CRAM file format
+supports such specific data read and HTSLib provides an option to use this.
+This can improve the performance on read operation.
+
+The hts_set_opt method does the selection of specified fields. There are flags
+indicating specific fields, like SAM_FLAG, SAM_SEQ, SAM_QNAME, in alignment
+data and a combination of flags for the required fields can be passed with
+CRAM_OPT_REQUIRED_FIELDS to this api.
+
+    ...
+    //select required field alone, this is useful for CRAM alone
+    if (hts_set_opt(infile, CRAM_OPT_REQUIRED_FIELDS, SAM_FLAG) < 0)
+        ... // error
+
+    //read header
+    in_samhdr = sam_hdr_read(infile);
+    ...
+    //read data, check flags and update count
+    while ((c = sam_read1(infile, in_samhdr, bamdata)) >= 0) {
+        if (bamdata->core.flag & BAM_FREAD1)
+            cntread1++;
+        ...
+Refer: flags_htsopt_field.c
+
+
+### Thread-pool to read / write
+
+The HTSLib api supports thread pooling for better performance. There are a few
+ways in which this can be used. The pool can be made specific for a file or a
+generic pool can be created and shared across multiple files. Thread pool can
+also be used to execute user defined tasks. The tasks are to be added to queue,
+threads in pool executes them and results can be queued back if required.
+
+To have a thread pool specific for a file, hts_set_opt api can be used with the
+file pointer, HTS_OPT_NTHREADS and the number of threads to be in the pool.
+Thread pool is released on closure of file. To have a thread pool which can be
+shared across different files, it needs to be initialized using hts_tpool_init
+api, passing number of threads as an argument. This thread pool can be
+associated with a file using hts_set_opt api. The file pointer,
+HTS_OPT_THREAD_POOL and the thread pool address are to be passed as arguments to
+the api. The thread pool has to be released with hts_tpool_destroy.
+
+The samples are trivial ones to showcase the usage of api. The number of threads
+to use for different tasks has to be identified based on complexity and
+parallelism of the task.
+
+Below excerpt shows file specific thread pool,
+
+    ...
+    //create file specific threads
+    if (hts_set_opt(infile, HTS_OPT_NTHREADS, 1) < 0 ||     //1 thread specific for reading
+    hts_set_opt(outfile1, HTS_OPT_NTHREADS, 1) < 0 ||       //1 thread specific for sam write
+    hts_set_opt(outfile2, HTS_OPT_NTHREADS, 2) < 0) {       //2 thread specific for bam write
+        printf("Failed to set thread options\n");
+        goto end;
+    }
+Refer: split_thread1.c
+
+Below excerpt shows a thread pool shared across files,
+
+    ...
+    //create a pool of 4 threads
+    if (!(tpool.pool = hts_tpool_init(4)))
+        ... // error
+    //share the pool with all the 3 files
+    if (hts_set_opt(infile, HTS_OPT_THREAD_POOL, &tpool) < 0 ||
+    hts_set_opt(outfile1, HTS_OPT_THREAD_POOL, &tpool) < 0 ||
+    hts_set_opt(outfile2, HTS_OPT_THREAD_POOL, &tpool) < 0) {
+        ... // error
+
+    ... // do something
+
+    //tidy up at end
+    if (tpool.pool)
+        hts_tpool_destroy(tpool.pool);
+    ...
+Refer: split_thread2.c
+
+Note that it is important to analyze the task in hand to decide the number of
+threads to be used. As an example, if the number of threads for reading is set
+to 2 and bam write to 1, keeping total number of threads the same, the
+performance may decrease as bam decoding is easier than encoding.
+
+Custom task / user defined functions can be performed on data using thread pool
+and for that, the task has to be scheduled to a queue. Thread pool associated
+with the queue will perform the task. There can be multiple pools and queues.
+The order of execution of threads are decided based on many factors and load on
+each task may vary, so the completion of the tasks may not be in the order of
+their queueing. The queues can be used in two different ways, one where the
+result is enqueued to queue again to be read in same order as initial queueing,
+second where the resuls are not enqueued and completed possibly in a different
+order than initial queueing. Explicitly created threads can also be used along
+with hts thread pool usage.
+
+hts_tpool_process_init initializes the queue / process, associates a queue with
+thread pool and reserves space for given number of tasks on queue. It takes a
+parameter indicating whether the result need to be enqueued for retrieval or
+not. If the result is enqueued, it is retrieved in the order of scheduling of
+task. Another parameter sets the maximum number of slots for tasks in queue,
+usually 2 times the number of threads are used. The input and output have their
+own queues and they grow as required upto the max set. hts_tpool_dispatch api
+enqueues the task to the queue. The api blocks when there is no space in queue.
+This behavior can be controlled with hts_tpool_dispatch2 api. The queue can be
+reset using hts_tpool_process_reset api where all tasks are discarded. The api
+hts_tpool_dispatch3 supports configuring cleanup routines which are to be run
+when reset occurs on the queue. hts_tpool_process_flush api can ensure that
+all the piled up tasks are processed, a possible case when the queueing and
+processing happen at different speeds. hts_tpool_process_shutdown api stops the
+processing of queue.
+
+There are a few apis which let the user to check the status of processing. The
+api hts_tpool_process_empty shows whether all the tasks are completed or not.
+The api hts_tpool_process_sz gives the number of tasks, at different states of
+processing. The api hts_tpool_process_len gives the number of results in output
+queue waiting to be collected.
+
+The order of execution of tasks depends on the number of threads involved and
+how the threads are scheduled by operating system. When the results are enqueued
+back to queue, they are read in same order of enqueueing of task and in that
+case the order of execution will not be noticed. When the results are not
+enqueued the results are available right away and the order of execution may be
+noticeable. Based on the nature of task and the need of order maintenance, users
+can select either of the queueing.
+
+Below excerpts shows the usage of queues and threads in both cases. In the 1st,
+alignments are updated with an aux tag indicating GC ratio. The order of data
+has to be maintained even after update, hence the result queueing is used to
+ensure same order as initial. A number of alignments are bunched together and
+reuse of allocated memory is made to make it perform better. A sentinel job is
+used to identify the completion of all tasks at the result collection side.
+    ...
+    void *thread_ordered_proc(void *args)
+    {
+        ...
+        for ( i = 0; i < bamdata->count; ++i) {
+            ...
+            for (pos = 0; pos < bamdata->bamarray[i]->core.l_qseq; ++pos)
+                count[bam_seqi(data,pos)]++;
+            ...
+            gcratio = (count[2] /*C*/ + count[4] /*G*/) / (float) (count[1] /*A*/ + count[8] /*T*/ + count[2] + count[4]);
+
+            if (bam_aux_append(bamdata->bamarray[i], "xr", 'f', sizeof(gcratio), (const uint8_t*)&gcratio) < 0) {
+
+    ...
+    void *threadfn_orderedwrite(void *args)
+    {
+        ...
+        //get result and write; wait if no result is in queue - until shutdown of queue
+        while (tdata->result == 0 &&
+            (r = hts_tpool_next_result_wait(tdata->queue)) != NULL) {
+            bamdata = (data*) hts_tpool_result_data(r);
+            ...
+            for (i = 0; i < bamdata->count; ++i) {
+                if (sam_write1(tdata->outfile, tdata->samhdr, bamdata->bamarray[i]) < 0) {
+                    ... // error
+            ...
+            hts_tpool_delete_result(r, 0);              //release the result memory
+            ...
+
+        // Shut down the process queue.  If we stopped early due to a write failure,
+        // this will signal to the other end that something has gone wrong.
+        hts_tpool_process_shutdown(tdata->queue);
+
+    ...
+    int main(int argc, char *argv[])
+    {
+        ...
+        if (!(pool = hts_tpool_init(cnt)))                  //thread pool
+            ... // error
+        tpool.pool = pool;      //to share the pool for file read and write as well
+        //queue to use with thread pool, for task and results
+        if (!(queue = hts_tpool_process_init(pool, cnt * 2, 0))) {
+    ...
+        //share the thread pool with i/o files
+        if (hts_set_opt(infile, HTS_OPT_THREAD_POOL, &tpool) < 0 ||
+            hts_set_opt(outfile, HTS_OPT_THREAD_POOL, &tpool) < 0)
+            ... // error
+        if (pthread_create(&thread, NULL, threadfn_orderedwrite, &twritedata))
+            ... // error
+        while (c >= 0) {
+            if (!(bamdata = getbamstorage(chunk, &bamcache)))
+                ... // error
+            for (cnt = 0; cnt < bamdata->maxsize; ++cnt) {
+                c = sam_read1(infile, in_samhdr, bamdata->bamarray[cnt]);
+                ...
+                if (hts_tpool_dispatch3(pool, queue, thread_ordered_proc, bamdata,
+                                        cleanup_bamstorage, cleanup_bamstorage,
+                                        0) == -1)
+                    ... // error
+        ...
+        if (queue) {
+            if (-1 == c) {
+                // EOF read, send a marker to tell the threadfn_orderedwrite()
+                // function to shut down.
+                if (hts_tpool_dispatch(pool, queue, thread_ordered_proc,
+                                    NULL) == -1) {
+                    ... // error
+                hts_tpool_process_shutdown(queue);
+
+        ...
+        // Wait for threadfn_orderedwrite to finish.
+        if (started_thread) {
+            pthread_join(thread, NULL);
+
+        ...
+        if (queue) {
+            // Once threadfn_orderedwrite has stopped, the queue can be
+            // cleaned up.
+            hts_tpool_process_destroy(queue);
+        }
+    ...
+Refer: qtask_ordered.c
+
+In this 2nd, the bases are counted and GC ratio of whole file is calculated.
+Order in which bases are counted is not relevant and no result queue required.
+The queue is created as input only.
+    ...
+    void *thread_unordered_proc(void *args)
+    {
+        ...
+        for ( i = 0; i < bamdata->count; ++i) {
+            data = bam_get_seq(bamdata->bamarray[i]);
+            for (pos = 0; pos < bamdata->bamarray[i]->core.l_qseq; ++pos)
+                counts[bam_seqi(data, pos)]++;
+
+        ...
+        //update result and add the memory block for reuse
+        pthread_mutex_lock(&bamdata->cache->lock);
+        for (i = 0; i < 16; i++) {
+            bamdata->bases->counts[i] += counts[i];
+        }
+
+        bamdata->next = bamdata->cache->list;
+        bamdata->cache->list = bamdata;
+        pthread_mutex_unlock(&bamdata->cache->lock);
+
+    ...
+    int main(int argc, char *argv[])
+    {
+        ...
+        if (!(queue = hts_tpool_process_init(pool, cnt * 2, 1)))
+            ... // error
+        c = 0;
+        while (c >= 0) {
+            ...
+            for (cnt = 0; cnt < bamdata->maxsize; ++cnt) {
+                c = sam_read1(infile, in_samhdr, bamdata->bamarray[cnt]);
+
+            ...
+            if (c >= -1 ) {
+                ...
+                if (hts_tpool_dispatch3(pool, queue, thread_unordered_proc, bamdata,
+                                        cleanup_bamstorage, cleanup_bamstorage,
+                                        0) == -1)
+                    ... // error
+        ...
+        if (-1 == c) {
+            // EOF read, ensure all are processed, waits for all to finish
+            if (hts_tpool_process_flush(queue) == -1) {
+                fprintf(stderr, "Failed to flush queue\n");
+            } else { //all done
+                //refer seq_nt16_str to find position of required bases
+                fprintf(stdout, "GCratio: %f\nBase counts:\n",
+                    (gccount.counts[2] /*C*/ + gccount.counts[4] /*G*/) / (float)
+                        (gccount.counts[1] /*A*/ + gccount.counts[8] /*T*/ +
+                            gccount.counts[2] + gccount.counts[4]));
+        ...
+        if (queue) {
+            hts_tpool_process_destroy(queue);
+        }
+Refer: qtask_unordered.c
+
+## More Information
+
+### CRAM reference files
+
+The cram reference data is required for the read of sequence data in CRAM
+format. The sequence data file may have it as embedded or as a reference to
+the actual file. When it is a reference, it is downloaded locally, in the
+cache directory for later usage. It will be stored in a directory structure
+based on the MD5 checksum in the cache directory.
+
+Each chromosome in a reference file gets saved as a separate file with md5sum
+as its path and name. The initial 4 numerals make the directory name and rest
+as the file name (<cache path>/<1st 2 of md5sum>/<2nd 2 of md5sum>/<rest of
+md5sum>).
+
+The download would be attempted from standard location, EBI ENA
+(https://www.ebi.ac.uk/ena).
+
+
+### Bam1_t
+
+This structure holds the sequence data in BAM format. There are fixed and
+variable size fields, basic and extended information on sequence
+data. Variable size data and extended information are kept together in a
+buffer, named data in bam1_t. Fields in the member named core, bam1_core_t,
+and a few macros together support the storage and handling of the whole
+sequence data.
+
+- core has a link to reference as a 0 based index in field tid. The mate /
+  reverse strand's link to reference is given by mtid.
+
+- Field pos and mpos gives the position in reference to which the sequence and
+  its mate / reverse strand match.
+
+- Field flag gives the properties of the given alignment. It shows the
+  alignment's orientation, mate status, read order etc.
+
+- Field qual gives the quality of the alignment read.
+
+- l_qname gives the length of the name of the alignment / read, l_extranul gives
+  the extra space used internally in the data field.
+
+- l_qseq gives the length of the alignment / read in the data field.
+
+-- n_cigar gives the number of CIGAR operations for the given alignment.
+
+- isize gives the insert size of the read / alignment.
+
+The bases in sequence data are stored by compressing 2 bases together in a
+byte.  When the reverse flag is set, the base data is reversed and
+complemented from the actual read (i.e. if the forward read is ACTG, the
+reverse read to be CAGT; it will be stored in SAM format with reversed and
+complemented format as ACTG with reverse flag set).
+
+Macros bam_get_qname, bam_get_seq, bam_get_qual, bam_get_aux, bam_get_l_aux,
+bam_seqi etc access the data field and retrieve the required data. The aux
+macros support the retrieval of auxiliary data from the data field.
+
+
+### Sam_hdr_t
+
+This structure holds the header information. This holds the number of targets
+/ SQ lines in the file, each one's length, name and reference count to this
+structure. It also has this information in an internal data structure for
+easier access of each field of this data.
+
+When this data is shared or assigned to another variable of a different scope
+or purpose, the reference count needs to be incremented to ensure that it is
+valid till the end of the variable's scope. sam_hdr_incr_ref and it needs to
+be destroyed as many times with sam_hdr_destroy api.
+
+
+### Index
+
+Indices need the data to be sorted by position.  They can be of different
+types with extension .bai, .csi or .tbi for compressed SAM/BAM/VCF files and
+.crai for CRAM files.  The index name can be passed along with the alignment
+file itself by appending a specific character sequence. The apis can detect this
+sequence and extract the index path. ##idx## is the sequence which separates
+the file path and index path.
+
+
+### Data files
+
+The data files can be a local file, a network file, a file accessible through
+the web or in cloud storage like google and amazon.  The data files can be
+represented with URIs like file://, file://localhost/.., ,ftp://..,
+gs+http[s].., s3+http[s]://
+
diff --git a/ext/htslib/samples/Makefile b/ext/htslib/samples/Makefile
new file mode 100644
index 0000000..ee632e3
--- /dev/null
+++ b/ext/htslib/samples/Makefile
@@ -0,0 +1,117 @@
+HTS_DIR = ../
+include $(HTS_DIR)/htslib_static.mk
+
+CC = gcc
+CFLAGS = -Wall -O2
+
+#to statically link to libhts
+LDFLAGS = $(HTS_DIR)/libhts.a -L$(HTS_DIR) $(HTSLIB_static_LDFLAGS) $(HTSLIB_static_LIBS)
+
+#to dynamically link to libhts
+#LDFLAGS = -L $(HTS_DIR) -lhts -Wl,-rpath,<installed lib path>
+
+PRGS = flags split split2 cram read_fast read_header read_ref read_bam \
+	read_aux dump_aux add_header rem_header update_header mod_bam mod_aux \
+	mod_aux_ba write_fast idx_on_write read_reg read_multireg pileup \
+	mpileup modstate pileup_mod flags_field split_t1 split_t2 \
+	read_fast_i qtask_ordered qtask_unordered index_fasta
+
+all: $(PRGS)
+
+flags: flags_demo.c
+	$(CC) $(CFLAGS) -I $(HTS_DIR) flags_demo.c -o $@  $(LDFLAGS)
+
+split: split.c
+	$(CC) $(CFLAGS) -I $(HTS_DIR) split.c -o $@  $(LDFLAGS)
+
+split2: split2.c
+	$(CC) $(CFLAGS) -I $(HTS_DIR) split2.c -o $@  $(LDFLAGS)
+
+cram: cram.c
+	$(CC) $(CFLAGS) -I $(HTS_DIR) cram.c -o $@  $(LDFLAGS)
+
+read_fast: read_fast.c
+	$(CC) $(CFLAGS) -I $(HTS_DIR) read_fast.c -o $@  $(LDFLAGS)
+
+read_header: read_header.c
+	$(CC) $(CFLAGS) -I $(HTS_DIR) read_header.c -o $@  $(LDFLAGS)
+
+read_ref: read_refname.c
+	$(CC) $(CFLAGS) -I $(HTS_DIR) read_refname.c -o $@  $(LDFLAGS)
+
+read_bam: read_bam.c
+	$(CC) $(CFLAGS) -I $(HTS_DIR) read_bam.c -o $@  $(LDFLAGS)
+
+read_aux: read_aux.c
+	$(CC) $(CFLAGS) -I $(HTS_DIR) read_aux.c -o $@  $(LDFLAGS)
+
+dump_aux: dump_aux.c
+	$(CC) $(CFLAGS) -I $(HTS_DIR) dump_aux.c -o $@  $(LDFLAGS)
+
+add_header: add_header.c
+	$(CC) $(CFLAGS) -I $(HTS_DIR) add_header.c -o $@  $(LDFLAGS)
+
+rem_header: rem_header.c
+	$(CC) $(CFLAGS) -I $(HTS_DIR) rem_header.c -o $@  $(LDFLAGS)
+
+update_header: update_header.c
+	$(CC) $(CFLAGS) -I $(HTS_DIR) update_header.c -o $@  $(LDFLAGS)
+
+mod_bam: mod_bam.c
+	$(CC) $(CFLAGS) -I $(HTS_DIR) mod_bam.c -o $@  $(LDFLAGS)
+
+mod_aux: mod_aux.c
+	$(CC) $(CFLAGS) -I $(HTS_DIR) mod_aux.c -o $@  $(LDFLAGS)
+
+mod_aux_ba: mod_aux_ba.c
+	$(CC) $(CFLAGS) -I $(HTS_DIR) mod_aux_ba.c -o $@  $(LDFLAGS)
+
+write_fast: write_fast.c
+	$(CC) $(CFLAGS) -I $(HTS_DIR) write_fast.c -o $@  $(LDFLAGS)
+
+idx_on_write: index_write.c
+	$(CC) $(CFLAGS) -I $(HTS_DIR) index_write.c -o $@  $(LDFLAGS)
+
+read_reg: index_reg_read.c
+	$(CC) $(CFLAGS) -I $(HTS_DIR) index_reg_read.c -o $@  $(LDFLAGS)
+
+read_multireg: index_multireg_read.c
+	$(CC) $(CFLAGS) -I $(HTS_DIR) index_multireg_read.c -o $@  $(LDFLAGS)
+
+read_fast_i: read_fast_index.c
+	$(CC) $(CFLAGS) -I $(HTS_DIR) read_fast_index.c -o $@  $(LDFLAGS)
+
+pileup: pileup.c
+	$(CC) $(CFLAGS) -I $(HTS_DIR) pileup.c -o $@  $(LDFLAGS)
+
+mpileup: mpileup.c
+	$(CC) $(CFLAGS) -I $(HTS_DIR) mpileup.c -o $@  $(LDFLAGS)
+
+modstate: modstate.c
+	$(CC) $(CFLAGS) -I $(HTS_DIR) modstate.c -o $@  $(LDFLAGS)
+
+pileup_mod: pileup_mod.c
+	$(CC) $(CFLAGS) -I $(HTS_DIR) pileup_mod.c -o $@  $(LDFLAGS)
+
+flags_field: flags_htsopt_field.c
+	$(CC) $(CFLAGS) -I $(HTS_DIR) flags_htsopt_field.c -o $@  $(LDFLAGS)
+
+split_t1: split_thread1.c
+	$(CC) $(CFLAGS) -I $(HTS_DIR) split_thread1.c -o $@  $(LDFLAGS)
+
+split_t2: split_thread2.c
+	$(CC) $(CFLAGS) -I $(HTS_DIR) split_thread2.c -o $@  $(LDFLAGS)
+
+index_fasta: index_fasta.c
+	$(CC) $(CFLAGS) -I $(HTS_DIR) index_fasta.c -o $@  $(LDFLAGS)
+
+qtask_ordered: qtask_ordered.c
+	$(CC) $(CFLAGS) -I $(HTS_DIR) qtask_ordered.c -o $@  $(LDFLAGS)
+
+qtask_unordered: qtask_unordered.c
+	$(CC) $(CFLAGS) -I $(HTS_DIR) qtask_unordered.c -o $@  $(LDFLAGS)
+
+clean:
+	find . -name "*.o" | xargs rm -rf
+	find . -name "*.dSYM" | xargs rm -rf
+	-rm -f $(PRGS)
diff --git a/ext/htslib/samples/add_header.c b/ext/htslib/samples/add_header.c
new file mode 100644
index 0000000..066b1d4
--- /dev/null
+++ b/ext/htslib/samples/add_header.c
@@ -0,0 +1,128 @@
+/*  add_header.c --  showcases the htslib api usage
+
+    Copyright (C) 2023 Genome Research Ltd.
+
+    Author: Vasudeva Sarma <vasudeva.sarma@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE
+
+*/
+
+/* The purpose of this code is to demonstrate the library apis and need proper error handling and optimisation */
+
+#include <getopt.h>
+#include <unistd.h>
+#include <htslib/sam.h>
+
+/// print_usage - print the usage
+/** @param fp pointer to the file / terminal to which usage to be dumped
+returns nothing
+*/
+static void print_usage(FILE *fp)
+{
+    fprintf(fp, "Usage: add_header infile\n\
+Adds new header lines of SQ, RG, PG and CO types\n");
+    return;
+}
+
+/// main_demo - start of the demo
+/** @param argc - count of arguments
+ *  @param argv - pointer to array of arguments
+returns 1 on failure 0 on success
+*/
+int main(int argc, char *argv[])
+{
+    const char *inname = NULL, sq[] = "@SQ\tSN:TR1\tLN:100\n@SQ\tSN:TR2\tLN:50";
+    int c = 0, ret = EXIT_FAILURE;
+    samFile *infile = NULL, *outfile = NULL;
+    sam_hdr_t *in_samhdr = NULL;
+    kstring_t data = KS_INITIALIZE;
+
+    //update_header infile header idval tag value
+    if (argc != 2) {
+        print_usage(stderr);
+        goto end;
+    }
+    inname = argv[1];
+
+    if (!(infile = sam_open(inname, "r"))) {
+        printf("Could not open %s\n", inname);
+        goto end;
+    }
+    if (!(outfile = sam_open("-", "w"))) {                  //use stdout as the output file for ease of display of update
+        printf("Could not open stdout\n");
+        goto end;
+    }
+
+    //read header
+    if (!(in_samhdr = sam_hdr_read(infile))) {
+        printf("Failed to read header from file!\n");
+        goto end;
+    }
+
+    //dump command line arguments for PG line
+    for (c = 0;  c < argc; ++c) {
+        kputs(argv[c], &data);
+        kputc(' ', &data);
+    }
+
+    //add SQ line with SN as TR1 and TR2
+    if (sam_hdr_add_lines(in_samhdr, &sq[0], 0)) {          //length as 0 for NULL terminated data
+        printf("Failed to add SQ lines\n");
+        goto end;
+    }
+
+    //add RG line with ID as RG1
+    if (sam_hdr_add_line(in_samhdr, "RG", "ID", "RG1", "LB", "Test", "SM", "S1", NULL)) {
+        printf("Failed to add RG line\n");
+        goto end;
+    }
+
+    //add pg line
+    if (sam_hdr_add_pg(in_samhdr, "add_header", "VN", "Test", "CL", data.s, NULL)) {            //NULL is to indicate end of args
+        printf("Failed to add PG line\n");
+        goto end;
+    }
+
+    if (sam_hdr_add_line(in_samhdr, "CO", "Test data", NULL)) {                                 //NULL is to indicate end of args
+        printf("Failed to add PG line\n");
+        goto end;
+    }
+
+    //write output
+    if (sam_hdr_write(outfile, in_samhdr) < 0) {
+        printf("Failed to write output\n");
+        goto end;
+    }
+    ret = EXIT_SUCCESS;
+    //bam data write to follow....
+end:
+    //cleanup
+    if (in_samhdr) {
+        sam_hdr_destroy(in_samhdr);
+    }
+    if (infile) {
+        sam_close(infile);
+    }
+    if (outfile) {
+        sam_close(outfile);
+    }
+    ks_free(&data);
+    return ret;
+}
diff --git a/ext/htslib/samples/cram.c b/ext/htslib/samples/cram.c
new file mode 100644
index 0000000..7b13423
--- /dev/null
+++ b/ext/htslib/samples/cram.c
@@ -0,0 +1,168 @@
+/*  cram.c --  showcases the htslib api usage
+
+    Copyright (C) 2023 Genome Research Ltd.
+
+    Author: Vasudeva Sarma <vasudeva.sarma@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE
+
+*/
+
+/* The purpose of this code is to demonstrate the library apis and need proper error handling and optimisation */
+
+#include <getopt.h>
+#include <unistd.h>
+#include <htslib/sam.h>
+
+/// print_usage - print the usage
+/** @param fp pointer to the file / terminal to which usage to be dumped
+returns nothing
+*/
+static void print_usage(FILE *fp)
+{
+    fprintf(fp, "Usage: cram infile reffile outdir\n\
+Dumps the input file alignments in cram format in given directory\n\
+1.cram has external reference\n\
+2.cram has reference embedded\n\
+3.cram has autogenerated reference\n\
+4.cram has no reference data in it\n");
+    return;
+}
+
+/// main_demo - start of the demo
+/** @param argc - count of arguments
+ *  @param argv - pointer to array of arguments
+returns 1 on failure 0 on success
+*/
+int main(int argc, char *argv[])
+{
+    const char *inname = NULL, *outdir = NULL, *reffile = NULL;
+    char *file1 = NULL, *file2 = NULL, *file3 = NULL, *file4 = NULL, *reffmt1 = NULL, *reffmt2 = NULL;
+    int c = 0, ret = EXIT_FAILURE, size1 = 0, size2 = 0, size3 = 0;
+    samFile *infile = NULL, *outfile1 = NULL, *outfile2 = NULL, *outfile3 = NULL, *outfile4 = NULL;
+    sam_hdr_t *in_samhdr = NULL;
+    bam1_t *bamdata = NULL;
+    htsFormat fmt1 = {0}, fmt2 = {0}, fmt3 = {0}, fmt4 = {0};
+
+    //cram infile reffile outdir
+    if (argc != 4) {
+        print_usage(stdout);
+        goto end;
+    }
+    inname = argv[1];
+    reffile = argv[2];
+    outdir = argv[3];
+
+    //allocate space for option string and output file names
+    size1 = sizeof(char) * (strlen(reffile) + sizeof("cram,reference=") + 1);
+    size2 = sizeof(char) * (strlen(reffile) + sizeof("cram,embed_ref=1,reference=") + 1);
+    size3 = sizeof(char) * (strlen(outdir) + sizeof("/1.cram") + 1);
+
+    reffmt1 = malloc(size1); reffmt2 = malloc(size2);
+    file1 = malloc(size3); file2 = malloc(size3);
+    file3 = malloc(size3); file4 = malloc(size3);
+
+    if (!file1 || !file2 || !file3 || !file4 || !reffmt1 || !reffmt2) {
+        printf("Failed to create buffers\n");
+        goto end;
+    }
+
+    snprintf(reffmt1, size1, "cram,reference=%s", reffile);
+    snprintf(reffmt2, size2, "cram,embed_ref=1,reference=%s", reffile);
+    snprintf(file1, size3, "%s/1.cram", outdir); snprintf(file2, size3, "%s/2.cram", outdir);
+    snprintf(file3, size3, "%s/3.cram", outdir); snprintf(file4, size3, "%s/4.cram", outdir);
+
+    if (hts_parse_format(&fmt1, reffmt1) == -1 ||                       //using external reference - uses the M5/UR tags to get reference data during read
+            hts_parse_format(&fmt2, reffmt2) == -1 ||                   //embed the reference internally
+            hts_parse_format(&fmt3, "cram,embed_ref=2") == -1 ||        //embed autogenerated reference
+            hts_parse_format(&fmt4, "cram,no_ref=1") == -1) {           //no reference data encoding at all
+        printf("Failed to set output option\n");
+        goto end;
+    }
+
+    //bam data storage
+    if (!(bamdata = bam_init1())) {
+        printf("Failed to initialize bamdata\n");
+        goto end;
+    }
+    //open input file - r reading
+    if (!(infile = sam_open(inname, "r"))) {
+        printf("Could not open %s\n", inname);
+        goto end;
+    }
+    //open output files - w write as SAM, wb write as BAM, wc as CRAM (equivalent to fmt3)
+    outfile1 = sam_open_format(file1, "wc", &fmt1); outfile2 = sam_open_format(file2, "wc", &fmt2);
+    outfile3 = sam_open_format(file3, "wc", &fmt3); outfile4 = sam_open_format(file4, "wc", &fmt4);
+    if (!outfile1 || !outfile2 || !outfile3 || !outfile4) {
+        printf("Could not open output file\n");
+        goto end;
+    }
+
+    //read header, required to resolve the target names to proper ids
+    if (!(in_samhdr = sam_hdr_read(infile))) {
+        printf("Failed to read header from file!\n");
+        goto end;
+    }
+    //write header
+    if ((sam_hdr_write(outfile1, in_samhdr) == -1) || (sam_hdr_write(outfile2, in_samhdr) == -1) ||
+            (sam_hdr_write(outfile3, in_samhdr) == -1) || (sam_hdr_write(outfile4, in_samhdr) == -1)) {
+        printf("Failed to write header\n");
+        goto end;
+    }
+
+    //check flags and write
+    while ((c = sam_read1(infile, in_samhdr, bamdata)) >= 0) {
+        if (sam_write1(outfile1, in_samhdr, bamdata) < 0 ||
+                sam_write1(outfile2, in_samhdr, bamdata) < 0 ||
+                sam_write1(outfile3, in_samhdr, bamdata) < 0 ||
+                sam_write1(outfile4, in_samhdr, bamdata) < 0) {
+            printf("Failed to write output data\n");
+            goto end;
+        }
+    }
+    if (-1 == c) {
+        //EOF
+        ret = EXIT_SUCCESS;
+    }
+    else {
+        printf("Error in reading data\n");
+    }
+end:
+#define IF_OL(X,Y) if((X)) {(Y);}           //if one liner
+    //cleanup
+    IF_OL(in_samhdr, sam_hdr_destroy(in_samhdr));
+    IF_OL(infile, sam_close(infile));
+    IF_OL(outfile1, sam_close(outfile1));
+    IF_OL(outfile2, sam_close(outfile2));
+    IF_OL(outfile3, sam_close(outfile3));
+    IF_OL(outfile4, sam_close(outfile4));
+    IF_OL(file1, free(file1));
+    IF_OL(file2, free(file2));
+    IF_OL(file3, free(file3));
+    IF_OL(file4, free(file4));
+    IF_OL(reffmt1, free(reffmt1));
+    IF_OL(reffmt2, free(reffmt2));
+    IF_OL(fmt1.specific, hts_opt_free(fmt1.specific));
+    IF_OL(fmt2.specific, hts_opt_free(fmt2.specific));
+    IF_OL(fmt3.specific, hts_opt_free(fmt3.specific));
+    IF_OL(fmt4.specific, hts_opt_free(fmt4.specific));
+    IF_OL(bamdata, bam_destroy1(bamdata));
+
+    return ret;
+}
diff --git a/ext/htslib/samples/dump_aux.c b/ext/htslib/samples/dump_aux.c
new file mode 100644
index 0000000..3caa160
--- /dev/null
+++ b/ext/htslib/samples/dump_aux.c
@@ -0,0 +1,188 @@
+/*  dump_aux.c --  showcases the htslib api usage
+
+    Copyright (C) 2023 Genome Research Ltd.
+
+    Author: Vasudeva Sarma <vasudeva.sarma@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE
+
+*/
+
+/* The purpose of this code is to demonstrate the library apis and need proper error handling and optimisation */
+
+#include <getopt.h>
+#include <unistd.h>
+#include <htslib/sam.h>
+
+/// print_usage - print the usage
+/** @param fp pointer to the file / terminal to which usage to be dumped
+returns nothing
+*/
+static void print_usage(FILE *fp)
+{
+    fprintf(fp, "Usage: dump_aux infile\n\
+Dump the aux tags from alignments\n");
+}
+
+/// printauxdata - prints aux data
+/** @param fp - file to which it to be printed - stdout or null
+ *  @param type - aux type
+ *  @param idx - index in array, -1 when not an array type
+ *  @param data - data
+ *  recurses when the data is array type
+returns 1 on failure 0 on success
+*/
+int printauxdata(FILE *fp, char type, int32_t idx, const uint8_t *data)
+{
+    uint32_t auxBcnt = 0;
+    int i = 0;
+    char auxBType = 'Z';
+
+    //the tag is already queried and ensured to exist and the type is retrieved from the tag data, also iterated within index for arrays, so no error is expected here.
+    //when these apis are used explicitly, these error conditions needs to be handled based on return value and errno
+    switch(type) {
+    case 'A':
+        fprintf(fp, "%c", bam_aux2A(data));                                                 //byte data
+        break;
+    case 'c':
+        fprintf(fp, "%d", (int8_t)(idx > -1 ? bam_auxB2i(data, idx) : bam_aux2i(data)));    //signed 1 byte data; bam_auxB2i - from array or bam_aux2i - non array data
+        break;
+    case 'C':
+        fprintf(fp, "%u", (uint8_t)(idx > -1 ? bam_auxB2i(data, idx) : bam_aux2i(data)));   //unsigned 1 byte data
+        break;
+    case 's':
+        fprintf(fp, "%d", (int16_t)(idx > -1 ? bam_auxB2i(data, idx) : bam_aux2i(data)));   //signed 2 byte data
+        break;
+    case 'S':
+        fprintf(fp, "%u", (uint16_t)(idx > -1 ? bam_auxB2i(data, idx) : bam_aux2i(data)));  //unsigned 2 byte data
+        break;
+    case 'i':
+        fprintf(fp, "%d", (int32_t)(idx > -1 ? bam_auxB2i(data, idx) : bam_aux2i(data)));   //signed 4 byte data
+        break;
+    case 'I':
+        fprintf(fp, "%u", (uint32_t)(idx > -1 ? bam_auxB2i(data, idx) : bam_aux2i(data)));  //unsigned 4 byte data
+        break;
+    case 'f':
+    case 'd':
+        fprintf(fp, "%g", (float)(idx > -1 ? bam_auxB2f(data, idx) : bam_aux2f(data)));     //floating point data, 4 bytes
+        break;
+    case 'H':
+    case 'Z':
+        fprintf(fp, "%s", bam_aux2Z(data));                                                 //array of char or hex data
+        break;
+    case 'B':                                                                               //array of char/int/float
+        auxBcnt = bam_auxB_len(data);                                                       //length of array
+        auxBType = bam_aux_type(data + 1);                                                  //type of element in array
+        fprintf(fp, "%c", auxBType);
+        for (i = 0; i < auxBcnt; ++i) {                                                     //iterate the array
+            fprintf(fp, ",");
+            //calling recursively  with index to reuse a few lines
+            if (printauxdata(fp, auxBType, i, data) == EXIT_FAILURE) {
+                return EXIT_FAILURE;
+            }
+        }
+        break;
+    default:
+        printf("Invalid aux tag?\n");
+        return EXIT_FAILURE;
+        break;
+    }
+    return EXIT_SUCCESS;
+}
+
+/// main_demo - start of the demo
+/** @param argc - count of arguments
+ *  @param argv - pointer to array of arguments
+returns 1 on failure 0 on success
+*/
+int main(int argc, char *argv[])
+{
+    const char *inname = NULL;
+    int ret = EXIT_FAILURE;
+    sam_hdr_t *in_samhdr = NULL;
+    samFile *infile = NULL;
+    int ret_r = 0;
+    bam1_t *bamdata = NULL;
+    uint8_t *data = NULL;
+
+    //dump_aux infile
+    if (argc != 2) {
+        print_usage(stderr);
+        goto end;
+    }
+    inname = argv[1];
+
+    if (!(bamdata = bam_init1())) {
+        printf("Failed to allocate data memory!\n");
+        goto end;
+    }
+
+    //open input file
+    if (!(infile = sam_open(inname, "r"))) {
+        printf("Could not open %s\n", inname);
+        goto end;
+    }
+
+    if (!(in_samhdr = sam_hdr_read(infile))) {
+        printf("Failed to read header from file!\n");
+        goto end;
+    }
+
+    while ((ret_r = sam_read1(infile, in_samhdr, bamdata)) >= 0) {
+        errno = 0;
+        data = NULL;
+        data = bam_aux_first(bamdata);                                              //get the first aux data
+        while (data) {
+            printf("%.2s:%c:", bam_aux_tag(data), NULL != strchr("cCsSiI", bam_aux_type(data)) ? 'i' : bam_aux_type(data));  //macros gets the tag and type of aux data
+            //dump the data
+            if (printauxdata(stdout, bam_aux_type(data), -1, data) == EXIT_FAILURE) {
+                printf("Failed to dump aux data\n");
+                goto end;
+            }
+            else {
+                printf(" ");
+            }
+            data = bam_aux_next(bamdata, data);                                     //get the next aux data
+        }
+        if (ENOENT != errno) {
+            printf("\nFailed to get aux data\n");
+            goto end;
+        }
+        printf("\n");
+    }
+    if (ret_r < -1) {
+        //read error
+        printf("Failed to read data\n");
+        goto end;
+    }
+
+    ret = EXIT_SUCCESS;
+end:
+    //cleanup
+    if (in_samhdr) {
+        sam_hdr_destroy(in_samhdr);
+    }
+    if (infile) {
+        sam_close(infile);
+    }
+    if (bamdata) {
+        bam_destroy1(bamdata);
+    }
+    return ret;
+}
diff --git a/ext/htslib/samples/flags_demo.c b/ext/htslib/samples/flags_demo.c
new file mode 100644
index 0000000..ac26be8
--- /dev/null
+++ b/ext/htslib/samples/flags_demo.c
@@ -0,0 +1,110 @@
+/*  flags_demo.c --  showcases the htslib api usage
+
+    Copyright (C) 2023 Genome Research Ltd.
+
+    Author: Vasudeva Sarma <vasudeva.sarma@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE
+
+*/
+
+/* The purpose of this code is to demonstrate the library apis and need proper error handling and optimisation */
+
+#include <getopt.h>
+#include <unistd.h>
+#include <htslib/sam.h>
+
+/// print_usage - show usage
+/** @param fp pointer to the file / terminal to which usage to be dumped
+returns nothing
+*/
+static void print_usage(FILE *fp)
+{
+    fprintf(fp, "Usage: flags <infile>\n\
+Shows the count of read1 and read2 alignments\n\
+This shows basic reading and alignment flag access\n");
+    return;
+}
+
+/// main_demo - start of the demo
+/** @param argc - count of arguments
+ *  @param argv - pointer to array of arguments
+returns 1 on failure 0 on success
+*/
+int main(int argc, char *argv[])
+{
+    const char *inname = NULL;              //input file name
+    int c = 0, ret = EXIT_FAILURE;
+    int64_t cntread1 = 0, cntread2 = 0;     //count
+    samFile *infile = NULL;                 //sam file
+    sam_hdr_t *in_samhdr = NULL;            //header of file
+    bam1_t *bamdata = NULL;                 //to hold the read data
+
+    if (argc != 2) {
+        print_usage(stdout);
+        goto end;
+    }
+    inname = argv[1];
+
+    //initialize
+    if (!(bamdata = bam_init1())) {
+        printf("Failed to initialize bamdata\n");
+        goto end;
+    }
+    //open input files - r reading
+    if (!(infile = sam_open(inname, "r"))) {
+        printf("Could not open %s\n", inname);
+        goto end;
+    }
+    //read header
+    if (!(in_samhdr = sam_hdr_read(infile))) {
+        printf( "Failed to read header from file\n");
+        goto end;
+    }
+
+    //read data, check flags and update count
+    while ((c = sam_read1(infile, in_samhdr, bamdata)) >= 0) {
+        if (bamdata->core.flag & BAM_FREAD1) {
+            cntread1++;
+        }
+        if (bamdata->core.flag & BAM_FREAD2) {
+            cntread2++;
+        }
+    }
+    if (c != -1) {
+        //error
+        printf("Failed to get data\n");
+        goto end;
+    }
+    //else -1 / EOF
+    printf("File %s has %"PRIhts_pos" read1 and %"PRIhts_pos" read2 alignments\n", inname, cntread1, cntread2);
+    ret = EXIT_SUCCESS;
+end:
+    //clean up
+    if (in_samhdr) {
+        sam_hdr_destroy(in_samhdr);
+    }
+    if (infile) {
+        sam_close(infile);
+    }
+    if (bamdata) {
+        bam_destroy1(bamdata);
+    }
+    return ret;
+}
diff --git a/ext/htslib/samples/flags_htsopt_field.c b/ext/htslib/samples/flags_htsopt_field.c
new file mode 100644
index 0000000..40a0aff
--- /dev/null
+++ b/ext/htslib/samples/flags_htsopt_field.c
@@ -0,0 +1,115 @@
+/*  flags_htsopt_field.c --  showcases the htslib api usage
+
+    Copyright (C) 2023 Genome Research Ltd.
+
+    Author: Vasudeva Sarma <vasudeva.sarma@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE
+
+*/
+
+/* The purpose of this code is to demonstrate the library apis and need proper error handling and optimisation */
+
+#include <getopt.h>
+#include <unistd.h>
+#include <htslib/sam.h>
+
+/// print_usage - show usage
+/** @param fp pointer to the file / terminal to which usage to be dumped
+returns nothing
+*/
+static void print_usage(FILE *fp)
+{
+    fprintf(fp, "Usage: flags_field <infile>\n\
+Shows the count of read1 and read2 alignments\n\
+This shows reading selected fields from CRAM file\n");
+    return;
+}
+
+/// main_demo - start of the demo
+/** @param argc - count of arguments
+ *  @param argv - pointer to array of arguments
+returns 1 on failure 0 on success
+*/
+int main(int argc, char *argv[])
+{
+    const char *inname = NULL;              //input file name
+    int c = 0, ret = EXIT_FAILURE;
+    int64_t cntread1 = 0, cntread2 = 0;     //count
+    samFile *infile = NULL;                 //sam file
+    sam_hdr_t *in_samhdr = NULL;            //header of file
+    bam1_t *bamdata = NULL;                 //to hold the read data
+
+    if (argc != 2) {
+        print_usage(stdout);
+        goto end;
+    }
+    inname = argv[1];
+
+    //initialize
+    if (!(bamdata = bam_init1())) {
+        printf("Failed to initialize bamdata\n");
+        goto end;
+    }
+    //open input files - r reading
+    if (!(infile = sam_open(inname, "r"))) {
+        printf("Could not open %s\n", inname);
+        goto end;
+    }
+    //select required field alone, this is useful for CRAM alone
+    if (hts_set_opt(infile, CRAM_OPT_REQUIRED_FIELDS, SAM_FLAG) < 0) {
+        printf("Failed to set htsoption\n");
+        goto end;
+    }
+    //read header
+    if (!(in_samhdr = sam_hdr_read(infile))) {
+        printf("Failed to read header from file\n");
+        goto end;
+    }
+
+    //read data, check flags and update count
+    while ((c = sam_read1(infile, in_samhdr, bamdata)) >= 0) {
+        if (bamdata->core.flag & BAM_FREAD1) {
+            cntread1++;
+        }
+        if (bamdata->core.flag & BAM_FREAD2) {
+            cntread2++;
+        }
+    }
+    if (c != -1) {
+        //error
+        printf("Failed to get data\n");
+        goto end;
+    }
+    //else -1 / EOF
+    printf("File %s has %"PRIhts_pos" read1 and %"PRIhts_pos" read2 alignments\n", inname, cntread1, cntread2);
+    ret = EXIT_SUCCESS;
+end:
+    //clean up
+    if (in_samhdr) {
+        sam_hdr_destroy(in_samhdr);
+    }
+    if (infile) {
+        sam_close(infile);
+    }
+    if (bamdata) {
+        bam_destroy1(bamdata);
+    }
+    return ret;
+}
diff --git a/ext/htslib/samples/index_fasta.c b/ext/htslib/samples/index_fasta.c
new file mode 100644
index 0000000..ba04890
--- /dev/null
+++ b/ext/htslib/samples/index_fasta.c
@@ -0,0 +1,72 @@
+/*  index_fasta.c --  showcases the htslib api usage
+
+    Copyright (C) 2024 Genome Research Ltd.
+
+    Author: Vasudeva Sarma <vasudeva.sarma@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE
+
+*/
+
+/* The purpose of this code is to demonstrate the library apis and need proper error handling and optimisation */
+
+#include <getopt.h>
+#include <unistd.h>
+#include <time.h>
+#include <htslib/sam.h>
+#include <htslib/faidx.h>
+
+/// print_usage - show usage
+/** @param fp pointer to the file / terminal to which usage to be dumped
+returns nothing
+*/
+static void print_usage(FILE *fp)
+{
+    fprintf(fp, "Usage: index_fasta <file>\n\
+Indexes a fasta/fastq file and saves along with source.\n");
+    return;
+}
+
+/// main - indexes fasta/fastq file
+/** @param argc - count of arguments
+ *  @param argv - pointer to array of arguments
+returns 1 on failure 0 on success
+*/
+int main(int argc, char *argv[])
+{
+    const char *filename = NULL;             //file name
+    int ret = EXIT_FAILURE;
+
+    if (argc != 2) {
+        print_usage(stdout);
+        goto end;
+    }
+    filename = argv[1];
+
+    // index the file
+    if (fai_build3(filename, NULL, NULL) == -1) {
+        printf("Indexing failed with %d\n", errno);
+        goto end;
+    }
+    //this creates an .fai file. If the file is bgzipped, a .gzi file will be created along with .fai
+    ret = EXIT_SUCCESS;
+end:
+    //clean up
+    return ret;
+}
diff --git a/ext/htslib/samples/index_multireg_read.c b/ext/htslib/samples/index_multireg_read.c
new file mode 100644
index 0000000..7bb8649
--- /dev/null
+++ b/ext/htslib/samples/index_multireg_read.c
@@ -0,0 +1,150 @@
+/*  index_multireg_read.c --  showcases the htslib api usage
+
+    Copyright (C) 2023 Genome Research Ltd.
+
+    Author: Vasudeva Sarma <vasudeva.sarma@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE
+
+*/
+
+/* The purpose of this code is to demonstrate the library apis and need proper error handling and optimisation */
+
+#include <getopt.h>
+#include <unistd.h>
+#include <htslib/sam.h>
+
+/// print_usage - print the usage
+/** @param fp pointer to the file / terminal to which usage to be dumped
+returns nothing
+*/
+static void print_usage(FILE *fp)
+{
+    fprintf(fp, "Usage: read_multireg infile count regspec_csv\n\
+    Reads alignment of a target matching to given region specifications\n\
+    read_multireg infile.sam 2 R1:10-100,R2:200");
+    return;
+}
+
+/// main_demo - start of the demo
+/** @param argc - count of arguments
+ *  @param argv - pointer to array of arguments
+returns 1 on failure 0 on success
+*/
+int main(int argc, char *argv[])
+{
+    const char *inname = NULL;
+    char *ptr = NULL;
+    int c = 0, ret = EXIT_FAILURE;
+    samFile *infile = NULL, *outfile = NULL;
+    sam_hdr_t *in_samhdr = NULL;
+    bam1_t *bamdata = NULL;
+    hts_idx_t *idx = NULL;
+    hts_itr_t *iter = NULL;
+    unsigned int regcnt = 0;
+    char **regions = NULL;
+
+    //read_multireg infile count regspec_csv
+    if (argc != 4) {
+        print_usage(stderr);
+        goto end;
+    }
+    inname = argv[1];
+    regcnt = atoi(argv[2]);
+    regions = calloc(regcnt, sizeof(char*));
+    //set each regspec as separate entry in region array
+    ptr = argv[3];
+    for (c = 0; ptr && (c < regcnt); ++c) {
+        regions[c] = ptr;
+        ptr = strchr(ptr, ',');
+        if (ptr) { *ptr = '\0'; ++ptr; }
+    }
+
+    if (regcnt == 0) {
+        printf("Region count can not be 0\n");
+        goto end;
+    }
+    //initialize bam data storage
+    if (!(bamdata = bam_init1())) {
+        printf("Failed to initialize bamdata\n");
+        goto end;
+    }
+    //open files, use stdout as output SAM file for ease of display
+    infile = sam_open(inname, "r");
+    outfile = sam_open("-", "w");
+    if (!outfile || !infile) {
+        printf("Could not open in/out files\n");
+        goto end;
+    }
+    //load index file, assume it to be present in same location
+    if (!(idx = sam_index_load(infile, inname))) {
+        printf("Failed to load the index\n");
+        goto end;
+    }
+    //read header
+    if (!(in_samhdr = sam_hdr_read(infile))) {
+        printf("Failed to read header from file!\n");
+        goto end;
+    }
+    //create iterator
+    if (!(iter = sam_itr_regarray(idx, in_samhdr, regions, regcnt))) {
+        printf("Failed to get iterator\n");
+        goto end;
+    }
+    if (regions) {
+        //can be freed as it is no longer required
+        free(regions);
+        regions = NULL;
+    }
+
+    //get required area
+    while ((c = sam_itr_multi_next(infile, iter, bamdata) >= 0)) {
+        //write to output
+        if (sam_write1(outfile, in_samhdr, bamdata) < 0) {
+            printf("Failed to write output\n");
+            goto end;
+        }
+    }
+    if (c != -1) {
+        printf("Error during read\n");
+        goto end;
+    }
+    ret = EXIT_SUCCESS;
+
+end:
+    //cleanup
+    if (in_samhdr) {
+        sam_hdr_destroy(in_samhdr);
+    }
+    if (infile) {
+        sam_close(infile);
+    }
+    if (outfile) {
+        sam_close(outfile);
+    }
+    if (bamdata) {
+        bam_destroy1(bamdata);
+    }
+    if (iter) {
+        sam_itr_destroy(iter);
+    }
+    if (idx)
+        hts_idx_destroy(idx);
+    return ret;
+}
diff --git a/ext/htslib/samples/index_reg_read.c b/ext/htslib/samples/index_reg_read.c
new file mode 100644
index 0000000..dec6849
--- /dev/null
+++ b/ext/htslib/samples/index_reg_read.c
@@ -0,0 +1,143 @@
+/*  index_reg_read.c --  showcases the htslib api usage
+
+    Copyright (C) 2023 Genome Research Ltd.
+
+    Author: Vasudeva Sarma <vasudeva.sarma@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE
+
+*/
+
+/* The purpose of this code is to demonstrate the library apis and need proper error handling and optimisation */
+
+#include <getopt.h>
+#include <unistd.h>
+#include <htslib/sam.h>
+
+/// print_usage - print the usage
+/** @param fp pointer to the file / terminal to which usage to be dumped
+returns nothing
+*/
+static void print_usage(FILE *fp)
+{
+    fprintf(fp, "Usage: read_reg infile idxfile region\n\
+Reads alignments matching to a specific region\n\
+\\. from start of file\n\
+\\* only unmapped reads\n\
+REFNAME all reads referring REFNAME\n\
+REFNAME:S all reads referring REFNAME and overlapping from S onwards\n\
+REFNAME:S-E all reads referring REFNAME overlapping from S to E\n\
+REFNAME:-E all reads referring REFNAME overlapping upto E\n");
+    return;
+}
+
+/// main_demo - start of the demo
+/** @param argc - count of arguments
+ *  @param argv - pointer to array of arguments
+returns 1 on failure 0 on success
+*/
+int main(int argc, char *argv[])
+{
+    const char *inname = NULL, *region = NULL;
+    char *idxfile = NULL;
+    int c = 0, ret = EXIT_FAILURE;
+    samFile *infile = NULL, *outfile = NULL;
+    sam_hdr_t *in_samhdr = NULL;
+    bam1_t *bamdata = NULL;
+    hts_idx_t *idx = NULL;
+    hts_itr_t *iter = NULL;
+
+    //readreg infile indexfile region
+    if (argc != 4) {
+        print_usage(stderr);
+        goto end;
+    }
+    inname = argv[1];
+    idxfile = argv[2];
+    region = argv[3];
+
+    //initialize bam data storage
+    if (!(bamdata = bam_init1())) {
+        printf("Failed to initialize bamdata\n");
+        goto end;
+    }
+
+    //open files
+    if (!(infile = sam_open(inname, "r"))) {
+        printf("Could not open input file\n");
+        goto end;
+    }
+    //using stdout as output file for ease of dumping data
+    if (!(outfile = sam_open("-", "w"))) {
+        printf("Could not open out file\n");
+        goto end;
+    }
+    //load index file
+    if (!(idx = sam_index_load2(infile, inname, idxfile))) {
+        printf("Failed to load the index\n");
+        goto end;
+    }
+    //can use sam_index_load if the index file is present in same location and follows standard naming conventions (i.e. <filename.ext>.<index type ext crai/bai/csi/tbi>)
+
+    //read header
+    if (!(in_samhdr = sam_hdr_read(infile))) {
+        printf("Failed to read header from file!\n");
+        goto end;
+    }
+    //create iterator
+    if (!(iter = sam_itr_querys(idx, in_samhdr, region))) {
+        printf("Failed to get iterator\n");
+        goto end;
+    }
+    //read using iterator
+    while ((c = sam_itr_next(infile, iter, bamdata)) >= 0) {
+        //write to output
+        if (sam_write1(outfile, in_samhdr, bamdata) < 0) {
+            printf("Failed to write output\n");
+            goto end;
+        }
+    }
+    if (c != -1) {
+        printf("Error during read\n");
+        goto end;
+    }
+    ret = EXIT_SUCCESS;
+
+end:
+    //cleanup
+    if (in_samhdr) {
+        sam_hdr_destroy(in_samhdr);
+    }
+    if (infile) {
+        sam_close(infile);
+    }
+    if (outfile) {
+        sam_close(outfile);
+    }
+    if (bamdata) {
+        bam_destroy1(bamdata);
+    }
+    if (iter) {
+        sam_itr_destroy(iter);
+    }
+    if (idx) {
+        hts_idx_destroy(idx);
+    }
+    return ret;
+}
diff --git a/ext/htslib/samples/index_write.c b/ext/htslib/samples/index_write.c
new file mode 100644
index 0000000..9ec63d4
--- /dev/null
+++ b/ext/htslib/samples/index_write.c
@@ -0,0 +1,166 @@
+/*  index_write.c --  showcases the htslib api usage
+
+    Copyright (C) 2023 Genome Research Ltd.
+
+    Author: Vasudeva Sarma <vasudeva.sarma@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE
+
+*/
+
+/* The purpose of this code is to demonstrate the library apis and need proper error handling and optimisation */
+
+#include <getopt.h>
+#include <libgen.h>
+#include <unistd.h>
+#include <htslib/sam.h>
+
+/// print_usage - print the usage
+/** @param fp pointer to the file / terminal to which usage to be dumped
+returns nothing
+*/
+static void print_usage(FILE *fp)
+{
+    fprintf(fp, "Usage: idx_on_write infile shiftsize outdir\n\
+Creates compressed sam file and index file for it in given directory\n");
+    return;
+}
+
+/// main_demo - start of the demo
+/** @param argc - count of arguments
+ *  @param argv - pointer to array of arguments
+returns 1 on failure 0 on success
+*/
+int main(int argc, char *argv[])
+{
+    const char *outdir = NULL;
+    char *inname = NULL, *fileidx = NULL, *outname = NULL, outmode[4] = "w";
+    int c = 0, ret = EXIT_FAILURE, size = 0;
+    samFile *infile = NULL, *outfile = NULL;
+    sam_hdr_t *in_samhdr = NULL;
+    bam1_t *bamdata = NULL;
+
+    //idx_on_write infile sizeshift outputdirectory
+    if (argc != 4) {
+        print_usage(stderr);
+        goto end;
+    }
+    inname = argv[1];
+    size = atoi(argv[2]);
+    outdir = argv[3];
+
+    //allocate space for output name - outdir/filename.ext.idxextNUL
+    c = strlen(basename(inname)) + strlen(outdir) + 10;
+    fileidx = malloc(sizeof(char) * c);
+    outname = malloc(sizeof(char) * c);
+    if (!fileidx || !outname) {
+        printf("Couldnt allocate memory\n");
+        goto end;
+    }
+    //initialize bam storage
+    if (!(bamdata = bam_init1())) {
+        printf("Failed to initialize bamdata\n");
+        goto end;
+    }
+
+    //open files
+    if ((infile = sam_open(inname, "r"))) {
+        //get file type and create output names
+        if (infile->format.format == cram) {
+            //set as crai
+            snprintf(fileidx, c, "%s/%s.crai", outdir, basename(inname));
+            snprintf(outname, c, "%s/%s", outdir, basename(inname));
+        }
+        else {
+            //set as either bai or csi based on interval
+            if (infile->format.format == sam && infile->format.compression == no_compression) {
+                //create as gzip compressed
+                snprintf(outname, c, "%s/%s.gz", outdir, basename(inname));
+                snprintf(fileidx, c, "%s/%s.gz.%s", outdir, basename(inname), !size ? "bai" : "csi");
+            }
+            else {
+                //with same name as input
+                snprintf(outname, c, "%s/%s", outdir, basename(inname));
+                snprintf(fileidx, c, "%s/%s.%s", outdir, basename(inname), !size ? "bai" : "csi");
+            }
+        }
+    }
+    c = 0;
+    sam_open_mode(outmode + 1, outname, NULL);          //set extra write options based on name
+    outfile = sam_open(outname, outmode);
+    if (!outfile || !infile) {
+        printf("Could not open files\n");
+        goto end;
+    }
+
+    //read header
+    if (!(in_samhdr = sam_hdr_read(infile))) {
+        printf("Failed to read header from file!\n");
+        goto end;
+    }
+    //write header
+    if (sam_hdr_write(outfile, in_samhdr)) {
+        printf("Failed to write header\n");
+        goto end;
+    }
+
+    // initialize indexing, before start of write
+    if (sam_idx_init(outfile, in_samhdr, size, fileidx)) {
+        printf("idx initialization failed\n");
+        goto end;
+    }
+    //read and write alignments
+    while ((c = sam_read1(infile, in_samhdr, bamdata)) >= 0) {
+        if (sam_write1(outfile, in_samhdr, bamdata) < 0) {
+            printf("Failed to write data\n");
+            goto end;
+        }
+    }
+    if (c != -1) {
+        printf("Error in reading data\n");
+        goto end;
+    }
+    //else EOF, save index
+    if (sam_idx_save(outfile)) {
+        printf("Could not save index\n");
+        goto end;
+    }
+    ret = EXIT_SUCCESS;
+end:
+    //cleanup
+    if (in_samhdr) {
+        sam_hdr_destroy(in_samhdr);
+    }
+    if (infile) {
+        sam_close(infile);
+    }
+    if (bamdata) {
+        bam_destroy1(bamdata);
+    }
+    if (fileidx) {
+        free(fileidx);
+    }
+    if (outname) {
+        free(outname);
+    }
+    if (outfile) {
+        sam_close(outfile);
+    }
+    return ret;
+}
diff --git a/ext/htslib/samples/mod_aux.c b/ext/htslib/samples/mod_aux.c
new file mode 100644
index 0000000..ae531b9
--- /dev/null
+++ b/ext/htslib/samples/mod_aux.c
@@ -0,0 +1,222 @@
+/*  mod_aux.c --  showcases the htslib api usage
+
+    Copyright (C) 2023 Genome Research Ltd.
+
+    Author: Vasudeva Sarma <vasudeva.sarma@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE
+
+*/
+
+/* The purpose of this code is to demonstrate the library apis and need proper error handling and optimisation */
+
+#include <getopt.h>
+#include <strings.h>
+#include <unistd.h>
+#include <htslib/sam.h>
+
+/// print_usage - print the usage
+/** @param fp pointer to the file / terminal to which usage to be dumped
+returns nothing
+*/
+static void print_usage(FILE *fp)
+{
+    fprintf(fp, "Usage: mod_aux infile QNAME tag type val\n\
+Add/update the given aux tag to all alignments\n\
+type A-char C-int F-float Z-string\n");
+}
+
+/// main_demo - start of the demo
+/** @param argc - count of arguments
+ *  @param argv - pointer to array of arguments
+returns 1 on failure 0 on success
+*/
+int main(int argc, char *argv[])
+{
+    const char *inname = NULL, *tag = NULL, *qname = NULL, *val = NULL;
+    char type = '\0';
+    int ret = EXIT_FAILURE, ret_r = 0, length = 0;
+    sam_hdr_t *in_samhdr = NULL;
+    samFile *infile = NULL, *outfile = NULL;
+    bam1_t *bamdata = NULL;
+    uint8_t *data = NULL;
+
+    //mod_aux infile QNAME tag type val
+    if (argc != 6) {
+        print_usage(stderr);
+        goto end;
+    }
+    inname = argv[1];
+    qname = argv[2];
+    tag = argv[3];
+    type = argv[4][0];
+    val = argv[5];
+
+    if (!(bamdata = bam_init1())) {
+        printf("Failed to allocate data memory!\n");
+        goto end;
+    }
+
+    //open input file
+    if (!(infile = sam_open(inname, "r"))) {
+        printf("Could not open %s\n", inname);
+        goto end;
+    }
+    //open output file
+    if (!(outfile = sam_open("-", "w"))) {
+        printf("Could not open std output\n");
+        goto end;
+    }
+
+    if (!(in_samhdr = sam_hdr_read(infile))) {
+        printf("Failed to read header from file!\n");
+        goto end;
+    }
+
+    if (sam_hdr_write(outfile, in_samhdr) == -1) {
+        printf("Failed to write header\n");
+        goto end;
+    }
+
+    while ((ret_r = sam_read1(infile, in_samhdr, bamdata)) >= 0) {
+        if (strcasecmp(bam_get_qname(bamdata), qname)) {
+            if (sam_write1(outfile, in_samhdr, bamdata) < 0) {
+                printf("Failed to write output\n");
+                goto end;
+            }
+            continue;   //not matching
+        }
+
+        errno = 0;
+        //matched to qname, update aux
+        if (!(data = bam_aux_get(bamdata, tag))) {
+            int i = 0; float f = 0;
+            //tag not present append
+            switch (type) {
+                case 'f':
+                case 'd':
+                    length = sizeof(float);
+                    f = atof(val);
+                    val = (const char*) &f;
+                    type = 'f';
+                break;
+                case 'C':
+                case 'S':
+                case 'I':
+                    length = sizeof(int);
+                    i = atoi(val);
+                    val = (const char*) &i;
+                break;
+                case 'Z':
+                    length = strlen(val) + 1;   //1 for NUL termination
+                break;
+                case 'A':
+                    length = 1;
+                break;
+                default:
+                    printf("Invalid type mentioned\n");
+                    goto end;
+                break;
+            }
+            if (bam_aux_append(bamdata, tag, type, length, (const uint8_t*)val)) {
+                printf("Failed to append aux data, errno: %d\n", errno);
+                goto end;
+            }
+        }
+        else {
+            char auxtype = bam_aux_type(data);
+            //update the tag with newer value
+            switch (type) {
+                case 'f':
+                case 'd':
+                    if (auxtype != 'f' && auxtype != 'd') {
+                        printf("Invalid aux type passed\n");
+                        goto end;
+                    }
+                    if (bam_aux_update_float(bamdata, tag, atof(val))) {
+                        printf("Failed to update float data, errno: %d\n", errno);
+                        goto end;
+                    }
+                break;
+                case 'C':
+                case 'S':
+                case 'I':
+                    if (auxtype != 'c' && auxtype != 'C' && auxtype != 's' && auxtype != 'S' && auxtype != 'i' && auxtype != 'I') {
+                        printf("Invalid aux type passed\n");
+                        goto end;
+                    }
+                    if (bam_aux_update_int(bamdata, tag, atoll(val))) {
+                        printf("Failed to update int data, errno: %d\n", errno);
+                        goto end;
+                    }
+                break;
+                case 'Z':
+                    if (auxtype != 'Z') {
+                        printf("Invalid aux type passed\n");
+                        goto end;
+                    }
+                    length = strlen(val) + 1;   //1 for NUL termination
+                    if (bam_aux_update_str(bamdata, tag, length, val)) {
+                        //with length as -1, length will be detected based on null terminated val data
+                        printf("Failed to update string data, errno: %d\n", errno);
+                        goto end;
+                    }
+                break;
+                case 'A':
+                    if (auxtype != 'A') {
+                        printf("Invalid aux type passed\n");
+                        goto end;
+                    }
+                    //update the char data directly on buffer
+                    *(data+1) = val[0];
+                break;
+                default:
+                    printf("Invalid data type\n");
+                    goto end;
+                break;
+            }
+        }
+        if (sam_write1(outfile, in_samhdr, bamdata) < 0) {
+            printf("Failed to write output\n");
+            goto end;
+        }
+    }
+    if (ret_r < -1) {
+        //read error
+        printf("Failed to read data\n");
+        goto end;
+    }
+
+    ret = EXIT_SUCCESS;
+end:
+    //cleanup
+    if (in_samhdr) {
+        sam_hdr_destroy(in_samhdr);
+    }
+    if (infile) {
+        sam_close(infile);
+    }
+    if (outfile) {
+        sam_close(outfile);
+    }
+    if (bamdata) {
+        bam_destroy1(bamdata);
+    }
+    return ret;
+}
diff --git a/ext/htslib/samples/mod_aux_ba.c b/ext/htslib/samples/mod_aux_ba.c
new file mode 100644
index 0000000..836a3d3
--- /dev/null
+++ b/ext/htslib/samples/mod_aux_ba.c
@@ -0,0 +1,147 @@
+/*  mod_aux_ba.c --  showcases the htslib api usage
+
+    Copyright (C) 2023 Genome Research Ltd.
+
+    Author: Vasudeva Sarma <vasudeva.sarma@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE
+
+*/
+
+/* The purpose of this code is to demonstrate the library apis and need proper error handling and optimisation */
+
+#include <getopt.h>
+#include <unistd.h>
+#include <htslib/sam.h>
+
+/// print_usage - print the usage
+/** @param fp pointer to the file / terminal to which usage to be dumped
+returns nothing
+*/
+static void print_usage(FILE *fp)
+{
+    fprintf(fp, "Usage: mod_aux_ba infile\n\
+Updates the count of bases as an aux array on all alignments\n\
+BA:B:I,count of ACTGN\n");
+}
+
+/// main_demo - start of the demo
+/** @param argc - count of arguments
+ *  @param argv - pointer to array of arguments
+returns 1 on failure 0 on success
+*/
+int main(int argc, char *argv[])
+{
+    const char *inname = NULL;
+    int i = 0, ret = EXIT_FAILURE, ret_r = 0;
+    uint32_t cnt[5] = {0};  //A C G T N
+    sam_hdr_t *in_samhdr = NULL;
+    samFile *infile = NULL, *outfile = NULL;
+    bam1_t *bamdata = NULL;
+
+    //mod_aux infile
+    if (argc != 2) {
+        print_usage(stderr);
+        goto end;
+    }
+    inname = argv[1];
+
+    if (!(bamdata = bam_init1())) {
+        printf("Failed to allocate data memory!\n");
+        goto end;
+    }
+
+    //open input file
+    if (!(infile = sam_open(inname, "r"))) {
+        printf("Could not open %s\n", inname);
+        goto end;
+    }
+
+    //open output file
+    if (!(outfile = sam_open("-", "w"))) {
+        printf("Could not open std output\n");
+        goto end;
+    }
+
+    if (!(in_samhdr = sam_hdr_read(infile))) {
+        printf("Failed to read header from file!\n");
+        goto end;
+    }
+
+    if (sam_hdr_write(outfile, in_samhdr) == -1) {
+        printf("Failed to write header\n");
+        goto end;
+    }
+
+    while ((ret_r = sam_read1(infile, in_samhdr, bamdata)) >= 0) {
+        errno = 0;
+        memset(cnt, 0, sizeof(cnt));
+        for (i = 0; i < bamdata->core.l_qseq; ++i) {
+            switch (seq_nt16_str[bam_seqi(bam_get_seq(bamdata),i)]) {
+                case 'A':
+                ++cnt[0];
+                break;
+                case 'C':
+                ++cnt[1];
+                break;
+                case 'G':
+                ++cnt[2];
+                break;
+                case 'T':
+                ++cnt[3];
+                break;
+                default:    //N
+                ++cnt[4];
+                break;
+            }
+        }
+
+        if (bam_aux_update_array(bamdata, "BA", 'I', sizeof(cnt)/sizeof(cnt[0]), cnt)) {
+            printf("Failed to update base array, errno %d", errno);
+            goto end;
+        }
+
+        if (sam_write1(outfile, in_samhdr, bamdata) < 0) {
+            printf("Failed to write output\n");
+            goto end;
+        }
+    }
+    if (ret_r < -1) {
+        //read error
+        printf("Failed to read data\n");
+        goto end;
+    }
+
+    ret = EXIT_SUCCESS;
+end:
+    //cleanup
+    if (in_samhdr) {
+        sam_hdr_destroy(in_samhdr);
+    }
+    if (infile) {
+        sam_close(infile);
+    }
+    if (outfile) {
+        sam_close(outfile);
+    }
+    if (bamdata) {
+        bam_destroy1(bamdata);
+    }
+    return ret;
+}
diff --git a/ext/htslib/samples/mod_bam.c b/ext/htslib/samples/mod_bam.c
new file mode 100644
index 0000000..6166396
--- /dev/null
+++ b/ext/htslib/samples/mod_bam.c
@@ -0,0 +1,230 @@
+/*  mod_bam.c --  showcases the htslib api usage
+
+    Copyright (C) 2023 Genome Research Ltd.
+
+    Author: Vasudeva Sarma <vasudeva.sarma@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE
+
+*/
+
+/* The purpose of this code is to demonstrate the library apis and need proper error handling and optimisation */
+
+#include <getopt.h>
+#include <strings.h>
+#include <unistd.h>
+#include <htslib/sam.h>
+
+/// print_usage - print the usage
+/** @param fp pointer to the file / terminal to which usage to be dumped
+returns nothing
+*/
+static void print_usage(FILE *fp)
+{
+    fprintf(fp, "Usage: mod_bam infile QNAME fieldpos newval\n\
+Modifies the alignment data field\n\
+fieldpos - 1 QNAME 2 FLAG 3 RNAME 4 POS 5 MAPQ 6 CIGAR 7 RNEXT 8 PNEXT 9 TLEN 10 SEQ 11 QUAL\n");
+}
+
+/// main_demo - start of the demo
+/** @param argc - count of arguments
+ *  @param argv - pointer to array of arguments
+returns 1 on failure 0 on success
+*/
+int main(int argc, char *argv[])
+{
+    const char *inname = NULL, *qname = NULL;
+    char *val = NULL;
+    int c = 0, ret = EXIT_FAILURE, field = 0;
+    sam_hdr_t *in_samhdr = NULL;
+    samFile *infile = NULL, *outfile = NULL;
+    int ret_r = 0, i = 0;
+    bam1_t *bamdata = NULL;
+
+    //mod_bam infile QNAME fieldpos newval
+    if (argc != 5) {
+        print_usage(stderr);
+        goto end;
+    }
+    inname = argv[1];
+    qname = argv[2];
+    //1 QNAME 2 FLAG 3 RNAME 4 POS 5 MAPQ 6 CIGAR 7 RNEXT 8 PNEXT 9 TLEN 10 SEQ 11 QUAL
+    field = atoi(argv[3]);
+    val = argv[4];
+
+    if (!(bamdata = bam_init1())) {
+        printf("Failed to allocate data memory!\n");
+        goto end;
+    }
+
+    //open input file
+    if (!(infile = sam_open(inname, "r")) || !(outfile = sam_open("-", "w"))) {
+        printf("Could not open input/output\n");
+        goto end;
+    }
+    //read header
+    if (!(in_samhdr = sam_hdr_read(infile))) {
+        printf("Failed to read header from file!\n");
+        goto end;
+    }
+
+    if (sam_hdr_write(outfile, in_samhdr) == -1) {
+        printf("Failed to write header\n");
+        goto end;
+    }
+
+    while ((ret_r = sam_read1(infile, in_samhdr, bamdata)) >= 0)
+    {
+        //QNAME FLAG RNAME POS MAPQ CIGAR RNEXT PNEXT TLEN SEQ QUAL [TAG:TYPE:VALUE]…
+        ret = 0;
+        if (!strcasecmp(qname, bam_get_qname(bamdata))) {
+            //the required one
+            switch(field) {
+            case 1:// QNAME
+                ret = bam_set_qname(bamdata, val);
+            break;
+            case 2:// FLAG
+                bamdata->core.flag = atol(val) & 0xFFFF;
+            break;
+            case 3:// RNAME
+            case 7:// RNEXT
+                if ((ret = sam_hdr_name2tid(in_samhdr, val)) < 0) {
+                    printf("Invalid reference name\n");
+                    ret = -1;
+                    break;
+                }
+                if (field == 3) {
+                    //reference
+                    bamdata->core.tid = ret;
+                }
+                else {
+                    //mate reference
+                    bamdata->core.mtid = ret;
+                }
+            break;
+            case 4:// POS
+                bamdata->core.pos = atoll(val);
+            break;
+            case 5:// MAPQ
+                bamdata->core.qual = atoi(val) & 0x0FF;
+            break;
+            case 6:// CIGAR
+            {
+                uint32_t *cigar = NULL;
+                size_t size = 0;
+                ssize_t ncigar = 0;
+                bam1_t *newbam = bam_init1();
+                if (!newbam) {
+                    printf("Failed to create new bam data\n");
+                    ret = -1;
+                    break;
+                }
+                //get cigar array and set all data in new bam record
+                if ((ncigar = sam_parse_cigar(val, NULL, &cigar, &size)) < 0) {
+                    printf("Failed to parse cigar\n");
+                    ret = -1;
+                    break;
+                }
+                if (bam_set1(newbam, bamdata->core.l_qname, bam_get_qname(bamdata), bamdata->core.flag, bamdata->core.tid, bamdata->core.pos, bamdata->core.qual,
+                    ncigar, cigar, bamdata->core.mtid, bamdata->core.mpos, bamdata->core.isize, bamdata->core.l_qseq, (const char*)bam_get_seq(bamdata), (const char*)bam_get_qual(bamdata), bam_get_l_aux(bamdata)) < 0) {
+                    printf("Failed to set bamdata\n");
+                    ret = -1;
+                    break;
+                }
+                //correct sequence data as input is expected in ascii format and not as compressed inside bam!
+                memcpy(bam_get_seq(newbam), bam_get_seq(bamdata), (bamdata->core.l_qseq + 1) / 2);
+                //copy the aux data
+                memcpy(bam_get_aux(newbam), bam_get_aux(bamdata), bam_get_l_aux(bamdata));
+
+                bam_destroy1(bamdata);
+                bamdata = newbam;
+            }
+            break;
+            case 8:// PNEXT
+                bamdata->core.mpos = atoll(val);
+            break;
+            case 9:// TLEN
+                bamdata->core.isize = atoll(val);
+            break;
+            case 10:// SEQ
+                i = strlen(val);
+                if (bamdata->core.l_qseq != i) {
+                    printf("SEQ length different\n");
+                    ret = -1;
+                    //as it is different, have to update quality data and cigar data as well and more info is required for it, which is not handled in this sample
+                    //accessing raw memory and moving is one option; creating and using new bam1_t object is another option.
+                    break;
+                }
+                for( c = 0; c < i; ++c) {
+                    bam_set_seqi(bam_get_seq(bamdata), c, seq_nt16_table[(unsigned char)val[c]]);
+                }
+            break;
+            case 11:// QUAL
+                i = strlen(val);
+                if (i != bamdata->core.l_qseq) {
+                    printf("Qual length different than sequence\n");
+                    ret = -1;
+                    break;
+                }
+                for (c = 0; c < i; ++c) {
+                    val[c] -= 33;               //phred score from ascii value
+                }
+                memcpy(bam_get_qual(bamdata), val, i);
+            break;
+            default:
+                printf("Invalid input\n");
+                goto end;
+            break;
+            }
+            if (ret < 0) {
+                printf("Failed to set new data\n");
+                ret = EXIT_FAILURE;
+                goto end;
+            }
+        }
+        if (sam_write1(outfile, in_samhdr, bamdata) < 0) {
+            printf("Failed to write bam data\n");
+            ret = EXIT_FAILURE;
+            goto end;
+        }
+    }
+
+    if (ret_r == -1 || ret != EXIT_FAILURE) {
+        // no error!
+        ret = EXIT_SUCCESS;
+    }
+    else {
+        printf("Failed to read data\n");
+    }
+end:
+    //cleanup
+    if (in_samhdr) {
+        sam_hdr_destroy(in_samhdr);
+    }
+    if (infile) {
+        sam_close(infile);
+    }
+    if (outfile) {
+        sam_close(outfile);
+    }
+    if (bamdata) {
+        bam_destroy1(bamdata);
+    }
+    return ret;
+}
diff --git a/ext/htslib/samples/modstate.c b/ext/htslib/samples/modstate.c
new file mode 100644
index 0000000..4d5f676
--- /dev/null
+++ b/ext/htslib/samples/modstate.c
@@ -0,0 +1,190 @@
+/*  modstate.c --  showcases the htslib api usage
+
+    Copyright (C) 2023 Genome Research Ltd.
+
+    Author: Vasudeva Sarma <vasudeva.sarma@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE
+
+*/
+
+/* The purpose of this code is to demonstrate the library apis and need proper error handling and optimisation */
+
+#include <getopt.h>
+#include <unistd.h>
+#include <htslib/sam.h>
+
+/// print_usage - print the usage
+/** @param fp pointer to the file / terminal to which usage to be dumped
+returns nothing
+*/
+static void print_usage(FILE *fp)
+{
+    fprintf(fp, "Usage: modstate infile option\n\
+Shows the base modifications on the alignment\n\
+Option can be 1 or 2 to select the api to use\n");
+}
+
+/// main_demo - start of the demo
+/** @param argc - count of arguments
+ *  @param argv - pointer to array of arguments
+returns 1 on failure 0 on success
+*/
+int main(int argc, char *argv[])
+{
+    const char *inname = NULL;
+    int ret = EXIT_FAILURE;
+    sam_hdr_t *in_samhdr = NULL;
+    samFile *infile = NULL;
+
+    int ret_r = 0, i = 0 , r = 0, j = 0, pos = 0, opt = 0, k = 0, cnt = 0, *bm = NULL;
+    bam1_t *bamdata = NULL;
+    uint8_t *data = NULL;
+    hts_base_mod_state *ms = NULL;
+
+
+    //modstate infile 1/2
+    if (argc != 3) {
+        print_usage(stderr);
+        goto end;
+    }
+    inname = argv[1];
+    opt = atoi(argv[2]) - 1;    //option 1 or 2?
+
+    if (!(bamdata = bam_init1())) {
+        printf("Failed to allocate data memory!\n");
+        goto end;
+    }
+
+    if (!(ms = hts_base_mod_state_alloc())) {
+        printf("Failed to allocate state memory\n");
+        goto end;
+    }
+
+    //open input file
+    if (!(infile = sam_open(inname, "r"))) {
+        printf("Could not open %s\n", inname);
+        goto end;
+    }
+    //read header
+    if (!(in_samhdr = sam_hdr_read(infile))) {
+        printf("Failed to read header from file!\n");
+        goto end;
+    }
+
+    while ((ret_r = sam_read1(infile, in_samhdr, bamdata)) >= 0)
+    {
+        i = 0;
+        data = bam_get_seq(bamdata);
+        if (bam_parse_basemod(bamdata, ms)) {
+            printf("Failed to parse the base mods\n");
+            goto end;
+        }
+        //dump the modifications
+        printf("Modifications:");
+        bm = bam_mods_recorded(ms, &cnt);
+        for (k = 0; k < cnt; ++k) {
+            printf("%c", bm[k]);
+        }
+        printf("\n");
+        hts_base_mod mod[5] = {0};  //for ATCGN
+        if (opt) {
+            //option 1
+            for (; i < bamdata->core.l_qseq; ++i) {
+                if ((r = bam_mods_at_next_pos(bamdata, ms, mod, sizeof(mod)/sizeof(mod[0]))) <= -1) {
+                    printf("Failed to get modifications\n");
+                    goto end;
+                }
+                else if (r > (sizeof(mod) / sizeof(mod[0]))) {
+                    printf("More modifications than this app can handle, update the app\n");
+                    goto end;
+                }
+                else if (!r) {
+                    //no modification at this pos
+                    printf("%c", seq_nt16_str[bam_seqi(data, i)]);
+                }
+                //modifications
+                for (j = 0; j < r; ++j) {
+                    printf("%c%c%c", mod[j].canonical_base, mod[j].strand ? '-' : '+', mod[j].modified_base);
+                }
+            }
+        }
+        else {
+            //option 2
+            while ((r = bam_next_basemod(bamdata, ms, mod, sizeof(mod)/sizeof(mod[0]), &pos)) >= 0) {
+                for (; i < bamdata->core.l_qseq && i < pos; ++i) {
+                    printf("%c", seq_nt16_str[bam_seqi(data, i)]);
+                }
+                //modifications
+                for (j = 0; j < r; ++j) {
+                    printf("%c%c%c", mod[j].canonical_base, mod[j].strand ? '-' : '+', mod[j].modified_base);
+                }
+                if (i == pos)
+                    i++;        //skip the modification already displayed
+                if (!r) {
+                    for (; i < bamdata->core.l_qseq; ++i) {
+                        printf("%c", seq_nt16_str[bam_seqi(data, i)]);
+                    }
+                    break;
+                }
+            }
+            if (r <= -1) {
+                printf("Failed to get modifications\n");
+                goto end;
+            }
+        }
+        printf("\n");
+    }
+
+    if (ret_r == -1) {
+        //check last alignment's base modification
+        int strand = 0, impl = 0;
+        char canonical = 0, modification[] = "mhfcgebaon";      //possible modifications
+        printf("\n\nLast alignment has \n");
+        for (k = 0; k < sizeof(modification) - 1; ++k) {        //avoiding NUL termination
+            if (bam_mods_query_type(ms, modification[k], &strand, &impl, &canonical)) {
+                printf ("No modification of %c type\n", modification[k]);
+            }
+            else {
+                printf("%s strand has %c modified with %c, can %sassume unlisted as unmodified\n", strand?"-/bottom/reverse":"+/top/forward", canonical, modification[k], impl?"" : "not " );
+            }
+        }
+        // no error!
+        ret = EXIT_SUCCESS;
+    }
+    else {
+        printf("Failed to read data\n");
+    }
+end:
+    //cleanup
+    if (in_samhdr) {
+        sam_hdr_destroy(in_samhdr);
+    }
+    if (infile) {
+        sam_close(infile);
+    }
+    if (bamdata) {
+        bam_destroy1(bamdata);
+    }
+
+    if (ms) {
+        hts_base_mod_state_free(ms);
+    }
+    return ret;
+}
diff --git a/ext/htslib/samples/mpileup.c b/ext/htslib/samples/mpileup.c
new file mode 100644
index 0000000..ecab705
--- /dev/null
+++ b/ext/htslib/samples/mpileup.c
@@ -0,0 +1,204 @@
+/*  mpileup.c --  showcases the htslib api usage
+
+    Copyright (C) 2023 Genome Research Ltd.
+
+    Author: Vasudeva Sarma <vasudeva.sarma@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE
+
+*/
+
+/* The purpose of this code is to demonstrate the library apis and need proper error handling and optimisation */
+
+#include <getopt.h>
+#include <unistd.h>
+#include <ctype.h>
+#include <htslib/sam.h>
+
+/// print_usage - show usage
+/** @param fp pointer to the file / terminal to which usage to be dumped
+returns nothing
+*/
+static void print_usage(FILE *fp)
+{
+    fprintf(fp, "Usage: mpileup infile ...\n\
+Shows the mpileup api usage.\n");
+    return;
+}
+
+typedef struct plpconf {
+    char *inname;
+    samFile *infile;
+    sam_hdr_t *in_samhdr;
+} plpconf;
+
+/// @brief plpconstructor
+/// @param data client data?
+/// @param b bam being loaded
+///     @param cd client data
+/// @return
+int plpconstructor(void *data, const bam1_t *b, bam_pileup_cd *cd) {
+    return 0;
+}
+
+int plpdestructor(void *data, const bam1_t *b, bam_pileup_cd *cd) {
+    return 0;
+}
+
+/// @brief bam_plp_auto_f reads alignment data for pileup operation
+/// @param data client callback data holding alignment file handle
+/// @param b bamdata read
+/// @return same as sam_read1
+int readdata(void *data, bam1_t *b)
+{
+    plpconf *conf = (plpconf*)data;
+    if (!conf || !conf->infile) {
+        return -2;  //cant read data
+    }
+
+    //read alignment and send
+    return sam_read1(conf->infile, conf->infile->bam_header, b);
+}
+
+/// main_demo - start of the demo
+/** @param argc - count of arguments
+ *  @param argv - pointer to array of arguments
+returns 1 on failure 0 on success
+*/
+int main(int argc, char *argv[])
+{
+    int ret = EXIT_FAILURE;
+    bam1_t *bamdata = NULL;
+    plpconf** conf = NULL;
+    bam_mplp_t mplpiter = NULL;
+    int tid = -1, input = 0, k = 0, dpt = 0, *depth = NULL;
+    hts_pos_t refpos = -1;
+    const bam_pileup1_t **plp = NULL;
+
+    //infile ...
+    if (argc < 2) {
+        print_usage(stderr);
+        goto end;
+    }
+    if ((conf = calloc(argc - 1, sizeof(plpconf*)))) {
+        for (input = 0; input < argc - 1; ++input) {
+            conf[input] = calloc(1, sizeof(plpconf));
+        }
+    }
+    depth = calloc(argc - 1, sizeof(int));
+    plp = calloc(argc - 1, sizeof(bam_pileup1_t*));
+    if (!conf || !depth || !plp) {
+        printf("Failed to allocate memory\n");
+        goto end;
+    }
+    for (input = 0; input < argc - 1; ++input) {
+        conf[input]->inname = argv[input+1];
+    }
+
+    //initialize
+    if (!(bamdata = bam_init1())) {
+        printf("Failed to initialize bamdata\n");
+        goto end;
+    }
+    //open input files
+    for(input = 0; input < argc - 1; ++input) {
+        if (!(conf[input]->infile = sam_open(conf[input]->inname, "r"))) {
+            printf("Could not open %s\n", conf[input]->inname);
+            goto end;
+        }
+        //read header
+        if (!(conf[input]->in_samhdr = sam_hdr_read(conf[input]->infile))) {
+            printf("Failed to read header from file!\n");
+            goto end;
+        }
+    }
+
+    if (!(mplpiter = bam_mplp_init(argc - 1, readdata, (void**) conf))) {
+        printf("Failed to initialize mpileup data\n");
+        goto end;
+    }
+
+    //set constructor destructor callbacks
+    bam_mplp_constructor(mplpiter, plpconstructor);
+    bam_mplp_destructor(mplpiter, plpdestructor);
+
+    while (bam_mplp64_auto(mplpiter, &tid, &refpos, depth, plp) > 0) {
+        printf("%d\t%"PRIhts_pos"\t", tid+1, refpos+1);
+
+        for (input = 0; input < argc - 1; ++input) {
+            for (dpt = 0; dpt  < depth[input]; ++dpt) {
+                if (plp[input][dpt].is_del || plp[input][dpt].is_refskip) {
+                    printf("*");
+                    continue;
+                }
+                //start and end are displayed in UPPER and rest on LOWER
+                printf("%c", plp[input][dpt].is_head ? toupper(seq_nt16_str[bam_seqi(bam_get_seq(plp[input][dpt].b), plp[input][dpt].qpos)]) :
+                                (plp[input]->is_tail ? toupper(seq_nt16_str[bam_seqi(bam_get_seq(plp[input][dpt].b), plp[input][dpt].qpos)]) : tolower(seq_nt16_str[bam_seqi(bam_get_seq(plp[input][dpt].b), plp[input][dpt].qpos)])));
+                if (plp[input][dpt].indel > 0) {
+                    //insertions, anyway not start or end
+                    printf("+%d", plp[input][dpt].indel);
+                    for (k = 0; k < plp[input][dpt].indel; ++k) {
+                        printf("%c", tolower(seq_nt16_str[bam_seqi(bam_get_seq(plp[input][dpt].b), plp[input][dpt].qpos + k + 1)]));
+                    }
+                }
+                else if (plp[input][dpt].indel < 0) {
+                    printf("%d", plp[input][dpt].indel);
+                    for (k = 0; k < -plp[input][dpt].indel; ++k) {
+                        printf("?");
+                    }
+                }
+            }
+            printf(" ");
+        }
+        printf("\n");
+        fflush(stdout);
+    }
+
+    ret = EXIT_SUCCESS;
+end:
+    //clean up
+    if (conf) {
+        for (input = 0; input < argc - 1; ++input) {
+            if (conf[input] && conf[input]->in_samhdr) {
+                sam_hdr_destroy(conf[input]->in_samhdr);
+            }
+            if (conf[input] && conf[input]->infile) {
+                sam_close(conf[input]->infile);
+            }
+            if (conf[input]) {
+                free(conf[input]);
+            }
+        }
+        free(conf);
+    }
+
+    if (bamdata) {
+        bam_destroy1(bamdata);
+    }
+    if (mplpiter) {
+        bam_mplp_destroy(mplpiter);
+    }
+    if (depth) {
+        free(depth);
+    }
+    if (plp) {
+        free(plp);
+    }
+    return ret;
+}
diff --git a/ext/htslib/samples/pileup.c b/ext/htslib/samples/pileup.c
new file mode 100644
index 0000000..be7aad8
--- /dev/null
+++ b/ext/htslib/samples/pileup.c
@@ -0,0 +1,183 @@
+/*  pileup.c --  showcases the htslib api usage
+
+    Copyright (C) 2023 Genome Research Ltd.
+
+    Author: Vasudeva Sarma <vasudeva.sarma@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE
+
+*/
+
+/* The purpose of this code is to demonstrate the library apis and need proper error handling and optimisation */
+
+#include <getopt.h>
+#include <unistd.h>
+#include <ctype.h>
+#include <htslib/sam.h>
+
+/// print_usage - show usage
+/** @param fp pointer to the file / terminal to which usage to be dumped
+returns nothing
+*/
+static void print_usage(FILE *fp)
+{
+    fprintf(fp, "Usage: pileup infile\n\
+Shows the pileup api usage.\n");
+    return;
+}
+
+typedef struct plpconf {
+    char *inname;
+    samFile *infile;
+    sam_hdr_t *in_samhdr;
+} plpconf;
+
+/// @brief plpconstructor
+/// @param data client data?
+/// @param b bam being loaded
+///     @param cd client data
+/// @return
+int plpconstructor(void *data, const bam1_t *b, bam_pileup_cd *cd) {
+    /*plpconf *conf= (plpconf*)data;
+    can access the data passed to pileup init from data
+    can do any alignment specific allocation / data storage here in param cd
+    it can hold either a float, 64 bit int or a pointer
+    when using cd, initialize and use as it will be reused after destructor*/
+    return 0;
+}
+
+int plpdestructor(void *data, const bam1_t *b, bam_pileup_cd *cd) {
+    /*plpconf *conf= (plpconf*)data;
+    can access the data passed to pileup init from data
+    deallocate any alignment specific allocation made in constructor and stored in param cd*/
+    return 0;
+}
+
+/// @brief bam_plp_auto_f reads alignment data for pileup operation
+/// @param data client callback data holding alignment file handle
+/// @param b bamdata read
+/// @return same as sam_read1
+int readdata(void *data, bam1_t *b)
+{
+    plpconf *conf = (plpconf*)data;
+    if (!conf || !conf->infile) {
+        return -2;  //cant read data
+    }
+
+    //read alignment and send
+    return sam_read1(conf->infile, conf->infile->bam_header, b);
+}
+
+/// main_demo - start of the demo
+/** @param argc - count of arguments
+ *  @param argv - pointer to array of arguments
+returns 1 on failure 0 on success
+*/
+int main(int argc, char *argv[])
+{
+    int ret = EXIT_FAILURE;
+    bam1_t *bamdata = NULL;
+    plpconf conf = {0};
+    bam_plp_t plpiter = NULL;
+    int tid = -1, n = -1, j = 0, k = 0;
+    int refpos = -1;
+    const bam_pileup1_t *plp = NULL;
+
+    //infile
+    if (argc != 2) {
+        print_usage(stderr);
+        goto end;
+    }
+    conf.inname = argv[1];
+
+    //initialize
+    if (!(bamdata = bam_init1())) {
+        printf("Failed to initialize bamdata\n");
+        goto end;
+    }
+    //open input files
+    if (!(conf.infile = sam_open(conf.inname, "r"))) {
+        printf("Could not open %s\n", conf.inname);
+        goto end;
+    }
+    //read header
+    if (!(conf.in_samhdr = sam_hdr_read(conf.infile))) {
+        printf("Failed to read header from file!\n");
+        goto end;
+    }
+
+    if (!(plpiter = bam_plp_init(readdata, &conf))) {
+        printf("Failed to initialize pileup data\n");
+        goto end;
+    }
+
+    //set constructor destructor callbacks
+    bam_plp_constructor(plpiter, plpconstructor);
+    bam_plp_destructor(plpiter, plpdestructor);
+
+    while ((plp = bam_plp_auto(plpiter, &tid, &refpos, &n))) {
+        printf("%d\t%d\t", tid+1, refpos+1);
+
+        for (j = 0; j < n; ++j) {
+            //doesnt detect succeeding insertion and deletion together here, only insertion is identified
+            //deletion is detected in plp->is_del as and when pos reaches the position
+            //if detection ahead is required, use bam_plp_insertion here which gives deletion length along with insertion
+            if (plp[j].is_del || plp[j].is_refskip) {
+                printf("*");
+                continue;
+            }
+            //start and end are displayed in UPPER and rest on LOWER
+            printf("%c", plp[j].is_head ? toupper(seq_nt16_str[bam_seqi(bam_get_seq(plp[j].b), plp[j].qpos)]) :
+                            (plp[j].is_tail ? toupper(seq_nt16_str[bam_seqi(bam_get_seq(plp[j].b), plp[j].qpos)]) : tolower(seq_nt16_str[bam_seqi(bam_get_seq(plp[j].b), plp[j].qpos)])));
+            if (plp[j].indel > 0) {
+                //insertions, anyway not start or end
+                printf("+%d", plp[j].indel);
+                for (k = 0; k < plp[j].indel; ++k) {
+                    printf("%c", tolower(seq_nt16_str[bam_seqi(bam_get_seq(plp[j].b), plp[j].qpos + k + 1)]));
+                }
+            }
+            else if (plp[j].indel < 0) {
+                printf("%d", plp[j].indel);
+                for (k = 0; k < -plp[j].indel; ++k) {
+                    printf("?");
+                }
+            }
+            printf(" ");
+        }
+        printf("\n");
+        fflush(stdout);
+    }
+
+    ret = EXIT_SUCCESS;
+end:
+    //clean up
+    if (conf.in_samhdr) {
+        sam_hdr_destroy(conf.in_samhdr);
+    }
+    if (conf.infile) {
+        sam_close(conf.infile);
+    }
+    if (bamdata) {
+        bam_destroy1(bamdata);
+    }
+    if (plpiter) {
+        bam_plp_destroy(plpiter);
+    }
+    return ret;
+}
diff --git a/ext/htslib/samples/pileup_mod.c b/ext/htslib/samples/pileup_mod.c
new file mode 100644
index 0000000..81ac5a5
--- /dev/null
+++ b/ext/htslib/samples/pileup_mod.c
@@ -0,0 +1,218 @@
+/*  pileup_mod.c --  showcases the htslib api usage
+
+    Copyright (C) 2023 Genome Research Ltd.
+
+    Author: Vasudeva Sarma <vasudeva.sarma@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE
+
+*/
+
+/* The purpose of this code is to demonstrate the library apis and need proper error handling and optimisation */
+
+#include <getopt.h>
+#include <unistd.h>
+#include <ctype.h>
+#include <htslib/sam.h>
+
+/// print_usage - show usage
+/** @param fp pointer to the file / terminal to which usage to be dumped
+returns nothing
+*/
+static void print_usage(FILE *fp)
+{
+    fprintf(fp, "Usage: pileup_mod infile\n\
+Shows the pileup api usage with base modification.\n");
+    return;
+}
+
+typedef struct plpconf {
+    char *inname;
+    samFile *infile;
+    sam_hdr_t *in_samhdr;
+} plpconf;
+
+/// @brief plpconstructor
+/// @param data client data?
+/// @param b bam being loaded
+///     @param cd client data
+/// @return
+int plpconstructor(void *data, const bam1_t *b, bam_pileup_cd *cd) {
+    //plpconf *conf= (plpconf*)data; can use this to access anything required from the data in pileup init
+
+    //when using cd, initialize and use as it will be reused after destructor
+    cd->p = hts_base_mod_state_alloc();
+    if (!cd->p) {
+        printf("Failed to allocate base modification state\n");
+        return 1;
+    }
+
+    //parse the bam data and gather modification data from MM tags
+    return (-1 == bam_parse_basemod(b, (hts_base_mod_state*)cd->p)) ? 1 : 0;
+}
+
+int plpdestructor(void *data, const bam1_t *b, bam_pileup_cd *cd) {
+    if (cd->p) {
+        hts_base_mod_state_free((hts_base_mod_state *)cd->p);
+        cd->p = NULL;
+    }
+    return 0;
+}
+
+/// @brief bam_plp_auto_f reads alignment data for pileup operation
+/// @param data client callback data holding alignment file handle
+/// @param b bamdata read
+/// @return same as sam_read1
+int readdata(void *data, bam1_t *b)
+{
+    plpconf *conf = (plpconf*)data;
+    if (!conf || !conf->infile) {
+        return -2;  //cant read data
+    }
+
+    //read alignment and send
+    return sam_read1(conf->infile, conf->infile->bam_header, b);
+}
+
+/// main_demo - start of the demo
+/** @param argc - count of arguments
+ *  @param argv - pointer to array of arguments
+returns 1 on failure 0 on success
+*/
+int main(int argc, char *argv[])
+{
+    int ret = EXIT_FAILURE;
+    bam1_t *bamdata = NULL;
+    plpconf conf = {0};
+    bam_plp_t plpiter = NULL;
+    int tid = -1, depth = -1, j = 0, k = 0, inslen = 0, dellen = 0, modlen = 0;
+    #define NMODS 5
+    hts_base_mod mods[NMODS] = {0};     //ACGT N
+    int refpos = -1;
+    const bam_pileup1_t *plp = NULL;
+    kstring_t insdata = KS_INITIALIZE;
+
+    //infile
+    if (argc != 2) {
+        print_usage(stderr);
+        goto end;
+    }
+    conf.inname = argv[1];
+
+    //initialize
+    if (!(bamdata = bam_init1())) {
+        printf("Failed to initialize bamdata\n");
+        goto end;
+    }
+    //open input files
+    if (!(conf.infile = sam_open(conf.inname, "r"))) {
+        printf("Could not open %s\n", conf.inname);
+        goto end;
+    }
+    //read header
+    if (!(conf.in_samhdr = sam_hdr_read(conf.infile))) {
+        printf("Failed to read header from file!\n");
+        goto end;
+    }
+
+    if (!(plpiter = bam_plp_init(readdata, &conf))) {
+        printf("Failed to initialize pileup data\n");
+        goto end;
+    }
+
+    //set constructor destructor callbacks
+    bam_plp_constructor(plpiter, plpconstructor);
+    bam_plp_destructor(plpiter, plpdestructor);
+
+    while ((plp = bam_plp_auto(plpiter, &tid, &refpos, &depth))) {
+        memset(&mods, 0, sizeof(mods));
+        printf("%d\t%d\t", tid+1, refpos+1);
+
+        for (j = 0; j < depth; ++j) {
+            dellen = 0;
+
+            if (plp[j].is_del || plp[j].is_refskip) {
+                printf("*");
+                continue;
+            }
+            /*invoke bam_mods_at_qpos before bam_plp_insertion_mod that the base modification
+            is retrieved before change in pileup pos thr' plp_insertion_mod call*/
+            if ((modlen = bam_mods_at_qpos(plp[j].b, plp[j].qpos, plp[j].cd.p, mods, NMODS)) == -1) {
+                printf("Failed to get modifications\n");
+                goto end;
+            }
+
+            //use plp_insertion/_mod to get insertion and del at the same position
+            if ((inslen = bam_plp_insertion_mod(&plp[j], (hts_base_mod_state*)plp[j].cd.p, &insdata, &dellen)) == -1) {
+                printf("Failed to get insertion status\n");
+                goto end;
+            }
+
+            //start and end are displayed in UPPER and rest on LOWER, only 1st modification considered
+            //base and modification
+            printf("%c%c%c", plp[j].is_head ? toupper(seq_nt16_str[bam_seqi(bam_get_seq(plp[j].b), plp[j].qpos)]) :
+                (plp[j].is_tail ? toupper(seq_nt16_str[bam_seqi(bam_get_seq(plp[j].b), plp[j].qpos)]) :
+                    tolower(seq_nt16_str[bam_seqi(bam_get_seq(plp[j].b), plp[j].qpos)])),
+                modlen > 0 ? mods[0].strand ? '-' : '+' : '\0',
+                modlen > 0 ? mods[0].modified_base : '\0');
+            //insertion and deletions
+            if (plp[j].indel > 0) {
+                //insertion
+                /*insertion data from plp_insertion_mod, note this shows the quality value as well
+                which is different from base and modification above;the lower case display is not attempted either*/
+                printf("+%d%s", plp[j].indel, insdata.s);
+                //handle deletion if any
+                if (dellen) {
+                    printf("-%d", dellen);
+                    for (k = 0; k < dellen; ++k) {
+                        printf("?");
+                    }
+                }
+            }
+            else if (plp[j].indel < 0) {
+                //deletion
+                printf("%d", plp[j].indel);
+                for (k = 0; k < -plp[j].indel; ++k) {
+                    printf("?");
+                }
+            }
+            printf(" ");
+        }
+        printf("\n");
+        fflush(stdout);
+    }
+
+    ret = EXIT_SUCCESS;
+end:
+    //clean up
+    if (conf.in_samhdr) {
+        sam_hdr_destroy(conf.in_samhdr);
+    }
+    if (conf.infile) {
+        sam_close(conf.infile);
+    }
+    if (bamdata) {
+        bam_destroy1(bamdata);
+    }
+    if (plpiter) {
+        bam_plp_destroy(plpiter);
+    }
+    ks_free(&insdata);
+    return ret;
+}
diff --git a/ext/htslib/samples/qtask_ordered.c b/ext/htslib/samples/qtask_ordered.c
new file mode 100644
index 0000000..a76d598
--- /dev/null
+++ b/ext/htslib/samples/qtask_ordered.c
@@ -0,0 +1,425 @@
+/*  qtask_ordered.c --  showcases the htslib api usage
+
+    Copyright (C) 2024 Genome Research Ltd.
+
+    Author: Vasudeva Sarma <vasudeva.sarma@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE
+
+*/
+
+/* The purpose of this code is to demonstrate the library apis and need proper error handling and optimisation */
+
+#include <getopt.h>
+#include <unistd.h>
+#include <pthread.h>
+#include <sys/time.h>
+#include <htslib/sam.h>
+#include <htslib/thread_pool.h>
+
+typedef struct data {
+    int count;                  //used up size
+    int maxsize;                //max size per data chunk
+    bam1_t **bamarray;          //bam1_t array for optimal queueing
+    struct data *next;          //pointer to next one - to reuse earlier allocations
+} data;
+
+typedef struct datacache
+{
+    pthread_mutex_t lock;       //synchronizes the access to cache
+    data *list;                 //data storage
+} datacache;
+
+typedef struct orderedwrite {
+    samFile *outfile;           //output file handle
+    sam_hdr_t *samhdr;          //header used to write data
+    hts_tpool_process *queue;   //queue from which results to be retrieved
+    datacache *cache;           //to re-use allocated storage
+    int result;                 //result code returned by writer thread
+} orderedwrite;
+
+/// print_usage - print the usage
+/** @param fp pointer to the file / terminal to which usage to be dumped
+returns nothing
+*/
+static void print_usage(FILE *fp)
+{
+    fprintf(fp, "Usage: qtask_ordered infile threadcount outdir [chunksize]\n\
+Calculates GC ratio - sum(G,C) / sum(A,T,C,G) - and adds to each alignment\n\
+as xr:f aux tag. Output is saved in outdir.\n\
+chunksize [4096] sets the number of alignments clubbed together to process.\n");
+    return;
+}
+
+/// getbamstorage - allocates storage for alignments to queue
+/** @param chunk number of bam data to allocate
+ * @param bamcache cached storage
+returns already allocated data storage if one is available, otherwise allocates new
+*/
+data* getbamstorage(int chunk, datacache *bamcache)
+{
+    int i = 0;
+    data *bamdata = NULL;
+
+    if (!bamcache) {
+        return NULL;
+    }
+    //get from cache if there is an already allocated storage
+    if (pthread_mutex_lock(&bamcache->lock)) {
+        return NULL;
+    }
+    if (bamcache->list) {                   //available
+        bamdata = bamcache->list;
+        bamcache->list = bamdata->next;     //remove and set next one as available
+        bamdata->next = NULL;               //remove link
+        bamdata->count = 0;
+        goto end;
+    }
+    //allocate and use
+    if (!(bamdata = malloc(sizeof(data)))) {
+        goto end;
+    }
+    bamdata->bamarray = malloc(chunk * sizeof(bam1_t*));
+    if (!bamdata->bamarray) {
+        free(bamdata);
+        bamdata = NULL;
+        goto end;
+    }
+    for (i = 0; i < chunk; ++i) {
+        bamdata->bamarray[i] = bam_init1();
+    }
+    bamdata->maxsize = chunk;
+    bamdata->count = 0;
+    bamdata->next = NULL;
+
+end:
+    pthread_mutex_unlock(&bamcache->lock);
+    return bamdata;
+}
+
+/// cleanup_bamstorage - frees a bamdata struct plus contents
+/** @param arg Pointer to data to free
+    @p arg has type void * so it can be used as a callback passed
+    to hts_tpool_dispatch3().
+ */
+void cleanup_bamstorage(void *arg)
+{
+    data *bamdata = (data *) arg;
+    if (!bamdata)
+        return;
+    if (bamdata->bamarray) {
+        int i;
+        for (i = 0; i < bamdata->maxsize; i++) {
+            bam_destroy1(bamdata->bamarray[i]);
+        }
+        free(bamdata->bamarray);
+    }
+    free(bamdata);
+}
+
+/// thread_ordered_proc - does the processing of task in queue and queues the output back
+/** @param args pointer to set of data to be processed
+returns the processed data
+the processing could be in any order based on the number of threads in use but read of output
+from queue will be in order
+a null data indicates the end of input and a null is returned to be added back to result queue
+*/
+void *thread_ordered_proc(void *args)
+{
+    int i = 0, pos = 0;
+    data *bamdata = (data*)args;
+    float gcratio = 0;
+    uint8_t *data = NULL;
+
+    if (bamdata == NULL)
+        return NULL; // Indicates no more input
+
+    for ( i = 0; i < bamdata->count; ++i) {
+        //add count
+        uint64_t count[16] = {0};
+        data = bam_get_seq(bamdata->bamarray[i]);
+        for (pos = 0; pos < bamdata->bamarray[i]->core.l_qseq; ++pos) {
+            count[bam_seqi(data,pos)]++;
+        }
+        /*it is faster to count all and use offset to get required counts rather than select
+        require ones inside the loop*/
+        gcratio = (count[2] /*C*/ + count[4] /*G*/) / (float) (count[1] /*A*/ + count[8] /*T*/ + count[2] + count[4]);
+
+        if (bam_aux_append(bamdata->bamarray[i], "xr", 'f', sizeof(gcratio), (const uint8_t*)&gcratio) < 0) {
+            fprintf(stderr, "Failed to add aux tag xr, errno: %d\n", errno);
+            break;
+        }
+    }
+    return bamdata;
+}
+
+/// threadfn_orderedwrite - thread that read the output from queue and writes
+/** @param args pointer to data specific for the thread
+returns NULL
+*/
+void *threadfn_orderedwrite(void *args)
+{
+    orderedwrite *tdata = (orderedwrite*)args;
+    hts_tpool_result *r = NULL;
+    data *bamdata = NULL;
+    int i = 0;
+
+    tdata->result = 0;
+
+    //get result and write; wait if no result is in queue - until shutdown of queue
+    while (tdata->result == 0 &&
+           (r = hts_tpool_next_result_wait(tdata->queue)) != NULL) {
+        bamdata = (data*) hts_tpool_result_data(r);
+
+        if (bamdata == NULL) {
+            // Indicator for no more input. Time to stop.
+            hts_tpool_delete_result(r, 0);
+            break;
+        }
+
+        for (i = 0; i < bamdata->count; ++i) {
+            if (sam_write1(tdata->outfile, tdata->samhdr, bamdata->bamarray[i]) < 0) {
+                fprintf(stderr, "Failed to write output data\n");
+                tdata->result = -1;
+                break;
+            }
+        }
+        hts_tpool_delete_result(r, 0);              //release the result memory
+
+        pthread_mutex_lock(&tdata->cache->lock);
+        bamdata->next = tdata->cache->list;         //make current list as next
+        tdata->cache->list = bamdata;               //set as current to reuse
+        pthread_mutex_unlock(&tdata->cache->lock);
+    }
+
+    // Shut down the process queue.  If we stopped early due to a write failure,
+    // this will signal to the other end that something has gone wrong.
+    hts_tpool_process_shutdown(tdata->queue);
+
+    return NULL;
+}
+
+/// main_demo - start of the demo
+/** @param argc - count of arguments
+ *  @param argv - pointer to array of arguments
+returns 1 on failure 0 on success
+*/
+int main(int argc, char *argv[])
+{
+    const char *inname = NULL, *outdir = NULL;
+    char *file = NULL;
+    int c = 0, ret = EXIT_FAILURE, cnt = 0, started_thread = 0, chunk = 0;
+    size_t size = 0;
+    samFile *infile = NULL, *outfile = NULL;
+    sam_hdr_t *in_samhdr = NULL;
+    pthread_t thread;
+    orderedwrite twritedata = {0};
+    hts_tpool *pool = NULL;
+    hts_tpool_process *queue = NULL;
+    htsThreadPool tpool = {NULL, 0};
+    data *bamdata = NULL;
+    datacache bamcache = {PTHREAD_MUTEX_INITIALIZER, NULL};
+
+    //qtask infile threadcount outdir [chunksize]
+    if (argc != 4 && argc != 5) {
+        print_usage(stdout);
+        goto end;
+    }
+    inname = argv[1];
+    cnt = atoi(argv[2]);
+    outdir = argv[3];
+    if (argc == 5) {    //chunk size present
+        chunk = atoi(argv[4]);
+    }
+    if (cnt < 1) {      //set proper thread count
+        cnt = 1;
+    }
+    if (chunk < 1) {    //set valid  chunk size
+        chunk = 4096;
+    }
+
+    //allocate space for output
+    size = (strlen(outdir) + sizeof("/out.bam") + 1);   //space for output file name and null termination
+    if (!(file = malloc(size))) {
+        fprintf(stderr, "Failed to set output path\n");
+        goto end;
+    }
+    snprintf(file, size, "%s/out.bam", outdir);         //output file name
+    if (!(pool = hts_tpool_init(cnt))) {                //thread pool
+        fprintf(stderr, "Failed to create thread pool\n");
+        goto end;
+    }
+    tpool.pool = pool;      //to share the pool for file read and write as well
+    //queue to use with thread pool, for task and results
+    if (!(queue = hts_tpool_process_init(pool, cnt * 2, 0))) {
+        fprintf(stderr, "Failed to create queue\n");
+        goto end;
+    }
+    //open input file - r reading
+    if (!(infile = sam_open(inname, "r"))) {
+        fprintf(stderr, "Could not open %s\n", inname);
+        goto end;
+    }
+    //open output files - w write as SAM, wb  write as BAM
+    if (!(outfile = sam_open(file, "wb"))) {
+        fprintf(stderr, "Could not open output file\n");
+        goto end;
+    }
+    //share the thread pool with i/o files
+    if (hts_set_opt(infile, HTS_OPT_THREAD_POOL, &tpool) < 0 ||
+          hts_set_opt(outfile, HTS_OPT_THREAD_POOL, &tpool) < 0) {
+        fprintf(stderr, "Failed to set threads to i/o files\n");
+        goto end;
+    }
+    //read header, required to resolve the target names to proper ids
+    if (!(in_samhdr = sam_hdr_read(infile))) {
+        fprintf(stderr, "Failed to read header from file!\n");
+        goto end;
+    }
+    //write header
+    if ((sam_hdr_write(outfile, in_samhdr) == -1)) {
+        fprintf(stderr, "Failed to write header\n");
+        goto end;
+    }
+
+    /* tasks are queued, worker threads get them and process in parallel;
+    the results are queued and they are to be removed in parallel as well */
+
+    // start output writer thread for ordered processing
+    twritedata.outfile = outfile;
+    twritedata.samhdr  = in_samhdr;
+    twritedata.result  = 0;
+    twritedata.queue   = queue;
+    twritedata.cache   = &bamcache;
+    if (pthread_create(&thread, NULL, threadfn_orderedwrite, &twritedata)) {
+        fprintf(stderr, "Failed to create writer thread\n");
+        goto end;
+    }
+    started_thread = 1;
+
+    c = 0;
+    while (c >= 0) {
+        if (!(bamdata = getbamstorage(chunk, &bamcache))) {
+            fprintf(stderr, "Failed to allocate memory\n");
+            break;
+        }
+        //read alignments, upto max size for this lot
+        for (cnt = 0; cnt < bamdata->maxsize; ++cnt) {
+            c = sam_read1(infile, in_samhdr, bamdata->bamarray[cnt]);
+            if (c < 0) {
+                break;      // EOF or failure
+            }
+        }
+        if (c >= -1 ) {
+            //max size data or reached EOF
+            bamdata->count = cnt;
+            // Queue the data for processing.  hts_tpool_dispatch3() is
+            // used here as it allows in-flight data to be cleaned up
+            // properly when stopping early due to errors.
+            if (hts_tpool_dispatch3(pool, queue, thread_ordered_proc, bamdata,
+                                    cleanup_bamstorage, cleanup_bamstorage,
+                                    0) == -1) {
+                fprintf(stderr, "Failed to schedule processing\n");
+                goto end;
+            }
+            bamdata = NULL;
+        } else {
+            fprintf(stderr, "Error in reading data\n");
+            break;
+        }
+    }
+
+    ret = EXIT_SUCCESS;
+
+ end:
+    // Tidy up after having dispatched all of the data.
+
+    // Note that the order here is important.  In particular, we need
+    // to join the thread that was started earlier before freeing anything
+    // to avoid any use-after-free errors.
+
+    // It's also possible to get here early due to various error conditions,
+    // so we need to carefully check which parts of the program state have
+    // been created before trying to clean them up.
+
+    if (queue) {
+        if (-1 == c) {
+            // EOF read, send a marker to tell the threadfn_orderedwrite()
+            // function to shut down.
+            if (hts_tpool_dispatch(pool, queue, thread_ordered_proc,
+                                   NULL) == -1) {
+                fprintf(stderr, "Failed to schedule sentinel job\n");
+                ret = EXIT_FAILURE;
+            }
+        } else {
+            // Error or we never wrote anything.  Shut down the queue to
+            // ensure threadfn_orderedwrite() wakes up and terminates.
+            hts_tpool_process_shutdown(queue);
+        }
+    }
+
+    // Wait for threadfn_orderedwrite to finish.
+    if (started_thread) {
+        pthread_join(thread, NULL);
+
+        // Once the writer thread has finished, check the result it sent back
+        if (twritedata.result != 0) {
+            ret = EXIT_FAILURE;
+        }
+    }
+
+    if (queue) {
+        // Once threadfn_orderedwrite has stopped, the queue can be
+        // cleaned up.
+        hts_tpool_process_destroy(queue);
+    }
+
+    if (in_samhdr) {
+        sam_hdr_destroy(in_samhdr);
+    }
+    if (infile) {
+        if (sam_close(infile) != 0) {
+            ret = EXIT_FAILURE;
+        }
+    }
+    if (outfile) {
+        if (sam_close(outfile) != 0) {
+            ret = EXIT_FAILURE;
+        }
+    }
+
+    pthread_mutex_lock(&bamcache.lock);
+    if (bamcache.list) {
+        struct data *tmp = NULL;
+        while (bamcache.list) {
+            tmp = bamcache.list;
+            bamcache.list = bamcache.list->next;
+            cleanup_bamstorage(tmp);
+        }
+    }
+    pthread_mutex_unlock(&bamcache.lock);
+
+    if (file) {
+        free(file);
+    }
+    if (pool) {
+        hts_tpool_destroy(pool);
+    }
+    return ret;
+}
diff --git a/ext/htslib/samples/qtask_unordered.c b/ext/htslib/samples/qtask_unordered.c
new file mode 100644
index 0000000..05fe503
--- /dev/null
+++ b/ext/htslib/samples/qtask_unordered.c
@@ -0,0 +1,320 @@
+/*  qtask_ordered.c --  showcases the htslib api usage
+
+    Copyright (C) 2024 Genome Research Ltd.
+
+    Author: Vasudeva Sarma <vasudeva.sarma@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE
+
+*/
+
+/* The purpose of this code is to demonstrate the library apis and need proper error handling and optimisation */
+
+#include <getopt.h>
+#include <unistd.h>
+#include <pthread.h>
+#include <sys/time.h>
+#include <htslib/sam.h>
+#include <htslib/thread_pool.h>
+
+struct datacache;
+
+typedef struct basecount {
+    uint64_t counts[16];        //count of all bases
+} basecount;
+
+typedef struct data {
+    int count;                  //used up size
+    int maxsize;                //max size per data chunk
+    bam1_t **bamarray;          //bam1_t array for optimal queueing
+
+    struct datacache *cache;
+    basecount *bases;           //count of all possible bases
+    struct data *next;          //pointer to next one - to reuse earlier allocations
+} data;
+
+typedef struct datacache
+{
+    pthread_mutex_t lock;       //synchronizes the access to cache
+    data *list;                 //data storage
+} datacache;
+
+/// print_usage - print the usage
+/** @param fp pointer to the file / terminal to which usage to be dumped
+returns nothing
+*/
+static void print_usage(FILE *fp)
+{
+    fprintf(fp, "Usage: qtask_unordered infile threadcount [chunksize]\n\
+Shows the base counts and calculates GC ratio - sum(G,C) / sum(A,T,C,G)\n\
+chunksize [4096] sets the number of alignments clubbed together to process.\n");
+    return;
+}
+
+/// getbamstorage - allocates storage for alignments to queue
+/** @param chunk number of bam data to allocate
+ * @param bases storage of result
+ * @param bamcache cached storage
+returns already allocated data storage if one is available, otherwise allocates new
+*/
+data* getbamstorage(int chunk, basecount *bases, datacache *bamcache)
+{
+    int i = 0;
+    data *bamdata = NULL;
+
+    if (!bamcache || !bases) {
+        return NULL;
+    }
+    //get from cache if there is an already allocated storage
+    if (pthread_mutex_lock(&bamcache->lock)) {
+        return NULL;
+    }
+    if (bamcache->list) {                   //available
+        bamdata = bamcache->list;
+        bamcache->list = bamdata->next;     //remove and set next one as available
+        bamdata->next = NULL;               //remove link
+        bamdata->count = 0;
+
+        bamdata->bases = bases;
+        bamdata->cache = bamcache;
+        goto end;
+    }
+    //allocate and use
+    if (!(bamdata = malloc(sizeof(data)))) {
+        goto end;
+    }
+    bamdata->bamarray = malloc(chunk * sizeof(bam1_t*));
+    if (!bamdata->bamarray) {
+        free(bamdata);
+        bamdata = NULL;
+        goto end;
+    }
+    for (i = 0; i < chunk; ++i) {
+        bamdata->bamarray[i] = bam_init1();
+    }
+    bamdata->maxsize = chunk;
+    bamdata->count = 0;
+    bamdata->next = NULL;
+
+    bamdata->bases = bases;
+    bamdata->cache = bamcache;
+
+end:
+    pthread_mutex_unlock(&bamcache->lock);
+    return bamdata;
+}
+
+/// cleanup_bamstorage - frees a bamdata struct plus contents
+/** @param arg Pointer to data to free
+    @p arg has type void * so it can be used as a callback passed
+    to hts_tpool_dispatch3().
+ */
+void cleanup_bamstorage(void *arg)
+{
+    data *bamdata = (data *) arg;
+    if (!bamdata)
+        return;
+    if (bamdata->bamarray) {
+        int i;
+        for (i = 0; i < bamdata->maxsize; i++) {
+            bam_destroy1(bamdata->bamarray[i]);
+        }
+        free(bamdata->bamarray);
+    }
+    free(bamdata);
+}
+
+/// thread_unordered_proc - does the processing of task in queue and updates result
+/** @param args pointer to set of data to be processed
+returns NULL
+the processing could be in any order based on the number of threads in use
+*/
+void *thread_unordered_proc(void *args)
+{
+    int i = 0;
+    data *bamdata = (data*)args;
+    uint64_t pos = 0;
+    uint8_t *data = NULL;
+    uint64_t counts[16] = {0};
+    for ( i = 0; i < bamdata->count; ++i) {
+        data = bam_get_seq(bamdata->bamarray[i]);
+        for (pos = 0; pos < bamdata->bamarray[i]->core.l_qseq; ++pos) {
+            /* it is faster to count all bases and select required ones later
+            compared to select and count here */
+            counts[bam_seqi(data, pos)]++;
+        }
+    }
+    //update result and add the memory block for reuse
+    pthread_mutex_lock(&bamdata->cache->lock);
+    for (i = 0; i < 16; i++) {
+        bamdata->bases->counts[i] += counts[i];
+    }
+
+    bamdata->next = bamdata->cache->list;
+    bamdata->cache->list = bamdata;
+    pthread_mutex_unlock(&bamdata->cache->lock);
+
+    return NULL;
+}
+
+/// main - start of the demo
+/** @param argc - count of arguments
+ *  @param argv - pointer to array of arguments
+returns 1 on failure 0 on success
+*/
+int main(int argc, char *argv[])
+{
+    const char *inname = NULL;
+    int c = 0, ret = EXIT_FAILURE, cnt = 0, chunk = 0;
+    samFile *infile = NULL;
+    sam_hdr_t *in_samhdr = NULL;
+    hts_tpool *pool = NULL;
+    hts_tpool_process *queue = NULL;
+    htsThreadPool tpool = {NULL, 0};
+    data *bamdata = NULL;
+    basecount gccount = {{0}};
+    datacache bamcache = {PTHREAD_MUTEX_INITIALIZER, NULL};
+
+    //qtask infile threadcount [chunksize]
+    if (argc != 3 && argc != 4) {
+        print_usage(stdout);
+        goto end;
+    }
+    inname = argv[1];
+    cnt = atoi(argv[2]);
+    if (argc == 4) {
+        chunk = atoi(argv[3]);
+    }
+    if (cnt < 1) {
+        cnt = 1;
+    }
+    if (chunk < 1) {
+        chunk = 4096;
+    }
+
+    if (!(pool = hts_tpool_init(cnt))) {
+        fprintf(stderr, "Failed to create thread pool\n");
+        goto end;
+    }
+    tpool.pool = pool;      //to share the pool for file read and write as well
+    //queue to use with thread pool, for tasks
+    if (!(queue = hts_tpool_process_init(pool, cnt * 2, 1))) {
+        fprintf(stderr, "Failed to create queue\n");
+        goto end;
+    }
+    //open input file - r reading
+    if (!(infile = sam_open(inname, "r"))) {
+        fprintf(stderr, "Could not open %s\n", inname);
+        goto end;
+    }
+    //share the thread pool with i/o files
+    if (hts_set_opt(infile, HTS_OPT_THREAD_POOL, &tpool) < 0) {
+        fprintf(stderr, "Failed to set threads to i/o files\n");
+        goto end;
+    }
+    //read header, required to resolve the target names to proper ids
+    if (!(in_samhdr = sam_hdr_read(infile))) {
+        fprintf(stderr, "Failed to read header from file!\n");
+        goto end;
+    }
+
+    /*tasks are queued, worker threads get them and process in parallel;
+    all bases are counted instead of counting atcg alone as it is faster*/
+
+    c = 0;
+    while (c >= 0) {
+        //use cached storage to avoid allocate/deallocate overheads
+        if (!(bamdata = getbamstorage(chunk, &gccount, &bamcache))) {
+            fprintf(stderr, "Failed to allocate memory\n");
+            break;
+        }
+        //read alignments, upto max size for this lot
+        for (cnt = 0; cnt < bamdata->maxsize; ++cnt) {
+            c = sam_read1(infile, in_samhdr, bamdata->bamarray[cnt]);
+            if (c < 0) {
+                break;      // EOF or failure
+            }
+        }
+        if (c >= -1 ) {
+            //max size data or reached EOF
+            bamdata->count = cnt;
+            // Queue the data for processing.  hts_tpool_dispatch3() is
+            // used here as it allows in-flight data to be cleaned up
+            // properly when stopping early due to errors.
+            if (hts_tpool_dispatch3(pool, queue, thread_unordered_proc, bamdata,
+                                    cleanup_bamstorage, cleanup_bamstorage,
+                                    0) == -1) {
+                fprintf(stderr, "Failed to schedule processing\n");
+                goto end;
+            }
+            bamdata = NULL;
+        } else {
+            fprintf(stderr, "Error in reading data\n");
+            break;
+        }
+    }
+
+     if (-1 == c) {
+        // EOF read, ensure all are processed, waits for all to finish
+        if (hts_tpool_process_flush(queue) == -1) {
+            fprintf(stderr, "Failed to flush queue\n");
+        } else { //all done
+            //refer seq_nt16_str to find position of required bases
+            fprintf(stdout, "GCratio: %f\nBase counts:\n",
+                (gccount.counts[2] /*C*/ + gccount.counts[4] /*G*/) / (float)
+                    (gccount.counts[1] /*A*/ + gccount.counts[8] /*T*/ +
+                        gccount.counts[2] + gccount.counts[4]));
+
+            for (cnt = 0; cnt < 16; ++cnt) {
+                fprintf(stdout, "%c: %"PRIu64"\n", seq_nt16_str[cnt], gccount.counts[cnt]);
+            }
+
+            ret = EXIT_SUCCESS;
+        }
+    }
+ end:
+    if (queue) {
+        hts_tpool_process_destroy(queue);
+    }
+
+    if (in_samhdr) {
+        sam_hdr_destroy(in_samhdr);
+    }
+    if (infile) {
+        if (sam_close(infile) != 0) {
+            ret = EXIT_FAILURE;
+        }
+    }
+
+    pthread_mutex_lock(&bamcache.lock);
+    if (bamcache.list) {
+        struct data *tmp = NULL;
+        while (bamcache.list) {
+            tmp = bamcache.list;
+            bamcache.list = bamcache.list->next;
+            cleanup_bamstorage(tmp);
+        }
+    }
+    pthread_mutex_unlock(&bamcache.lock);
+
+    if (pool) {
+        hts_tpool_destroy(pool);
+    }
+    return ret;
+}
diff --git a/ext/htslib/samples/read_aux.c b/ext/htslib/samples/read_aux.c
new file mode 100644
index 0000000..efd6f36
--- /dev/null
+++ b/ext/htslib/samples/read_aux.c
@@ -0,0 +1,207 @@
+/*  read_aux.c --  showcases the htslib api usage
+
+    Copyright (C) 2023 Genome Research Ltd.
+
+    Author: Vasudeva Sarma <vasudeva.sarma@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE
+
+*/
+
+/* The purpose of this code is to demonstrate the library apis and need proper error handling and optimisation */
+
+#include <getopt.h>
+#include <unistd.h>
+#include <htslib/sam.h>
+
+/// print_usage - print the usage
+/** @param fp pointer to the file / terminal to which usage to be dumped
+returns nothing
+*/
+static void print_usage(FILE *fp)
+{
+    fprintf(fp, "Usage: read_aux infile tag\n\
+Read the given aux tag from alignments either as SAM string or as raw data\n");
+}
+
+/// printauxdata - prints aux data
+/** @param fp - file to which it to be printed - stdout or null
+ *  @param type - aux type
+ *  @param idx - index in array, -1 when not an array type
+ *  @param data - data
+ *  recurses when the data is array type
+returns 1 on failure 0 on success
+*/
+int printauxdata(FILE *fp, char type, int32_t idx, const uint8_t *data)
+{
+    uint32_t auxBcnt = 0;
+    int i = 0;
+    char auxBType = 'Z';
+
+    //the tag is already queried and ensured to exist and the type is retrieved from the tag data, also iterated within index for arrays, so no error is expected here.
+    //when these apis are used explicitly, these error conditions needs to be handled based on return value and errno
+    switch(type) {
+    case 'A':
+        fprintf(fp, "%c", bam_aux2A(data));                                                 //byte data
+        break;
+    case 'c':
+        fprintf(fp, "%d", (int8_t)(idx > -1 ? bam_auxB2i(data, idx) : bam_aux2i(data)));    //signed 1 byte data; bam_auxB2i - from array or bam_aux2i - non array data
+        break;
+    case 'C':
+        fprintf(fp, "%u", (uint8_t)(idx > -1 ? bam_auxB2i(data, idx) : bam_aux2i(data)));   //unsigned 1 byte data
+        break;
+    case 's':
+        fprintf(fp, "%d", (int16_t)(idx > -1 ? bam_auxB2i(data, idx) : bam_aux2i(data)));   //signed 2 byte data
+        break;
+    case 'S':
+        fprintf(fp, "%u", (uint16_t)(idx > -1 ? bam_auxB2i(data, idx) : bam_aux2i(data)));  //unsigned 2 byte data
+        break;
+    case 'i':
+        fprintf(fp, "%d", (int32_t)(idx > -1 ? bam_auxB2i(data, idx) : bam_aux2i(data)));   //signed 4 byte data
+        break;
+    case 'I':
+        fprintf(fp, "%u", (uint32_t)(idx > -1 ? bam_auxB2i(data, idx) : bam_aux2i(data)));  //unsigned 4 byte data
+        break;
+    case 'f':
+    case 'd':
+        fprintf(fp, "%g", (float)(idx > -1 ? bam_auxB2f(data, idx) : bam_aux2f(data)));     //floating point data, 4 bytes
+        break;
+    case 'H':
+    case 'Z':
+        fprintf(fp, "%s", bam_aux2Z(data));                                                 //array of char or hex data
+        break;
+    case 'B':                                                                               //array of char/int/float
+        auxBcnt = bam_auxB_len(data);                                                       //length of array
+        auxBType = bam_aux_type(data + 1);                                                  //type of element in array
+        fprintf(fp, "%c", auxBType);
+        for (i = 0; i < auxBcnt; ++i) {                                                     //iterate the array
+            fprintf(fp, ",");
+            //calling recursively  with index to reuse a few lines
+            if (printauxdata(fp, auxBType, i, data) == EXIT_FAILURE) {
+                return EXIT_FAILURE;
+            }
+        }
+        break;
+    default:
+        printf("Invalid aux tag?\n");
+        return EXIT_FAILURE;
+        break;
+    }
+    return EXIT_SUCCESS;
+}
+
+/// main_demo - start of the demo
+/** @param argc - count of arguments
+ *  @param argv - pointer to array of arguments
+returns 1 on failure 0 on success
+*/
+int main(int argc, char *argv[])
+{
+    const char *inname = NULL, *tag = NULL;
+    int c = 0, ret = EXIT_FAILURE, ret_r = 0, i = 0;
+    sam_hdr_t *in_samhdr = NULL;
+    samFile *infile = NULL;
+    bam1_t *bamdata = NULL;
+    uint8_t *data = NULL;
+    kstring_t sdata = KS_INITIALIZE;
+
+    //read_aux infile tag
+    if (argc != 3) {
+        print_usage(stderr);
+        goto end;
+    }
+    inname = argv[1];
+    tag = argv[2];
+
+    if (!(bamdata = bam_init1())) {
+        printf("Failed to allocate data memory!\n");
+        goto end;
+    }
+
+    //open input file
+    if (!(infile = sam_open(inname, "r"))) {
+        printf("Could not open %s\n", inname);
+        goto end;
+    }
+
+    if (!(in_samhdr = sam_hdr_read(infile))) {
+        printf("Failed to read header from file!\n");
+        goto end;
+    }
+
+    while ((ret_r = sam_read1(infile, in_samhdr, bamdata)) >= 0) {
+        errno = 0; i++;
+        ks_clear(&sdata);
+        if (i % 2) {    //use options alternatively to demonstrate both
+            //option 1 - get data as string with tag and type
+            if ((c = bam_aux_get_str(bamdata, tag, &sdata)) == 1) {
+                printf("%s\n",sdata.s);
+            }
+            else if (c == 0 && errno == ENOENT) {
+                //tag not present
+                printf("Tag not present\n");
+            }
+            else {
+                //error
+                printf("Failed to get tag\n");
+                goto end;
+            }
+        }
+        else {
+            //option 2 - get raw data
+            if (!(data = bam_aux_get(bamdata, tag))) {
+                //tag data not returned, errno gives the reason
+                if (errno == ENOENT) {
+                    printf("Tag not present\n");
+                }
+                else {
+                    printf("Invalid aux data\n");
+                }
+            }
+            else {
+                //got the tag, read and print
+                if (printauxdata(stdout, bam_aux_type(data), -1, data) == EXIT_FAILURE) {
+                    printf("Failed to read aux data\n");
+                    goto end;
+                }
+                printf("\n");
+            }
+        }
+    }
+    if (ret_r < -1) {
+        //read error
+        printf("Failed to read data\n");
+        goto end;
+    }
+
+    ret = EXIT_SUCCESS;
+end:
+    //cleanup
+    if (in_samhdr) {
+        sam_hdr_destroy(in_samhdr);
+    }
+    if (infile) {
+        sam_close(infile);
+    }
+    if (bamdata) {
+        bam_destroy1(bamdata);
+    }
+    ks_free(&sdata);
+    return ret;
+}
diff --git a/ext/htslib/samples/read_bam.c b/ext/htslib/samples/read_bam.c
new file mode 100644
index 0000000..30bedf8
--- /dev/null
+++ b/ext/htslib/samples/read_bam.c
@@ -0,0 +1,139 @@
+/*  read_bam.c --  showcases the htslib api usage
+
+    Copyright (C) 2023 Genome Research Ltd.
+
+    Author: Vasudeva Sarma <vasudeva.sarma@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE
+
+*/
+
+/* The purpose of this code is to demonstrate the library apis and need proper error handling and optimisation */
+
+#include <getopt.h>
+#include <unistd.h>
+#include <htslib/sam.h>
+
+/// print_usage - print the usage
+/** @param fp pointer to the file / terminal to which usage to be dumped
+returns nothing
+*/
+static void print_usage(FILE *fp)
+{
+    fprintf(fp, "Usage: read_bam infile\n\
+Shows the alignment data from file\n");
+}
+
+/// main_demo - start of the demo
+/** @param argc - count of arguments
+ *  @param argv - pointer to array of arguments
+returns 1 on failure 0 on success
+*/
+int main(int argc, char *argv[])
+{
+    const char *inname = NULL, *tidname = NULL, *flags = NULL;
+    int ret = EXIT_FAILURE;
+    sam_hdr_t *in_samhdr = NULL;
+    samFile *infile = NULL;
+
+    int ret_r = 0, i = 0;
+    bam1_t *bamdata = NULL;
+    uint8_t *data = NULL;
+    uint32_t *cigar = NULL;
+
+
+    //read_bam infile
+    if (argc != 2) {
+        print_usage(stderr);
+        goto end;
+    }
+    inname = argv[1];
+
+    if (!(bamdata = bam_init1())) {
+        printf("Failed to allocate data memory!\n");
+        goto end;
+    }
+
+    //open input file
+    if (!(infile = sam_open(inname, "r"))) {
+        printf("Could not open %s\n", inname);
+        goto end;
+    }
+    //read header
+    if (!(in_samhdr = sam_hdr_read(infile))) {
+        printf("Failed to read header from file!\n");
+        goto end;
+    }
+
+    while ((ret_r = sam_read1(infile, in_samhdr, bamdata)) >= 0)
+    {
+        //QNAME FLAG RNAME POS MAPQ CIGAR RNEXT PNEXT TLEN SEQ QUAL [TAG:TYPE:VALUE]…
+        printf("NAME: %s\n", bam_get_qname(bamdata));                                   //get the query name using the macro
+        flags = bam_flag2str(bamdata->core.flag);                                       //flags as string
+        printf("FLG: %d - %s\n", bamdata->core.flag, flags);                            //flag is available in core structure
+        free((void*)flags);
+        tidname = sam_hdr_tid2name(in_samhdr, bamdata->core.tid);
+        printf("RNAME/TID: %d - %s\n", bamdata->core.tid, tidname? tidname: "" );       //retrieves the target name using the value in bam and by referring the header
+        printf("POS: %"PRIhts_pos"\n", bamdata->core.pos + 1);                          //internally position is 0 based and on text output / SAM it is 1 based
+        printf("MQUAL: %d\n", bamdata->core.qual);                                      //map quality value
+
+        cigar = bam_get_cigar(bamdata);                                                 //retrieves the cigar data
+        printf("CGR: ");
+        for (i = 0; i < bamdata->core.n_cigar; ++i) {                                   //no. of cigar data entries
+            printf("%d%c", bam_cigar_oplen(cigar[i]), bam_cigar_opchr(cigar[i]));       //the macros gives the count of operation and the symbol of operation for given cigar entry
+        }
+        printf("\nTLEN/ISIZE: %"PRIhts_pos"\n", bamdata->core.isize);
+
+        data = bam_get_seq(bamdata);                                                    //get the sequence data
+        if (bamdata->core.l_qseq != bam_cigar2qlen(bamdata->core.n_cigar, cigar)) {     //checks the length with CIGAR and query
+            printf("\nLength doesnt matches to cigar data\n");
+            goto end;
+        }
+
+        printf("SEQ: ");
+        for (i = 0; i < bamdata->core.l_qseq ; ++i) {                                   //sequence length
+            printf("%c", seq_nt16_str[bam_seqi(data, i)]);                              //retrieves the base from (internal compressed) sequence data
+        }
+        printf("\nQUAL: ");
+        for (int i = 0; i < bamdata->core.l_qseq ; ++i) {
+            printf("%c", bam_get_qual(bamdata)[i]+33);                                  //retrives the quality value
+        }
+        printf("\n\n");
+    }
+
+    if (ret_r == -1) {
+        // no error!
+        ret = EXIT_SUCCESS;
+    }
+    else {
+        printf("Failed to read data\n");
+    }
+end:
+    //cleanup
+    if (in_samhdr) {
+        sam_hdr_destroy(in_samhdr);
+    }
+    if (infile) {
+        sam_close(infile);
+    }
+    if (bamdata) {
+        bam_destroy1(bamdata);
+    }
+    return ret;
+}
diff --git a/ext/htslib/samples/read_fast.c b/ext/htslib/samples/read_fast.c
new file mode 100644
index 0000000..10f807b
--- /dev/null
+++ b/ext/htslib/samples/read_fast.c
@@ -0,0 +1,119 @@
+/*  read_fast.c --  showcases the htslib api usage
+
+    Copyright (C) 2023 Genome Research Ltd.
+
+    Author: Vasudeva Sarma <vasudeva.sarma@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE
+
+*/
+
+/* The purpose of this code is to demonstrate the library apis and need proper error handling and optimisation */
+
+#include <getopt.h>
+#include <unistd.h>
+#include <htslib/sam.h>
+
+/// print_usage - show usage
+/** @param fp pointer to the file / terminal to which usage to be dumped
+returns nothing
+*/
+static void print_usage(FILE *fp)
+{
+    fprintf(fp, "Usage: read_fast <infile>\n\
+Reads the fasta/fastq file and shows the content.\n");
+    return;
+}
+
+/// main_demo - start of the demo
+/** @param argc - count of arguments
+ *  @param argv - pointer to array of arguments
+returns 1 on failure 0 on success
+*/
+int main(int argc, char *argv[])
+{
+    const char *inname = NULL;              //input file name
+    int c = 0, ret = EXIT_FAILURE;
+    samFile *infile = NULL;                 //sam file
+    sam_hdr_t *in_samhdr = NULL;            //header of file
+    bam1_t *bamdata = NULL;                 //to hold the read data
+
+    if (argc != 2) {
+        print_usage(stdout);
+        goto end;
+    }
+    inname = argv[1];
+
+    //initialize
+    if (!(bamdata = bam_init1())) {
+        printf("Failed to initialize bamdata\n");
+        goto end;
+    }
+    //open input files - r reading
+    if (!(infile = sam_open(inname, "r"))) {
+        printf("Could not open %s\n", inname);
+        goto end;
+    }
+    if (infile->format.format != fasta_format && infile->format.format != fastq_format) {
+        printf("Invalid file specified\n");
+        goto end;
+    }
+
+    //read header
+    if (!(in_samhdr = sam_hdr_read(infile))) {
+        printf( "Failed to read header from file\n");
+        goto end;
+    }
+
+    //read data
+    while ((c = sam_read1(infile, in_samhdr, bamdata)) >= 0) {
+        printf("\nname: ");
+        printf("%s", bam_get_qname(bamdata));
+        printf("\nsequence: ");
+        for (c = 0; c < bamdata->core.l_qseq; ++c) {
+            printf("%c", seq_nt16_str[bam_seqi(bam_get_seq(bamdata), c)]);
+        }
+        if (infile->format.format == fastq_format) {
+            printf("\nquality: ");
+            for (c = 0; c < bamdata->core.l_qseq; ++c) {
+                printf("%c", bam_get_qual(bamdata)[c] + 33);
+            }
+        }
+    }
+    printf("\n");
+    if (c != -1) {
+        //error
+        printf("Failed to get data\n");
+        goto end;
+    }
+    //else -1 / EOF
+    ret = EXIT_SUCCESS;
+end:
+    //clean up
+    if (in_samhdr) {
+        sam_hdr_destroy(in_samhdr);
+    }
+    if (infile) {
+        sam_close(infile);
+    }
+    if (bamdata) {
+        bam_destroy1(bamdata);
+    }
+    return ret;
+}
diff --git a/ext/htslib/samples/read_fast_index.c b/ext/htslib/samples/read_fast_index.c
new file mode 100644
index 0000000..9707663
--- /dev/null
+++ b/ext/htslib/samples/read_fast_index.c
@@ -0,0 +1,163 @@
+/*  read_fast_index.c --  showcases the htslib api usage
+
+    Copyright (C) 2023 Genome Research Ltd.
+
+    Author: Vasudeva Sarma <vasudeva.sarma@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE
+
+*/
+
+/* The purpose of this code is to demonstrate the library apis and need proper error handling and optimisation */
+
+#include <getopt.h>
+#include <unistd.h>
+#include <htslib/sam.h>
+#include <htslib/faidx.h>
+
+/// print_usage - show usage
+/** @param fp pointer to the file / terminal to which usage to be dumped
+returns nothing
+*/
+static void print_usage(FILE *fp)
+{
+    fprintf(fp, "Usage: read_fast_i <infile> A/Q 0/1 regiondef\n\
+Reads the fasta/fastq file using index and shows the content.\n\
+For fasta files use A and Q for fastq files.\n\
+Region can be 1 or more of <reference name>[:start-end] entries separated by comma.\n\
+For single region, give regcount as 0 and non 0 for multi-regions.\n");
+    return;
+}
+
+/// main_demo - start of the demo
+/** @param argc - count of arguments
+ *  @param argv - pointer to array of arguments
+returns 1 on failure 0 on success
+*/
+int main(int argc, char *argv[])
+{
+    const char *inname = NULL, *region = NULL, *data = NULL, *remaining = NULL;
+    int ret = EXIT_FAILURE, tid = -1, usemulti = 0;
+    faidx_t *idx = NULL;
+    enum fai_format_options fmt = FAI_FASTA;
+    hts_pos_t len = 0, beg = 0, end = 0;
+
+    //read_fast_i infile A/Q regcount region
+    if (argc != 5) {
+        print_usage(stdout);
+        goto end;
+    }
+    inname = argv[1];
+    if (argv[2][0] == 'Q') {
+        fmt = FAI_FASTQ;
+    }
+    usemulti = atoi(argv[3]);
+    region = argv[4];
+
+    //load index
+    if (!(idx = fai_load3_format(inname, NULL, NULL, FAI_CREATE, fmt))) {
+        printf("Failed to load index\n");
+        goto end;
+    }
+
+    if (!usemulti) {
+        //get data from given region
+        if (!(data = fai_fetch64(idx, region, &len))) {
+            if (-1 == len) {
+                printf("Failed to get data\n");                 //failure
+                goto end;
+            }
+            else {
+                printf("Data not found for given region\n");    //no data
+            }
+        }
+        else {
+            printf("Data: %"PRId64" %s\n", len, data);
+            free((void*)data);
+            //get quality for fastq type
+            if (fmt == FAI_FASTQ) {
+                if (!(data = fai_fetchqual64(idx, region, &len))) {
+                    if (len == -1) {
+                        printf("Failed to get data\n");
+                        goto end;
+                    }
+                    else {
+                        printf("Data not found for given region\n");
+                    }
+                }
+                else {
+                    printf("Qual: %"PRId64" %s\n", len, data);
+                    free((void*)data);
+                }
+            }
+        }
+    }
+    else {
+        //parse, get each region and get data for each
+        while ((remaining = fai_parse_region(idx, region, &tid, &beg, &end, HTS_PARSE_LIST))) {     //here expects regions as csv
+            //parsed the region, correct end points based on actual data
+            if (fai_adjust_region(idx, tid, &beg, &end) == -1) {
+                printf("Error in adjusting region for tid %d\n", tid);
+                goto end;
+            }
+            //get data for given region
+            if (!(data = faidx_fetch_seq64(idx, faidx_iseq(idx, tid), beg, end, &len))) {
+                if (len == -1) {
+                    printf("Failed to get data\n");                 //failure
+                    goto end;
+                }
+                else {
+                    printf("No data found for given region\n");     //no data
+                }
+            }
+            else {
+                printf("Data: %"PRIhts_pos" %s\n", len, data);
+                free((void*)data);
+                data = NULL;
+
+                //get quality data for fastq
+                if (fmt == FAI_FASTQ) {
+                    if (!(data = faidx_fetch_qual64(idx, faidx_iseq(idx, tid), beg, end, &len))) {
+                        if (len == -1) {
+                            printf("Failed to get qual data\n");
+                            goto end;
+                        }
+                        else {
+                            printf("No data found for given region\n");
+                        }
+                    }
+                    else {
+                        printf("Qual: %"PRIhts_pos" %s\n", len, data);
+                        free((void*)data);
+                        data = NULL;
+                    }
+                }
+            }
+            region = remaining;                                     //parse remaining region defs
+        }
+    }
+
+    ret = EXIT_SUCCESS;
+end:
+    //clean up
+    if (idx) {
+        fai_destroy(idx);
+    }
+    return ret;
+}
diff --git a/ext/htslib/samples/read_header.c b/ext/htslib/samples/read_header.c
new file mode 100644
index 0000000..54b07e7
--- /dev/null
+++ b/ext/htslib/samples/read_header.c
@@ -0,0 +1,173 @@
+/*  read_header.c --  showcases the htslib api usage
+
+    Copyright (C) 2023 Genome Research Ltd.
+
+    Author: Vasudeva Sarma <vasudeva.sarma@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE
+
+*/
+
+/* The purpose of this code is to demonstrate the library apis and need proper error handling and optimisation */
+
+#include <getopt.h>
+#include <unistd.h>
+#include <htslib/sam.h>
+
+/// print_usage - print the usage
+/** @param fp pointer to the file / terminal to which susage to be dumped
+returns nothing
+*/
+static void print_usage(FILE *fp)
+{
+    fprintf(fp, "Usage: read_header infile header [id val] [tag]\n\
+This shows given tag from given header or the whole line\n");
+    return;
+}
+
+/// main_demo - start of the demo
+/** @param argc - count of arguments
+ *  @param argv - pointer to array of arguments
+returns 1 on failure 0 on success
+*/
+int main(int argc, char *argv[])
+{
+    const char *inname = NULL, *header = NULL, *tag = NULL, *idval = NULL;
+    char *id = NULL;
+    int c = 0, ret = EXIT_FAILURE, linecnt = 0;
+    samFile *infile = NULL;
+    sam_hdr_t *in_samhdr = NULL;
+    kstring_t data = KS_INITIALIZE;
+
+    //read_header infile header tag
+    if (argc < 3 || argc > 6) {
+        print_usage(stderr);
+        goto end;
+    }
+    inname = argv[1];
+    header = argv[2];
+    if (argc == 4) {            //header and tag
+        tag = argv[3];
+        //find unique identifier field name for requested header type
+        if (header[0] == 'H' && header[1] == 'D') {
+            id = NULL;
+        }
+        else if (header[0] == 'S' && header[1] == 'Q') {
+            id = "SN";
+        }
+        else if (header[0] == 'R' && header[1] == 'G') {
+            id = "ID";
+        }
+        else if (header[0] == 'P' && header[1] == 'G') {
+            id = "ID";
+        }
+        else if (header[0] == 'C' && header[1] == 'O') {
+            id = "";
+        }
+        else {
+            printf("Invalid header type\n");
+            goto end;
+        }
+    }
+    else if (argc == 5) {       //header id val
+        id = argv[3];
+        idval = argv[4];
+    }
+    else if (argc == 6) {       //header id val tag
+        id = argv[3];
+        idval = argv[4];
+        tag = argv[5];
+    }
+
+    //open input files
+    if (!(infile = sam_open(inname, "r"))) {
+        printf("Could not open %s\n", inname);
+        goto end;
+    }
+
+    //read header
+    if (!(in_samhdr = sam_hdr_read(infile))) {
+        printf("Failed to read header from file!\n");
+        goto end;
+    }
+
+    if (id && idval) {
+        if (tag) {
+            ret = sam_hdr_find_tag_id(in_samhdr, header, id, idval, tag, &data);
+        }
+        else {
+            ret = sam_hdr_find_line_id(in_samhdr, header, id, idval, &data);
+        }
+
+        if (ret == 0) {
+            printf("%s\n", data.s);
+        }
+        else  if (ret == -1) {
+            printf("No matching tag found\n");
+            goto end;
+        }
+        else {
+            printf("Failed to find header line\n");
+            goto end;
+        }
+    }
+    else {
+        //get count of given header type
+        linecnt = sam_hdr_count_lines(in_samhdr, header);
+        if (linecnt == 0) {
+            printf("No matching line found\n");
+            goto end;
+        }
+        for (c = 0; c < linecnt; ++c ) {
+            if (tag) {
+                //non CO, get the tag requested
+                ret = sam_hdr_find_tag_pos(in_samhdr, header, c, tag, &data);
+            }
+            else {
+                //CO header, there are no tags but the whole line
+                ret = sam_hdr_find_line_pos(in_samhdr, header, c, &data);
+            }
+
+            if (ret == 0) {
+                printf("%s\n", data.s);
+                continue;
+            }
+            else if (ret == -1) {
+                printf("Tag not present\n");
+                continue;
+            }
+            else {
+                printf("Failed to get tag\n");
+                goto end;
+            }
+        }
+    }
+    ret = EXIT_SUCCESS;
+
+end:
+    //cleanup
+    if (in_samhdr) {
+        sam_hdr_destroy(in_samhdr);
+    }
+    if (infile) {
+        sam_close(infile);
+    }
+    ks_free(&data);
+    return ret;
+}
diff --git a/ext/htslib/samples/read_refname.c b/ext/htslib/samples/read_refname.c
new file mode 100644
index 0000000..9b4918d
--- /dev/null
+++ b/ext/htslib/samples/read_refname.c
@@ -0,0 +1,125 @@
+/*  read_refname.c --  showcases the htslib api usage
+
+    Copyright (C) 2023 Genome Research Ltd.
+
+    Author: Vasudeva Sarma <vasudeva.sarma@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE
+
+*/
+
+/* The purpose of this code is to demonstrate the library apis and need proper error handling and optimisation */
+
+#include <getopt.h>
+#include <unistd.h>
+#include <htslib/sam.h>
+
+/// print_usage - print the usage
+/** @param fp pointer to the file / terminal to which usage to be dumped
+returns nothing
+*/
+static void print_usage(FILE *fp)
+{
+    fprintf(fp, "Usage: read_refname infile minsize\n\
+This shows name of references which has length above the given size\n");
+    return;
+}
+
+/// main_demo - start of the demo
+/** @param argc - count of arguments
+ *  @param argv - pointer to array of arguments
+returns 1 on failure 0 on success
+*/
+int main(int argc, char *argv[])
+{
+    const char *inname = NULL, *id = NULL;
+    int c = 0, ret = EXIT_FAILURE, linecnt = 0, pos = 0;
+    samFile *infile = NULL;
+    sam_hdr_t *in_samhdr = NULL;
+    kstring_t data = KS_INITIALIZE;
+    int64_t minsize = 0, size = 0;
+
+    if (argc != 3 && argc != 2) {
+        print_usage(stdout);
+        goto end;
+    }
+    inname = argv[1];
+    if (argc == 3) {
+        minsize = atoll(argv[2]);
+    }
+
+    //open input files
+    if (!(infile = sam_open(inname, "r"))) {
+        printf("Could not open %s\n", inname);
+        goto end;
+    }
+
+    //read header
+    if (!(in_samhdr = sam_hdr_read(infile))) {
+        printf("Failed to read header from file!\n");
+        goto end;
+    }
+
+    linecnt = sam_hdr_count_lines(in_samhdr, "SQ"); //get reference count
+    if (linecnt <= 0) {
+        if (!linecnt) {
+            printf("No reference line present\n");
+        }
+        else {
+            printf("Failed to get reference line count\n");
+        }
+        goto end;
+    }
+    //iterate and check each reference's length
+    for (pos = 1, c = 0; c < linecnt; ++c) {
+        if ((ret = sam_hdr_find_tag_pos(in_samhdr, "SQ", c, "LN", &data) == -2)) {
+            printf("Failed to get length\n");
+            goto end;
+        }
+        else if (ret == -1) {
+            //length not present, ignore
+            continue;
+        }
+        //else have length
+        size = atoll(data.s);
+        if (size < minsize) {
+            //not required
+            continue;
+        }
+        if (!(id = sam_hdr_line_name(in_samhdr, "SQ", c))) {    //sam_hdr_find_tag_pos(in_samhdr, "SQ", c, "SN", &data) can also do the same!
+            printf("Failed to get id for reference data\n");
+            goto end;
+        }
+        printf("%d,%s,%s\n", pos, id, data.s);
+        pos++;
+    }
+
+    ret = EXIT_SUCCESS;
+
+end:
+    //cleanup
+    if (in_samhdr) {
+        sam_hdr_destroy(in_samhdr);
+    }
+    if (infile) {
+        sam_close(infile);
+    }
+    ks_free(&data);
+    return ret;
+}
diff --git a/ext/htslib/samples/rem_header.c b/ext/htslib/samples/rem_header.c
new file mode 100644
index 0000000..852d5f0
--- /dev/null
+++ b/ext/htslib/samples/rem_header.c
@@ -0,0 +1,138 @@
+/*  rem_header.c --  showcases the htslib api usage
+
+    Copyright (C) 2023 Genome Research Ltd.
+
+    Author: Vasudeva Sarma <vasudeva.sarma@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE
+
+*/
+
+/* The purpose of this code is to demonstrate the library apis and need proper error handling and optimisation */
+
+#include <getopt.h>
+#include <unistd.h>
+#include <htslib/sam.h>
+
+/// print_usage - print the usage
+/** @param fp pointer to the file / terminal to which usage to be dumped
+returns nothing
+*/
+static void print_usage(FILE *fp)
+{
+    fprintf(fp, "Usage: rem_header infile header [id]\n\
+Removes header line of given type and id\n");
+    return;
+}
+
+/// main_demo - start of the demo
+/** @param argc - count of arguments
+ *  @param argv - pointer to array of arguments
+returns 1 on failure 0 on success
+*/
+int main(int argc, char *argv[])
+{
+    const char *inname = NULL, *header = NULL, *idval = NULL;
+    char *id = NULL;
+    int ret = EXIT_FAILURE;
+    samFile *infile = NULL, *outfile = NULL;
+    sam_hdr_t *in_samhdr = NULL;
+
+    //update_header infile header idval tag value
+    if (argc <3 || argc > 4) {
+        //3 & 4 are ok, 3-> all of given header type, 4->given id of given header type to be removed
+        print_usage(stderr);
+        goto end;
+    }
+    inname = argv[1];
+    header = argv[2];
+    if (argc == 4) {
+        idval = argv[3];
+    }
+
+    //unique identifier for each of the header types
+    if (header[0] == 'H' && header[1] == 'D') {
+        id = NULL;
+    }
+    else if (header[0] == 'S' && header[1] == 'Q') {
+        id = "SN";
+    }
+    else if (header[0] == 'R' && header[1] == 'G') {
+        id = "ID";
+    }
+    else if (header[0] == 'P' && header[1] == 'G') {
+        id = "ID";
+    }
+    else if (header[0] == 'C' && header[1] == 'O') {
+        //CO field can be removed using the position of it using sam_hdr_remove_line_pos
+        id = "";
+    }
+    else {
+        printf("Invalid header type\n");
+        goto end;
+    }
+
+    if (!(infile = sam_open(inname, "r"))) {
+        printf("Could not open %s\n", inname);
+        goto end;
+    }
+    if (!(outfile = sam_open("-", "w"))) {      //use stdout as the output file for ease of display of update
+        printf("Could not open stdout\n");
+        goto end;
+    }
+
+    //read header
+    if (!(in_samhdr = sam_hdr_read(infile))) {
+        printf("Failed to read header from file!\n");
+        goto end;
+    }
+    if (idval) {
+        //remove specific line
+        if (sam_hdr_remove_line_id(in_samhdr, header, id, idval)) {
+            printf("Failed to remove header line\n");
+            goto end;
+        }
+    }
+    else {
+        //remove multiple lines of a header type
+        if (sam_hdr_remove_lines(in_samhdr, header, id, NULL)) {
+            printf("Failed to remove header line\n");
+            goto end;
+        }
+    }
+    //write output
+    if (sam_hdr_write(outfile, in_samhdr) < 0) {
+        printf("Failed to write output\n");
+        goto end;
+    }
+    ret = EXIT_SUCCESS;
+    //bam data write to follow....
+end:
+    //cleanup
+    if (in_samhdr) {
+        sam_hdr_destroy(in_samhdr);
+    }
+    if (infile) {
+        sam_close(infile);
+    }
+    if (outfile) {
+        sam_close(outfile);
+    }
+    return ret;
+}
diff --git a/ext/htslib/samples/sample.bed b/ext/htslib/samples/sample.bed
new file mode 100644
index 0000000..2ae458f
--- /dev/null
+++ b/ext/htslib/samples/sample.bed
@@ -0,0 +1,4 @@
+T1	1	2
+T1	30	35
+T2	10	15
+T2	30	40
diff --git a/ext/htslib/samples/sample.ref.fa b/ext/htslib/samples/sample.ref.fa
new file mode 100644
index 0000000..5789e8c
--- /dev/null
+++ b/ext/htslib/samples/sample.ref.fa
@@ -0,0 +1,4 @@
+>T1 T1:1-40
+AAAAACTGAAAACCCCTTTTGGGGACTGTTAACAGTTTTT
+>T2 T2:1:40
+TTTTCCCCACTGAAAACCCCTTTTGGGGACTGTTAACAGT
diff --git a/ext/htslib/samples/sample.ref.fq b/ext/htslib/samples/sample.ref.fq
new file mode 100644
index 0000000..18b2b96
--- /dev/null
+++ b/ext/htslib/samples/sample.ref.fq
@@ -0,0 +1,16 @@
+@T1
+AAAAACTGAAAACCCCTTTTGGGGACTGTTAACAGTTTTT
++
+AAAAACTGAAAACCCCTTTTGGGGACTGTTAACAGTTTTT
+@T2
+TTTTCCCCACTGAAAACCCCTTTTGGGGACTGTTAACAGT
++
+TTTTCCCCACTGAAAACCCCTTTTGGGGACTGTTAACAGT
+@T3
+TTTTGGGGACTGTTAACAGT
++
+TTTTGGGGACTGTTAACAGT
+@T4
+TTTTCCCCACTGAAAACCCCTTTTGGGGACTGTTAACAGTTTTTCCCCACTGAAAACCCCTTTTGGGGACTGTTAACAGTTTTTGGGGACTGTTAACAGT
++
+TTTTCCCCACTGAAAACCCCTTTTGGGGACTGTTAACAGTTTTTCCCCACTGAAAACCCCTTTTGGGGACTGTTAACAGTTTTTGGGGACTGTTAACAGT
diff --git a/ext/htslib/samples/sample.sam b/ext/htslib/samples/sample.sam
new file mode 100644
index 0000000..58515c9
--- /dev/null
+++ b/ext/htslib/samples/sample.sam
@@ -0,0 +1,29 @@
+@HD	VN:1.17	SO:unknown
+@SQ	SN:T1	LN:40
+@SQ	SN:T2	LN:40
+@CO	@SQ SN* LN* AH AN AS DS  M5 SP TP UR
+@CO	@RG ID* BC CN DS DT FO KS LB PG PI PL PM PU SM
+@CO	@PG ID* PN CL PP DS VN
+@CO	this is a dummy alignment file to demonstrate different abilities of hts apis
+@CO	QNAME FLAG RNAME POS MAPQ CIGAR RNEXT PNEXT TLEN SEQ QUAL [TAG:TYPE:VALUE]…
+@CO	1234567890123456789012345678901234567890
+@CO	AAAAACTGAAAACCCCTTTTGGGGACTGTTAACAGTTTTT	T1
+@CO	TTTTCCCCACTGAAAACCCCTTTTGGGGACTGTTAACAGT	T2
+@CO	ITR1-ITR2M, ITR2-ITR2M are proper pairs in T1 and T2, UNMP1 is partly mapped and pair is unmapped, UNMP2 & 3 are unmapped
+@CO	A1-A2, A4-A3 are proper pairs with A4-A3 in different read order. A5 is secondary alignment
+ITR1	99	T1	5	40	4M	=	33	10	ACTG	()()
+ITR2	147	T2	23	49	2M	=	35	-10	TT	**
+ITR2M	99	T2	35	51	2M	=	23	10	AA	&&
+ITR1M	147	T1	33	37	4M	=	5	-10	ACTG	$$$$
+UNMP1	73	T1	21	40	3M	*	0	5	GGG	&&1
+UNMP2	141	*	0	0	*	*	0	7	AA	&&
+UNMP3	77	*	0	0	*	*	0	5	GGG	&&2
+A1	99	T1	25	35	6M	=	31	8	ACTGTT	******
+A2	147	T1	31	33	6M	=	25	-8	ACTGTT	()()()
+A3	147	T2	23	47	2M1X	=	12	-5	TTG	(((
+A4	99	T2	12	50	3M	=	23	5	GAA	()(
+A5	355	T1	25	35	4M	=	33	5	ACTG	PPPP
+B1	99	T1	25	35	6M	=	31	8	GCTATT	******
+B3	147	T2	23	47	2M1X	=	12	-5	TAG	(((
+B4	99	T2	12	50	3M	=	23	5	GAT	()(
+B5	355	T1	25	35	4M	=	33	5	AGTG	PPPP
diff --git a/ext/htslib/samples/split.c b/ext/htslib/samples/split.c
new file mode 100644
index 0000000..c51dbd3
--- /dev/null
+++ b/ext/htslib/samples/split.c
@@ -0,0 +1,153 @@
+/*  split.c --  showcases the htslib api usage
+
+    Copyright (C) 2023 Genome Research Ltd.
+
+    Author: Vasudeva Sarma <vasudeva.sarma@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE
+
+*/
+
+/* The purpose of this code is to demonstrate the library apis and need proper error handling and optimisation */
+
+#include <getopt.h>
+#include <unistd.h>
+#include <htslib/sam.h>
+
+/// print_usage - print the usage
+/** @param fp pointer to the file / terminal to which usage to be dumped
+returns nothing
+*/
+static void print_usage(FILE *fp)
+{
+    fprintf(fp, "Usage: split infile outdir\n\
+Splits the input file alignments to read1 and read2 and saves as 1.sam and 2.bam in given directory\n\
+Shows the basic writing of output\n");
+    return;
+}
+
+/// main_demo - start of the demo
+/** @param argc - count of arguments
+ *  @param argv - pointer to array of arguments
+returns 1 on failure 0 on success
+*/
+int main(int argc, char *argv[])
+{
+    const char *inname = NULL, *outdir = NULL;
+    char *file1 = NULL, *file2 = NULL;
+    int c = 0, ret = EXIT_FAILURE, size = 0;
+    samFile *infile = NULL, *outfile1 = NULL, *outfile2 = NULL;
+    sam_hdr_t *in_samhdr = NULL;
+    bam1_t *bamdata = NULL;
+
+    if (argc != 3) {
+        print_usage(stdout);
+        goto end;
+    }
+    inname = argv[1];
+    outdir = argv[2];
+
+    //allocate space for output
+    size = sizeof(char) * (strlen(outdir) + sizeof("/1.sam") + 1); //space for output file name and null termination
+    file1 = malloc(size);
+    file2 = malloc(size);
+    if (!file1 || !file2) {
+        printf("Failed to set output path\n");
+        goto end;
+    }
+
+    //output file names
+    snprintf(file1, size, "%s/1.sam", outdir);  //for SAM output
+    snprintf(file2, size, "%s/2.bam", outdir);  //for BAM output
+    //bam data storage
+    if (!(bamdata = bam_init1())) {
+        printf("Failed to initialize bamdata\n");
+        goto end;
+    }
+    //open input file - r reading
+    if (!(infile = sam_open(inname, "r"))) {
+        printf("Could not open %s\n", inname);
+        goto end;
+    }
+    //open output files - w write as SAM, wb  write as BAM
+    outfile1 = sam_open(file1, "w");            //as SAM
+    outfile2 = sam_open(file2, "wb");           //as BAM
+    if (!outfile1 || !outfile2) {
+        printf("Could not open output file\n");
+        goto end;
+    }
+
+    //read header, required to resolve the target names to proper ids
+    if (!(in_samhdr = sam_hdr_read(infile))) {
+        printf("Failed to read header from file!\n");
+        goto end;
+    }
+    //write header
+    if ((sam_hdr_write(outfile1, in_samhdr) == -1) || (sam_hdr_write(outfile2, in_samhdr) == -1)) {
+        printf("Failed to write header\n");
+        goto end;
+    }
+
+    //check flags and write
+    while ((c = sam_read1(infile, in_samhdr, bamdata)) >= 0) {
+        if (bamdata->core.flag & BAM_FREAD1) {
+            if (sam_write1(outfile1, in_samhdr, bamdata) < 0) {
+                printf("Failed to write output data\n");
+                goto end;
+            }
+        }
+        else if (bamdata->core.flag & BAM_FREAD2) {
+            if (sam_write1(outfile2, in_samhdr, bamdata) < 0) {
+                printf("Failed to write output data\n");
+                goto end;
+            }
+        }
+    }
+    if (-1 == c) {
+        //EOF
+        ret = EXIT_SUCCESS;
+    }
+    else {
+        printf("Error in reading data\n");
+    }
+end:
+    //cleanup
+    if (in_samhdr) {
+        sam_hdr_destroy(in_samhdr);
+    }
+    if (infile) {
+        sam_close(infile);
+    }
+    if (bamdata) {
+        bam_destroy1(bamdata);
+    }
+    if (file1) {
+        free(file1);
+    }
+    if (file2) {
+        free(file2);
+    }
+    if (outfile1) {
+        sam_close(outfile1);
+    }
+    if (outfile2) {
+        sam_close(outfile2);
+    }
+    return ret;
+}
diff --git a/ext/htslib/samples/split2.c b/ext/htslib/samples/split2.c
new file mode 100644
index 0000000..33fabbd
--- /dev/null
+++ b/ext/htslib/samples/split2.c
@@ -0,0 +1,158 @@
+/*  split2.c --  showcases the htslib api usage
+
+    Copyright (C) 2023 Genome Research Ltd.
+
+    Author: Vasudeva Sarma <vasudeva.sarma@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE
+
+*/
+
+/* The purpose of this code is to demonstrate the library apis and need proper error handling and optimisation */
+
+#include <getopt.h>
+#include <unistd.h>
+#include <htslib/sam.h>
+
+/// print_usage - print the usage
+/** @param fp pointer to the file / terminal to which usage to be dumped
+returns nothing
+*/
+static void print_usage(FILE *fp)
+{
+    fprintf(fp, "Usage: split2 infile outdir\n\
+Splits the input file alignments to read1 and read2 and saves as 1.sam and 2.bam in given directory\n\
+Shows file type selection through name and format api\n");
+    return;
+}
+
+/// main_demo - start of the demo
+/** @param argc - count of arguments
+ *  @param argv - pointer to array of arguments
+returns 1 on failure 0 on success
+*/
+int main(int argc, char *argv[])
+{
+    const char *inname = NULL, *outdir = NULL;
+    char *file1 = NULL, *file2 = NULL, mode1[5] = "w", mode2[5] = "w";
+    int c = 0, ret = EXIT_FAILURE, size = 0;
+    samFile *infile = NULL, *outfile1 = NULL, *outfile2 = NULL;
+    sam_hdr_t *in_samhdr = NULL;
+    bam1_t *bamdata = NULL;
+
+    if (argc != 3) {
+        print_usage(stdout);
+        goto end;
+    }
+    inname = argv[1];
+    outdir = argv[2];
+
+    //allocate space for output
+    size = sizeof(char) * (strlen(outdir) + sizeof("/1.sam.gz") + 1); //space for output file name and null termination
+    file1 = malloc(size);
+    file2 = malloc(size);
+    if (!file1 || !file2) {
+        printf("Failed to set output path\n");
+        goto end;
+    }
+
+    //output file names
+    snprintf(file1, size, "%s/1.sam.gz", outdir);   //name of Read1 file
+    snprintf(file2, size, "%s/2.sam", outdir);      //name of Read2 file
+    //bam data storage
+    if (!(bamdata = bam_init1())) {
+        printf("Failed to initialize bamdata\n");
+        goto end;
+    }
+    //set file open mode based on file name for 1st and as explicit for 2nd
+    if ((sam_open_mode(mode1+1, file1, NULL) == -1) || (sam_open_mode(mode2+1, file2, "sam.gz") == -1)) {
+        printf("Failed to set open mode\n");
+        goto end;
+    }
+    //open input file
+    if (!(infile = sam_open(inname, "r"))) {
+        printf("Could not open %s\n", inname);
+        goto end;
+    }
+    //open output files
+    outfile1 = sam_open(file1, mode1);                          //as compressed SAM through sam_open
+    outfile2 = sam_open_format(file2, mode2, NULL);             //as compressed SAM through sam_open_format
+    if (!outfile1 || !outfile2) {
+        printf("Could not open output file\n");
+        goto end;
+    }
+
+    //read header, required to resolve the target names to proper ids
+    if (!(in_samhdr = sam_hdr_read(infile))) {
+        printf("Failed to read header from file!\n");
+        goto end;
+    }
+    //write header
+    if ((sam_hdr_write(outfile1, in_samhdr) == -1) || (sam_hdr_write(outfile2, in_samhdr) == -1)) {
+        printf("Failed to write header\n");
+        goto end;
+    }
+
+    //check flags and write
+    while ((c = sam_read1(infile, in_samhdr, bamdata)) >= 0) {
+        if (bamdata->core.flag & BAM_FREAD1) {
+            if (sam_write1(outfile1, in_samhdr, bamdata) < 0) {
+                printf("Failed to write output data\n");
+                goto end;
+            }
+        }
+        else if (bamdata->core.flag & BAM_FREAD2) {
+            if (sam_write1(outfile2, in_samhdr, bamdata) < 0) {
+                printf("Failed to write output data\n");
+                goto end;
+            }
+        }
+    }
+    if (-1 == c) {
+        //EOF
+        ret = EXIT_SUCCESS;
+    }
+    else {
+        printf("Error in reading data\n");
+    }
+end:
+    //cleanup
+    if (in_samhdr) {
+        sam_hdr_destroy(in_samhdr);
+    }
+    if (infile) {
+        sam_close(infile);
+    }
+    if (bamdata) {
+        bam_destroy1(bamdata);
+    }
+    if (file1) {
+        free(file1);
+    }
+    if (file2) {
+        free(file2);
+    }
+    if (outfile1) {
+        sam_close(outfile1);
+    }
+    if (outfile2) {
+        sam_close(outfile2);
+    }
+    return ret;
+}
diff --git a/ext/htslib/samples/split_thread1.c b/ext/htslib/samples/split_thread1.c
new file mode 100644
index 0000000..551c7f0
--- /dev/null
+++ b/ext/htslib/samples/split_thread1.c
@@ -0,0 +1,161 @@
+/*  split_thread1.c --  showcases the htslib api usage
+
+    Copyright (C) 2023 Genome Research Ltd.
+
+    Author: Vasudeva Sarma <vasudeva.sarma@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE
+
+*/
+
+/* The purpose of this code is to demonstrate the library apis and need proper error handling and optimisation */
+
+#include <getopt.h>
+#include <unistd.h>
+#include <htslib/sam.h>
+
+/// print_usage - print the usage
+/** @param fp pointer to the file / terminal to which usage to be dumped
+returns nothing
+*/
+static void print_usage(FILE *fp)
+{
+    fprintf(fp, "Usage: split_t1 infile outdir\n\
+Splits the input file alignments to read1 and read2 and saves as 1.sam and 2.bam in given directory\n\
+Shows the usage of basic thread in htslib\n");
+    return;
+}
+
+/// main_demo - start of the demo
+/** @param argc - count of arguments
+ *  @param argv - pointer to array of arguments
+returns 1 on failure 0 on success
+*/
+int main(int argc, char *argv[])
+{
+    const char *inname = NULL, *outdir = NULL;
+    char *file1 = NULL, *file2 = NULL;
+    int c = 0, ret = EXIT_FAILURE, size = 0;
+    samFile *infile = NULL, *outfile1 = NULL, *outfile2 = NULL;
+    sam_hdr_t *in_samhdr = NULL;
+    bam1_t *bamdata = NULL;
+
+    if (argc != 3) {
+        print_usage(stdout);
+        goto end;
+    }
+    inname = argv[1];
+    outdir = argv[2];
+
+    //allocate space for output
+    size = sizeof(char) * (strlen(outdir) + sizeof("/1.sam") + 1); //space for output file name and null termination
+    file1 = malloc(size);
+    file2 = malloc(size);
+    if (!file1 || !file2) {
+        printf("Failed to set output path\n");
+        goto end;
+    }
+
+    //output file names
+    snprintf(file1, size, "%s/1.sam", outdir);  //for SAM output
+    snprintf(file2, size, "%s/2.bam", outdir);  //for BAM output
+    //bam data storage
+    if (!(bamdata = bam_init1())) {
+        printf("Failed to initialize bamdata\n");
+        goto end;
+    }
+    //open input file - r reading
+    if (!(infile = sam_open(inname, "r"))) {
+        printf("Could not open %s\n", inname);
+        goto end;
+    }
+    //open output files - w write as SAM, wb  write as BAM
+    outfile1 = sam_open(file1, "w");            //as SAM
+    outfile2 = sam_open(file2, "wb");           //as BAM
+    if (!outfile1 || !outfile2) {
+        printf("Could not open output file\n");
+        goto end;
+    }
+
+    //create file specific threads
+    if (hts_set_opt(infile, HTS_OPT_NTHREADS, 1) < 0 ||     //1 thread specific for reading
+    hts_set_opt(outfile1, HTS_OPT_NTHREADS, 1) < 0 ||       //1 thread specific for sam write
+    hts_set_opt(outfile2, HTS_OPT_NTHREADS, 2) < 0) {       //2 thread specific for bam write
+        printf("Failed to set thread options\n");
+        goto end;
+    }
+
+    //read header, required to resolve the target names to proper ids
+    if (!(in_samhdr = sam_hdr_read(infile))) {
+        printf("Failed to read header from file!\n");
+        goto end;
+    }
+    //write header
+    if ((sam_hdr_write(outfile1, in_samhdr) == -1) || (sam_hdr_write(outfile2, in_samhdr) == -1)) {
+        printf("Failed to write header\n");
+        goto end;
+    }
+
+    //check flags and write
+    while ((c = sam_read1(infile, in_samhdr, bamdata)) >= 0) {
+        if (bamdata->core.flag & BAM_FREAD1) {
+            if (sam_write1(outfile1, in_samhdr, bamdata) < 0) {
+                printf("Failed to write output data\n");
+                goto end;
+            }
+        }
+        else if (bamdata->core.flag & BAM_FREAD2) {
+            if (sam_write1(outfile2, in_samhdr, bamdata) < 0) {
+                printf("Failed to write output data\n");
+                goto end;
+            }
+        }
+    }
+    if (-1 == c) {
+        //EOF
+        ret = EXIT_SUCCESS;
+    }
+    else {
+        printf("Error in reading data\n");
+    }
+end:
+    //cleanup
+    if (in_samhdr) {
+        sam_hdr_destroy(in_samhdr);
+    }
+    if (infile) {
+        sam_close(infile);
+    }
+    if (bamdata) {
+        bam_destroy1(bamdata);
+    }
+    if (file1) {
+        free(file1);
+    }
+    if (file2) {
+        free(file2);
+    }
+    if (outfile1) {
+        sam_close(outfile1);
+    }
+    if (outfile2) {
+        sam_close(outfile2);
+    }
+    return ret;
+}
diff --git a/ext/htslib/samples/split_thread2.c b/ext/htslib/samples/split_thread2.c
new file mode 100644
index 0000000..dc8bc9f
--- /dev/null
+++ b/ext/htslib/samples/split_thread2.c
@@ -0,0 +1,171 @@
+/*  split_thread2.c --  showcases the htslib api usage
+
+    Copyright (C) 2023 Genome Research Ltd.
+
+    Author: Vasudeva Sarma <vasudeva.sarma@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE
+
+*/
+
+/* The purpose of this code is to demonstrate the library apis and need proper error handling and optimisation */
+
+#include <getopt.h>
+#include <unistd.h>
+#include <htslib/sam.h>
+#include <htslib/thread_pool.h>
+
+/// print_usage - print the usage
+/** @param fp pointer to the file / terminal to which usage to be dumped
+returns nothing
+*/
+static void print_usage(FILE *fp)
+{
+    fprintf(fp, "Usage: split_t2 infile outdir\n\
+Splits the input file alignments to read1 and read2 and saves as 1.sam and 2.bam in given directory\n\
+Shows the usage of thread pool\n");
+    return;
+}
+
+/// main_demo - start of the demo
+/** @param argc - count of arguments
+ *  @param argv - pointer to array of arguments
+returns 1 on failure 0 on success
+*/
+int main(int argc, char *argv[])
+{
+    const char *inname = NULL, *outdir = NULL;
+    char *file1 = NULL, *file2 = NULL;
+    int c = 0, ret = EXIT_FAILURE, size = 0;
+    samFile *infile = NULL, *outfile1 = NULL, *outfile2 = NULL;
+    sam_hdr_t *in_samhdr = NULL;
+    bam1_t *bamdata = NULL;
+    htsThreadPool tpool = {NULL, 0};
+
+    if (argc != 3) {
+        print_usage(stdout);
+        goto end;
+    }
+    inname = argv[1];
+    outdir = argv[2];
+
+    //allocate space for output
+    size = sizeof(char) * (strlen(outdir) + sizeof("/1.sam") + 1); //space for output file name and null termination
+    file1 = malloc(size);
+    file2 = malloc(size);
+    if (!file1 || !file2) {
+        printf("Failed to set output path\n");
+        goto end;
+    }
+
+    //output file names
+    snprintf(file1, size, "%s/1.sam", outdir);  //for SAM output
+    snprintf(file2, size, "%s/2.bam", outdir);  //for BAM output
+    //bam data storage
+    if (!(bamdata = bam_init1())) {
+        printf("Failed to initialize bamdata\n");
+        goto end;
+    }
+    //open input file - r reading
+    if (!(infile = sam_open(inname, "r"))) {
+        printf("Could not open %s\n", inname);
+        goto end;
+    }
+    //open output files - w write as SAM, wb  write as BAM
+    outfile1 = sam_open(file1, "w");            //as SAM
+    outfile2 = sam_open(file2, "wb");           //as BAM
+    if (!outfile1 || !outfile2) {
+        printf("Could not open output file\n");
+        goto end;
+    }
+
+    //create a pool of 4 threads
+    if (!(tpool.pool = hts_tpool_init(4))) {
+        printf("Failed to initialize the thread pool\n");
+        goto end;
+    }
+    //share the pool with all the 3 files
+    if (hts_set_opt(infile, HTS_OPT_THREAD_POOL, &tpool) < 0 ||
+    hts_set_opt(outfile1, HTS_OPT_THREAD_POOL, &tpool) < 0 ||
+    hts_set_opt(outfile2, HTS_OPT_THREAD_POOL, &tpool) < 0) {
+        printf("Failed to set thread options\n");
+        goto end;
+    }
+
+    //read header, required to resolve the target names to proper ids
+    if (!(in_samhdr = sam_hdr_read(infile))) {
+        printf("Failed to read header from file!\n");
+        goto end;
+    }
+    //write header
+    if ((sam_hdr_write(outfile1, in_samhdr) == -1) || (sam_hdr_write(outfile2, in_samhdr) == -1)) {
+        printf("Failed to write header\n");
+        goto end;
+    }
+
+    //check flags and write
+    while ((c = sam_read1(infile, in_samhdr, bamdata)) >= 0) {
+        if (bamdata->core.flag & BAM_FREAD1) {
+            if (sam_write1(outfile1, in_samhdr, bamdata) < 0) {
+                printf("Failed to write output data\n");
+                goto end;
+            }
+        }
+        else if (bamdata->core.flag & BAM_FREAD2) {
+            if (sam_write1(outfile2, in_samhdr, bamdata) < 0) {
+                printf("Failed to write output data\n");
+                goto end;
+            }
+        }
+    }
+    if (-1 == c) {
+        //EOF
+        ret = EXIT_SUCCESS;
+    }
+    else {
+        printf("Error in reading data\n");
+    }
+end:
+    //cleanup
+    if (in_samhdr) {
+        sam_hdr_destroy(in_samhdr);
+    }
+    if (infile) {
+        sam_close(infile);
+    }
+    if (bamdata) {
+        bam_destroy1(bamdata);
+    }
+    if (file1) {
+        free(file1);
+    }
+    if (file2) {
+        free(file2);
+    }
+    if (outfile1) {
+        sam_close(outfile1);
+    }
+    if (outfile2) {
+        sam_close(outfile2);
+    }
+    if (tpool.pool) {
+        hts_tpool_destroy(tpool.pool);
+    }
+    return ret;
+}
diff --git a/ext/htslib/samples/update_header.c b/ext/htslib/samples/update_header.c
new file mode 100644
index 0000000..237d5c4
--- /dev/null
+++ b/ext/htslib/samples/update_header.c
@@ -0,0 +1,131 @@
+/*  update_header.c --  showcases the htslib api usage
+
+    Copyright (C) 2023 Genome Research Ltd.
+
+    Author: Vasudeva Sarma <vasudeva.sarma@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE
+
+*/
+
+/* The purpose of this code is to demonstrate the library apis and need proper error handling and optimisation */
+
+#include <getopt.h>
+#include <unistd.h>
+#include <htslib/sam.h>
+
+/// print_usage - print the usage
+/** @param fp pointer to the file / terminal to which usage to be dumped
+returns nothing
+*/
+static void print_usage(FILE *fp)
+{
+    fprintf(fp, "Usage: update_header infile header idval tag value\n\
+Updates the tag's value on line given in id on header of given type\n");
+    return;
+}
+
+/// main_demo - start of the demo
+/** @param argc - count of arguments
+ *  @param argv - pointer to array of arguments
+returns 1 on failure 0 on success
+*/
+int main(int argc, char *argv[])
+{
+    const char *inname = NULL, *tag = NULL, *idval = NULL, *val = NULL, *header = NULL;
+    char *id = NULL;
+    int ret = EXIT_FAILURE;
+    samFile *infile = NULL, *outfile = NULL;
+    sam_hdr_t *in_samhdr = NULL;
+
+    //update_header infile header idval tag value
+    if (argc != 6) {
+        print_usage(stderr);
+        goto end;
+    }
+    inname = argv[1];
+    header = argv[2];
+    idval = argv[3];
+    tag = argv[4];
+    val = argv[5];
+
+    //unique identifier for each of the header types
+    if (header[0] == 'H' && header[1] == 'D') {
+        id = NULL;
+        printf("This sample doesnt not support modifying HD fields\n");
+    }
+    else if (header[0] == 'S' && header[1] == 'Q') {
+        id = "SN";
+    }
+    else if (header[0] == 'R' && header[1] == 'G') {
+        id = "ID";
+    }
+    else if (header[0] == 'P' && header[1] == 'G') {
+        id = "ID";
+    }
+    else if (header[0] == 'C' && header[1] == 'O') {
+        tag = NULL;
+        id = "";
+        printf("This sample doesnt not support modifying CO fields\n");
+    }
+    else {
+        printf("Invalid header type\n");
+        goto end;
+    }
+
+    if (!(infile = sam_open(inname, "r"))) {
+        printf("Could not open %s\n", inname);
+        goto end;
+    }
+    if (!(outfile = sam_open("-", "w"))) {      //use stdout as the output file for ease of display of update
+        printf("Could not open stdout\n");
+        goto end;
+    }
+
+    //read header
+    if (!(in_samhdr = sam_hdr_read(infile))) {
+        printf("Failed to read header from file!\n");
+        goto end;
+    }
+
+    //update with new data
+    if (sam_hdr_update_line(in_samhdr, header, id, idval, tag, val, NULL) < 0) {
+        printf("Failed to update data\n");
+        goto end;
+    }
+    //write output
+    if (sam_hdr_write(outfile, in_samhdr) < 0) {
+        printf("Failed to write output\n");
+        goto end;
+    }
+    ret = EXIT_SUCCESS;
+    //bam data write to follow....
+end:
+    //cleanup
+    if (in_samhdr) {
+        sam_hdr_destroy(in_samhdr);
+    }
+    if (infile) {
+        sam_close(infile);
+    }
+    if (outfile) {
+        sam_close(outfile);
+    }
+    return ret;
+}
diff --git a/ext/htslib/samples/write_fast.c b/ext/htslib/samples/write_fast.c
new file mode 100644
index 0000000..95d919f
--- /dev/null
+++ b/ext/htslib/samples/write_fast.c
@@ -0,0 +1,116 @@
+/*  write_fast.c --  showcases the htslib api usage
+
+    Copyright (C) 2023 Genome Research Ltd.
+
+    Author: Vasudeva Sarma <vasudeva.sarma@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE
+
+*/
+
+/* The purpose of this code is to demonstrate the library apis and need proper error handling and optimisation */
+
+#include <getopt.h>
+#include <unistd.h>
+#include <time.h>
+#include <htslib/sam.h>
+#include <htslib/faidx.h>
+
+/// print_usage - show usage
+/** @param fp pointer to the file / terminal to which usage to be dumped
+returns nothing
+*/
+static void print_usage(FILE *fp)
+{
+    fprintf(fp, "Usage: write_fast <file> <sequence> [<qualities]\n\
+Appends a fasta/fastq file.\n");
+    return;
+}
+
+/// main_demo - start of the demo
+/** @param argc - count of arguments
+ *  @param argv - pointer to array of arguments
+returns 1 on failure 0 on success
+*/
+int main(int argc, char *argv[])
+{
+    const char *outname = NULL;             //output file name
+    int ret = EXIT_FAILURE;
+    samFile *outfile = NULL;                //sam file
+    bam1_t *bamdata = NULL;                 //to hold the read data
+    char mode[4] = "a";
+    const char *data = NULL, *qual = NULL;  //ref data and quality
+    char name[256] = {0};
+
+    if (argc > 4 || argc < 3) {
+        print_usage(stdout);
+        goto end;
+    }
+    outname = argv[1];
+    data = argv[2];
+    if (argc == 4) {    //fastq data
+        qual = argv[3];
+        if (strlen(data) != strlen(qual)) {     //check for proper length of data and quality values
+            printf("Incorrect reference and quality data\n");
+            goto end;
+        }
+    }
+
+    //initialize
+    if (!(bamdata = bam_init1())) {
+        printf("Failed to initialize bamdata\n");
+        goto end;
+    }
+    if (sam_open_mode(mode + 1, outname, NULL) < 0) {
+        printf("Invalid file name\n");
+        goto end;
+    }
+    //open output file
+    if (!(outfile = sam_open(outname, mode))) {         //expects the name to have correct extension!
+        printf("Could not open %s\n", outname);
+        goto end;
+    }
+    /* if the file name extension is not appropriate to the content, inconsistent data will be present in output.
+    if required, htsFormat and sam_open_format can be explicitly used to ensure appropriateness of content.
+    htsFormat fmt = {sequence_data, fastq_format / fasta_format};
+    sam_open_format(outname, mode, fmt);
+    */
+
+    snprintf(name, sizeof(name), "Test_%ld", (long) time(NULL));
+    //data
+    if (bam_set1(bamdata, strlen(name), name, BAM_FUNMAP, -1, -1, 0, 0, NULL, -1, -1, 0, strlen(data), data, qual, 0) < 0) {
+        printf("Failed to set data\n");
+        goto end;
+    }
+    //as we write only FASTA/FASTQ, we can get away without providing headers
+    if (sam_write1(outfile, NULL, bamdata) < 0) {
+        printf("Failed to write data\n");
+        goto end;
+    }
+    ret = EXIT_SUCCESS;
+end:
+    //clean up
+    if (outfile) {
+        sam_close(outfile);
+    }
+    if (bamdata) {
+        bam_destroy1(bamdata);
+    }
+    return ret;
+}
diff --git a/ext/htslib/simd.c b/ext/htslib/simd.c
new file mode 100644
index 0000000..865dd88
--- /dev/null
+++ b/ext/htslib/simd.c
@@ -0,0 +1,222 @@
+/*  simd.c -- SIMD optimised versions of various internal functions.
+
+    Copyright (C) 2024 Genome Research Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+
+// These must be defined before the first system include to ensure that legacy
+// BSD types needed by <sys/sysctl.h> remain defined when _XOPEN_SOURCE is set.
+#if defined __APPLE__
+#define _DARWIN_C_SOURCE
+#elif defined __NetBSD__
+#define _NETBSD_SOURCE
+#endif
+
+#include "htslib/sam.h"
+#include "sam_internal.h"
+
+#if defined __x86_64__
+#include <immintrin.h>
+#elif defined __ARM_NEON
+#include <arm_neon.h>
+#endif
+
+#if defined __arm__ || defined __aarch64__
+
+#if defined __linux__ || defined __FreeBSD__
+#include <sys/auxv.h>
+#elif defined __APPLE__
+#include <sys/types.h>
+#include <sys/sysctl.h>
+#elif defined __NetBSD__
+#include <stddef.h>
+#include <sys/param.h>
+#include <sys/sysctl.h>
+#ifdef __aarch64__
+#include <aarch64/armreg.h>
+#else
+#include <arm/armreg.h>
+#endif
+#elif defined _WIN32
+#include <processthreadsapi.h>
+#endif
+
+static inline int cpu_supports_neon(void) {
+#if defined __linux__ && defined __arm__ && defined HWCAP_NEON
+    return (getauxval(AT_HWCAP) & HWCAP_NEON) != 0;
+#elif defined __linux__ && defined __arm__ && defined HWCAP_ARM_NEON
+    return (getauxval(AT_HWCAP) & HWCAP_ARM_NEON) != 0;
+#elif defined __linux__ && defined __aarch64__ && defined HWCAP_ASIMD
+    return (getauxval(AT_HWCAP) & HWCAP_ASIMD) != 0;
+#elif defined __APPLE__ && defined __aarch64__
+    int32_t ctl;
+    size_t ctlsize = sizeof ctl;
+    if (sysctlbyname("hw.optional.AdvSIMD", &ctl, &ctlsize, NULL, 0) != 0) return 0;
+    if (ctlsize != sizeof ctl) return 0;
+    return ctl;
+#elif defined __FreeBSD__ && defined __arm__ && defined HWCAP_NEON
+    unsigned long cap;
+    if (elf_aux_info(AT_HWCAP, &cap, sizeof cap) != 0) return 0;
+    return (cap & HWCAP_NEON) != 0;
+#elif defined __FreeBSD__ && defined __aarch64__ && defined HWCAP_ASIMD
+    unsigned long cap;
+    if (elf_aux_info(AT_HWCAP, &cap, sizeof cap) != 0) return 0;
+    return (cap & HWCAP_ASIMD) != 0;
+#elif defined __NetBSD__ && defined __arm__ && defined ARM_MVFR0_ASIMD_MASK
+    uint32_t buf[16];
+    size_t buflen = sizeof buf;
+    if (sysctlbyname("machdep.id_mvfr", buf, &buflen, NULL, 0) != 0) return 0;
+    if (buflen < sizeof(uint32_t)) return 0;
+    return (buf[0] & ARM_MVFR0_ASIMD_MASK) == 0x00000002;
+#elif defined __NetBSD__ && defined __aarch64__ && defined ID_AA64PFR0_EL1_ADVSIMD
+    struct aarch64_sysctl_cpu_id buf;
+    size_t buflen = sizeof buf;
+    if (sysctlbyname("machdep.cpu0.cpu_id", &buf, &buflen, NULL, 0) != 0) return 0;
+    if (buflen < offsetof(struct aarch64_sysctl_cpu_id, ac_aa64pfr0) + sizeof(uint64_t)) return 0;
+    return (buf.ac_aa64pfr0 & ID_AA64PFR0_EL1_ADVSIMD & 0x00e00000) == 0;
+#elif defined _WIN32
+    return IsProcessorFeaturePresent(PF_ARM_V8_INSTRUCTIONS_AVAILABLE) != 0;
+#else
+    return 0;
+#endif
+}
+
+#endif
+
+#ifdef BUILDING_SIMD_NIBBLE2BASE
+
+void (*htslib_nibble2base)(uint8_t *nib, char *seq, int len) = nibble2base_default;
+
+#if defined __x86_64__
+
+/*
+ * Convert a nibble encoded BAM sequence to a string of bases.
+ *
+ * Using SSSE3 instructions, 16 codepoints that hold 2 bases each can be
+ * unpacked into 32 indexes from 0-15. Using the pshufb instruction these can
+ * be converted to the IUPAC characters.
+ * It falls back on the nibble2base_default function for the remainder.
+ */
+
+__attribute__((target("ssse3")))
+static void nibble2base_ssse3(uint8_t *nib, char *seq, int len) {
+    const char *seq_end_ptr = seq + len;
+    char *seq_cursor = seq;
+    uint8_t *nibble_cursor = nib;
+    const char *seq_vec_end_ptr = seq_end_ptr - (2 * sizeof(__m128i) - 1);
+    __m128i nuc_lookup_vec = _mm_lddqu_si128((__m128i *)seq_nt16_str);
+    /* Nucleotides are encoded 4-bits per nucleotide and stored in 8-bit bytes
+       as follows: |AB|CD|EF|GH|. The 4-bit codes (going from 0-15) can be used
+       together with the pshufb instruction as a lookup table. The most efficient
+       way is to use bitwise AND and shift to create two vectors. One with all
+       the upper codes (|A|C|E|G|) and one with the lower codes (|B|D|F|H|).
+       The lookup can then be performed and the resulting vectors can be
+       interleaved again using the unpack instructions. */
+    while (seq_cursor < seq_vec_end_ptr) {
+        __m128i encoded = _mm_lddqu_si128((__m128i *)nibble_cursor);
+        __m128i encoded_upper = _mm_srli_epi64(encoded, 4);
+        encoded_upper = _mm_and_si128(encoded_upper, _mm_set1_epi8(15));
+        __m128i encoded_lower = _mm_and_si128(encoded, _mm_set1_epi8(15));
+        __m128i nucs_upper = _mm_shuffle_epi8(nuc_lookup_vec, encoded_upper);
+        __m128i nucs_lower = _mm_shuffle_epi8(nuc_lookup_vec, encoded_lower);
+        __m128i first_nucleotides = _mm_unpacklo_epi8(nucs_upper, nucs_lower);
+        __m128i second_nucleotides = _mm_unpackhi_epi8(nucs_upper, nucs_lower);
+        _mm_storeu_si128((__m128i *)seq_cursor, first_nucleotides);
+        _mm_storeu_si128((__m128i *)(seq_cursor + sizeof(__m128i)),
+                         second_nucleotides);
+        nibble_cursor += sizeof(__m128i);
+        seq_cursor += 2 * sizeof(__m128i);
+    }
+    nibble2base_default(nibble_cursor, seq_cursor, seq_end_ptr - seq_cursor);
+}
+
+__attribute__((constructor))
+static void nibble2base_resolve(void) {
+    if (__builtin_cpu_supports("ssse3")) {
+        htslib_nibble2base = nibble2base_ssse3;
+    }
+}
+
+#elif defined __ARM_NEON
+
+static void nibble2base_neon(uint8_t *nib, char *seq0, int len) {
+    uint8x16_t low_nibbles_mask = vdupq_n_u8(0x0f);
+    uint8x16_t nuc_lookup_vec = vld1q_u8((const uint8_t *) seq_nt16_str);
+#ifndef __aarch64__
+    uint8x8x2_t nuc_lookup_vec2 = {{ vget_low_u8(nuc_lookup_vec), vget_high_u8(nuc_lookup_vec) }};
+#endif
+
+    uint8_t *seq = (uint8_t *) seq0;
+    int blocks;
+
+    for (blocks = len / 32; blocks > 0; --blocks) {
+        uint8x16_t encoded = vld1q_u8(nib);
+        nib += 16;
+
+        /* Translate the high and low nibbles to nucleotide letters separately,
+           then interleave them back together via vzipq for writing. */
+
+        uint8x16_t high_nibbles = vshrq_n_u8(encoded, 4);
+        uint8x16_t low_nibbles  = vandq_u8(encoded, low_nibbles_mask);
+
+#ifdef __aarch64__
+        uint8x16_t high_nucleotides = vqtbl1q_u8(nuc_lookup_vec, high_nibbles);
+        uint8x16_t low_nucleotides  = vqtbl1q_u8(nuc_lookup_vec, low_nibbles);
+#else
+        uint8x8_t high_low  = vtbl2_u8(nuc_lookup_vec2, vget_low_u8(high_nibbles));
+        uint8x8_t high_high = vtbl2_u8(nuc_lookup_vec2, vget_high_u8(high_nibbles));
+        uint8x16_t high_nucleotides = vcombine_u8(high_low, high_high);
+
+        uint8x8_t low_low  = vtbl2_u8(nuc_lookup_vec2, vget_low_u8(low_nibbles));
+        uint8x8_t low_high = vtbl2_u8(nuc_lookup_vec2, vget_high_u8(low_nibbles));
+        uint8x16_t low_nucleotides = vcombine_u8(low_low, low_high);
+#endif
+
+#ifdef __aarch64__
+        vst1q_u8_x2(seq, vzipq_u8(high_nucleotides, low_nucleotides));
+#else
+        // Avoid vst1q_u8_x2 as GCC erroneously omits it on 32-bit ARM
+        uint8x16x2_t nucleotides = {{ high_nucleotides, low_nucleotides }};
+        vst2q_u8(seq, nucleotides);
+#endif
+        seq += 32;
+    }
+
+    if (len % 32 != 0)
+        nibble2base_default(nib, (char *) seq, len % 32);
+}
+
+static __attribute__((constructor)) void nibble2base_resolve(void) {
+    if (cpu_supports_neon()) htslib_nibble2base = nibble2base_neon;
+}
+
+#endif
+
+#endif // BUILDING_SIMD_NIBBLE2BASE
+
+// Potentially useful diagnostic, and prevents "empty translation unit" errors
+const char htslib_simd[] =
+    "SIMD functions present:"
+#ifdef BUILDING_SIMD_NIBBLE2BASE
+    " nibble2base"
+#endif
+    ".";
diff --git a/ext/htslib/synced_bcf_reader.c b/ext/htslib/synced_bcf_reader.c
new file mode 100644
index 0000000..1835ea2
--- /dev/null
+++ b/ext/htslib/synced_bcf_reader.c
@@ -0,0 +1,1520 @@
+/*  synced_bcf_reader.c -- stream through multiple VCF files.
+
+    Copyright (C) 2012-2023 Genome Research Ltd.
+
+    Author: Petr Danecek <pd3@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+
+#include <stdlib.h>
+#include <assert.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <string.h>
+#include <strings.h>
+#include <limits.h>
+#include <inttypes.h>
+#include <errno.h>
+#include <sys/stat.h>
+#include "htslib/synced_bcf_reader.h"
+#include "htslib/kseq.h"
+#include "htslib/khash_str2int.h"
+#include "htslib/bgzf.h"
+#include "htslib/thread_pool.h"
+#include "bcf_sr_sort.h"
+
+#define REQUIRE_IDX_      1
+#define ALLOW_NO_IDX_     2
+
+// Maximum indexable coordinate of .csi, for default min_shift of 14.
+// This comes out to about 17 Tbp.  Limiting factor is the bin number,
+// which is a uint32_t in CSI.  The highest number of levels compatible
+// with this is 10 (needs 31 bits).
+#define MAX_CSI_COOR ((1LL << (14 + 30)) - 1)
+
+typedef struct
+{
+    hts_pos_t start, end;   // records are marked for skipping have start>end
+}
+region1_t;
+
+typedef struct bcf_sr_region_t
+{
+    region1_t *regs;            // regions will sorted and merged, redundant records marked for skipping have start>end
+    int nregs, mregs, creg;     // creg: the current active region
+}
+region_t;
+
+#define BCF_SR_AUX(x) ((aux_t*)((x)->aux))
+typedef struct
+{
+    sr_sort_t sort;
+    int regions_overlap, targets_overlap;
+}
+aux_t;
+
+static bcf_sr_regions_t *bcf_sr_regions_alloc(void);
+static int _regions_add(bcf_sr_regions_t *reg, const char *chr, hts_pos_t start, hts_pos_t end);
+static bcf_sr_regions_t *_regions_init_string(const char *str);
+static int _regions_match_alleles(bcf_sr_regions_t *reg, int als_idx, bcf1_t *rec);
+static void _regions_sort_and_merge(bcf_sr_regions_t *reg);
+static int _bcf_sr_regions_overlap(bcf_sr_regions_t *reg, const char *seq, hts_pos_t start, hts_pos_t end, int missed_reg_handler);
+static void bcf_sr_seek_start(bcf_srs_t *readers);
+
+char *bcf_sr_strerror(int errnum)
+{
+    switch (errnum)
+    {
+        case open_failed:
+            return strerror(errno);
+        case not_bgzf:
+            return "not compressed with bgzip";
+        case idx_load_failed:
+            return "could not load index";
+        case file_type_error:
+            return "unknown file type";
+        case api_usage_error:
+            return "API usage error";
+        case header_error:
+            return "could not parse header";
+        case no_eof:
+            return "no BGZF EOF marker; file may be truncated";
+        case no_memory:
+            return "Out of memory";
+        case vcf_parse_error:
+            return "VCF parse error";
+        case bcf_read_error:
+            return "BCF read error";
+        case noidx_error:
+            return "merge of unindexed files failed";
+        default: return "";
+    }
+}
+
+int bcf_sr_set_opt(bcf_srs_t *readers, bcf_sr_opt_t opt, ...)
+{
+    va_list args;
+    switch (opt)
+    {
+        case BCF_SR_REQUIRE_IDX:
+            readers->require_index = REQUIRE_IDX_;
+            return 0;
+
+        case BCF_SR_ALLOW_NO_IDX:
+            readers->require_index = ALLOW_NO_IDX_;
+            return 0;
+
+        case BCF_SR_PAIR_LOGIC:
+            va_start(args, opt);
+            BCF_SR_AUX(readers)->sort.pair = va_arg(args, int);
+            return 0;
+
+        case BCF_SR_REGIONS_OVERLAP:
+            va_start(args, opt);
+            BCF_SR_AUX(readers)->regions_overlap = va_arg(args, int);
+            if ( readers->regions ) readers->regions->overlap = BCF_SR_AUX(readers)->regions_overlap;
+            return 0;
+
+        case BCF_SR_TARGETS_OVERLAP:
+            va_start(args, opt);
+            BCF_SR_AUX(readers)->targets_overlap = va_arg(args, int);
+            if ( readers->targets ) readers->targets->overlap = BCF_SR_AUX(readers)->targets_overlap;
+            return 0;
+
+        default:
+            break;
+    }
+    return 1;
+}
+
+static int *init_filters(bcf_hdr_t *hdr, const char *filters, int *nfilters)
+{
+    kstring_t str = {0,0,0};
+    const char *tmp = filters, *prev = filters;
+    int nout = 0, *out = NULL;
+    while ( 1 )
+    {
+        if ( *tmp==',' || !*tmp )
+        {
+            int *otmp = (int*) realloc(out, (nout+1)*sizeof(int));
+            if (!otmp)
+                goto err;
+            out = otmp;
+            if ( tmp-prev==1 && *prev=='.' )
+            {
+                out[nout] = -1;
+                nout++;
+            }
+            else
+            {
+                str.l = 0;
+                kputsn(prev, tmp-prev, &str);
+                out[nout] = bcf_hdr_id2int(hdr, BCF_DT_ID, str.s);
+                if ( out[nout]>=0 ) nout++;
+            }
+            if ( !*tmp ) break;
+            prev = tmp+1;
+        }
+        tmp++;
+    }
+    if ( str.m ) free(str.s);
+    *nfilters = nout;
+    return out;
+
+ err:
+    if (str.m) free(str.s);
+    free(out);
+    return NULL;
+}
+
+int bcf_sr_set_regions(bcf_srs_t *readers, const char *regions, int is_file)
+{
+    if ( readers->nreaders || readers->regions )
+    {
+        if ( readers->regions ) bcf_sr_regions_destroy(readers->regions);
+        readers->regions = bcf_sr_regions_init(regions,is_file,0,1,-2);
+        bcf_sr_seek_start(readers);
+        return 0;
+    }
+
+    readers->regions = bcf_sr_regions_init(regions,is_file,0,1,-2);
+    if ( !readers->regions ) return -1;
+    readers->explicit_regs = 1;
+    readers->require_index = REQUIRE_IDX_;
+    readers->regions->overlap = BCF_SR_AUX(readers)->regions_overlap;
+    return 0;
+}
+
+int bcf_sr_set_targets(bcf_srs_t *readers, const char *targets, int is_file, int alleles)
+{
+    if ( readers->nreaders || readers->targets )
+    {
+        hts_log_error("Must call bcf_sr_set_targets() before bcf_sr_add_reader()");
+        return -1;
+    }
+    if ( targets[0]=='^' )
+    {
+        readers->targets_exclude = 1;
+        targets++;
+    }
+    readers->targets = bcf_sr_regions_init(targets,is_file,0,1,-2);
+    if ( !readers->targets ) return -1;
+    readers->targets_als = alleles;
+    readers->targets->overlap = BCF_SR_AUX(readers)->targets_overlap;
+    return 0;
+}
+
+int bcf_sr_set_threads(bcf_srs_t *files, int n_threads)
+{
+    if (!(files->n_threads = n_threads))
+        return 0;
+
+    files->p = calloc(1, sizeof(*files->p));
+    if (!files->p) {
+        files->errnum = no_memory;
+        return -1;
+    }
+    if (!(files->p->pool = hts_tpool_init(n_threads)))
+        return -1;
+
+    return 0;
+}
+
+void bcf_sr_destroy_threads(bcf_srs_t *files) {
+    if (!files->p)
+        return;
+
+    if (files->p->pool)
+        hts_tpool_destroy(files->p->pool);
+    free(files->p);
+}
+
+int bcf_sr_add_reader(bcf_srs_t *files, const char *fname)
+{
+    char fmode[5];
+    strcpy(fmode, "r");
+    vcf_open_mode(fmode+1, fname, NULL);
+    htsFile* file_ptr = hts_open(fname, fmode);
+    if ( ! file_ptr ) {
+        files->errnum = open_failed;
+        return 0;
+    }
+
+    files->has_line = (int*) realloc(files->has_line, sizeof(int)*(files->nreaders+1));
+    files->has_line[files->nreaders] = 0;
+    files->readers  = (bcf_sr_t*) realloc(files->readers, sizeof(bcf_sr_t)*(files->nreaders+1));
+    bcf_sr_t *reader = &files->readers[files->nreaders++];
+    memset(reader,0,sizeof(bcf_sr_t));
+
+    reader->file = file_ptr;
+
+    files->errnum = 0;
+
+    if ( reader->file->format.compression==bgzf )
+    {
+        BGZF *bgzf = hts_get_bgzfp(reader->file);
+        if ( bgzf && bgzf_check_EOF(bgzf) == 0 ) {
+            files->errnum = no_eof;
+            hts_log_warning("No BGZF EOF marker; file '%s' may be truncated", fname);
+        }
+        if (files->p)
+            bgzf_thread_pool(bgzf, files->p->pool, files->p->qsize);
+    }
+
+    if ( files->require_index==REQUIRE_IDX_ )
+    {
+        if ( reader->file->format.format==vcf )
+        {
+            if ( reader->file->format.compression!=bgzf )
+            {
+                files->errnum = not_bgzf;
+                return 0;
+            }
+
+            reader->tbx_idx = tbx_index_load(fname);
+            if ( !reader->tbx_idx )
+            {
+                files->errnum = idx_load_failed;
+                return 0;
+            }
+
+            reader->header = bcf_hdr_read(reader->file);
+        }
+        else if ( reader->file->format.format==bcf )
+        {
+            if ( reader->file->format.compression!=bgzf )
+            {
+                files->errnum = not_bgzf;
+                return 0;
+            }
+
+            reader->header = bcf_hdr_read(reader->file);
+
+            reader->bcf_idx = bcf_index_load(fname);
+            if ( !reader->bcf_idx )
+            {
+                files->errnum = idx_load_failed;
+                return 0;
+            }
+        }
+        else
+        {
+            files->errnum = file_type_error;
+            return 0;
+        }
+    }
+    else
+    {
+        if ( reader->file->format.format==bcf || reader->file->format.format==vcf )
+        {
+            reader->header = bcf_hdr_read(reader->file);
+        }
+        else
+        {
+            files->errnum = file_type_error;
+            return 0;
+        }
+        files->streaming = 1;
+    }
+    if ( files->streaming && files->nreaders>1 )
+    {
+        static int no_index_warned = 0;
+        if ( files->require_index==ALLOW_NO_IDX_ && !no_index_warned )
+        {
+            hts_log_warning("Using multiple unindexed files may produce errors, make sure chromosomes are in the same order!");
+            no_index_warned = 1;
+        }
+        if ( files->require_index!=ALLOW_NO_IDX_ )
+        {
+            files->errnum = api_usage_error;
+            hts_log_error("Must set require_index when the number of readers is greater than one");
+            return 0;
+        }
+    }
+    if ( files->streaming && files->regions )
+    {
+        files->errnum = api_usage_error;
+        hts_log_error("Cannot tabix-jump in streaming mode");
+        return 0;
+    }
+    if ( !reader->header )
+    {
+        files->errnum = header_error;
+        return 0;
+    }
+
+    reader->fname = strdup(fname);
+    if ( files->apply_filters )
+        reader->filter_ids = init_filters(reader->header, files->apply_filters, &reader->nfilter_ids);
+
+    // Update list of chromosomes
+    if ( !files->explicit_regs && !files->streaming )
+    {
+        int n = 0, i;
+        const char **names;
+
+        if ( !files->regions )
+        {
+            files->regions = bcf_sr_regions_alloc();
+            if ( !files->regions )
+            {
+                hts_log_error("Cannot allocate regions data structure");
+                return 0;
+            }
+        }
+
+        names = reader->tbx_idx ? tbx_seqnames(reader->tbx_idx, &n) : bcf_hdr_seqnames(reader->header, &n);
+        for (i=0; i<n; i++)
+        {
+            _regions_add(files->regions, names[i], -1, -1);
+        }
+        free(names);
+        _regions_sort_and_merge(files->regions);
+    }
+
+    if ( files->require_index==ALLOW_NO_IDX_ && files->nreaders > 1 )
+    {
+        bcf_hdr_t *hdr0 = files->readers[0].header;
+        bcf_hdr_t *hdr1 = reader->header;
+        if ( hdr0->n[BCF_DT_CTG]!=hdr1->n[BCF_DT_CTG] )
+        {
+            files->errnum = noidx_error;
+            hts_log_error("Different number of sequences in the header, refusing to stream multiple unindexed files");
+            return 0;
+        }
+        int i;
+        for (i=0; i<hdr0->n[BCF_DT_CTG]; i++)
+        {
+            if ( strcmp(bcf_hdr_id2name(hdr0,i),bcf_hdr_id2name(hdr1,i)) )
+            {
+                files->errnum = noidx_error;
+                hts_log_error("Sequences in the header appear in different order, refusing to stream multiple unindexed files");
+                return 0;
+            }
+        }
+    }
+
+    return 1;
+}
+
+bcf_srs_t *bcf_sr_init(void)
+{
+    bcf_srs_t *files = (bcf_srs_t*) calloc(1,sizeof(bcf_srs_t));
+    files->aux = (aux_t*) calloc(1,sizeof(aux_t));
+    bcf_sr_sort_init(&BCF_SR_AUX(files)->sort);
+    bcf_sr_set_opt(files,BCF_SR_REGIONS_OVERLAP,1);
+    bcf_sr_set_opt(files,BCF_SR_TARGETS_OVERLAP,0);
+    return files;
+}
+
+static void bcf_sr_destroy1(bcf_sr_t *reader)
+{
+    free(reader->fname);
+    if ( reader->tbx_idx ) tbx_destroy(reader->tbx_idx);
+    if ( reader->bcf_idx ) hts_idx_destroy(reader->bcf_idx);
+    bcf_hdr_destroy(reader->header);
+    hts_close(reader->file);
+    if ( reader->itr ) tbx_itr_destroy(reader->itr);
+    int j;
+    for (j=0; j<reader->mbuffer; j++)
+        bcf_destroy1(reader->buffer[j]);
+    free(reader->buffer);
+    free(reader->samples);
+    free(reader->filter_ids);
+}
+
+void bcf_sr_destroy(bcf_srs_t *files)
+{
+    int i;
+    for (i=0; i<files->nreaders; i++)
+        bcf_sr_destroy1(&files->readers[i]);
+    free(files->has_line);
+    free(files->readers);
+    for (i=0; i<files->n_smpl; i++) free(files->samples[i]);
+    free(files->samples);
+    if (files->targets) bcf_sr_regions_destroy(files->targets);
+    if (files->regions) bcf_sr_regions_destroy(files->regions);
+    if (files->tmps.m) free(files->tmps.s);
+    if (files->n_threads) bcf_sr_destroy_threads(files);
+    bcf_sr_sort_destroy(&BCF_SR_AUX(files)->sort);
+    free(files->aux);
+    free(files);
+}
+
+void bcf_sr_remove_reader(bcf_srs_t *files, int i)
+{
+    assert( !files->samples );  // not ready for this yet
+    bcf_sr_sort_remove_reader(files, &BCF_SR_AUX(files)->sort, i);
+    bcf_sr_destroy1(&files->readers[i]);
+    if ( i+1 < files->nreaders )
+    {
+        memmove(&files->readers[i], &files->readers[i+1], (files->nreaders-i-1)*sizeof(bcf_sr_t));
+        memmove(&files->has_line[i], &files->has_line[i+1], (files->nreaders-i-1)*sizeof(int));
+    }
+    files->nreaders--;
+}
+
+#if DEBUG_SYNCED_READER
+void debug_buffer(FILE *fp, bcf_sr_t *reader)
+{
+    int j;
+    for (j=0; j<=reader->nbuffer; j++)
+    {
+        bcf1_t *line = reader->buffer[j];
+        fprintf(fp,"\t%p\t%s%s\t%s:%"PRIhts_pos"\t%s ", (void*)line,reader->fname,j==0?"*":" ",reader->header->id[BCF_DT_CTG][line->rid].key,line->pos+1,line->n_allele?line->d.allele[0]:"");
+        int k;
+        for (k=1; k<line->n_allele; k++) fprintf(fp," %s", line->d.allele[k]);
+        fprintf(fp,"\n");
+    }
+}
+
+void debug_buffers(FILE *fp, bcf_srs_t *files)
+{
+    int i;
+    for (i=0; i<files->nreaders; i++)
+    {
+        fprintf(fp, "has_line: %d\t%s\n", bcf_sr_has_line(files,i),files->readers[i].fname);
+        debug_buffer(fp, &files->readers[i]);
+    }
+    fprintf(fp,"\n");
+}
+#endif
+
+static inline int has_filter(bcf_sr_t *reader, bcf1_t *line)
+{
+    int i, j;
+    if ( !line->d.n_flt )
+    {
+        for (j=0; j<reader->nfilter_ids; j++)
+            if ( reader->filter_ids[j]<0 ) return 1;
+        return 0;
+    }
+    for (i=0; i<line->d.n_flt; i++)
+    {
+        for (j=0; j<reader->nfilter_ids; j++)
+            if ( line->d.flt[i]==reader->filter_ids[j] ) return 1;
+    }
+    return 0;
+}
+
+static int _reader_seek(bcf_sr_t *reader, const char *seq, hts_pos_t start, hts_pos_t end)
+{
+    if ( end>=MAX_CSI_COOR )
+    {
+        hts_log_error("The coordinate is out of csi index limit: %"PRIhts_pos, end+1);
+        exit(1);
+    }
+    if ( reader->itr )
+    {
+        hts_itr_destroy(reader->itr);
+        reader->itr = NULL;
+    }
+    reader->nbuffer = 0;
+    if ( reader->tbx_idx )
+    {
+        int tid = tbx_name2id(reader->tbx_idx, seq);
+        if ( tid==-1 ) return -1;    // the sequence not present in this file
+        reader->itr = tbx_itr_queryi(reader->tbx_idx,tid,start,end+1);
+    }
+    else
+    {
+        int tid = bcf_hdr_name2id(reader->header, seq);
+        if ( tid==-1 ) return -1;    // the sequence not present in this file
+        reader->itr = bcf_itr_queryi(reader->bcf_idx,tid,start,end+1);
+    }
+    if (!reader->itr) {
+        hts_log_error("Could not seek: %s:%"PRIhts_pos"-%"PRIhts_pos, seq, start + 1, end + 1);
+        abort();
+    }
+    return 0;
+}
+
+/*
+ *  _readers_next_region() - jumps to next region if necessary
+ *  Returns 0 on success or -1 when there are no more regions left
+ */
+static int _readers_next_region(bcf_srs_t *files)
+{
+    // Need to open new chromosome? Check number of lines in all readers' buffers
+    int i, eos = 0;
+    for (i=0; i<files->nreaders; i++)
+        if ( !files->readers[i].itr && !files->readers[i].nbuffer ) eos++;
+
+    if ( eos!=files->nreaders )
+    {
+        // Some of the readers still has buffered lines
+        return 0;
+    }
+
+    // No lines in the buffer, need to open new region or quit.
+    int prev_iseq = files->regions->iseq;
+    hts_pos_t prev_end = files->regions->end;
+    if ( bcf_sr_regions_next(files->regions)<0 ) return -1;
+    files->regions->prev_end = prev_iseq==files->regions->iseq ? prev_end : -1;
+
+    for (i=0; i<files->nreaders; i++)
+        _reader_seek(&files->readers[i],files->regions->seq_names[files->regions->iseq],files->regions->start,files->regions->end);
+
+    return 0;
+}
+
+static void _set_variant_boundaries(bcf1_t *rec, hts_pos_t *beg, hts_pos_t *end)
+{
+    hts_pos_t off;
+    if ( rec->n_allele )
+    {
+        off = rec->rlen;
+        bcf_unpack(rec, BCF_UN_STR);
+        int i;
+        for (i=1; i<rec->n_allele; i++)
+        {
+            // Make symbolic alleles start at POS, although this is not strictly true for
+            // <DEL>,<INS> where POS should be the position BEFORE the deletion/insertion.
+            // However, since arbitrary symbolic alleles can be defined by the user, we
+            // will simplify the interpretation of --targets-overlap and --region-overlap.
+            int j = 0;
+            char *ref = rec->d.allele[0];
+            char *alt = rec->d.allele[i];
+            while ( ref[j] && alt[j] && ref[j]==alt[j] ) j++;
+            if ( off > j ) off = j;
+            if ( !off ) break;
+        }
+    }
+    else
+        off = 0;
+
+    *beg = rec->pos + off;
+    *end = rec->pos + rec->rlen - 1;
+}
+
+/*
+ *  _reader_fill_buffer() - buffers all records with the same coordinate
+ */
+static int _reader_fill_buffer(bcf_srs_t *files, bcf_sr_t *reader)
+{
+    // Return if the buffer is full: the coordinate of the last buffered record differs
+    if ( reader->nbuffer && reader->buffer[reader->nbuffer]->pos != reader->buffer[1]->pos ) return 0;
+
+    // No iterator (sequence not present in this file) and not streaming
+    if ( !reader->itr && !files->streaming ) return 0;
+
+    // Fill the buffer with records starting at the same position
+    int i, ret = 0;
+    while (1)
+    {
+        if ( reader->nbuffer+1 >= reader->mbuffer )
+        {
+            // Increase buffer size
+            reader->mbuffer += 8;
+            reader->buffer = (bcf1_t**) realloc(reader->buffer, sizeof(bcf1_t*)*reader->mbuffer);
+            for (i=8; i>0; i--)     // initialize
+            {
+                reader->buffer[reader->mbuffer-i] = bcf_init1();
+                reader->buffer[reader->mbuffer-i]->max_unpack = files->max_unpack;
+                reader->buffer[reader->mbuffer-i]->pos = -1;    // for rare cases when VCF starts from 1
+            }
+        }
+        if ( files->streaming )
+        {
+            if ( reader->file->format.format==vcf )
+            {
+                ret = hts_getline(reader->file, KS_SEP_LINE, &files->tmps);
+                if ( ret < -1 ) files->errnum = bcf_read_error;
+                if ( ret < 0 ) break; // no more lines or an error
+                ret = vcf_parse1(&files->tmps, reader->header, reader->buffer[reader->nbuffer+1]);
+                if ( ret<0 ) { files->errnum = vcf_parse_error; break; }
+            }
+            else if ( reader->file->format.format==bcf )
+            {
+                ret = bcf_read1(reader->file, reader->header, reader->buffer[reader->nbuffer+1]);
+                if ( ret < -1 ) files->errnum = bcf_read_error;
+                if ( ret < 0 ) break; // no more lines or an error
+            }
+            else
+            {
+                hts_log_error("Fixme: not ready for this");
+                exit(1);
+            }
+        }
+        else if ( reader->tbx_idx )
+        {
+            ret = tbx_itr_next(reader->file, reader->tbx_idx, reader->itr, &files->tmps);
+            if ( ret < -1 ) files->errnum = bcf_read_error;
+            if ( ret < 0 ) break; // no more lines or an error
+            ret = vcf_parse1(&files->tmps, reader->header, reader->buffer[reader->nbuffer+1]);
+            if ( ret<0 ) { files->errnum = vcf_parse_error; break; }
+        }
+        else
+        {
+            ret = bcf_itr_next(reader->file, reader->itr, reader->buffer[reader->nbuffer+1]);
+            if ( ret < -1 ) files->errnum = bcf_read_error;
+            if ( ret < 0 ) break; // no more lines or an error
+            bcf_subset_format(reader->header,reader->buffer[reader->nbuffer+1]);
+        }
+
+        // Prevent creation of duplicates from records overlapping multiple regions
+        // and recognize true variant overlaps vs record overlaps (e.g. TA>T vs A>-)
+        if ( files->regions )
+        {
+            hts_pos_t beg, end;
+            if ( BCF_SR_AUX(files)->regions_overlap==0 )
+                beg = end = reader->buffer[reader->nbuffer+1]->pos;
+            else if ( BCF_SR_AUX(files)->regions_overlap==1 )
+            {
+                beg = reader->buffer[reader->nbuffer+1]->pos;
+                end = reader->buffer[reader->nbuffer+1]->pos + reader->buffer[reader->nbuffer+1]->rlen - 1;
+            }
+            else if ( BCF_SR_AUX(files)->regions_overlap==2 )
+                _set_variant_boundaries(reader->buffer[reader->nbuffer+1], &beg,&end);
+            else
+            {
+                hts_log_error("This should never happen, just to keep clang compiler happy: %d",BCF_SR_AUX(files)->targets_overlap);
+                exit(1);
+            }
+            if ( beg <= files->regions->prev_end || end < files->regions->start || beg > files->regions->end ) continue;
+        }
+
+        // apply filter
+        if ( !reader->nfilter_ids )
+            bcf_unpack(reader->buffer[reader->nbuffer+1], BCF_UN_STR);
+        else
+        {
+            bcf_unpack(reader->buffer[reader->nbuffer+1], BCF_UN_STR|BCF_UN_FLT);
+            if ( !has_filter(reader, reader->buffer[reader->nbuffer+1]) ) continue;
+        }
+        reader->nbuffer++;
+
+        if ( reader->buffer[reader->nbuffer]->rid != reader->buffer[1]->rid ) break;
+        if ( reader->buffer[reader->nbuffer]->pos != reader->buffer[1]->pos ) break;    // the buffer is full
+    }
+    if ( ret<0 )
+    {
+        // done for this region
+        tbx_itr_destroy(reader->itr);
+        reader->itr = NULL;
+    }
+    if ( files->require_index==ALLOW_NO_IDX_ && reader->buffer[reader->nbuffer]->rid < reader->buffer[1]->rid )
+    {
+         hts_log_error("Sequences out of order, cannot stream multiple unindexed files: %s", reader->fname);
+         exit(1);
+    }
+    return 0; // FIXME: Check for more errs in this function
+}
+
+/*
+ *  _readers_shift_buffer() - removes the first line
+ */
+static void _reader_shift_buffer(bcf_sr_t *reader)
+{
+    if ( !reader->nbuffer ) return;
+    int i;
+    bcf1_t *tmp = reader->buffer[1];
+    for (i=2; i<=reader->nbuffer; i++)
+        reader->buffer[i-1] = reader->buffer[i];
+    if ( reader->nbuffer > 1 )
+        reader->buffer[reader->nbuffer] = tmp;
+    reader->nbuffer--;
+}
+
+static int next_line(bcf_srs_t *files)
+{
+    const char *chr = NULL;
+    hts_pos_t min_pos = HTS_POS_MAX;
+
+    // Loop until next suitable line is found or all readers have finished
+    while ( 1 )
+    {
+        // Get all readers ready for the next region.
+        if ( files->regions && _readers_next_region(files)<0 ) break;
+
+        // Fill buffers and find the minimum chromosome
+        int i, min_rid = INT32_MAX;
+        for (i=0; i<files->nreaders; i++)
+        {
+            _reader_fill_buffer(files, &files->readers[i]);
+            if ( files->require_index==ALLOW_NO_IDX_ )
+            {
+                if ( !files->readers[i].nbuffer ) continue;
+                if ( min_rid > files->readers[i].buffer[1]->rid ) min_rid = files->readers[i].buffer[1]->rid;
+            }
+        }
+
+        for (i=0; i<files->nreaders; i++)
+        {
+            if ( !files->readers[i].nbuffer ) continue;
+            if ( files->require_index==ALLOW_NO_IDX_ && min_rid != files->readers[i].buffer[1]->rid ) continue;
+
+            // Update the minimum coordinate
+            if ( min_pos > files->readers[i].buffer[1]->pos )
+            {
+                min_pos = files->readers[i].buffer[1]->pos;
+                chr = bcf_seqname(files->readers[i].header, files->readers[i].buffer[1]);
+                assert(chr);
+                bcf_sr_sort_set_active(&BCF_SR_AUX(files)->sort, i);
+            }
+            else if ( min_pos==files->readers[i].buffer[1]->pos )
+                bcf_sr_sort_add_active(&BCF_SR_AUX(files)->sort, i);
+        }
+        if ( min_pos==HTS_POS_MAX )
+        {
+            if ( !files->regions ) break;
+            continue;
+        }
+
+        // Skip this position if not present in targets
+        if ( files->targets )
+        {
+            int match = 0;
+            for (i=0; i<files->nreaders; i++)
+            {
+                if ( !files->readers[i].nbuffer || files->readers[i].buffer[1]->pos!=min_pos ) continue;
+                hts_pos_t beg, end;
+                if ( BCF_SR_AUX(files)->targets_overlap==0 )
+                    beg = end = min_pos;
+                else if ( BCF_SR_AUX(files)->targets_overlap==1 )
+                {
+                    beg = min_pos;
+                    end = min_pos + files->readers[i].buffer[1]->rlen - 1;
+                }
+                else if ( BCF_SR_AUX(files)->targets_overlap==2 )
+                    _set_variant_boundaries(files->readers[i].buffer[1], &beg,&end);
+                else
+                {
+                    hts_log_error("This should never happen, just to keep clang compiler happy: %d",BCF_SR_AUX(files)->targets_overlap);
+                    exit(1);
+                }
+                int overlap = bcf_sr_regions_overlap(files->targets, chr, beg, end)==0 ? 1 : 0;
+                if ( (!files->targets_exclude && !overlap) || (files->targets_exclude && overlap) )
+                    _reader_shift_buffer(&files->readers[i]);
+                else
+                    match = 1;
+            }
+            if ( !match )
+            {
+                min_pos = HTS_POS_MAX;
+                chr = NULL;
+                continue;
+            }
+        }
+        break;  // done: chr and min_pos are set
+    }
+    if ( !chr ) return 0;
+
+    return bcf_sr_sort_next(files, &BCF_SR_AUX(files)->sort, chr, min_pos);
+}
+
+int bcf_sr_next_line(bcf_srs_t *files)
+{
+    if ( !files->targets_als )
+        return next_line(files);
+
+    while (1)
+    {
+        int i, ret = next_line(files);
+        if ( !ret ) return ret;
+
+        for (i=0; i<files->nreaders; i++)
+            if ( files->has_line[i] ) break;
+
+        if ( _regions_match_alleles(files->targets, files->targets_als-1, files->readers[i].buffer[0]) ) return ret;
+
+        // Check if there are more duplicate lines in the buffers. If not, return this line as if it
+        // matched the targets, even if there is a type mismatch
+        for (i=0; i<files->nreaders; i++)
+        {
+            if ( !files->has_line[i] ) continue;
+            if ( files->readers[i].nbuffer==0 || files->readers[i].buffer[1]->pos!=files->readers[i].buffer[0]->pos ) continue;
+            break;
+        }
+        if ( i==files->nreaders ) return ret;   // no more lines left, output even if target alleles are not of the same type
+    }
+}
+
+static void bcf_sr_seek_start(bcf_srs_t *readers)
+{
+    bcf_sr_regions_t *reg = readers->regions;
+    int i;
+    for (i=0; i<reg->nseqs; i++)
+        reg->regs[i].creg = -1;
+    reg->iseq = 0;
+    reg->start = -1;
+    reg->end   = -1;
+    reg->prev_seq = -1;
+    reg->prev_start = -1;
+    reg->prev_end   = -1;
+}
+
+
+int bcf_sr_seek(bcf_srs_t *readers, const char *seq, hts_pos_t pos)
+{
+    if ( !readers->regions ) return 0;
+    bcf_sr_sort_reset(&BCF_SR_AUX(readers)->sort);
+    if ( !seq && !pos )
+    {
+        // seek to start
+        bcf_sr_seek_start(readers);
+        return 0;
+    }
+
+    int i, nret = 0;
+
+    // Need to position both the readers and the regions. The latter is a bit of a mess
+    // because we can have in memory or external regions. The safe way is:
+    //  - reset all regions as if they were not read from at all (bcf_sr_seek_start)
+    //  - find the requested iseq (stored in the seq_hash)
+    //  - position regions to the requested position (bcf_sr_regions_overlap)
+    bcf_sr_seek_start(readers);
+    if ( khash_str2int_get(readers->regions->seq_hash, seq, &i)>=0 ) readers->regions->iseq = i;
+    _bcf_sr_regions_overlap(readers->regions, seq, pos, pos, 0);
+
+    for (i=0; i<readers->nreaders; i++)
+    {
+        nret += _reader_seek(&readers->readers[i],seq,pos,MAX_CSI_COOR-1);
+    }
+    return nret;
+}
+
+int bcf_sr_set_samples(bcf_srs_t *files, const char *fname, int is_file)
+{
+    int i, j, nsmpl, free_smpl = 0;
+    char **smpl = NULL;
+
+    void *exclude = (fname[0]=='^') ? khash_str2int_init() : NULL;
+    if ( exclude || strcmp("-",fname) ) // "-" stands for all samples
+    {
+        smpl = hts_readlist(fname, is_file, &nsmpl);
+        if ( !smpl )
+        {
+            hts_log_error("Could not read the file: \"%s\"", fname);
+            return 0;
+        }
+        if ( exclude )
+        {
+            for (i=0; i<nsmpl; i++)
+                khash_str2int_inc(exclude, smpl[i]);
+        }
+        free_smpl = 1;
+    }
+    if ( !smpl )
+    {
+        smpl  = files->readers[0].header->samples;   // intersection of all samples
+        nsmpl = bcf_hdr_nsamples(files->readers[0].header);
+    }
+
+    files->samples = NULL;
+    files->n_smpl  = 0;
+    for (i=0; i<nsmpl; i++)
+    {
+        if ( exclude && khash_str2int_has_key(exclude,smpl[i])  ) continue;
+
+        int n_isec = 0;
+        for (j=0; j<files->nreaders; j++)
+        {
+            if ( bcf_hdr_id2int(files->readers[j].header, BCF_DT_SAMPLE, smpl[i])<0 ) break;
+            n_isec++;
+        }
+        if ( n_isec!=files->nreaders )
+        {
+            hts_log_warning("The sample \"%s\" was not found in %s, skipping",
+                smpl[i], files->readers[n_isec].fname);
+            continue;
+        }
+
+        files->samples = (char**) realloc(files->samples, (files->n_smpl+1)*sizeof(const char*));
+        files->samples[files->n_smpl++] = strdup(smpl[i]);
+    }
+
+    if ( exclude ) khash_str2int_destroy(exclude);
+    if ( free_smpl )
+    {
+        for (i=0; i<nsmpl; i++) free(smpl[i]);
+        free(smpl);
+    }
+
+    if ( !files->n_smpl )
+    {
+        if ( files->nreaders>1 )
+            hts_log_warning("No samples in common");
+        return 0;
+    }
+    for (i=0; i<files->nreaders; i++)
+    {
+        bcf_sr_t *reader = &files->readers[i];
+        reader->samples  = (int*) malloc(sizeof(int)*files->n_smpl);
+        reader->n_smpl   = files->n_smpl;
+        for (j=0; j<files->n_smpl; j++)
+            reader->samples[j] = bcf_hdr_id2int(reader->header, BCF_DT_SAMPLE, files->samples[j]);
+    }
+    return 1;
+}
+
+// Allocate a new region list structure.
+static bcf_sr_regions_t *bcf_sr_regions_alloc(void)
+{
+    bcf_sr_regions_t *reg = (bcf_sr_regions_t *) calloc(1, sizeof(bcf_sr_regions_t));
+    if ( !reg ) return NULL;
+
+    reg->start = reg->end = -1;
+    reg->prev_start = reg->prev_end = reg->prev_seq = -1;
+    return reg;
+}
+
+// Add a new region into a list. On input the coordinates are 1-based, inclusive, then stored 0-based,
+// inclusive. Sorting and merging step needed afterwards: qsort(..,cmp_regions) and merge_regions().
+static int _regions_add(bcf_sr_regions_t *reg, const char *chr, hts_pos_t start, hts_pos_t end)
+{
+    if ( start==-1 && end==-1 )
+    {
+        start = 0; end = MAX_CSI_COOR-1;
+    }
+    else
+    {
+        start--; end--; // store 0-based coordinates
+    }
+
+    if ( !reg->seq_hash )
+         reg->seq_hash = khash_str2int_init();
+
+    int iseq;
+    if ( khash_str2int_get(reg->seq_hash, chr, &iseq)<0 )
+    {
+        // the chromosome block does not exist
+        iseq = reg->nseqs++;
+        reg->seq_names = (char**) realloc(reg->seq_names,sizeof(char*)*reg->nseqs);
+        reg->regs = (region_t*) realloc(reg->regs,sizeof(region_t)*reg->nseqs);
+        memset(&reg->regs[reg->nseqs-1],0,sizeof(region_t));
+        reg->seq_names[iseq] = strdup(chr);
+        reg->regs[iseq].creg = -1;
+        khash_str2int_set(reg->seq_hash,reg->seq_names[iseq],iseq);
+    }
+
+    region_t *creg = &reg->regs[iseq];
+    hts_expand(region1_t,creg->nregs+1,creg->mregs,creg->regs);
+    creg->regs[creg->nregs].start = start;
+    creg->regs[creg->nregs].end   = end;
+    creg->nregs++;
+
+    return 0; // FIXME: check for errs in this function
+}
+
+static int regions_cmp(const void *aptr, const void *bptr)
+{
+    region1_t *a = (region1_t*)aptr;
+    region1_t *b = (region1_t*)bptr;
+    if ( a->start < b->start ) return -1;
+    if ( a->start > b->start ) return 1;
+    if ( a->end < b->end ) return -1;
+    if ( a->end > b->end ) return 1;
+    return 0;
+}
+static void regions_merge(region_t *reg)
+{
+    int i = 0, j;
+    while ( i<reg->nregs )
+    {
+        j = i + 1;
+        while ( j<reg->nregs && reg->regs[i].end >= reg->regs[j].start )
+        {
+            if ( reg->regs[i].end < reg->regs[j].end ) reg->regs[i].end = reg->regs[j].end;
+            reg->regs[j].start = 1;  reg->regs[j].end = 0;  // if beg>end, this region marked for skipping
+            j++;
+        }
+        i = j;
+    }
+}
+void _regions_sort_and_merge(bcf_sr_regions_t *reg)
+{
+    if ( !reg ) return;
+
+    int i;
+    for (i=0; i<reg->nseqs; i++)
+    {
+        qsort(reg->regs[i].regs, reg->regs[i].nregs, sizeof(*reg->regs[i].regs), regions_cmp);
+        regions_merge(&reg->regs[i]);
+    }
+}
+
+// File name or a list of genomic locations. If file name, NULL is returned.
+// Recognises regions in the form chr, chr:pos, chr:beg-end, chr:beg-, {weird-chr-name}:pos.
+// Cannot use hts_parse_region() as that requires the header and if header is not present,
+// wouldn't learn the chromosome name.
+static bcf_sr_regions_t *_regions_init_string(const char *str)
+{
+    bcf_sr_regions_t *reg = bcf_sr_regions_alloc();
+    if ( !reg ) return NULL;
+
+    kstring_t tmp = {0,0,0};
+    const char *sp = str, *ep = str;
+    hts_pos_t from, to;
+    while ( 1 )
+    {
+        tmp.l = 0;
+        if ( *ep=='{' )
+        {
+            while ( *ep && *ep!='}' ) ep++;
+            if ( !*ep )
+            {
+                hts_log_error("Could not parse the region, mismatching braces in: \"%s\"", str);
+                goto exit_nicely;
+            }
+            ep++;
+            kputsn(sp+1,ep-sp-2,&tmp);
+        }
+        else
+        {
+            while ( *ep && *ep!=',' && *ep!=':' ) ep++;
+            kputsn(sp,ep-sp,&tmp);
+        }
+        if ( *ep==':' )
+        {
+            sp = ep+1;
+            from = hts_parse_decimal(sp,(char**)&ep,0);
+            if ( sp==ep )
+            {
+                hts_log_error("Could not parse the region(s): %s", str);
+                goto exit_nicely;
+            }
+            if ( !*ep || *ep==',' )
+            {
+                _regions_add(reg, tmp.s, from, from);
+                sp = ep;
+                continue;
+            }
+            if ( *ep!='-' )
+            {
+                hts_log_error("Could not parse the region(s): %s", str);
+                goto exit_nicely;
+            }
+            ep++;
+            sp = ep;
+            to = hts_parse_decimal(sp,(char**)&ep,0);
+            if ( *ep && *ep!=',' )
+            {
+                hts_log_error("Could not parse the region(s): %s", str);
+                goto exit_nicely;
+            }
+            if ( sp==ep ) to = MAX_CSI_COOR-1;
+            _regions_add(reg, tmp.s, from, to);
+            if ( !*ep ) break;
+            sp = ep;
+        }
+        else if ( !*ep || *ep==',' )
+        {
+            if ( tmp.l ) _regions_add(reg, tmp.s, -1, -1);
+            if ( !*ep ) break;
+            sp = ++ep;
+        }
+        else
+        {
+            hts_log_error("Could not parse the region(s): %s", str);
+            goto exit_nicely;
+        }
+    }
+    free(tmp.s);
+    return reg;
+
+exit_nicely:
+    bcf_sr_regions_destroy(reg);
+    free(tmp.s);
+    return NULL;
+}
+
+// ichr,ifrom,ito are 0-based;
+// returns -1 on error, 0 if the line is a comment line, 1 on success
+static int _regions_parse_line(char *line, int ichr, int ifrom, int ito, char **chr, char **chr_end, hts_pos_t *from, hts_pos_t *to)
+{
+    if (ifrom < 0 || ito < 0) return -1;
+    *chr_end = NULL;
+
+    if ( line[0]=='#' ) return 0;
+
+    int k,l;    // index of the start and end column of the tab-delimited file
+    if ( ifrom <= ito )
+        k = ifrom, l = ito;
+    else
+        l = ifrom, k = ito;
+
+    int i;
+    char *se = line, *ss = NULL; // start and end
+    char *tmp;
+    for (i=0; i<=k && *se; i++)
+    {
+        ss = i==0 ? se++ : ++se;
+        while (*se && *se!='\t') se++;
+    }
+    if ( i<=k ) return -1;
+    if ( k==l )
+    {
+        *from = *to = hts_parse_decimal(ss, &tmp, 0);
+        if ( tmp==ss || (*tmp && *tmp!='\t') ) return -1;
+    }
+    else
+    {
+        if ( k==ifrom )
+            *from = hts_parse_decimal(ss, &tmp, 0);
+        else
+            *to = hts_parse_decimal(ss, &tmp, 0);
+        if ( ss==tmp || (*tmp && *tmp!='\t') ) return -1;
+
+        for (i=k; i<l && *se; i++)
+        {
+            ss = ++se;
+            while (*se && *se!='\t') se++;
+        }
+        if ( i<l ) return -1;
+        if ( k==ifrom )
+            *to = hts_parse_decimal(ss, &tmp, 0);
+        else
+            *from = hts_parse_decimal(ss, &tmp, 0);
+        if ( ss==tmp || (*tmp && *tmp!='\t') ) return -1;
+    }
+
+    ss = se = line;
+    for (i=0; i<=ichr && *se; i++)
+    {
+        if ( i>0 ) ss = ++se;
+        while (*se && *se!='\t') se++;
+    }
+    if ( i<=ichr ) return -1;
+    *chr_end = se;
+    *chr = ss;
+    return 1;
+}
+
+bcf_sr_regions_t *bcf_sr_regions_init(const char *regions, int is_file, int ichr, int ifrom, int ito)
+{
+    bcf_sr_regions_t *reg;
+    if ( !is_file )
+    {
+        reg = _regions_init_string(regions);
+        _regions_sort_and_merge(reg);
+        return reg;
+    }
+
+    reg = bcf_sr_regions_alloc();
+    if ( !reg ) return NULL;
+
+    reg->file = hts_open(regions, "rb");
+    if ( !reg->file )
+    {
+        hts_log_error("Could not open file: %s", regions);
+        free(reg);
+        return NULL;
+    }
+
+    reg->tbx = tbx_index_load3(regions, NULL, HTS_IDX_SAVE_REMOTE|HTS_IDX_SILENT_FAIL);
+    if ( !reg->tbx )
+    {
+        size_t iline = 0;
+        int len = strlen(regions);
+        int is_bed  = strcasecmp(".bed",regions+len-4) ? 0 : 1;
+        if ( !is_bed && !strcasecmp(".bed.gz",regions+len-7) ) is_bed = 1;
+
+        if ( reg->file->format.format==vcf ) ito = 1;
+
+        // read the whole file, tabix index is not present
+        while ( hts_getline(reg->file, KS_SEP_LINE, &reg->line) > 0 )
+        {
+            iline++;
+            char *chr, *chr_end;
+            hts_pos_t from, to;
+            int ret;
+            ret = _regions_parse_line(reg->line.s, ichr,ifrom,abs(ito), &chr,&chr_end,&from,&to);
+            if ( ret < 0 )
+            {
+                if ( ito<0 )
+                    ret = _regions_parse_line(reg->line.s, ichr,ifrom,ifrom, &chr,&chr_end,&from,&to);
+                if ( ret<0 )
+                {
+                    hts_log_error("Could not parse %zu-th line of file %s, using the columns %d,%d[,%d]",
+                        iline, regions,ichr+1,ifrom+1,ito+1);
+                    hts_close(reg->file); reg->file = NULL; free(reg);
+                    return NULL;
+                }
+                ito = ifrom;
+            }
+            else if ( ito<0 )
+                ito = abs(ito);
+            if ( !ret ) continue;
+            if ( is_bed ) from++;
+            *chr_end = 0;
+            _regions_add(reg, chr, from, to);
+            *chr_end = '\t';
+        }
+        hts_close(reg->file); reg->file = NULL;
+        if ( !reg->nseqs ) { free(reg); return NULL; }
+        _regions_sort_and_merge(reg);
+        return reg;
+    }
+
+    reg->seq_names = (char**) tbx_seqnames(reg->tbx, &reg->nseqs);
+    if ( !reg->seq_hash )
+        reg->seq_hash = khash_str2int_init();
+    int i;
+    for (i=0; i<reg->nseqs; i++)
+    {
+        khash_str2int_set(reg->seq_hash,reg->seq_names[i],i);
+    }
+    reg->fname  = strdup(regions);
+    reg->is_bin = 1;
+    return reg;
+}
+
+void bcf_sr_regions_destroy(bcf_sr_regions_t *reg)
+{
+    int i;
+    free(reg->fname);
+    if ( reg->itr ) tbx_itr_destroy(reg->itr);
+    if ( reg->tbx ) tbx_destroy(reg->tbx);
+    if ( reg->file ) hts_close(reg->file);
+    if ( reg->als ) free(reg->als);
+    if ( reg->als_str.s ) free(reg->als_str.s);
+    free(reg->line.s);
+    if ( reg->regs )
+    {
+         // free only in-memory names, tbx names are const
+        for (i=0; i<reg->nseqs; i++)
+        {
+            free(reg->seq_names[i]);
+            free(reg->regs[i].regs);
+        }
+    }
+    free(reg->regs);
+    free(reg->seq_names);
+    khash_str2int_destroy(reg->seq_hash);
+    free(reg);
+}
+
+int bcf_sr_regions_seek(bcf_sr_regions_t *reg, const char *seq)
+{
+    reg->iseq = reg->start = reg->end = -1;
+    if ( khash_str2int_get(reg->seq_hash, seq, &reg->iseq) < 0 ) return -1;  // sequence seq not in regions
+
+    // using in-memory regions
+    if ( reg->regs )
+    {
+        reg->regs[reg->iseq].creg = -1;
+        return 0;
+    }
+
+    // reading regions from tabix
+    if ( reg->itr ) tbx_itr_destroy(reg->itr);
+    reg->itr = tbx_itr_querys(reg->tbx, seq);
+    if ( reg->itr ) return 0;
+
+    return -1;
+}
+
+// Returns 0 on success, -1 when done
+static int advance_creg(region_t *reg)
+{
+    int i = reg->creg + 1;
+    while ( i<reg->nregs && reg->regs[i].start > reg->regs[i].end ) i++;    // regions with start>end are marked to skip by merge_regions()
+    reg->creg = i;
+    if ( i>=reg->nregs ) return -1;
+    return 0;
+}
+
+int bcf_sr_regions_next(bcf_sr_regions_t *reg)
+{
+    if ( reg->iseq<0 ) return -1;
+    reg->start = reg->end = -1;
+    reg->nals = 0;
+
+    // using in-memory regions
+    if ( reg->regs )
+    {
+        while ( reg->iseq < reg->nseqs )
+        {
+            if ( advance_creg(&reg->regs[reg->iseq])==0 ) break;    // a valid record was found
+            reg->iseq++;
+        }
+        if ( reg->iseq >= reg->nseqs ) { reg->iseq = -1; return -1; } // no more regions left
+        region1_t *creg = &reg->regs[reg->iseq].regs[reg->regs[reg->iseq].creg];
+        reg->start = creg->start;
+        reg->end   = creg->end;
+        return 0;
+    }
+
+    // reading from tabix
+    char *chr, *chr_end;
+    int ichr = 0, ifrom = 1, ito = 2, is_bed = 0;
+    hts_pos_t from, to;
+    if ( reg->tbx )
+    {
+        ichr   = reg->tbx->conf.sc-1;
+        ifrom  = reg->tbx->conf.bc-1;
+        ito    = reg->tbx->conf.ec-1;
+        if ( ito<0 ) ito = ifrom;
+        is_bed = reg->tbx->conf.preset==TBX_UCSC ? 1 : 0;
+    }
+
+    int ret = 0;
+    while ( !ret )
+    {
+        if ( reg->itr )
+        {
+            // tabix index present, reading a chromosome block
+            ret = tbx_itr_next(reg->file, reg->tbx, reg->itr, &reg->line);
+            if ( ret<0 ) { reg->iseq = -1; return -1; }
+        }
+        else
+        {
+            if ( reg->is_bin )
+            {
+                // Waited for seek which never came. Reopen in text mode and stream
+                // through the regions, otherwise hts_getline would fail
+                hts_close(reg->file);
+                reg->file = hts_open(reg->fname, "r");
+                if ( !reg->file )
+                {
+                    hts_log_error("Could not open file: %s", reg->fname);
+                    reg->file = NULL;
+                    bcf_sr_regions_destroy(reg);
+                    return -1;
+                }
+                reg->is_bin = 0;
+            }
+
+            // tabix index absent, reading the whole file
+            ret = hts_getline(reg->file, KS_SEP_LINE, &reg->line);
+            if ( ret<0 ) { reg->iseq = -1; return -1; }
+        }
+        ret = _regions_parse_line(reg->line.s, ichr,ifrom,ito, &chr,&chr_end,&from,&to);
+        if ( ret<0 )
+        {
+            hts_log_error("Could not parse the file %s, using the columns %d,%d,%d",
+                reg->fname,ichr+1,ifrom+1,ito+1);
+            return -1;
+        }
+    }
+    if ( is_bed ) from++;
+
+    *chr_end = 0;
+    if ( khash_str2int_get(reg->seq_hash, chr, &reg->iseq)<0 )
+    {
+        hts_log_error("Broken tabix index? The sequence \"%s\" not in dictionary [%s]",
+            chr, reg->line.s);
+        exit(1);
+    }
+    *chr_end = '\t';
+
+    reg->start = from - 1;
+    reg->end   = to - 1;
+    return 0;
+}
+
+static int _regions_match_alleles(bcf_sr_regions_t *reg, int als_idx, bcf1_t *rec)
+{
+    if ( reg->regs )
+    {
+        // payload is not supported for in-memory regions, switch to regidx instead in future
+        hts_log_error("Compressed and indexed targets file is required");
+        exit(1);
+    }
+
+    int i = 0, max_len = 0;
+    if ( !reg->nals )
+    {
+        char *ss = reg->line.s;
+        while ( i<als_idx && *ss )
+        {
+            if ( *ss=='\t' ) i++;
+            ss++;
+        }
+        char *se = ss;
+        reg->nals = 1;
+        while ( *se && *se!='\t' )
+        {
+            if ( *se==',' ) reg->nals++;
+            se++;
+        }
+        ks_resize(&reg->als_str, se-ss+1+reg->nals);
+        reg->als_str.l = 0;
+        hts_expand(char*,reg->nals,reg->mals,reg->als);
+        reg->nals = 0;
+
+        se = ss;
+        while ( *(++se) )
+        {
+            if ( *se=='\t' ) break;
+            if ( *se!=',' ) continue;
+            reg->als[reg->nals] = &reg->als_str.s[reg->als_str.l];
+            kputsn(ss,se-ss,&reg->als_str);
+            if ( &reg->als_str.s[reg->als_str.l] - reg->als[reg->nals] > max_len ) max_len = &reg->als_str.s[reg->als_str.l] - reg->als[reg->nals];
+            reg->als_str.l++;
+            reg->nals++;
+            ss = ++se;
+        }
+        reg->als[reg->nals] = &reg->als_str.s[reg->als_str.l];
+        kputsn(ss,se-ss,&reg->als_str);
+        if ( &reg->als_str.s[reg->als_str.l] - reg->als[reg->nals] > max_len ) max_len = &reg->als_str.s[reg->als_str.l] - reg->als[reg->nals];
+        reg->nals++;
+        reg->als_type = max_len > 1 ? VCF_INDEL : VCF_SNP;  // this is a simplified check, see vcf.c:bcf_set_variant_types
+    }
+    int type = bcf_get_variant_types(rec);
+    if ( reg->als_type & VCF_INDEL )
+        return type & VCF_INDEL ? 1 : 0;
+    return !(type & VCF_INDEL) ? 1 : 0;
+}
+
+int bcf_sr_regions_overlap(bcf_sr_regions_t *reg, const char *seq, hts_pos_t start, hts_pos_t end)
+{
+    return _bcf_sr_regions_overlap(reg,seq,start,end,1);
+}
+
+static int _bcf_sr_regions_overlap(bcf_sr_regions_t *reg, const char *seq, hts_pos_t start, hts_pos_t end, int missed_reg_handler)
+{
+    int iseq;
+    if ( khash_str2int_get(reg->seq_hash, seq, &iseq)<0 ) return -1;    // no such sequence
+    if ( missed_reg_handler && !reg->missed_reg_handler ) missed_reg_handler = 0;
+
+    if ( reg->prev_seq==-1 || iseq!=reg->prev_seq || reg->prev_start > start ) // new chromosome or after a seek
+    {
+        // flush regions left on previous chromosome
+        if ( missed_reg_handler && reg->prev_seq!=-1 && reg->iseq!=-1 )
+            bcf_sr_regions_flush(reg);
+
+        bcf_sr_regions_seek(reg, seq);
+        reg->start = reg->end = -1;
+    }
+    if ( reg->prev_seq==iseq && reg->iseq!=iseq ) return -2;    // no more regions on this chromosome
+    reg->prev_seq = reg->iseq;
+    reg->prev_start = start;
+
+    while ( iseq==reg->iseq && reg->end < start )
+    {
+        if ( bcf_sr_regions_next(reg) < 0 ) return -2;  // no more regions left
+        if ( reg->iseq != iseq ) return -1; // does not overlap any regions
+        if ( missed_reg_handler && reg->end < start ) reg->missed_reg_handler(reg, reg->missed_reg_data);
+    }
+    if ( reg->start <= end ) return 0;    // region overlap
+    return -1;  // no overlap
+}
+
+int bcf_sr_regions_flush(bcf_sr_regions_t *reg)
+{
+    if ( !reg->missed_reg_handler || reg->prev_seq==-1 ) return 0;
+    while ( !bcf_sr_regions_next(reg) ) reg->missed_reg_handler(reg, reg->missed_reg_data);
+    return 0; // FIXME: check for errs in this function
+}
+
diff --git a/ext/htslib/tabix.1 b/ext/htslib/tabix.1
new file mode 100644
index 0000000..f0dc7b5
--- /dev/null
+++ b/ext/htslib/tabix.1
@@ -0,0 +1,209 @@
+.TH tabix 1 "12 September 2024" "htslib-1.21" "Bioinformatics tools"
+.SH NAME
+.PP
+tabix \- Generic indexer for TAB-delimited genome position files
+.\"
+.\" Copyright (C) 2009-2011 Broad Institute.
+.\" Copyright (C) 2014, 2016, 2018, 2020, 2022, 2024 Genome Research Ltd.
+.\"
+.\" Author: Heng Li <lh3@sanger.ac.uk>
+.\"
+.\" Permission is hereby granted, free of charge, to any person obtaining a
+.\" copy of this software and associated documentation files (the "Software"),
+.\" to deal in the Software without restriction, including without limitation
+.\" the rights to use, copy, modify, merge, publish, distribute, sublicense,
+.\" and/or sell copies of the Software, and to permit persons to whom the
+.\" Software is furnished to do so, subject to the following conditions:
+.\"
+.\" The above copyright notice and this permission notice shall be included in
+.\" all copies or substantial portions of the Software.
+.\"
+.\" THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+.\" IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+.\" FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+.\" THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+.\" LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+.\" FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+.\" DEALINGS IN THE SOFTWARE.
+.\"
+.SH SYNOPSIS
+.PP
+.B tabix
+.RB [ -0lf ]
+.RB [ -p
+gff|bed|sam|vcf]
+.RB [ -s
+.IR seqCol ]
+.RB [ -b
+.IR begCol ]
+.RB [ -e
+.IR endCol ]
+.RB [ -S
+.IR lineSkip ]
+.RB [ -c
+.IR metaChar ]
+.I in.tab.bgz
+.RI [ "region1 " [ "region2 " [ ... "]]]"
+
+.SH DESCRIPTION
+.PP
+Tabix indexes a TAB-delimited genome position file
+.I in.tab.bgz
+and creates an index file
+.RI ( in.tab.bgz.tbi
+or 
+.IR in.tab.bgz.csi )
+when
+.I region
+is absent from the command-line. The input data file must be position
+sorted and compressed by
+.B bgzip
+which has a
+.BR gzip (1)
+like interface.
+
+After indexing, tabix is able to quickly retrieve data lines overlapping
+.I regions
+specified in the format "chr:beginPos-endPos".
+(Coordinates specified in this region format are 1-based and inclusive.)
+
+Fast data retrieval also
+works over network if URI is given as a file name and in this case the
+index file will be downloaded if it is not present locally.
+
+The tabix
+.RI ( .tbi )
+and BAI index formats can handle individual chromosomes up to 512 Mbp
+(2^29 bases) in length.
+If your input file might contain data lines with begin or end positions
+greater than that, you will need to use a CSI index.
+
+Multiple threads can be used for operations except listing of sequence names.
+
+.SH INDEXING OPTIONS
+.TP 10
+.B -0, --zero-based
+Specify that the position in the data file is 0-based half-open
+(e.g. UCSC files) rather than 1-based.
+.TP
+.BI "-b, --begin " INT
+Column of start chromosomal position. [4]
+.TP
+.BI "-c, --comment " CHAR
+Skip lines started with character CHAR. [#]
+.TP
+.BI "-C, --csi"
+Produce CSI format index instead of classical tabix or BAI style indices.
+.TP
+.BI "-e, --end " INT
+Column of end chromosomal position. The end column can be the same as the
+start column. [5]
+.TP
+.B "-f, --force "
+Force to overwrite the index file if it is present.
+.TP
+.BI "-m, --min-shift " INT
+Set minimal interval size for CSI indices to 2^INT [14]
+.TP
+.BI "-p, --preset " STR
+Input format for indexing. Valid values are: gff, bed, sam, vcf.
+This option should not be applied together with any of
+.BR -s ", " -b ", " -e ", " -c " and " -0 ;
+it is not used for data retrieval because this setting is stored in
+the index file. [gff]
+.TP
+.BI "-s, --sequence " INT
+Column of sequence name. Option
+.BR -s ", " -b ", " -e ", " -S ", " -c " and " -0
+are all stored in the index file and thus not used in data retrieval. [1]
+.TP
+.BI "-S, --skip-lines " INT
+Skip first INT lines in the data file. [0]
+
+.SH QUERYING AND OTHER OPTIONS
+.TP
+.B "-h, --print-header "
+Print also the header/meta lines.
+.TP
+.B "-H, --only-header "
+Print only the header/meta lines.
+.TP
+.B "-l, --list-chroms "
+List the sequence names stored in the index file.
+.TP
+.BI "-r, --reheader " FILE
+Replace the header with the content of FILE
+.TP
+.BI "-R, --regions " FILE
+Restrict to regions listed in the FILE. The FILE can be BED file (requires .bed, .bed.gz, .bed.bgz 
+file name extension) or a TAB-delimited file with CHROM, POS, and,  optionally,
+POS_TO columns, where positions are 1-based and inclusive.  When this option is in use, the input
+file may not be sorted. 
+.TP
+.BI "-T, --targets " FILE
+Similar to 
+.B -R
+but the entire input will be read sequentially and regions not listed in FILE will be skipped.
+.TP
+.BI "-D "
+Do not download the index file before opening it. Valid for remote files only.
+.TP
+.BI "--cache " INT
+Set the BGZF block cache size to INT megabytes. [10]
+
+This is of most benefit when the
+.B -R
+option is used, which can cause blocks to be read more than once.
+Setting the size to 0 will disable the cache.
+.TP
+.B --separate-regions
+This option can be used when multiple regions are supplied in the command line
+and the user needs to quickly see which file records belong to which region.
+For this, a line with the name of the region, preceded by the file specific
+comment symbol, is inserted in the output before its corresponding group of
+records.
+.TP
+.BI "--verbosity " INT
+Set verbosity of logging messages printed to stderr.
+The default is 3, which turns on error and warning messages;
+2 reduces warning messages;
+1 prints only error messages and 0 is mostly silent.
+Values higher than 3 produce additional informational and debugging messages.
+.TP
+.BI "-@, --threads " INT
+Set number of threads to use for the operation.
+The default is 0, where no extra threads are in use.
+.PP
+.SH EXAMPLE
+(grep "^#" in.gff; grep -v "^#" in.gff | sort -t"`printf '\(rst'`" -k1,1 -k4,4n) | bgzip > sorted.gff.gz;
+
+tabix -p gff sorted.gff.gz;
+
+tabix sorted.gff.gz chr1:10,000,000-20,000,000;
+
+.SH NOTES
+It is straightforward to achieve overlap queries using the standard
+B-tree index (with or without binning) implemented in all SQL databases,
+or the R-tree index in PostgreSQL and Oracle. But there are still many
+reasons to use tabix. Firstly, tabix directly works with a lot of widely
+used TAB-delimited formats such as GFF/GTF and BED. We do not need to
+design database schema or specialized binary formats. Data do not need
+to be duplicated in different formats, either. Secondly, tabix works on
+compressed data files while most SQL databases do not. The GenCode
+annotation GTF can be compressed down to 4%.  Thirdly, tabix is
+fast. The same indexing algorithm is known to work efficiently for an
+alignment with a few billion short reads. SQL databases probably cannot
+easily handle data at this scale. Last but not the least, tabix supports
+remote data retrieval. One can put the data file and the index at an FTP
+or HTTP server, and other users or even web services will be able to get
+a slice without downloading the entire file.
+
+.SH AUTHOR
+.PP
+Tabix was written by Heng Li. The BGZF library was originally
+implemented by Bob Handsaker and modified by Heng Li for remote file
+access and in-memory caching.
+
+.SH SEE ALSO
+.IR bgzip (1),
+.IR samtools (1)
diff --git a/ext/htslib/tabix.c b/ext/htslib/tabix.c
new file mode 100644
index 0000000..2fb5d4b
--- /dev/null
+++ b/ext/htslib/tabix.c
@@ -0,0 +1,848 @@
+/*  tabix.c -- Generic indexer for TAB-delimited genome position files.
+
+    Copyright (C) 2009-2011 Broad Institute.
+    Copyright (C) 2010-2012, 2014-2020, 2024 Genome Research Ltd.
+
+    Author: Heng Li <lh3@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#include <config.h>
+
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <strings.h>
+#include <getopt.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <errno.h>
+#include "htslib/tbx.h"
+#include "htslib/sam.h"
+#include "htslib/vcf.h"
+#include "htslib/kseq.h"
+#include "htslib/bgzf.h"
+#include "htslib/hts.h"
+#include "htslib/regidx.h"
+#include "htslib/hts_defs.h"
+#include "htslib/hts_log.h"
+#include "htslib/thread_pool.h"
+
+//for easy coding
+#define RELEASE_TPOOL(X) { hts_tpool *ptr = (hts_tpool*)(X); if (ptr) { hts_tpool_destroy(ptr); } }
+#define bam_index_build3(fn, min_shift, nthreads) (sam_index_build3((fn), NULL, (min_shift), (nthreads)))
+
+typedef struct
+{
+    char *regions_fname, *targets_fname;
+    int print_header, header_only, cache_megs, download_index, separate_regs, threads;
+}
+args_t;
+
+static void HTS_FORMAT(HTS_PRINTF_FMT, 1, 2) HTS_NORETURN
+error(const char *format, ...)
+{
+    va_list ap;
+    fflush(stdout);
+    va_start(ap, format);
+    vfprintf(stderr, format, ap);
+    va_end(ap);
+    fflush(stderr);
+    exit(EXIT_FAILURE);
+}
+
+static void HTS_FORMAT(HTS_PRINTF_FMT, 1, 2) HTS_NORETURN
+error_errno(const char *format, ...)
+{
+    va_list ap;
+    int eno = errno;
+    fflush(stdout);
+    if (format) {
+        va_start(ap, format);
+        vfprintf(stderr, format, ap);
+        va_end(ap);
+    }
+    if (eno) {
+        fprintf(stderr, "%s%s\n", format ? ": " : "", strerror(eno));
+    } else {
+        fprintf(stderr, "\n");
+    }
+    fflush(stderr);
+    exit(EXIT_FAILURE);
+}
+
+
+#define IS_GFF  (1<<0)
+#define IS_BED  (1<<1)
+#define IS_SAM  (1<<2)
+#define IS_VCF  (1<<3)
+#define IS_BCF  (1<<4)
+#define IS_BAM  (1<<5)
+#define IS_CRAM (1<<6)
+#define IS_GAF  (1<<7)
+#define IS_TXT  (IS_GFF|IS_BED|IS_SAM|IS_VCF)
+
+int file_type(const char *fname)
+{
+    int l = strlen(fname);
+    if (l>=7 && strcasecmp(fname+l-7, ".gff.gz") == 0) return IS_GFF;
+    else if (l>=7 && strcasecmp(fname+l-7, ".bed.gz") == 0) return IS_BED;
+    else if (l>=7 && strcasecmp(fname+l-7, ".sam.gz") == 0) return IS_SAM;
+    else if (l>=7 && strcasecmp(fname+l-7, ".vcf.gz") == 0) return IS_VCF;
+    else if (l>=4 && strcasecmp(fname+l-4, ".bcf") == 0) return IS_BCF;
+    else if (l>=4 && strcasecmp(fname+l-4, ".bam") == 0) return IS_BAM;
+    else if (l>=4 && strcasecmp(fname+l-5, ".cram") == 0) return IS_CRAM;
+    else if (l>=7 && strcasecmp(fname+l-7, ".gaf.gz") == 0) return IS_GAF;
+
+    htsFile *fp = hts_open(fname,"r");
+    if (!fp) {
+        if (errno == ENOEXEC) {
+            // hts_open() uses this to report that it didn't understand the
+            // file format.
+            error("Couldn't understand format of \"%s\"\n", fname);
+        } else {
+            error_errno("Couldn't open \"%s\"", fname);
+        }
+    }
+    enum htsExactFormat format = hts_get_format(fp)->format;
+    hts_close(fp);
+    if ( format == bcf ) return IS_BCF;
+    if ( format == bam ) return IS_BAM;
+    if ( format == cram ) return IS_CRAM;
+    if ( format == vcf ) return IS_VCF;
+
+    return 0;
+}
+
+static char **parse_regions(char *regions_fname, char **argv, int argc, int *nregs)
+{
+    kstring_t str = {0,0,0};
+    int iseq = 0, ireg = 0;
+    char **regs = NULL;
+    *nregs = argc;
+
+    if ( regions_fname )
+    {
+        // improve me: this is a too heavy machinery for parsing regions...
+
+        regidx_t *idx = regidx_init(regions_fname, NULL, NULL, 0, NULL);
+        if ( !idx ) {
+            error_errno("Could not build region list for \"%s\"", regions_fname);
+        }
+        regitr_t *itr = regitr_init(idx);
+        if ( !itr ) {
+            error_errno("Could not initialize an iterator over \"%s\"",
+                        regions_fname);
+        }
+
+        (*nregs) += regidx_nregs(idx);
+        regs = (char**) malloc(sizeof(char*)*(*nregs));
+        if (!regs) error_errno(NULL);
+
+        int nseq;
+        char **seqs = regidx_seq_names(idx, &nseq);
+        for (iseq=0; iseq<nseq; iseq++)
+        {
+            if (regidx_overlap(idx, seqs[iseq], 0, HTS_POS_MAX, itr) < 0)
+                error_errno("Failed to build overlapping regions list");
+
+            while ( regitr_overlap(itr) )
+            {
+                str.l = 0;
+                if (ksprintf(&str, "%s:%"PRIhts_pos"-%"PRIhts_pos, seqs[iseq], itr->beg+1, itr->end+1) < 0) {
+                    error_errno(NULL);
+                }
+                regs[ireg] = strdup(str.s);
+                if (!regs[ireg]) error_errno(NULL);
+                ireg++;
+            }
+        }
+        regidx_destroy(idx);
+        regitr_destroy(itr);
+    }
+    free(str.s);
+
+    if ( !ireg )
+    {
+        if ( argc )
+        {
+            regs = (char**) malloc(sizeof(char*)*argc);
+            if (!regs) error_errno(NULL);
+        }
+        else
+        {
+            regs = (char**) malloc(sizeof(char*));
+            if (!regs) error_errno(NULL);
+            regs[0] = strdup(".");
+            if (!regs[0]) error_errno(NULL);
+            *nregs = 1;
+        }
+    }
+
+    for (iseq=0; iseq<argc; iseq++, ireg++) {
+        regs[ireg] = strdup(argv[iseq]);
+        if (!regs[ireg]) error_errno(NULL);
+    }
+    return regs;
+}
+static int query_regions(args_t *args, tbx_conf_t *conf, char *fname, char **regs, int nregs)
+{
+    int i;
+    htsThreadPool tpool = {NULL, 0};
+    htsFile *fp = hts_open(fname,"r");
+    if ( !fp ) error_errno("Could not open \"%s\"", fname);
+    enum htsExactFormat format = hts_get_format(fp)->format;
+    if (args->cache_megs)
+        hts_set_cache_size(fp, args->cache_megs * 1048576);
+
+    //set threads if needed, errors are logged and ignored
+    if (args->threads >= 1) {
+        if (!(tpool.pool = hts_tpool_init(args->threads))) {
+            hts_log_info("Could not initialize thread pool!");
+        }
+        if (hts_set_thread_pool(fp, &tpool) < 0) {
+            hts_log_info("Could not set thread pool!");
+        }
+    }
+
+    regidx_t *reg_idx = NULL;
+    if ( args->targets_fname )
+    {
+        reg_idx = regidx_init(args->targets_fname, NULL, NULL, 0, NULL);
+        if (!reg_idx) {
+            RELEASE_TPOOL(tpool.pool);
+            error_errno("Could not build region list for \"%s\"",
+                        args->targets_fname);
+        }
+    }
+
+    if ( format == bcf )
+    {
+        htsFile *out = hts_open("-","w");
+        if ( !out ) {
+            RELEASE_TPOOL(tpool.pool);
+            error_errno("Could not open stdout");
+        }
+        if (hts_set_thread_pool(out, &tpool) < 0) {
+            hts_log_info("Could not set thread pool to output file!");
+        }
+        hts_idx_t *idx = bcf_index_load3(fname, NULL, args->download_index ? HTS_IDX_SAVE_REMOTE : 0);
+        if ( !idx ) {
+            RELEASE_TPOOL(tpool.pool);
+            error_errno("Could not load .csi index of \"%s\"", fname);
+        }
+
+        bcf_hdr_t *hdr = bcf_hdr_read(fp);
+        if ( !hdr ) {
+            RELEASE_TPOOL(tpool.pool);
+            error_errno("Could not read the header from \"%s\"", fname);
+        }
+
+        if ( args->print_header ) {
+            if ( bcf_hdr_write(out,hdr)!=0 ) {
+                RELEASE_TPOOL(tpool.pool);
+                error_errno("Failed to write to stdout");
+            }
+        }
+        if ( !args->header_only )
+        {
+            assert(regs != NULL);
+            bcf1_t *rec = bcf_init();
+            if (!rec) {
+                RELEASE_TPOOL(tpool.pool);
+                error_errno(NULL);
+            }
+            for (i=0; i<nregs; i++)
+            {
+                int ret, found = 0;
+                hts_itr_t *itr = bcf_itr_querys(idx,hdr,regs[i]);
+                if (!itr) continue;
+                while ((ret = bcf_itr_next(fp, itr, rec)) >=0 )
+                {
+                    if ( reg_idx )
+                    {
+                        const char *chr = bcf_seqname(hdr,rec);
+                        if (!chr) {
+                            RELEASE_TPOOL(tpool.pool);
+                            error("Bad BCF record in \"%s\" : "
+                                  "Invalid CONTIG id %d\n",
+                                  fname, rec->rid);
+                        }
+                        if ( !regidx_overlap(reg_idx,chr,rec->pos,rec->pos+rec->rlen-1, NULL) ) continue;
+                    }
+                    if (!found) {
+                        if (args->separate_regs) printf("%c%s\n", conf->meta_char, regs[i]);
+                        found = 1;
+                    }
+                    if ( bcf_write(out,hdr,rec)!=0 ) {
+                        RELEASE_TPOOL(tpool.pool);
+                        error_errno("Failed to write to stdout");
+                    }
+                }
+
+                if (ret < -1) {
+                    RELEASE_TPOOL(tpool.pool);
+                    error_errno("Reading \"%s\" failed", fname);
+                }
+                bcf_itr_destroy(itr);
+            }
+            bcf_destroy(rec);
+        }
+        if ( hts_close(out) ) {
+            RELEASE_TPOOL(tpool.pool);
+            error_errno("hts_close returned non-zero status for stdout");
+        }
+
+        bcf_hdr_destroy(hdr);
+        hts_idx_destroy(idx);
+    }
+    else if ( format==vcf || format==sam || format==bed || format==text_format || format==unknown_format )
+    {
+        tbx_t *tbx = tbx_index_load3(fname, NULL, args->download_index ? HTS_IDX_SAVE_REMOTE : 0);
+        if ( !tbx ) {
+            RELEASE_TPOOL(tpool.pool);
+            error_errno("Could not load .tbi/.csi index of %s", fname);
+        }
+        kstring_t str = {0,0,0};
+        if ( args->print_header )
+        {
+            int ret;
+            while ((ret = hts_getline(fp, KS_SEP_LINE, &str)) >= 0)
+            {
+                if ( !str.l || str.s[0]!=tbx->conf.meta_char ) break;
+                if (puts(str.s) < 0) {
+                    RELEASE_TPOOL(tpool.pool);
+                    error_errno("Error writing to stdout");
+                }
+            }
+            if (ret < -1) {
+                RELEASE_TPOOL(tpool.pool);
+                error_errno("Reading \"%s\" failed", fname);
+            }
+        }
+        if ( !args->header_only )
+        {
+            int nseq;
+            const char **seq = NULL;
+            if ( reg_idx ) {
+                seq = tbx_seqnames(tbx, &nseq);
+                if (!seq) {
+                    RELEASE_TPOOL(tpool.pool);
+                    error_errno("Failed to get sequence names list");
+                }
+            }
+            for (i=0; i<nregs; i++)
+            {
+                int ret, found = 0;
+                hts_itr_t *itr = tbx_itr_querys(tbx, regs[i]);
+                if ( !itr ) continue;
+                while ((ret = tbx_itr_next(fp, tbx, itr, &str)) >= 0)
+                {
+                    if ( reg_idx && !regidx_overlap(reg_idx,seq[itr->curr_tid],itr->curr_beg,itr->curr_end-1, NULL) ) continue;
+                    if (!found) {
+                        if (args->separate_regs) printf("%c%s\n", conf->meta_char, regs[i]);
+                        found = 1;
+                    }
+                    if (puts(str.s) < 0) {
+                        RELEASE_TPOOL(tpool.pool);
+                        error_errno("Failed to write to stdout");
+                    }
+                }
+                if (ret < -1) {
+                    RELEASE_TPOOL(tpool.pool);
+                    error_errno("Reading \"%s\" failed", fname);
+                }
+                tbx_itr_destroy(itr);
+            }
+            free(seq);
+        }
+        free(str.s);
+        tbx_destroy(tbx);
+    }
+    else if ( format==bam ) {
+        RELEASE_TPOOL(tpool.pool);
+        error("Please use \"samtools view\" for querying BAM files.\n");
+    }
+
+    if ( reg_idx ) regidx_destroy(reg_idx);
+    if ( hts_close(fp) ) {
+        RELEASE_TPOOL(tpool.pool);
+        error_errno("hts_close returned non-zero status: %s", fname);
+    }
+
+    for (i=0; i<nregs; i++) free(regs[i]);
+    free(regs);
+    RELEASE_TPOOL(tpool.pool);
+    return 0;
+}
+static int query_chroms(char *fname, int download)
+{
+    const char **seq;
+    int i, nseq, ftype = file_type(fname);
+    if ( ftype & IS_TXT || !ftype )
+    {
+        tbx_t *tbx = tbx_index_load3(fname, NULL, download ? HTS_IDX_SAVE_REMOTE : 0);
+        if ( !tbx ) error_errno("Could not load .tbi index of %s", fname);
+        seq = tbx_seqnames(tbx, &nseq);
+        if (!seq) error_errno("Couldn't get list of sequence names");
+        for (i=0; i<nseq; i++) {
+            if (printf("%s\n", seq[i]) < 0)
+                error_errno("Couldn't write to stdout");
+        }
+        free(seq);
+        tbx_destroy(tbx);
+    }
+    else if ( ftype==IS_BCF )
+    {
+        htsFile *fp = hts_open(fname,"r");
+        if ( !fp ) error_errno("Could not open \"%s\"", fname);
+        bcf_hdr_t *hdr = bcf_hdr_read(fp);
+        if ( !hdr ) error_errno("Could not read the header: \"%s\"", fname);
+        hts_close(fp);
+        hts_idx_t *idx = bcf_index_load3(fname, NULL, download ? HTS_IDX_SAVE_REMOTE : 0);
+        if ( !idx ) error_errno("Could not load .csi index of \"%s\"", fname);
+        seq = bcf_index_seqnames(idx, hdr, &nseq);
+        if (!seq) error_errno("Couldn't get list of sequence names");
+        for (i=0; i<nseq; i++) {
+            if (printf("%s\n", seq[i]) < 0)
+                error_errno("Couldn't write to stdout");
+        }
+        free(seq);
+        bcf_hdr_destroy(hdr);
+        hts_idx_destroy(idx);
+    }
+    else if ( ftype==IS_BAM )   // todo: BAM
+        error("BAM: todo\n");
+    return 0;
+}
+
+int reheader_file(const char *fname, const char *header, int ftype, tbx_conf_t *conf, int threads)
+{
+    hts_tpool *tpool = NULL;
+    if (threads >= 1) {
+        if (!(tpool = hts_tpool_init(threads))) {
+            hts_log_info("Could not initialize thread pool!");
+        }
+    }
+    if ( ftype & IS_TXT || !ftype )
+    {
+        BGZF *fp = bgzf_open(fname,"r");
+        if (!fp) {
+            RELEASE_TPOOL(tpool);
+            return -1;
+        }
+        if (bgzf_thread_pool(fp, tpool, 0) < 0) {
+            hts_log_info("Could not set thread pool!");
+        }
+        if (bgzf_read_block(fp) != 0 || !fp->block_length ) {
+            RELEASE_TPOOL(tpool);
+            return -1;
+        }
+
+        char *buffer = fp->uncompressed_block;
+        int skip_until = 0;
+
+        // Skip the header: find out the position of the data block
+        if ( buffer[0]==conf->meta_char )
+        {
+            skip_until = 1;
+            while (1)
+            {
+                if ( buffer[skip_until]=='\n' )
+                {
+                    skip_until++;
+                    if ( skip_until>=fp->block_length )
+                    {
+                        if ( bgzf_read_block(fp) != 0 || !fp->block_length ) {
+                            RELEASE_TPOOL(tpool);
+                            error("FIXME: No body in the file: %s\n", fname);
+                        }
+                        skip_until = 0;
+                    }
+                    // The header has finished
+                    if ( buffer[skip_until]!=conf->meta_char ) break;
+                }
+                skip_until++;
+                if ( skip_until>=fp->block_length )
+                {
+                    if (bgzf_read_block(fp) != 0 || !fp->block_length) {
+                        RELEASE_TPOOL(tpool);
+                        error("FIXME: No body in the file: %s\n", fname);
+                    }
+                    skip_until = 0;
+                }
+            }
+        }
+
+        // Output the new header
+        FILE *hdr  = fopen(header,"r");
+        if ( !hdr ) {
+            RELEASE_TPOOL(tpool);
+            error("%s: %s", header,strerror(errno));
+        }
+        const size_t page_size = 32768;
+        char *buf = malloc(page_size);
+        BGZF *bgzf_out = bgzf_open("-", "w");
+        ssize_t nread;
+
+        if (!buf) {
+            RELEASE_TPOOL(tpool);
+            error("%s\n", strerror(errno));
+        }
+        if (!bgzf_out) {
+            RELEASE_TPOOL(tpool);
+            error_errno("Couldn't open output stream");
+        }
+        if (bgzf_thread_pool(bgzf_out, tpool, 0) < 0) {
+            hts_log_info("Could not set thread pool to output file!");
+        }
+        while ( (nread=fread(buf,1,page_size-1,hdr))>0 )
+        {
+            if ( nread<page_size-1 && buf[nread-1]!='\n' ) buf[nread++] = '\n';
+            if (bgzf_write(bgzf_out, buf, nread) < 0) {
+                RELEASE_TPOOL(tpool);
+                error_errno("Write error %d", bgzf_out->errcode);
+            }
+        }
+        if ( ferror(hdr) ) {
+            RELEASE_TPOOL(tpool);
+            error_errno("Failed to read \"%s\"", header);
+        }
+        if ( fclose(hdr) ) {
+            RELEASE_TPOOL(tpool);
+            error_errno("Closing \"%s\" failed", header);
+        }
+
+        // Output all remaining data read with the header block
+        if ( fp->block_length - skip_until > 0 )
+        {
+            if (bgzf_write(bgzf_out, buffer+skip_until, fp->block_length-skip_until) < 0) {
+                RELEASE_TPOOL(tpool);
+                error_errno("Write error %d",fp->errcode);
+            }
+        }
+        if (bgzf_flush(bgzf_out) < 0) {
+            RELEASE_TPOOL(tpool);
+            error_errno("Write error %d", bgzf_out->errcode);
+        }
+
+        while (1)
+        {
+            nread = bgzf_raw_read(fp, buf, page_size);
+            if ( nread<=0 ) break;
+
+            int count = bgzf_raw_write(bgzf_out, buf, nread);
+            if (count != nread) {
+                RELEASE_TPOOL(tpool);
+                error_errno("Write failed, wrote %d instead of %d bytes", count,(int)nread);
+            }
+        }
+        if (nread < 0) {
+            RELEASE_TPOOL(tpool);
+            error_errno("Error reading \"%s\"", fname);
+        }
+        if (bgzf_close(bgzf_out) < 0) {
+            RELEASE_TPOOL(tpool);
+            error_errno("Error %d closing output", bgzf_out->errcode);
+        }
+        if (bgzf_close(fp) < 0) {
+            RELEASE_TPOOL(tpool);
+            error_errno("Error %d closing \"%s\"", bgzf_out->errcode, fname);
+        }
+        free(buf);
+    }
+    else {
+        RELEASE_TPOOL(tpool);
+        error("todo: reheader BCF, BAM\n");  // BCF is difficult, records contain pointers to the header.
+    }
+    RELEASE_TPOOL(tpool);
+    return 0;
+}
+
+static int usage(FILE *fp, int status)
+{
+    fprintf(fp, "\n");
+    fprintf(fp, "Version: %s\n", hts_version());
+    fprintf(fp, "Usage:   tabix [OPTIONS] [FILE] [REGION [...]]\n");
+    fprintf(fp, "\n");
+    fprintf(fp, "Indexing Options:\n");
+    fprintf(fp, "   -0, --zero-based           coordinates are zero-based\n");
+    fprintf(fp, "   -b, --begin INT            column number for region start [4]\n");
+    fprintf(fp, "   -c, --comment CHAR         skip comment lines starting with CHAR [null]\n");
+    fprintf(fp, "   -C, --csi                  generate CSI index for VCF (default is TBI)\n");
+    fprintf(fp, "   -e, --end INT              column number for region end (if no end, set INT to -b) [5]\n");
+    fprintf(fp, "   -f, --force                overwrite existing index without asking\n");
+    fprintf(fp, "   -m, --min-shift INT        set minimal interval size for CSI indices to 2^INT [14]\n");
+    fprintf(fp, "   -p, --preset STR           gff, bed, sam, vcf, gaf\n");
+    fprintf(fp, "   -s, --sequence INT         column number for sequence names (suppressed by -p) [1]\n");
+    fprintf(fp, "   -S, --skip-lines INT       skip first INT lines [0]\n");
+    fprintf(fp, "\n");
+    fprintf(fp, "Querying and other options:\n");
+    fprintf(fp, "   -h, --print-header         print also the header lines\n");
+    fprintf(fp, "   -H, --only-header          print only the header lines\n");
+    fprintf(fp, "   -l, --list-chroms          list chromosome names\n");
+    fprintf(fp, "   -r, --reheader FILE        replace the header with the content of FILE\n");
+    fprintf(fp, "   -R, --regions FILE         restrict to regions listed in the file\n");
+    fprintf(fp, "   -T, --targets FILE         similar to -R but streams rather than index-jumps\n");
+    fprintf(fp, "   -D                         do not download the index file\n");
+    fprintf(fp, "       --cache INT            set cache size to INT megabytes (0 disables) [10]\n");
+    fprintf(fp, "       --separate-regions     separate the output by corresponding regions\n");
+    fprintf(fp, "       --verbosity INT        set verbosity [3]\n");
+    fprintf(fp, "   -@, --threads INT          number of additional threads to use [0]\n");
+    fprintf(fp, "\n");
+    return status;
+}
+
+int main(int argc, char *argv[])
+{
+    int c, detect = 1, min_shift = 0, is_force = 0, list_chroms = 0, do_csi = 0;
+    tbx_conf_t conf = tbx_conf_gff;
+    char *reheader = NULL;
+    args_t args;
+    memset(&args,0,sizeof(args_t));
+    args.cache_megs = 10;
+    args.download_index = 1;
+    int32_t new_line_skip = -1;
+
+    static const struct option loptions[] =
+    {
+        {"help", no_argument, NULL, 2},
+        {"regions", required_argument, NULL, 'R'},
+        {"targets", required_argument, NULL, 'T'},
+        {"csi", no_argument, NULL, 'C'},
+        {"zero-based", no_argument, NULL, '0'},
+        {"print-header", no_argument, NULL, 'h'},
+        {"only-header", no_argument, NULL, 'H'},
+        {"begin", required_argument, NULL, 'b'},
+        {"comment", required_argument, NULL, 'c'},
+        {"end", required_argument, NULL, 'e'},
+        {"force", no_argument, NULL, 'f'},
+        {"min-shift", required_argument, NULL, 'm'},
+        {"preset", required_argument, NULL, 'p'},
+        {"sequence", required_argument, NULL, 's'},
+        {"skip-lines", required_argument, NULL, 'S'},
+        {"list-chroms", no_argument, NULL, 'l'},
+        {"reheader", required_argument, NULL, 'r'},
+        {"version", no_argument, NULL, 1},
+        {"verbosity", required_argument, NULL, 3},
+        {"cache", required_argument, NULL, 4},
+        {"separate-regions", no_argument, NULL, 5},
+        {"threads", required_argument, NULL, '@'},
+        {NULL, 0, NULL, 0}
+    };
+
+    char *tmp;
+    while ((c = getopt_long(argc, argv, "hH?0b:c:e:fm:p:s:S:lr:CR:T:D@:", loptions,NULL)) >= 0)
+    {
+        switch (c)
+        {
+            case 'R': args.regions_fname = optarg; break;
+            case 'T': args.targets_fname = optarg; break;
+            case 'C': do_csi = 1; break;
+            case 'r': reheader = optarg; break;
+            case 'h': args.print_header = 1; break;
+            case 'H': args.print_header = 1; args.header_only = 1; break;
+            case 'l': list_chroms = 1; break;
+            case '0': conf.preset |= TBX_UCSC; detect = 0; break;
+            case 'b':
+                conf.bc = strtol(optarg,&tmp,10);
+                if ( *tmp ) error("Could not parse argument: -b %s\n", optarg);
+                detect = 0;
+                break;
+            case 'e':
+                conf.ec = strtol(optarg,&tmp,10);
+                if ( *tmp ) error("Could not parse argument: -e %s\n", optarg);
+                detect = 0;
+                break;
+            case 'c': conf.meta_char = *optarg; detect = 0; break;
+            case 'f': is_force = 1; break;
+            case 'm':
+                min_shift = strtol(optarg,&tmp,10);
+                if ( *tmp ) error("Could not parse argument: -m %s\n", optarg);
+                break;
+            case 'p':
+                detect = 0;
+                if (strcmp(optarg, "gff") == 0) conf = tbx_conf_gff;
+                else if (strcmp(optarg, "bed") == 0) conf = tbx_conf_bed;
+                else if (strcmp(optarg, "sam") == 0) conf = tbx_conf_sam;
+                else if (strcmp(optarg, "vcf") == 0) conf = tbx_conf_vcf;
+                else if (strcmp(optarg, "gaf") == 0) conf = tbx_conf_gaf;
+                else if (strcmp(optarg, "bcf") == 0) detect = 1; // bcf is autodetected, preset is not needed
+                else if (strcmp(optarg, "bam") == 0) detect = 1; // same as bcf
+                else error("The preset string not recognised: '%s'\n", optarg);
+                break;
+            case 's':
+                conf.sc = strtol(optarg,&tmp,10);
+                if ( *tmp ) error("Could not parse argument: -s %s\n", optarg);
+                detect = 0;
+                break;
+            case 'S':
+                new_line_skip = strtol(optarg,&tmp,10);
+                if ( *tmp ) error("Could not parse argument: -S %s\n", optarg);
+                detect = 0;
+                break;
+            case 'D':
+                args.download_index = 0;
+                break;
+            case 1:
+                printf(
+"tabix (htslib) %s\n"
+"Copyright (C) 2024 Genome Research Ltd.\n", hts_version());
+                return EXIT_SUCCESS;
+            case 2:
+                return usage(stdout, EXIT_SUCCESS);
+            case 3: {
+                int v = atoi(optarg);
+                if (v < 0) v = 0;
+                hts_set_log_level(v);
+                break;
+            }
+            case 4:
+                args.cache_megs = atoi(optarg);
+                if (args.cache_megs < 0) {
+                    args.cache_megs = 0;
+                } else if (args.cache_megs >= INT_MAX / 1048576) {
+                    args.cache_megs = INT_MAX / 1048576;
+                }
+                break;
+            case 5:
+                args.separate_regs = 1;
+                break;
+            case '@':   //thread count
+                args.threads = atoi(optarg);
+                break;
+            default: return usage(stderr, EXIT_FAILURE);
+        }
+    }
+
+    if (new_line_skip >= 0)
+        conf.line_skip = new_line_skip;
+
+    if ( optind==argc ) return usage(stderr, EXIT_FAILURE);
+
+    if ( list_chroms )
+        return query_chroms(argv[optind], args.download_index);
+
+    char *fname = argv[optind];
+    int ftype = file_type(fname);
+    if ( detect )  // no preset given
+    {
+        if ( ftype==IS_GFF ) conf = tbx_conf_gff;
+        else if ( ftype==IS_BED ) conf = tbx_conf_bed;
+        else if ( ftype==IS_GAF ) conf = tbx_conf_gaf;
+        else if ( ftype==IS_SAM ) conf = tbx_conf_sam;
+        else if ( ftype==IS_VCF )
+        {
+            conf = tbx_conf_vcf;
+            if ( !min_shift && do_csi ) min_shift = 14;
+        }
+        else if ( ftype==IS_BCF )
+        {
+            if ( !min_shift ) min_shift = 14;
+        }
+        else if ( ftype==IS_BAM )
+        {
+            if ( !min_shift ) min_shift = 14;
+        }
+    }
+    if ( argc > optind+1 || args.header_only || args.regions_fname || args.targets_fname )
+    {
+        int nregs = 0;
+        char **regs = NULL;
+        if ( !args.header_only )
+            regs = parse_regions(args.regions_fname, argv+optind+1, argc-optind-1, &nregs);
+        return query_regions(&args, &conf, fname, regs, nregs);
+    }
+    if ( do_csi )
+    {
+        if ( !min_shift ) min_shift = 14;
+        min_shift *= do_csi;  // positive for CSIv2, negative for CSIv1
+    }
+    if ( min_shift!=0 && !do_csi ) do_csi = 1;
+
+    if ( reheader )
+        return reheader_file(fname, reheader, ftype, &conf, args.threads);
+
+    char *suffix = ".tbi";
+    if ( do_csi ) suffix = ".csi";
+    else if ( ftype==IS_BAM ) suffix = ".bai";
+    else if ( ftype==IS_CRAM ) suffix = ".crai";
+
+    char *idx_fname = calloc(strlen(fname) + 6, 1);
+    if (!idx_fname) error("%s\n", strerror(errno));
+    strcat(strcpy(idx_fname, fname), suffix);
+
+    struct stat stat_tbi, stat_file;
+    if ( !is_force && stat(idx_fname, &stat_tbi)==0 )
+    {
+        // Before complaining about existing index, check if the VCF file isn't
+        // newer. This is a common source of errors, people tend not to notice
+        // that tabix failed
+        stat(fname, &stat_file);
+        if ( stat_file.st_mtime <= stat_tbi.st_mtime )
+            error("[tabix] the index file exists. Please use '-f' to overwrite.\n");
+    }
+    free(idx_fname);
+
+    int ret;
+    if ( ftype==IS_CRAM )
+    {
+        if ( bam_index_build3(fname, min_shift, args.threads)!=0 ) error("bam_index_build failed: %s\n", fname);
+        return 0;
+    }
+    else if ( do_csi )
+    {
+        if ( ftype==IS_BCF )
+        {
+            if ( bcf_index_build3(fname, NULL, min_shift, args.threads)!=0 ) error("bcf_index_build failed: %s\n", fname);
+            return 0;
+        }
+        if ( ftype==IS_BAM )
+        {
+            if ( bam_index_build3(fname, min_shift, args.threads)!=0 ) error("bam_index_build failed: %s\n", fname);
+            return 0;
+        }
+
+        switch (ret = tbx_index_build3(fname, NULL, min_shift, args.threads, &conf))
+        {
+            case 0:
+                return 0;
+            case -2:
+                error("[tabix] the compression of '%s' is not BGZF\n", fname);
+            default:
+                error("tbx_index_build3 failed: %s\n", fname);
+        }
+    }
+    else    // TBI index
+    {
+        switch (ret = tbx_index_build3(fname, NULL, min_shift, args.threads, &conf))
+        {
+            case 0:
+                return 0;
+            case -2:
+                error("[tabix] the compression of '%s' is not BGZF\n", fname);
+            default:
+                error("tbx_index_build3 failed: %s\n", fname);
+        }
+    }
+
+    return 0;
+}
diff --git a/ext/htslib/tbx.c b/ext/htslib/tbx.c
new file mode 100644
index 0000000..6625005
--- /dev/null
+++ b/ext/htslib/tbx.c
@@ -0,0 +1,529 @@
+/*  tbx.c -- tabix API functions.
+
+    Copyright (C) 2009, 2010, 2012-2015, 2017-2020, 2022-2023 Genome Research Ltd.
+    Copyright (C) 2010-2012 Broad Institute.
+
+    Author: Heng Li <lh3@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <assert.h>
+#include <errno.h>
+#include "htslib/tbx.h"
+#include "htslib/bgzf.h"
+#include "htslib/hts_endian.h"
+#include "hts_internal.h"
+
+#include "htslib/khash.h"
+KHASH_DECLARE(s2i, kh_cstr_t, int64_t)
+
+HTSLIB_EXPORT
+const tbx_conf_t tbx_conf_gff = { 0, 1, 4, 5, '#', 0 };
+
+HTSLIB_EXPORT
+const tbx_conf_t tbx_conf_bed = { TBX_UCSC, 1, 2, 3, '#', 0 };
+
+HTSLIB_EXPORT
+const tbx_conf_t tbx_conf_psltbl = { TBX_UCSC, 15, 17, 18, '#', 0 };
+
+HTSLIB_EXPORT
+const tbx_conf_t tbx_conf_sam = { TBX_SAM, 3, 4, 0, '@', 0 };
+
+HTSLIB_EXPORT
+const tbx_conf_t tbx_conf_vcf = { TBX_VCF, 1, 2, 0, '#', 0 };
+const tbx_conf_t tbx_conf_gaf = { TBX_GAF, 1, 6, 0, '#', 0 };
+
+typedef struct {
+    int64_t beg, end;
+    char *ss, *se;
+    int tid;
+} tbx_intv_t;
+
+static inline int get_tid(tbx_t *tbx, const char *ss, int is_add)
+{
+    khint_t k;
+    khash_t(s2i) *d;
+    if ((tbx->conf.preset&0xffff) == TBX_GAF) return(0);
+    if (tbx->dict == 0) tbx->dict = kh_init(s2i);
+    if (!tbx->dict) return -1; // Out of memory
+    d = (khash_t(s2i)*)tbx->dict;
+    if (is_add) {
+        int absent;
+        k = kh_put(s2i, d, ss, &absent);
+        if (absent < 0) {
+            return -1; // Out of memory
+        } else if (absent) {
+            char *ss_dup = strdup(ss);
+            if (ss_dup) {
+                kh_key(d, k) = ss_dup;
+                kh_val(d, k) = kh_size(d) - 1;
+            } else {
+                kh_del(s2i, d, k);
+                return -1; // Out of memory
+            }
+        }
+    } else k = kh_get(s2i, d, ss);
+    return k == kh_end(d)? -1 : kh_val(d, k);
+}
+
+int tbx_name2id(tbx_t *tbx, const char *ss)
+{
+    return get_tid(tbx, ss, 0);
+}
+
+int tbx_parse1(const tbx_conf_t *conf, size_t len, char *line, tbx_intv_t *intv)
+{
+    size_t i, b = 0;
+    int id = 1;
+    char *s;
+    intv->ss = intv->se = 0; intv->beg = intv->end = -1;
+    for (i = 0; i <= len; ++i) {
+        if (line[i] == '\t' || line[i] == 0) {
+            if (id == conf->sc) {
+                intv->ss = line + b; intv->se = line + i;
+            } else if (id == conf->bc) {
+                // here ->beg is 0-based.
+                if ((conf->preset&0xffff) == TBX_GAF){
+                    // if gaf find the smallest and largest node id
+                    char *t;
+                    int64_t nodeid = -1;
+                    for (s = line + b + 1; s < line + i;) {
+                        nodeid = strtoll(s, &t, 0);
+                        if(intv->beg == -1){
+                            intv->beg = intv->end = nodeid;
+                        } else {
+                            if(nodeid < intv->beg){
+                                intv->beg = nodeid;
+                            }
+
+                            if(nodeid > intv->end){
+                                intv->end = nodeid;
+                            }
+                        }
+                        s = t + 1;
+                    }
+                } else {
+                    intv->beg = strtoll(line + b, &s, 0);
+
+                    if (conf->bc <= conf->ec) // don't overwrite an already set end point
+                        intv->end = intv->beg;
+
+                    if ( s==line+b ) return -1; // expected int
+
+                    if (!(conf->preset&TBX_UCSC))
+                        --intv->beg;
+                    else if (conf->bc <= conf->ec)
+                        ++intv->end;
+
+                    if (intv->beg < 0) {
+                        hts_log_warning("Coordinate <= 0 detected. "
+                                        "Did you forget to use the -0 option?");
+                        intv->beg = 0;
+                    }
+                    if (intv->end < 1) intv->end = 1;
+                }
+            } else {
+                if ((conf->preset&0xffff) == TBX_GENERIC) {
+                    if (id == conf->ec)
+                    {
+                        intv->end = strtoll(line + b, &s, 0);
+                        if ( s==line+b ) return -1; // expected int
+                    }
+                } else if ((conf->preset&0xffff) == TBX_SAM) {
+                    if (id == 6) { // CIGAR
+                        int l = 0;
+                        char *t;
+                        for (s = line + b; s < line + i;) {
+                            long x = strtol(s, &t, 10);
+                            char op = toupper_c(*t);
+                            if (op == 'M' || op == 'D' || op == 'N') l += x;
+                            s = t + 1;
+                        }
+                        if (l == 0) l = 1;
+                        intv->end = intv->beg + l;
+                    }
+                } else if ((conf->preset&0xffff) == TBX_VCF) {
+                    if (id == 4) {
+                        if (b < i) intv->end = intv->beg + (i - b);
+                    } else if (id == 8) { // look for "END="
+                        int c = line[i];
+                        line[i] = 0;
+                        s = strstr(line + b, "END=");
+                        if (s == line + b) s += 4;
+                        else if (s) {
+                            s = strstr(line + b, ";END=");
+                            if (s) s += 5;
+                        }
+                        if (s && *s != '.') {
+                            long long end = strtoll(s, &s, 0);
+                            if (end <= intv->beg) {
+                                static int reported = 0;
+                                if (!reported) {
+                                    int l = intv->ss ? (int) (intv->se - intv->ss) : 0;
+                                    hts_log_warning("VCF INFO/END=%lld is smaller than POS at %.*s:%"PRIhts_pos"\n"
+                                                    "This tag will be ignored. "
+                                                    "Note: only one invalid END tag will be reported.",
+                                                    end, l >= 0 ? l : 0,
+                                                    intv->ss ? intv->ss : "",
+                                                    intv->beg);
+                                    reported = 1;
+                                }
+                            } else {
+                                intv->end = end;
+                            }
+                        }
+                        line[i] = c;
+                    }
+                }
+            }
+            b = i + 1;
+            ++id;
+        }
+    }
+    if (intv->ss == 0 || intv->se == 0 || intv->beg < 0 || intv->end < 0) return -1;
+    return 0;
+}
+
+static inline int get_intv(tbx_t *tbx, kstring_t *str, tbx_intv_t *intv, int is_add)
+{
+    if (tbx_parse1(&tbx->conf, str->l, str->s, intv) == 0) {
+        int c = *intv->se;
+        *intv->se = '\0';
+        if ((tbx->conf.preset&0xffff) == TBX_GAF){
+            intv->tid = 0;
+        } else {
+            intv->tid = get_tid(tbx, intv->ss, is_add);
+        }
+        *intv->se = c;
+        if (intv->tid < 0) return -2;  // get_tid out of memory
+        return (intv->beg >= 0 && intv->end >= 0)? 0 : -1;
+    } else {
+        char *type = NULL;
+        switch (tbx->conf.preset&0xffff)
+        {
+            case TBX_SAM: type = "TBX_SAM"; break;
+            case TBX_VCF: type = "TBX_VCF"; break;
+            case TBX_GAF: type = "TBX_GAF"; break;
+            case TBX_UCSC: type = "TBX_UCSC"; break;
+            default: type = "TBX_GENERIC"; break;
+        }
+        if (hts_is_utf16_text(str))
+            hts_log_error("Failed to parse %s: offending line appears to be encoded as UTF-16", type);
+        else
+            hts_log_error("Failed to parse %s: was wrong -p [type] used?\nThe offending line was: \"%s\"",
+                type, str->s);
+        return -1;
+    }
+}
+
+/*
+ * Called by tabix iterator to read the next record.
+ * Returns    >=  0 on success
+ *               -1 on EOF
+ *            <= -2 on error
+ */
+int tbx_readrec(BGZF *fp, void *tbxv, void *sv, int *tid, hts_pos_t *beg, hts_pos_t *end)
+{
+    tbx_t *tbx = (tbx_t *) tbxv;
+    kstring_t *s = (kstring_t *) sv;
+    int ret;
+    if ((ret = bgzf_getline(fp, '\n', s)) >= 0) {
+        tbx_intv_t intv;
+        if (get_intv(tbx, s, &intv, 0) < 0)
+            return -2;
+        *tid = intv.tid; *beg = intv.beg; *end = intv.end;
+    }
+    return ret;
+}
+
+static int tbx_set_meta(tbx_t *tbx)
+{
+    int i, l = 0, l_nm;
+    uint32_t x[7];
+    char **name;
+    uint8_t *meta;
+    khint_t k;
+    khash_t(s2i) *d = (khash_t(s2i)*)tbx->dict;
+
+    memcpy(x, &tbx->conf, 24);
+    name = (char**)malloc(sizeof(char*) * kh_size(d));
+    if (!name) return -1;
+    for (k = kh_begin(d), l = 0; k != kh_end(d); ++k) {
+        if (!kh_exist(d, k)) continue;
+        name[kh_val(d, k)] = (char*)kh_key(d, k);
+        l += strlen(kh_key(d, k)) + 1; // +1 to include '\0'
+    }
+    l_nm = x[6] = l;
+    meta = (uint8_t*)malloc(l_nm + 28);
+    if (!meta) { free(name); return -1; }
+    if (ed_is_big())
+        for (i = 0; i < 7; ++i)
+            x[i] = ed_swap_4(x[i]);
+    memcpy(meta, x, 28);
+    for (l = 28, i = 0; i < (int)kh_size(d); ++i) {
+        int x = strlen(name[i]) + 1;
+        memcpy(meta + l, name[i], x);
+        l += x;
+    }
+    free(name);
+    hts_idx_set_meta(tbx->idx, l, meta, 0);
+    return 0;
+}
+
+// Minimal effort parser to extract reference length out of VCF header line
+// This is used only used to adjust the number of levels if necessary,
+// so not a major problem if it doesn't always work.
+static void adjust_max_ref_len_vcf(const char *str, int64_t *max_ref_len)
+{
+    const char *ptr;
+    int64_t len;
+    if (strncmp(str, "##contig", 8) != 0) return;
+    ptr = strstr(str + 8, "length");
+    if (!ptr) return;
+    for (ptr += 6; *ptr == ' ' || *ptr == '='; ptr++) {}
+    len = strtoll(ptr, NULL, 10);
+    if (*max_ref_len < len) *max_ref_len = len;
+}
+
+// Same for sam files
+static void adjust_max_ref_len_sam(const char *str, int64_t *max_ref_len)
+{
+    const char *ptr;
+    int64_t len;
+    if (strncmp(str, "@SQ", 3) != 0) return;
+    ptr = strstr(str + 3, "\tLN:");
+    if (!ptr) return;
+    ptr += 4;
+    len = strtoll(ptr, NULL, 10);
+    if (*max_ref_len < len) *max_ref_len = len;
+}
+
+// Adjusts number of levels if not big enough.  This can happen for
+// files with very large contigs.
+static int adjust_n_lvls(int min_shift, int n_lvls, int64_t max_len)
+{
+    int64_t s = hts_bin_maxpos(min_shift, n_lvls);
+    max_len += 256;
+    for (; max_len > s; ++n_lvls, s <<= 3) {}
+    return n_lvls;
+}
+
+tbx_t *tbx_index(BGZF *fp, int min_shift, const tbx_conf_t *conf)
+{
+    tbx_t *tbx;
+    kstring_t str;
+    int ret, first = 0, n_lvls, fmt;
+    int64_t lineno = 0;
+    uint64_t last_off = 0;
+    tbx_intv_t intv;
+    int64_t max_ref_len = 0;
+
+    str.s = 0; str.l = str.m = 0;
+    tbx = (tbx_t*)calloc(1, sizeof(tbx_t));
+    if (!tbx) return NULL;
+    tbx->conf = *conf;
+    if (min_shift > 0) n_lvls = (TBX_MAX_SHIFT - min_shift + 2) / 3, fmt = HTS_FMT_CSI;
+    else min_shift = 14, n_lvls = 5, fmt = HTS_FMT_TBI;
+    while ((ret = bgzf_getline(fp, '\n', &str)) >= 0) {
+        ++lineno;
+        if (str.s[0] == tbx->conf.meta_char && fmt == HTS_FMT_CSI) {
+            switch (tbx->conf.preset) {
+                case TBX_SAM:
+                    adjust_max_ref_len_sam(str.s, &max_ref_len); break;
+                case TBX_VCF:
+                    adjust_max_ref_len_vcf(str.s, &max_ref_len); break;
+                default:
+                    break;
+            }
+        }
+        if (lineno <= tbx->conf.line_skip || str.s[0] == tbx->conf.meta_char) {
+            last_off = bgzf_tell(fp);
+            continue;
+        }
+        if (first == 0) {
+            if (fmt == HTS_FMT_CSI) {
+                if (!max_ref_len)
+                    max_ref_len = (int64_t)100*1024*1024*1024; // 100G default
+                n_lvls = adjust_n_lvls(min_shift, n_lvls, max_ref_len);
+            }
+            tbx->idx = hts_idx_init(0, fmt, last_off, min_shift, n_lvls);
+            if (!tbx->idx) goto fail;
+            first = 1;
+        }
+        ret = get_intv(tbx, &str, &intv, 1);
+        if (ret < -1) goto fail;  // Out of memory
+        if (ret < 0) continue; // Skip unparsable lines
+        if (hts_idx_push(tbx->idx, intv.tid, intv.beg, intv.end,
+                         bgzf_tell(fp), 1) < 0) {
+            goto fail;
+        }
+    }
+    if (ret < -1) goto fail;
+    if ( !tbx->idx ) tbx->idx = hts_idx_init(0, fmt, last_off, min_shift, n_lvls);   // empty file
+    if (!tbx->idx) goto fail;
+    if ( !tbx->dict ) tbx->dict = kh_init(s2i);
+    if (!tbx->dict) goto fail;
+    if (hts_idx_finish(tbx->idx, bgzf_tell(fp)) != 0) goto fail;
+    if (tbx_set_meta(tbx) != 0) goto fail;
+    free(str.s);
+    return tbx;
+
+ fail:
+    free(str.s);
+    tbx_destroy(tbx);
+    return NULL;
+}
+
+void tbx_destroy(tbx_t *tbx)
+{
+    khash_t(s2i) *d = (khash_t(s2i)*)tbx->dict;
+    if (d != NULL)
+    {
+        khint_t k;
+        for (k = kh_begin(d); k != kh_end(d); ++k)
+            if (kh_exist(d, k)) free((char*)kh_key(d, k));
+    }
+    hts_idx_destroy(tbx->idx);
+    kh_destroy(s2i, d);
+    free(tbx);
+}
+
+int tbx_index_build3(const char *fn, const char *fnidx, int min_shift, int n_threads, const tbx_conf_t *conf)
+{
+    tbx_t *tbx;
+    BGZF *fp;
+    int ret;
+    if ((fp = bgzf_open(fn, "r")) == 0) return -1;
+    if ( n_threads ) bgzf_mt(fp, n_threads, 256);
+    if ( bgzf_compression(fp) != bgzf ) { bgzf_close(fp); return -2; }
+    tbx = tbx_index(fp, min_shift, conf);
+    bgzf_close(fp);
+    if ( !tbx ) return -1;
+    ret = hts_idx_save_as(tbx->idx, fn, fnidx, min_shift > 0? HTS_FMT_CSI : HTS_FMT_TBI);
+    tbx_destroy(tbx);
+    return ret;
+}
+
+int tbx_index_build2(const char *fn, const char *fnidx, int min_shift, const tbx_conf_t *conf)
+{
+    return tbx_index_build3(fn, fnidx, min_shift, 0, conf);
+}
+
+int tbx_index_build(const char *fn, int min_shift, const tbx_conf_t *conf)
+{
+    return tbx_index_build3(fn, NULL, min_shift, 0, conf);
+}
+
+static tbx_t *index_load(const char *fn, const char *fnidx, int flags)
+{
+    tbx_t *tbx;
+    uint8_t *meta;
+    char *nm, *p;
+    uint32_t l_meta, l_nm;
+    tbx = (tbx_t*)calloc(1, sizeof(tbx_t));
+    if (!tbx)
+        return NULL;
+    tbx->idx = hts_idx_load3(fn, fnidx, HTS_FMT_TBI, flags);
+    if ( !tbx->idx )
+    {
+        free(tbx);
+        return NULL;
+    }
+    meta = hts_idx_get_meta(tbx->idx, &l_meta);
+    if ( !meta || l_meta < 28) goto invalid;
+
+    tbx->conf.preset = le_to_i32(&meta[0]);
+    tbx->conf.sc = le_to_i32(&meta[4]);
+    tbx->conf.bc = le_to_i32(&meta[8]);
+    tbx->conf.ec = le_to_i32(&meta[12]);
+    tbx->conf.meta_char = le_to_i32(&meta[16]);
+    tbx->conf.line_skip = le_to_i32(&meta[20]);
+    l_nm = le_to_u32(&meta[24]);
+    if (l_nm > l_meta - 28) goto invalid;
+
+    p = nm = (char*)meta + 28;
+    // This assumes meta is NUL-terminated, so we can merrily strlen away.
+    // hts_idx_load_local() assures this for us by adding a NUL on the end
+    // of whatever it reads.
+    for (; p - nm < l_nm; p += strlen(p) + 1) {
+        if (get_tid(tbx, p, 1) < 0) {
+            hts_log_error("%s", strerror(errno));
+            goto fail;
+        }
+    }
+    return tbx;
+
+ invalid:
+    hts_log_error("Invalid index header for %s", fnidx ? fnidx : fn);
+
+ fail:
+    tbx_destroy(tbx);
+    return NULL;
+}
+
+tbx_t *tbx_index_load3(const char *fn, const char *fnidx, int flags)
+{
+    return index_load(fn, fnidx, flags);
+}
+
+tbx_t *tbx_index_load2(const char *fn, const char *fnidx)
+{
+    return index_load(fn, fnidx, 1);
+}
+
+tbx_t *tbx_index_load(const char *fn)
+{
+    return index_load(fn, NULL, 1);
+}
+
+const char **tbx_seqnames(tbx_t *tbx, int *n)
+{
+    khash_t(s2i) *d = (khash_t(s2i)*)tbx->dict;
+    if (d == NULL)
+    {
+        *n = 0;
+        return calloc(1, sizeof(char *));
+    }
+    int tid, m = kh_size(d);
+    const char **names = (const char**) calloc(m,sizeof(const char*));
+    khint_t k;
+    if (!names) {
+        *n = 0;
+        return NULL;
+    }
+    for (k=kh_begin(d); k<kh_end(d); k++)
+    {
+        if ( !kh_exist(d,k) ) continue;
+        tid = kh_val(d,k);
+        assert( tid<m );
+        names[tid] = kh_key(d,k);
+    }
+    // sanity check: there should be no gaps
+    for (tid=0; tid<m; tid++)
+        assert(names[tid]);
+    *n = m;
+    return names;
+}
+
diff --git a/ext/htslib/textutils.c b/ext/htslib/textutils.c
new file mode 100644
index 0000000..b2c29a8
--- /dev/null
+++ b/ext/htslib/textutils.c
@@ -0,0 +1,497 @@
+/*  textutils.c -- non-bioinformatics utility routines for text etc.
+
+    Copyright (C) 2016, 2018-2020 Genome Research Ltd.
+
+    Author: John Marshall <jm18@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+
+#include <stdio.h>
+#include <string.h>
+
+#include "htslib/hfile.h"
+#include "htslib/kstring.h"
+#include "htslib/sam.h"  // For stringify_argv() declaration
+
+#include "hts_internal.h"
+
+static int dehex(char c)
+{
+    if (c >= 'a' && c <= 'f') return c - 'a' + 10;
+    else if (c >= 'A' && c <= 'F') return c - 'A' + 10;
+    else if (c >= '0' && c <= '9') return c - '0';
+    else return -1;  // Hence dehex('\0') = -1
+}
+
+int hts_decode_percent(char *dest, size_t *destlen, const char *s)
+{
+    char *d = dest;
+    int hi, lo;
+
+    while (*s) {
+        if (*s == '%' && (hi = dehex(s[1])) >= 0 && (lo = dehex(s[2])) >= 0) {
+            *d++ = (hi << 4) | lo;
+            s += 3;
+        }
+        else *d++ = *s++;
+    }
+
+    *d = '\0';
+    *destlen = d - dest;
+    return 0;
+}
+
+static int debase64(char c)
+{
+    if (c >= 'a' && c <= 'z') return c - 'a' + 26;
+    else if (c >= 'A' && c <= 'Z') return c - 'A';
+    else if (c >= '0' && c <= '9') return c - '0' + 52;
+    else if (c == '/') return 63;
+    else if (c == '+') return 62;
+    else return -1;  // Hence debase64('\0') = -1
+}
+
+size_t hts_base64_decoded_length(size_t len)
+{
+    size_t nquartets = (len + 2) / 4;
+    return 3 * nquartets;
+}
+
+int hts_decode_base64(char *dest, size_t *destlen, const char *s)
+{
+    char *d = dest;
+    int x0, x1, x2, x3;
+
+    while (1) {
+        x0 = debase64(*s++);
+        x1 = (x0 >= 0)? debase64(*s++) : -1;
+        x2 = (x1 >= 0)? debase64(*s++) : -1;
+        x3 = (x2 >= 0)? debase64(*s++) : -1;
+        if (x3 < 0) break;
+
+        *d++ = (x0 << 2) | (x1 >> 4);
+        *d++ = (x1 << 4) | (x2 >> 2);
+        *d++ = (x2 << 6) | x3;
+    }
+
+    if (x1 >= 0) *d++ = (x0 << 2) | (x1 >> 4);
+    if (x2 >= 0) *d++ = (x1 << 4) | (x2 >> 2);
+
+    *destlen = d - dest;
+    return 0;
+}
+
+static char *encode_utf8(char *s, unsigned x)
+{
+    if (x >= 0x10000) {
+        *s++ = 0xF0 | (x >> 18);
+        *s++ = 0x80 | ((x >> 12) & 0x3F);
+        *s++ = 0x80 | ((x >> 6) & 0x3F);
+        *s++ = 0x80 | (x & 0x3F);
+    }
+    else if (x >= 0x800) {
+        *s++ = 0xE0 | (x >> 12);
+        *s++ = 0x80 | ((x >> 6) & 0x3F);
+        *s++ = 0x80 | (x & 0x3F);
+    }
+    else if (x >= 0x80) {
+        *s++ = 0xC0 | (x >> 6);
+        *s++ = 0x80 | (x & 0x3F);
+    }
+    else *s++ = x;
+
+    return s;
+}
+
+static char *sscan_string(char *s)
+{
+    char *d = s;
+    int d1, d2, d3, d4;
+
+    for (;;) switch (*s) {
+    case '\\':
+        switch (s[1]) {
+        case '\0': *d = '\0'; return s+1;
+        case 'b': *d++ = '\b'; s += 2; break;
+        case 'f': *d++ = '\f'; s += 2; break;
+        case 'n': *d++ = '\n'; s += 2; break;
+        case 'r': *d++ = '\r'; s += 2; break;
+        case 't': *d++ = '\t'; s += 2; break;
+        default:  *d++ = s[1]; s += 2; break;
+        case 'u':
+            if ((d1 = dehex(s[2])) >= 0 && (d2 = dehex(s[3])) >= 0 &&
+                (d3 = dehex(s[4])) >= 0 && (d4 = dehex(s[5])) >= 0) {
+                d = encode_utf8(d, d1 << 12 | d2 << 8 | d3 << 4 | d4);
+                s += 6;
+            }
+            break;
+        }
+        break;
+
+    case '"':
+        *d = '\0';
+        return s+1;
+
+    case '\0':
+        *d = '\0';
+        return s;
+
+    default:
+        *d++ = *s++;
+        break;
+    }
+}
+
+static int fscan_string(hFILE *fp, kstring_t *d)
+{
+    int c, d1, d2, d3, d4;
+    uint32_t e = 0;
+
+    while ((c = hgetc(fp)) != EOF) switch (c) {
+    case '\\':
+        if ((c = hgetc(fp)) == EOF) return e == 0 ? 0 : -1;
+        switch (c) {
+        case 'b': e |= kputc('\b', d) < 0; break;
+        case 'f': e |= kputc('\f', d) < 0; break;
+        case 'n': e |= kputc('\n', d) < 0; break;
+        case 'r': e |= kputc('\r', d) < 0; break;
+        case 't': e |= kputc('\t', d) < 0; break;
+        default:  e |= kputc(c,    d) < 0; break;
+        case 'u':
+            if ((c = hgetc(fp)) != EOF && (d1 = dehex(c)) >= 0 &&
+                (c = hgetc(fp)) != EOF && (d2 = dehex(c)) >= 0 &&
+                (c = hgetc(fp)) != EOF && (d3 = dehex(c)) >= 0 &&
+                (c = hgetc(fp)) != EOF && (d4 = dehex(c)) >= 0) {
+                char buf[8];
+                char *lim = encode_utf8(buf, d1 << 12 | d2 << 8 | d3 << 4 | d4);
+                e |= kputsn(buf, lim - buf, d) < 0;
+            }
+            break;
+        }
+        break;
+
+    case '"':
+        return e == 0 ? 0 : -1;
+
+    default:
+        e |= kputc(c, d) < 0;
+        break;
+    }
+    return e == 0 ? 0 : -1;
+}
+
+static char token_type(hts_json_token *token)
+{
+    const char *s = token->str;
+
+    switch (*s) {
+    case 'f':
+        return (strcmp(s, "false") == 0)? 'b' : '?';
+    case 'n':
+        return (strcmp(s, "null") == 0)? '.' : '?';
+    case 't':
+        return (strcmp(s, "true") == 0)? 'b' : '?';
+    case '-':
+    case '0': case '1': case '2': case '3': case '4':
+    case '5': case '6': case '7': case '8': case '9':
+        return 'n';
+    default:
+        return '?';
+    }
+}
+
+HTSLIB_EXPORT
+hts_json_token * hts_json_alloc_token(void) {
+    return calloc(1, sizeof(hts_json_token));
+}
+
+HTSLIB_EXPORT
+char hts_json_token_type(hts_json_token *token) {
+    return token->type;
+}
+
+HTSLIB_EXPORT
+void hts_json_free_token(hts_json_token *token) {
+    free(token);
+}
+
+HTSLIB_EXPORT
+char *hts_json_token_str(hts_json_token *token) {
+    return token->str;
+}
+
+HTSLIB_EXPORT
+char hts_json_snext(char *str, size_t *state, hts_json_token *token)
+{
+    char *s = &str[*state >> 2];
+    int hidden = *state & 3;
+
+    if (hidden) {
+        *state &= ~3;
+        return token->type = "?}]?"[hidden];
+    }
+
+#define STATE(s,h)  (((s) - str) << 2 | (h))
+
+    for (;;) switch (*s) {
+    case ' ':
+    case '\t':
+    case '\r':
+    case '\n':
+    case ',':
+    case ':':
+        s++;
+        continue;
+
+    case '\0':
+        return token->type = '\0';
+
+    case '{':
+    case '[':
+    case '}':
+    case ']':
+        *state = STATE(s+1, 0);
+        return token->type = *s;
+
+    case '"':
+        token->str = s+1;
+        *state = STATE(sscan_string(s+1), 0);
+        return token->type = 's';
+
+    default:
+        token->str = s;
+        s += strcspn(s, " \t\r\n,]}");
+        hidden = (*s == '}')? 1 : (*s == ']')? 2 : 0;
+        if (*s != '\0') *s++ = '\0';
+        *state = STATE(s, hidden);
+        return token->type = token_type(token);
+    }
+
+#undef STATE
+}
+
+HTSLIB_EXPORT
+char hts_json_fnext(struct hFILE *fp, hts_json_token *token, kstring_t *kstr)
+{
+    char peek;
+    int c;
+
+    for (;;) switch (c = hgetc(fp)) {
+    case ' ':
+    case '\t':
+    case '\r':
+    case '\n':
+    case ',':
+    case ':':
+        continue;
+
+    case EOF:
+        return token->type = '\0';
+
+    case '{':
+    case '[':
+    case '}':
+    case ']':
+        return token->type = c;
+
+    case '"':
+        kstr->l = 0;
+        fscan_string(fp, kstr);
+        if (kstr->l == 0) kputsn("", 0, kstr);
+        token->str = kstr->s;
+        return token->type = 's';
+
+    default:
+        kstr->l = 0;
+        kputc(c, kstr);
+        while (hpeek(fp, &peek, 1) == 1 && !strchr(" \t\r\n,]}", peek)) {
+            if ((c = hgetc(fp)) == EOF) break;
+            kputc(c, kstr);
+        }
+        token->str = kstr->s;
+        return token->type = token_type(token);
+    }
+}
+
+
+typedef char hts_json_nextfn(void *arg1, void *arg2, hts_json_token *token);
+
+static char skip_value(char type, hts_json_nextfn *next, void *arg1, void *arg2)
+{
+    hts_json_token token;
+    int level;
+
+    switch (type? type : next(arg1, arg2, &token)) {
+    case '\0':
+        return '\0';
+
+    case '?':
+    case '}':
+    case ']':
+        return '?';
+
+    case '{':
+    case '[':
+        level = 1;
+        break;
+
+    default:
+        return 'v';
+    }
+
+    while (level > 0)
+        switch (next(arg1, arg2, &token)) {
+        case '\0':
+            return '\0';
+
+        case '?':
+            return '?';
+
+        case '{':
+        case '[':
+            level++;
+            break;
+
+        case '}':
+        case ']':
+            --level;
+            break;
+
+        default:
+            break;
+        }
+
+    return 'v';
+}
+
+static char snext(void *arg1, void *arg2, hts_json_token *token)
+{
+    return hts_json_snext(arg1, arg2, token);
+}
+
+HTSLIB_EXPORT
+char hts_json_sskip_value(char *str, size_t *state, char type)
+{
+    return skip_value(type, snext, str, state);
+}
+
+static char fnext(void *arg1, void *arg2, hts_json_token *token)
+{
+    return hts_json_fnext(arg1, token, arg2);
+}
+
+HTSLIB_EXPORT
+char hts_json_fskip_value(struct hFILE *fp, char type)
+{
+    kstring_t str = { 0, 0, NULL };
+    char ret = skip_value(type, fnext, fp, &str);
+    free(str.s);
+    return ret;
+}
+
+/*
+ * A function to help with construction of CL tags in @PG records.
+ * Takes an argc, argv pair and returns a single space-separated string.
+ * This string should be deallocated by the calling function.
+ *
+ * Returns malloced char * on success
+ *         NULL on failure
+ */
+char *stringify_argv(int argc, char *argv[]) {
+    char *str, *cp;
+    size_t nbytes = 1;
+    int i, j;
+
+    /* Allocate */
+    for (i = 0; i < argc; i++) {
+        if (i > 0) nbytes += 1;
+        nbytes += strlen(argv[i]);
+    }
+    if (!(str = malloc(nbytes)))
+        return NULL;
+
+    /* Copy */
+    cp = str;
+    for (i = 0; i < argc; i++) {
+        if (i > 0) *cp++ = ' ';
+        j = 0;
+        while (argv[i][j]) {
+            if (argv[i][j] == '\t')
+                *cp++ = ' ';
+            else
+                *cp++ = argv[i][j];
+            j++;
+        }
+    }
+    *cp++ = 0;
+
+    return str;
+}
+
+/* Utility function for printing possibly malicious text data
+ */
+const char *
+hts_strprint(char *buf, size_t buflen, char quote, const char *s, size_t len)
+{
+    const char *slim = (len < SIZE_MAX)? &s[len] : NULL;
+    char *t = buf, *bufend = buf + buflen;
+
+    size_t qlen = quote? 1 : 0;
+    if (quote) *t++ = quote;
+
+    for (; slim? (s < slim) : (*s); s++) {
+        char c;
+        size_t clen;
+        switch (*s) {
+        case '\n': c = 'n'; clen = 2; break;
+        case '\r': c = 'r'; clen = 2; break;
+        case '\t': c = 't'; clen = 2; break;
+        case '\0': c = '0'; clen = 2; break;
+        case '\\': c = '\\'; clen = 2; break;
+        default:
+            c = *s;
+            if (c == quote) clen = 2;
+            else clen = isprint_c(c)? 1 : 4;
+            break;
+        }
+
+        if (t-buf + clen + qlen >= buflen) {
+            while (t-buf + 3 + qlen >= buflen) t--;
+            if (quote) *t++ = quote;
+            strcpy(t, "...");
+            return buf;
+        }
+
+        if (clen == 4) {
+            snprintf(t, bufend - t, "\\x%02X", (unsigned char) c);
+            t += clen;
+        }
+        else {
+            if (clen == 2) *t++ = '\\';
+            *t++ = c;
+        }
+    }
+
+    if (quote) *t++ = quote;
+    *t = '\0';
+    return buf;
+}
diff --git a/ext/htslib/textutils_internal.h b/ext/htslib/textutils_internal.h
new file mode 100644
index 0000000..faa1d4d
--- /dev/null
+++ b/ext/htslib/textutils_internal.h
@@ -0,0 +1,426 @@
+/* textutils_internal.h -- non-bioinformatics utility routines for text etc.
+
+   Copyright (C) 2016,2018-2020, 2024 Genome Research Ltd.
+
+   Author: John Marshall <jm18@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#ifndef HTSLIB_TEXTUTILS_INTERNAL_H
+#define HTSLIB_TEXTUTILS_INTERNAL_H
+
+/* N.B. These interfaces may be used by plug-ins */
+
+#include <ctype.h>
+#include <stdlib.h>
+#include "htslib/kstring.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/// Decode percent-encoded (URL-encoded) text
+/** On input, _dest_ should be a buffer at least the same size as _s_,
+    and may be equal to _s_ to decode in place.  On output, _dest_ will be
+    NUL-terminated and the number of characters written (not including the
+    NUL) is stored in _destlen_.
+*/
+int hts_decode_percent(char *dest, size_t *destlen, const char *s);
+
+/// Return decoded data length given length of base64-encoded text
+/** This gives an upper bound, as it overestimates by a byte or two when
+    the encoded text ends with (possibly omitted) `=` padding characters.
+*/
+size_t hts_base64_decoded_length(size_t len);
+
+/// Decode base64-encoded data
+/** On input, _dest_ should be a sufficient buffer (see `hts_base64_length()`),
+    and may be equal to _s_ to decode in place.  On output, the number of
+    bytes written is stored in _destlen_.
+*/
+int hts_decode_base64(char *dest, size_t *destlen, const char *s);
+
+/// Token structure returned by JSON lexing functions
+/** Structure is defined in hts_internal.h
+ */
+
+typedef struct hts_json_token hts_json_token;
+
+/// Allocate an empty JSON token structure, for use with hts_json_* functions
+/** @return An empty token on success; NULL on failure
+ */
+HTSLIB_EXPORT
+hts_json_token *hts_json_alloc_token(void);
+
+/// Free a JSON token
+HTSLIB_EXPORT
+void hts_json_free_token(hts_json_token *token);
+
+/// Accessor function to get JSON token type
+/** @param  token Pointer to JSON token
+    @return Character indicating the token type
+
+Token types correspond to scalar JSON values and selected punctuation
+as follows:
+  - `s` string
+  - `n` number
+  - `b` boolean literal
+  - `.` null literal
+  - `{`, `}`, `[`, `]` object and array delimiters
+  - `?` lexing error
+  - `!` other errors (e.g. out of memory)
+  - `\0` terminator at end of input
+*/
+HTSLIB_EXPORT
+char hts_json_token_type(hts_json_token *token);
+
+/// Accessor function to get JSON token in string form
+/** @param  token Pointer to JSON token
+    @return String representation of the JSON token; NULL if unset
+
+If the token was parsed from a string using hts_json_snext(), the return value
+will point into the string passed as the first parameter to hts_json_snext().
+If the token was parsed from a file using hts_json_fnext(), the return value
+will point at the kstring_t buffer passed as the third parameter to
+hts_json_fnext().  In that case, the value will only be valid until the
+next call to hts_json_fnext().
+ */
+HTSLIB_EXPORT
+char *hts_json_token_str(hts_json_token *token);
+
+/// Read one JSON token from a string
+/** @param str    The input C string
+    @param state  The input string state
+    @param token  On return, filled in with the token read
+    @return  The type of the token read
+
+On return, `token->str` points into the supplied input string, which
+is modified by having token-terminating characters overwritten as NULs.
+The `state` argument records the current position within `str` after each
+`hts_json_snext()` call, and should be set to 0 before the first call.
+*/
+HTSLIB_EXPORT
+char hts_json_snext(char *str, size_t *state, hts_json_token *token);
+
+/// Read and discard a complete JSON value from a string
+/** @param str    The input C string
+    @param state  The input string state, as per `hts_json_snext()`
+    @param type   If the first token of the value to be discarded has already
+                  been read, provide its type; otherwise `'\0'`
+    @return  One of `v` (success), `\0` (end of string), and `?` (lexing error)
+
+Skips a complete JSON value, which may be a single token or an entire object
+or array.
+*/
+HTSLIB_EXPORT
+char hts_json_sskip_value(char *str, size_t *state, char type);
+
+struct hFILE;
+
+/// Read one JSON token from a file
+/** @param fp     The file stream
+    @param token  On return, filled in with the token read
+    @param kstr   Buffer used to store the token string returned
+    @return  The type of the token read
+
+The `kstr` buffer is used to store the string value of the token read,
+so `token->str` is only valid until the next time `hts_json_fnext()` is
+called with the same `kstr` argument.
+*/
+HTSLIB_EXPORT
+char hts_json_fnext(struct hFILE *fp, hts_json_token *token, kstring_t *kstr);
+
+/// Read and discard a complete JSON value from a file
+/** @param fp    The file stream
+    @param type  If the first token of the value to be discarded has already
+                 been read, provide its type; otherwise `'\0'`
+    @return  One of `v` (success), `\0` (EOF), and `?` (lexing error)
+
+Skips a complete JSON value, which may be a single token or an entire object
+or array.
+*/
+HTSLIB_EXPORT
+char hts_json_fskip_value(struct hFILE *fp, char type);
+
+// The <ctype.h> functions operate on ints such as are returned by fgetc(),
+// i.e., characters represented as unsigned-char-valued ints, or EOF.
+// To operate on plain chars (and to avoid warnings on some platforms),
+// technically one must cast to unsigned char everywhere (see CERT STR37-C)
+// or less painfully use these *_c() functions that operate on plain chars
+// (but not EOF, which must be considered separately where it is applicable).
+// TODO We may eventually wish to implement these functions directly without
+// using their <ctype.h> equivalents, and thus make them immune to locales.
+static inline int isalnum_c(char c) { return isalnum((unsigned char) c); }
+static inline int isalpha_c(char c) { return isalpha((unsigned char) c); }
+static inline int isdigit_c(char c) { return isdigit((unsigned char) c); }
+static inline int isgraph_c(char c) { return isgraph((unsigned char) c); }
+static inline int islower_c(char c) { return islower((unsigned char) c); }
+static inline int isprint_c(char c) { return isprint((unsigned char) c); }
+static inline int ispunct_c(char c) { return ispunct((unsigned char) c); }
+static inline int isspace_c(char c) { return isspace((unsigned char) c); }
+static inline int isupper_c(char c) { return isupper((unsigned char) c); }
+static inline int isxdigit_c(char c) { return isxdigit((unsigned char) c); }
+static inline char tolower_c(char c) { return tolower((unsigned char) c); }
+static inline char toupper_c(char c) { return toupper((unsigned char) c); }
+
+/// Copy possibly malicious text data to a buffer
+/** @param buf     Destination buffer
+    @param buflen  Size of the destination buffer (>= 4; >= 6 when quotes used)
+    @param quote   Quote character (or '\0' for no quoting of the output)
+    @param s       String to be copied
+    @param len     Length of the input string, or SIZE_MAX to copy until '\0'
+    @return The destination buffer, @a buf.
+
+Copies the source text string (escaping any unprintable characters) to the
+destination buffer. The destination buffer will always be NUL-terminated;
+the text will be truncated (and "..." appended) if necessary to make it fit.
+ */
+const char *hts_strprint(char *buf, size_t buflen, char quote,
+                         const char *s, size_t len);
+
+// Faster replacements for strtol, for use when parsing lots of numbers.
+// Note that these only handle base 10 and do not skip leading whitespace
+
+/// Convert a string to a signed integer, with overflow detection
+/** @param[in]  in     Input string
+    @param[out] end    Returned end pointer
+    @param[in]  bits   Bits available for the converted value
+    @param[out] failed Location of overflow flag
+    @return String value converted to an int64_t
+
+Converts a signed decimal string to an int64_t.  The string should
+consist of an optional '+' or '-' sign followed by one or more of
+the digits 0 to 9.  The output value will be limited to fit in the
+given number of bits (including the sign bit).  If the value is too big,
+the largest possible value will be returned and *failed will be set to 1.
+
+The address of the first character following the converted number will
+be stored in *end.
+
+Both end and failed must be non-NULL.
+ */
+static inline int64_t hts_str2int(const char *in, char **end, int bits,
+                                    int *failed) {
+    uint64_t n = 0, limit = (1ULL << (bits - 1)) - 1;
+    uint32_t fast = (bits - 1) * 1000 / 3322 + 1; // log(10)/log(2) ~= 3.322
+    const unsigned char *v = (const unsigned char *) in;
+    const unsigned int ascii_zero = '0'; // Prevents conversion to signed
+    unsigned int d;
+
+    int neg;
+    switch(*v) {
+    case '-':
+        limit++;
+        neg=1;
+        v++;
+        // See "dup" comment below
+        while (--fast && *v>='0' && *v<='9')
+            n = n*10 + *v++ - ascii_zero;
+        break;
+
+    case '+':
+        v++;
+        // fall through
+
+    default:
+        neg = 0;
+        // dup of above.  This is somewhat unstable and mainly for code
+        // size cheats to prevent instruction cache lines spanning 32-byte
+        // blocks in the sam_parse_B_vals calling code.  It's been tested
+        // on gcc7, gcc13, clang10 and clang16 with -O2 and -O3.  While
+        // not exhaustive, this code duplication gives stable fast results
+        // while a single copy does not.
+        // (NB: system was "seq4d", so quite old)
+        while (--fast && *v>='0' && *v<='9')
+            n = n*10 + *v++ - ascii_zero;
+        break;
+    }
+
+    // NB gcc7 is slow with (unsigned)(*v - ascii_zero) < 10,
+    // while gcc13 prefers it.
+    if (*v>='0' && !fast) { // rejects ',' and tab
+        uint64_t limit_d_10 = limit / 10;
+        uint64_t limit_m_10 = limit - 10 * limit_d_10;
+        while ((d = *v - ascii_zero) < 10) {
+            if (n < limit_d_10 || (n == limit_d_10 && d <= limit_m_10)) {
+                n = n*10 + d;
+                v++;
+            } else {
+                do { v++; } while (*v - ascii_zero < 10);
+                n = limit;
+                *failed = 1;
+                break;
+            }
+        }
+    }
+
+    *end = (char *)v;
+
+    return neg ? (int64_t)-n : (int64_t)n;
+}
+
+/// Convert a string to an unsigned integer, with overflow detection
+/** @param[in]  in     Input string
+    @param[out] end    Returned end pointer
+    @param[in]  bits   Bits available for the converted value
+    @param[out] failed Location of overflow flag
+    @return String value converted to a uint64_t
+
+Converts an unsigned decimal string to a uint64_t.  The string should
+consist of an optional '+' sign followed by one or more of the digits 0
+to 9.  The output value will be limited to fit in the given number of bits.
+If the value is too big, the largest possible value will be returned
+and *failed will be set to 1.
+
+The address of the first character following the converted number will
+be stored in *end.
+
+Both end and failed must be non-NULL.
+ */
+
+static inline uint64_t hts_str2uint(const char *in, char **end, int bits,
+                                    int *failed) {
+    uint64_t n = 0, limit = (bits < 64 ? (1ULL << bits) : 0) - 1;
+    const unsigned char *v = (const unsigned char *) in;
+    const unsigned int ascii_zero = '0'; // Prevents conversion to signed
+    uint32_t fast = bits * 1000 / 3322 + 1; // log(10)/log(2) ~= 3.322
+    unsigned int d;
+
+    if (*v == '+')
+        v++;
+
+    while (--fast && *v>='0' && *v<='9')
+        n = n*10 + *v++ - ascii_zero;
+
+    if ((unsigned)(*v - ascii_zero) < 10 && !fast) {
+        uint64_t limit_d_10 = limit / 10;
+        uint64_t limit_m_10 = limit - 10 * limit_d_10;
+        while ((d = *v - ascii_zero) < 10) {
+            if (n < limit_d_10 || (n == limit_d_10 && d <= limit_m_10)) {
+                n = n*10 + d;
+                v++;
+            } else {
+                do { v++; } while (*v - ascii_zero < 10);
+                n = limit;
+                *failed = 1;
+                break;
+            }
+        }
+    }
+
+    *end = (char *)v;
+    return n;
+}
+
+/// Convert a string to a double, with overflow detection
+/** @param[in]  in     Input string
+    @param[out] end    Returned end pointer
+    @param[out] failed Location of overflow flag
+    @return String value converted to a double
+
+Converts a floating point value string to a double.  The string should
+have the format [+-]?[0-9]*[.]?[0-9]* with at least one and no more than 15
+digits.  Strings that do not match (inf, nan, values with exponents) will
+be passed on to strtod() for processing.
+
+If the value is too big, the largest possible value will be returned;
+if it is too small to be represented in a double zero will be returned.
+In both cases errno will be set to ERANGE.
+
+If no characters could be converted, *failed will be set to 1.
+
+The address of the first character following the converted number will
+be stored in *end.
+
+Both end and failed must be non-NULL.
+ */
+
+static inline double hts_str2dbl(const char *in, char **end, int *failed) {
+    uint64_t n = 0;
+    int max_len = 15;
+    const unsigned char *v = (const unsigned char *) in;
+    const unsigned int ascii_zero = '0'; // Prevents conversion to signed
+    int neg = 0, point = -1;
+    double d;
+    static double D[] = {1,1, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7,
+                         1e8, 1e9, 1e10,1e11,1e12,1e13,1e14,1e15,
+                         1e16,1e17,1e18,1e19,1e20};
+
+    while (isspace(*v))
+        v++;
+
+    if (*v == '-') {
+        neg = 1;
+        v++;
+    } else if (*v == '+') {
+        v++;
+    }
+
+    switch(*v) {
+    case '1': case '2': case '3': case '4':
+    case '5': case '6': case '7': case '8': case '9':
+        break;
+
+    case '0':
+        if (v[1] != 'x' && v[1] != 'X') break;
+        // else fall through - hex number
+
+    default:
+        // Non numbers, like NaN, Inf
+        d = strtod(in, end);
+        if (*end == in)
+            *failed = 1;
+        return d;
+    }
+
+    while (*v == '0') ++v;
+
+    const unsigned char *start = v;
+
+    while (--max_len && *v>='0' && *v<='9')
+        n = n*10 + *v++ - ascii_zero;
+    if (max_len && *v == '.') {
+        point = v - start;
+        v++;
+        while (--max_len && *v>='0' && *v<='9')
+            n = n*10 + *v++ - ascii_zero;
+    }
+    if (point < 0)
+        point = v - start;
+
+    // Outside the scope of this quick and dirty parser.
+    if (!max_len || *v == 'e' || *v == 'E') {
+        d = strtod(in, end);
+        if (*end == in)
+            *failed = 1;
+        return d;
+    }
+
+    *end = (char *)v;
+    d = n / D[v - start - point];
+
+    return neg ? -d : d;
+}
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/ext/htslib/thread_pool.c b/ext/htslib/thread_pool.c
new file mode 100644
index 0000000..252a9d2
--- /dev/null
+++ b/ext/htslib/thread_pool.c
@@ -0,0 +1,1535 @@
+/*  thread_pool.c -- A pool of generic worker threads
+
+    Copyright (c) 2013-2020 Genome Research Ltd.
+
+    Author: James Bonfield <jkb@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#ifndef TEST_MAIN
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+#endif
+
+#include <stdlib.h>
+#include <inttypes.h>
+#include <signal.h>
+#include <errno.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/time.h>
+#include <assert.h>
+#include <stdarg.h>
+#include <unistd.h>
+#include <limits.h>
+
+#include "thread_pool_internal.h"
+#include "htslib/hts_log.h"
+
+// Minimum stack size for threads.  Required for some rANS codecs
+// that use over 2Mbytes of stack for encoder / decoder state
+#define HTS_MIN_THREAD_STACK (3 * 1024 * 1024)
+
+static void hts_tpool_process_detach_locked(hts_tpool *p,
+                                            hts_tpool_process *q);
+
+//#define DEBUG
+
+#ifdef DEBUG
+static int worker_id(hts_tpool *p) {
+    int i;
+    pthread_t s = pthread_self();
+    for (i = 0; i < p->tsize; i++) {
+        if (pthread_equal(s, p->t[i].tid))
+            return i;
+    }
+    return -1;
+}
+
+void DBG_OUT(FILE *fp, char *fmt, ...) {
+    va_list args;
+    va_start(args, fmt);
+    vfprintf(fp, fmt, args);
+    va_end(args);
+}
+#else
+#define DBG_OUT(...) do{}while(0)
+#endif
+
+/* ----------------------------------------------------------------------------
+ * A process-queue to hold results from the thread pool.
+ *
+ * Each thread pool may have jobs of multiple types being queued up and
+ * interleaved, so we attach several job process-queues to a single pool.
+ *
+ * The jobs themselves are expected to push their results onto their
+ * appropriate results queue.
+ */
+
+/*
+ * Adds a result to the end of the process result queue.
+ *
+ * Returns 0 on success;
+ *        -1 on failure
+ */
+static int hts_tpool_add_result(hts_tpool_job *j, void *data) {
+    hts_tpool_process *q = j->q;
+    hts_tpool_result *r;
+
+    pthread_mutex_lock(&q->p->pool_m);
+
+    DBG_OUT(stderr, "%d: Adding result to queue %p, serial %"PRId64", %d of %d\n",
+            worker_id(j->p), q, j->serial, q->n_output+1, q->qsize);
+
+    if (--q->n_processing == 0)
+        pthread_cond_signal(&q->none_processing_c);
+
+    /* No results queue is fine if we don't want any results back */
+    if (q->in_only) {
+        pthread_mutex_unlock(&q->p->pool_m);
+        return 0;
+    }
+
+    if (!(r = malloc(sizeof(*r)))) {
+        pthread_mutex_unlock(&q->p->pool_m);
+        hts_tpool_process_shutdown(q);
+        return -1;
+    }
+
+    r->next = NULL;
+    r->data = data;
+    r->result_cleanup = j->result_cleanup;
+    r->serial = j->serial;
+
+    q->n_output++;
+    if (q->output_tail) {
+        q->output_tail->next = r;
+        q->output_tail = r;
+    } else {
+        q->output_head = q->output_tail = r;
+    }
+
+    assert(r->serial >= q->next_serial    // Or it will never be dequeued ...
+           || q->next_serial == INT_MAX); // ... unless flush in progress.
+    if (r->serial == q->next_serial) {
+        DBG_OUT(stderr, "%d: Broadcasting result_avail (id %"PRId64")\n",
+                worker_id(j->p), r->serial);
+        pthread_cond_broadcast(&q->output_avail_c);
+        DBG_OUT(stderr, "%d: Broadcast complete\n", worker_id(j->p));
+    }
+
+    pthread_mutex_unlock(&q->p->pool_m);
+
+    return 0;
+}
+
+static void wake_next_worker(hts_tpool_process *q, int locked);
+
+/* Core of hts_tpool_next_result() */
+static hts_tpool_result *hts_tpool_next_result_locked(hts_tpool_process *q) {
+    hts_tpool_result *r, *last;
+
+    if (q->shutdown)
+        return NULL;
+
+    for (last = NULL, r = q->output_head; r; last = r, r = r->next) {
+        if (r->serial == q->next_serial)
+            break;
+    }
+
+    if (r) {
+        // Remove r from out linked list
+        if (q->output_head == r)
+            q->output_head = r->next;
+        else
+            last->next = r->next;
+
+        if (q->output_tail == r)
+            q->output_tail = last;
+
+        if (!q->output_head)
+            q->output_tail = NULL;
+
+        q->next_serial++;
+        q->n_output--;
+
+        if (q->qsize && q->n_output < q->qsize) {
+            // Not technically input full, but can guarantee there is
+            // room for the input to go somewhere so we still signal.
+            // The waiting code will then check the condition again.
+            if (q->n_input < q->qsize)
+                pthread_cond_signal(&q->input_not_full_c);
+            if (!q->shutdown)
+                wake_next_worker(q, 1);
+        }
+    }
+
+    return r;
+}
+
+/*
+ * Pulls the next item off the process result queue.  The caller should free
+ * it (and any internals as appropriate) after use.  This doesn't wait for a
+ * result to be present.
+ *
+ * Results will be returned in strict order.
+ *
+ * Returns hts_tpool_result pointer if a result is ready.
+ *         NULL if not.
+ */
+hts_tpool_result *hts_tpool_next_result(hts_tpool_process *q) {
+    hts_tpool_result *r;
+
+    DBG_OUT(stderr, "Requesting next result on queue %p\n", q);
+
+    pthread_mutex_lock(&q->p->pool_m);
+    r = hts_tpool_next_result_locked(q);
+    pthread_mutex_unlock(&q->p->pool_m);
+
+    DBG_OUT(stderr, "(q=%p) Found %p\n", q, r);
+
+    return r;
+}
+
+/*
+ * Pulls the next item off the process result queue.  The caller should free
+ * it (and any internals as appropriate) after use.  This will wait for
+ * a result to be present if none are currently available.
+ *
+ * Results will be returned in strict order.
+ *
+ * Returns hts_tpool_result pointer if a result is ready.
+ *         NULL on error or during shutdown.
+ */
+hts_tpool_result *hts_tpool_next_result_wait(hts_tpool_process *q) {
+    hts_tpool_result *r;
+
+    pthread_mutex_lock(&q->p->pool_m);
+    while (!(r = hts_tpool_next_result_locked(q))) {
+        /* Possible race here now avoided via _locked() call, but in case... */
+        struct timeval now;
+        struct timespec timeout;
+
+        gettimeofday(&now, NULL);
+        timeout.tv_sec = now.tv_sec + 10;
+        timeout.tv_nsec = now.tv_usec * 1000;
+
+        q->ref_count++;
+        if (q->shutdown) {
+            int rc = --q->ref_count;
+            pthread_mutex_unlock(&q->p->pool_m);
+            if (rc == 0)
+                hts_tpool_process_destroy(q);
+            return NULL;
+        }
+        pthread_cond_timedwait(&q->output_avail_c, &q->p->pool_m, &timeout);
+
+        q->ref_count--;
+    }
+    pthread_mutex_unlock(&q->p->pool_m);
+
+    return r;
+}
+
+/*
+ * Returns true if there are no items in the process results queue and
+ * also none still pending.
+ */
+int hts_tpool_process_empty(hts_tpool_process *q) {
+    int empty;
+
+    pthread_mutex_lock(&q->p->pool_m);
+    empty = q->n_input == 0 && q->n_processing == 0 && q->n_output == 0;
+    pthread_mutex_unlock(&q->p->pool_m);
+
+    return empty;
+}
+
+void hts_tpool_process_ref_incr(hts_tpool_process *q) {
+    pthread_mutex_lock(&q->p->pool_m);
+    q->ref_count++;
+    pthread_mutex_unlock(&q->p->pool_m);
+}
+
+void hts_tpool_process_ref_decr(hts_tpool_process *q) {
+    pthread_mutex_lock(&q->p->pool_m);
+    if (--q->ref_count <= 0) {
+        pthread_mutex_unlock(&q->p->pool_m);
+        hts_tpool_process_destroy(q);
+        return;
+    }
+
+    // maybe also call destroy here if needed?
+    pthread_mutex_unlock(&q->p->pool_m);
+}
+
+/*
+ * Returns the number of completed jobs in the process results queue.
+ */
+int hts_tpool_process_len(hts_tpool_process *q) {
+    int len;
+
+    pthread_mutex_lock(&q->p->pool_m);
+    len = q->n_output;
+    pthread_mutex_unlock(&q->p->pool_m);
+
+    return len;
+}
+
+/*
+ * Returns the number of completed jobs in the process results queue plus the
+ * number running and queued up to run.
+ */
+int hts_tpool_process_sz(hts_tpool_process *q) {
+    int len;
+
+    pthread_mutex_lock(&q->p->pool_m);
+    len = q->n_output + q->n_input + q->n_processing;
+    pthread_mutex_unlock(&q->p->pool_m);
+
+    return len;
+}
+
+/*
+ * Shutdown a process.
+ *
+ * This sets the shutdown flag and wakes any threads waiting on process
+ * condition variables.
+ */
+static void hts_tpool_process_shutdown_locked(hts_tpool_process *q) {
+    q->shutdown = 1;
+    pthread_cond_broadcast(&q->output_avail_c);
+    pthread_cond_broadcast(&q->input_not_full_c);
+    pthread_cond_broadcast(&q->input_empty_c);
+    pthread_cond_broadcast(&q->none_processing_c);
+}
+
+void hts_tpool_process_shutdown(hts_tpool_process *q) {
+    pthread_mutex_lock(&q->p->pool_m);
+    hts_tpool_process_shutdown_locked(q);
+    pthread_mutex_unlock(&q->p->pool_m);
+}
+
+int hts_tpool_process_is_shutdown(hts_tpool_process *q) {
+    pthread_mutex_lock(&q->p->pool_m);
+    int r = q->shutdown;
+    pthread_mutex_unlock(&q->p->pool_m);
+    return r;
+}
+
+/*
+ * Frees a result 'r' and if free_data is true also frees
+ * the internal r->data result too.
+ */
+void hts_tpool_delete_result(hts_tpool_result *r, int free_data) {
+    if (!r)
+        return;
+
+    if (free_data && r->data)
+        free(r->data);
+
+    free(r);
+}
+
+/*
+ * Returns the data portion of a hts_tpool_result, corresponding
+ * to the actual "result" itself.
+ */
+void *hts_tpool_result_data(hts_tpool_result *r) {
+    return r->data;
+}
+
+/*
+ * Initialises a thread process-queue.
+ *
+ * In_only, if true, indicates that the process generates does not need to
+ * hold any output.  Otherwise an output queue is used to store the results
+ * of processing each input job.
+ *
+ * Results hts_tpool_process pointer on success;
+ *         NULL on failure
+ */
+hts_tpool_process *hts_tpool_process_init(hts_tpool *p, int qsize, int in_only) {
+    hts_tpool_process *q = malloc(sizeof(*q));
+    if (!q)
+        return NULL;
+
+    pthread_cond_init(&q->output_avail_c,   NULL);
+    pthread_cond_init(&q->input_not_full_c, NULL);
+    pthread_cond_init(&q->input_empty_c,    NULL);
+    pthread_cond_init(&q->none_processing_c,NULL);
+
+    q->p           = p;
+    q->input_head  = NULL;
+    q->input_tail  = NULL;
+    q->output_head = NULL;
+    q->output_tail = NULL;
+    q->next_serial = 0;
+    q->curr_serial = 0;
+    q->no_more_input = 0;
+    q->n_input     = 0;
+    q->n_output    = 0;
+    q->n_processing= 0;
+    q->qsize       = qsize;
+    q->in_only     = in_only;
+    q->shutdown    = 0;
+    q->wake_dispatch = 0;
+    q->ref_count   = 1;
+
+    q->next        = NULL;
+    q->prev        = NULL;
+
+    hts_tpool_process_attach(p, q);
+
+    return q;
+}
+
+/* Deallocates memory for a thread process-queue.
+ * Must be called before the thread pool is destroyed.
+ */
+void hts_tpool_process_destroy(hts_tpool_process *q) {
+    DBG_OUT(stderr, "Destroying results queue %p\n", q);
+
+    if (!q)
+        return;
+
+    // Prevent dispatch from queuing up any more jobs.
+    // We want to reset (and flush) the queue here, before
+    // we set the shutdown flag, but we need to avoid races
+    // with queue more input during reset.
+    pthread_mutex_lock(&q->p->pool_m);
+    q->no_more_input = 1;
+    pthread_mutex_unlock(&q->p->pool_m);
+
+    // Ensure it's fully drained before destroying the queue
+    hts_tpool_process_reset(q, 0);
+    pthread_mutex_lock(&q->p->pool_m);
+    hts_tpool_process_detach_locked(q->p, q);
+    hts_tpool_process_shutdown_locked(q);
+
+    // Maybe a worker is scanning this queue, so delay destruction
+    if (--q->ref_count > 0) {
+        pthread_mutex_unlock(&q->p->pool_m);
+        return;
+    }
+
+    pthread_cond_destroy(&q->output_avail_c);
+    pthread_cond_destroy(&q->input_not_full_c);
+    pthread_cond_destroy(&q->input_empty_c);
+    pthread_cond_destroy(&q->none_processing_c);
+    pthread_mutex_unlock(&q->p->pool_m);
+
+    free(q);
+
+    DBG_OUT(stderr, "Destroyed results queue %p\n", q);
+}
+
+
+/*
+ * Attach and detach a thread process-queue with / from the thread pool
+ * scheduler.
+ *
+ * We need to do attach after making a thread process, but may also wish
+ * to temporarily detach if we wish to stop running jobs on a specific
+ * process while permitting other process to continue.
+ */
+void hts_tpool_process_attach(hts_tpool *p, hts_tpool_process *q) {
+    pthread_mutex_lock(&p->pool_m);
+    if (p->q_head) {
+        q->next = p->q_head;
+        q->prev = p->q_head->prev;
+        p->q_head->prev->next = q;
+        p->q_head->prev = q;
+    } else {
+        q->next = q;
+        q->prev = q;
+    }
+    p->q_head = q;
+    assert(p->q_head && p->q_head->prev && p->q_head->next);
+    pthread_mutex_unlock(&p->pool_m);
+}
+
+static void hts_tpool_process_detach_locked(hts_tpool *p,
+                                            hts_tpool_process *q) {
+    if (!p->q_head || !q->prev || !q->next)
+        return;
+
+    hts_tpool_process *curr = p->q_head, *first = curr;
+    do {
+        if (curr == q) {
+            q->next->prev = q->prev;
+            q->prev->next = q->next;
+            p->q_head = q->next;
+            q->next = q->prev = NULL;
+
+            // Last one
+            if (p->q_head == q)
+                p->q_head = NULL;
+            break;
+        }
+
+        curr = curr->next;
+    } while (curr != first);
+}
+
+void hts_tpool_process_detach(hts_tpool *p, hts_tpool_process *q) {
+    pthread_mutex_lock(&p->pool_m);
+    hts_tpool_process_detach_locked(p, q);
+    pthread_mutex_unlock(&p->pool_m);
+}
+
+
+/* ----------------------------------------------------------------------------
+ * The thread pool.
+ */
+
+#define TDIFF(t2,t1) ((t2.tv_sec-t1.tv_sec)*1000000 + t2.tv_usec-t1.tv_usec)
+
+/*
+ * A worker thread.
+ *
+ * Once woken, each thread checks each process-queue in the pool in turn,
+ * looking for input jobs that also have room for the output (if it requires
+ * storing).  If found, we execute it and repeat.
+ *
+ * If we checked all input queues and find no such job, then we wait until we
+ * are signalled to check again.
+ */
+static void *tpool_worker(void *arg) {
+    hts_tpool_worker *w = (hts_tpool_worker *)arg;
+    hts_tpool *p = w->p;
+    hts_tpool_job *j;
+
+    pthread_mutex_lock(&p->pool_m);
+    while (!p->shutdown) {
+        // Pop an item off the pool queue
+
+        assert(p->q_head == 0 || (p->q_head->prev && p->q_head->next));
+
+        int work_to_do = 0;
+        hts_tpool_process *first = p->q_head, *q = first;
+        do {
+            // Iterate over queues, finding one with jobs and also
+            // room to put the result.
+            //if (q && q->input_head && !hts_tpool_process_output_full(q)) {
+            if (q && q->input_head
+                && q->qsize - q->n_output > q->n_processing
+                && !q->shutdown) {
+                work_to_do = 1;
+                break;
+            }
+
+            if (q) q = q->next;
+        } while (q && q != first);
+
+        if (!work_to_do) {
+            // We scanned all queues and cannot process any, so we wait.
+            p->nwaiting++;
+
+            // Push this thread to the top of the waiting stack
+            if (p->t_stack_top == -1 || p->t_stack_top > w->idx)
+                p->t_stack_top = w->idx;
+
+            p->t_stack[w->idx] = 1;
+//            printf("%2d: no work.  In=%d Proc=%d Out=%d  full=%d\n",
+//                   w->idx, p->q_head->n_input, p->q_head->n_processing, p->q_head->n_output,
+//                   hts_tpool_process_output_full(p->q_head));
+            pthread_cond_wait(&w->pending_c, &p->pool_m);
+            p->t_stack[w->idx] = 0;
+
+            /* Find new t_stack_top */
+            int i;
+            p->t_stack_top = -1;
+            for (i = 0; i < p->tsize; i++) {
+                if (p->t_stack[i]) {
+                    p->t_stack_top = i;
+                    break;
+                }
+            }
+
+            p->nwaiting--;
+            continue; // To outer loop.
+        }
+
+        // Otherwise work_to_do, so process as many items in this queue as
+        // possible before switching to another queue.  This means threads
+        // often end up being dedicated to one type of work.
+        q->ref_count++;
+        while (q->input_head && q->qsize - q->n_output > q->n_processing) {
+            if (p->shutdown)
+                goto shutdown;
+
+            if (q->shutdown)
+                // Queue shutdown, but there may be other queues
+                break;
+
+            j = q->input_head;
+            assert(j->p == p);
+
+            if (!(q->input_head = j->next))
+                q->input_tail = NULL;
+
+            // Transitioning from full queue to not-full means we can wake up
+            // any blocked dispatch threads.  We broadcast this as it's only
+            // happening once (on the transition) rather than every time we
+            // are below qsize.
+            // (I wish I could remember why io_lib rev 3660 changed this from
+            //  == to >=, but keeping it just in case!)
+            q->n_processing++;
+            if (q->n_input-- >= q->qsize)
+                pthread_cond_broadcast(&q->input_not_full_c);
+
+            if (q->n_input == 0)
+                pthread_cond_signal(&q->input_empty_c);
+
+            p->njobs--; // Total number of jobs; used to adjust to CPU scaling
+
+            pthread_mutex_unlock(&p->pool_m);
+
+            DBG_OUT(stderr, "%d: Processing queue %p, serial %"PRId64"\n",
+                    worker_id(j->p), q, j->serial);
+
+            if (hts_tpool_add_result(j, j->func(j->arg)) < 0)
+                goto err;
+            //memset(j, 0xbb, sizeof(*j));
+            free(j);
+
+            pthread_mutex_lock(&p->pool_m);
+        }
+        if (--q->ref_count == 0) { // we were the last user
+            hts_tpool_process_destroy(q);
+        } else {
+            // Out of jobs on this queue, so restart search from next one.
+            // This is equivalent to "work-stealing".
+            if (p->q_head)
+                p->q_head = p->q_head->next;
+        }
+    }
+
+ shutdown:
+    pthread_mutex_unlock(&p->pool_m);
+#ifdef DEBUG
+    fprintf(stderr, "%d: Shutting down\n", worker_id(p));
+#endif
+    return NULL;
+
+ err:
+#ifdef DEBUG
+    fprintf(stderr, "%d: Failed to add result\n", worker_id(p));
+#endif
+    // Hard failure, so shutdown all queues
+    pthread_mutex_lock(&p->pool_m);
+    hts_tpool_process *first = p->q_head, *q = first;
+    if (q) {
+        do {
+            hts_tpool_process_shutdown_locked(q);
+            q->shutdown = 2; // signify error.
+            q = q->next;
+        } while (q != first);
+    }
+    pthread_mutex_unlock(&p->pool_m);
+    return NULL;
+}
+
+static void wake_next_worker(hts_tpool_process *q, int locked) {
+    if (!q) return;
+    hts_tpool *p = q->p;
+    if (!locked)
+        pthread_mutex_lock(&p->pool_m);
+
+    // Update the q_head to be this queue so we'll start processing
+    // the queue we know to have results.
+    assert(q->prev && q->next); // attached
+    p->q_head = q;
+
+    // Wake up if we have more jobs waiting than CPUs. This partially combats
+    // CPU frequency scaling effects.  Starting too many threads and then
+    // running out of jobs can cause each thread to have lots of start/stop
+    // cycles, which then translates often to CPU frequency scaling
+    // adjustments.  Instead it is better to only start as many threads as we
+    // need to keep the throughput up, meaning some threads run flat out and
+    // others are idle.
+    //
+    // This isn't perfect as we need to know how many can actually start,
+    // rather than how many are waiting.  A limit on output queue size makes
+    // these two figures different.
+    assert(p->njobs >= q->n_input);
+
+    int running = p->tsize - p->nwaiting;
+    int sig = p->t_stack_top >= 0 && p->njobs > p->tsize - p->nwaiting
+        && (q->n_processing < q->qsize - q->n_output);
+
+//#define AVG_USAGE
+#ifdef AVG_USAGE
+    // Track average number of running threads and try to keep close.
+    // We permit this to change, but slowly.  This avoids "boom and bust" cycles
+    // where we read a lot of data, start a lot of jobs, then become idle again.
+    // This way some threads run steadily and others dormant, which is better
+    // for throughput.
+    //
+    // It's 50:50 if this is a good thing.  It helps some tasks quite significantly
+    // while slightly hindering other (perhaps more usual) jobs.
+
+    if (++p->n_count == 256) {
+        p->n_count >>= 1;
+        p->n_running >>= 1;
+    }
+    p->n_running += running;
+    // Built in lag to avoid see-sawing.  Is this safe in all cases?
+    if (sig && p->n_count >= 128 && running*p->n_count > p->n_running+1) sig=0;
+#endif
+
+    if (0) {
+        printf("%d waiting, %d running, %d output, %d, arun %d => %d\t", p->njobs,
+               running, q->n_output, q->qsize - q->n_output,
+               p->n_running/p->n_count, sig);
+        int i;
+        for (i = 0; i < p->tsize; i++)
+            putchar("x "[p->t_stack[i]]);
+        putchar('\n');
+    }
+
+    if (sig)
+        pthread_cond_signal(&p->t[p->t_stack_top].pending_c);
+
+    if (!locked)
+        pthread_mutex_unlock(&p->pool_m);
+}
+
+/*
+ * Creates a worker pool with n worker threads.
+ *
+ * Returns pool pointer on success;
+ *         NULL on failure
+ */
+hts_tpool *hts_tpool_init(int n) {
+    int t_idx = 0;
+    size_t stack_size = 0;
+    pthread_attr_t pattr;
+    int pattr_init_done = 0;
+    hts_tpool *p = malloc(sizeof(*p));
+    if (!p)
+        return NULL;
+    p->tsize = n;
+    p->njobs = 0;
+    p->nwaiting = 0;
+    p->shutdown = 0;
+    p->q_head = NULL;
+    p->t_stack = NULL;
+    p->n_count = 0;
+    p->n_running = 0;
+    p->t = malloc(n * sizeof(p->t[0]));
+    if (!p->t) {
+        free(p);
+        return NULL;
+    }
+    p->t_stack = malloc(n * sizeof(*p->t_stack));
+    if (!p->t_stack) {
+        free(p->t);
+        free(p);
+        return NULL;
+    }
+    p->t_stack_top = -1;
+
+    pthread_mutexattr_t attr;
+    pthread_mutexattr_init(&attr);
+    pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE);
+    pthread_mutex_init(&p->pool_m, &attr);
+    pthread_mutexattr_destroy(&attr);
+
+    pthread_mutex_lock(&p->pool_m);
+
+    // Ensure new threads have a reasonably large stack.  On some platforms,
+    // for example MacOS which defaults to 512Kb, this is not big enough
+    // for some of the rANS codecs.
+
+    if (pthread_attr_init(&pattr) < 0)
+        goto cleanup;
+    pattr_init_done = 1;
+    if (pthread_attr_getstacksize(&pattr, &stack_size) < 0)
+        goto cleanup;
+    if (stack_size < HTS_MIN_THREAD_STACK) {
+        if (pthread_attr_setstacksize(&pattr, HTS_MIN_THREAD_STACK) < 0)
+            goto cleanup;
+    }
+
+    for (t_idx = 0; t_idx < n; t_idx++) {
+        hts_tpool_worker *w = &p->t[t_idx];
+        p->t_stack[t_idx] = 0;
+        w->p = p;
+        w->idx = t_idx;
+        pthread_cond_init(&w->pending_c, NULL);
+        if (0 != pthread_create(&w->tid, &pattr, tpool_worker, w))
+            goto cleanup;
+    }
+
+    pthread_mutex_unlock(&p->pool_m);
+    pthread_attr_destroy(&pattr);
+
+    return p;
+
+ cleanup: {
+        // Any threads started will be waiting for p->pool_m, so we can
+        // stop them cleanly by setting p->shutdown, releasing the mutex and
+        // waiting for them to finish.
+        int j;
+        int save_errno = errno;
+        hts_log_error("Couldn't start thread pool worker : %s",
+                      strerror(errno));
+        p->shutdown = 1;
+        pthread_mutex_unlock(&p->pool_m);
+        for (j = 0; j < t_idx; j++) {
+            pthread_join(p->t[j].tid, NULL);
+            pthread_cond_destroy(&p->t[j].pending_c);
+        }
+        pthread_mutex_destroy(&p->pool_m);
+        if (pattr_init_done)
+            pthread_attr_destroy(&pattr);
+        free(p->t_stack);
+        free(p->t);
+        free(p);
+        errno = save_errno;
+        return NULL;
+    }
+}
+
+/*
+ * Returns the number of requested threads for a pool.
+ */
+int hts_tpool_size(hts_tpool *p) {
+    return p->tsize;
+}
+
+/*
+ * Adds an item to the work pool.
+ *
+ * Returns 0 on success
+ *        -1 on failure
+ */
+int hts_tpool_dispatch(hts_tpool *p, hts_tpool_process *q,
+                    void *(*func)(void *arg), void *arg) {
+    return hts_tpool_dispatch3(p, q, func, arg, NULL, NULL, 0);
+}
+
+/*
+ * As above but optional non-block flag.
+ *
+ * nonblock  0 => block if input queue is full
+ * nonblock +1 => don't block if input queue is full, but do not add task
+ * nonblock -1 => add task regardless of whether queue is full (over-size)
+ */
+int hts_tpool_dispatch2(hts_tpool *p, hts_tpool_process *q,
+                        void *(*func)(void *arg), void *arg, int nonblock) {
+    return hts_tpool_dispatch3(p, q, func, arg, NULL, NULL, nonblock);
+}
+
+int hts_tpool_dispatch3(hts_tpool *p, hts_tpool_process *q,
+                        void *(*exec_func)(void *arg), void *arg,
+                        void (*job_cleanup)(void *arg),
+                        void (*result_cleanup)(void *data),
+                        int nonblock) {
+    hts_tpool_job *j;
+
+    pthread_mutex_lock(&p->pool_m);
+
+    DBG_OUT(stderr, "Dispatching job for queue %p, serial %"PRId64"\n",
+            q, q->curr_serial);
+
+    if ((q->no_more_input || q->n_input >= q->qsize) && nonblock == 1) {
+        pthread_mutex_unlock(&p->pool_m);
+        errno = EAGAIN;
+        return -1;
+    }
+
+    if (!(j = malloc(sizeof(*j)))) {
+        pthread_mutex_unlock(&p->pool_m);
+        return -1;
+    }
+    j->func = exec_func;
+    j->arg = arg;
+    j->job_cleanup = job_cleanup;
+    j->result_cleanup = result_cleanup;
+    j->next = NULL;
+    j->p = p;
+    j->q = q;
+    j->serial = q->curr_serial++;
+
+    if (nonblock == 0) {
+        while ((q->no_more_input || q->n_input >= q->qsize) &&
+               !q->shutdown && !q->wake_dispatch) {
+            pthread_cond_wait(&q->input_not_full_c, &q->p->pool_m);
+        }
+        if (q->no_more_input || q->shutdown) {
+            free(j);
+            pthread_mutex_unlock(&p->pool_m);
+            return -1;
+        }
+        if (q->wake_dispatch) {
+            //fprintf(stderr, "Wake => non-block for this operation\n");
+            q->wake_dispatch = 0;
+        }
+    }
+
+    p->njobs++;    // total across all queues
+    q->n_input++;  // queue specific
+
+    if (q->input_tail) {
+        q->input_tail->next = j;
+        q->input_tail = j;
+    } else {
+        q->input_head = q->input_tail = j;
+    }
+
+    DBG_OUT(stderr, "Dispatched (serial %"PRId64")\n", j->serial);
+
+    // Let a worker know we have data.
+    // Keep incoming queue at 1 per running thread, so there is always
+    // something waiting when they end their current task.  If we go above
+    // this signal to start more threads (if available). This has the effect
+    // of concentrating jobs to fewer cores when we are I/O bound, which in
+    // turn benefits systems with auto CPU frequency scaling.
+    if (!q->shutdown)
+        wake_next_worker(q, 1);
+
+    pthread_mutex_unlock(&p->pool_m);
+
+    return 0;
+}
+
+/*
+ * Wakes up a single thread stuck in dispatch and make it return with
+ * errno EAGAIN.
+ */
+void hts_tpool_wake_dispatch(hts_tpool_process *q) {
+    pthread_mutex_lock(&q->p->pool_m);
+    q->wake_dispatch = 1;
+    pthread_cond_signal(&q->input_not_full_c);
+    pthread_mutex_unlock(&q->p->pool_m);
+}
+
+/*
+ * Flushes the process-queue, but doesn't exit. This simply drains the queue
+ * and ensures all worker threads have finished their current tasks
+ * associated with this process.
+ *
+ * NOT: This does not mean the worker threads are not executing jobs in
+ * another process-queue.
+ *
+ * Returns 0 on success;
+ *        -1 on failure
+ */
+int hts_tpool_process_flush(hts_tpool_process *q) {
+    int i;
+    hts_tpool *p = q->p;
+
+    DBG_OUT(stderr, "Flushing pool %p\n", p);
+
+    // Drains the queue
+    pthread_mutex_lock(&p->pool_m);
+
+    // Wake up everything for the final sprint!
+    for (i = 0; i < p->tsize; i++)
+        if (p->t_stack[i])
+            pthread_cond_signal(&p->t[i].pending_c);
+
+    // Ensure there is room for the final sprint.
+    // Ideally we shouldn't get here, but the "q->qsize - q->n_output >
+    // n_processing" check in tpool_worker means we can trigger a
+    // deadlock there.  This negates that possibility.
+    if (q->qsize < q->n_output + q->n_input + q->n_processing)
+        q->qsize = q->n_output + q->n_input + q->n_processing;
+
+    // When shutdown, we won't be launching more, but we can still
+    // wait for any processing jobs complete.
+    if (q->shutdown) {
+        while (q->n_processing)
+            pthread_cond_wait(&q->none_processing_c, &p->pool_m);
+    }
+
+    // Wait for n_input and n_processing to hit zero.
+    while (!q->shutdown && (q->n_input || q->n_processing)) {
+        struct timeval now;
+        struct timespec timeout;
+
+        while (q->n_input && !q->shutdown) {
+            gettimeofday(&now, NULL);
+            timeout.tv_sec = now.tv_sec + 1;
+            timeout.tv_nsec = now.tv_usec * 1000;
+            pthread_cond_timedwait(&q->input_empty_c, &p->pool_m, &timeout);
+        }
+
+        // Note: even if q->shutdown is set, we still have to wait until
+        // q->n_processing is zero as we cannot terminate while things are
+        // running otherwise we free up the data being worked on.
+        while (q->n_processing) {
+            gettimeofday(&now, NULL);
+            timeout.tv_sec = now.tv_sec + 1;
+            timeout.tv_nsec = now.tv_usec * 1000;
+            pthread_cond_timedwait(&q->none_processing_c, &p->pool_m,
+                                   &timeout);
+        }
+        if (q->shutdown) break;
+    }
+
+    pthread_mutex_unlock(&p->pool_m);
+
+    DBG_OUT(stderr, "Flushed complete for pool %p, queue %p\n", p, q);
+
+    return 0;
+}
+
+/*
+ * Resets a process to the initial state.
+ *
+ * This removes any queued up input jobs, disables any notification of
+ * new results/output, flushes what is left and then discards any
+ * queued output.  Anything consumer stuck in a wait on results to
+ * appear should stay stuck and will only wake up when new data is
+ * pushed through the queue.
+ *
+ * Returns 0 on success;
+ *        -1 on failure
+ */
+int hts_tpool_process_reset(hts_tpool_process *q, int free_results) {
+    hts_tpool_job *j, *jn, *j_head;
+    hts_tpool_result *r, *rn, *r_head;
+
+    pthread_mutex_lock(&q->p->pool_m);
+    // prevent next_result from returning data during our flush
+    q->next_serial = INT_MAX;
+
+    // Remove any queued input not yet being acted upon
+    j_head = q->input_head;
+    q->input_head = q->input_tail = NULL;
+    q->n_input = 0;
+
+    // Remove any queued output, thus ensuring we have room to flush.
+    r_head = q->output_head;
+    q->output_head = q->output_tail = NULL;
+    q->n_output = 0;
+    pthread_mutex_unlock(&q->p->pool_m);
+
+    // Release memory.  This can be done unlocked now the lists have been
+    // removed from the queue
+    for (j = j_head; j; j = jn) {
+        jn = j->next;
+        if (j->job_cleanup) j->job_cleanup(j->arg);
+        free(j);
+    }
+
+    for (r = r_head; r; r = rn) {
+        rn = r->next;
+        if (r->result_cleanup) {
+            r->result_cleanup(r->data);
+            r->data = NULL;
+        }
+        hts_tpool_delete_result(r, free_results);
+    }
+
+    // Wait for any jobs being processed to complete.
+    // (TODO: consider how to cancel any currently processing jobs.
+    // Probably this is too hard.)
+    if (hts_tpool_process_flush(q) != 0)
+        return -1;
+
+    // Remove any new output.
+    pthread_mutex_lock(&q->p->pool_m);
+    r_head = q->output_head;
+    q->output_head = q->output_tail = NULL;
+    q->n_output = 0;
+
+    // Finally reset the serial back to the starting point.
+    q->next_serial = q->curr_serial = 0;
+    pthread_cond_signal(&q->input_not_full_c);
+    pthread_mutex_unlock(&q->p->pool_m);
+
+    // Discard unwanted output
+    for (r = r_head; r; r = rn) {
+        //fprintf(stderr, "Discard output %d\n", r->serial);
+        rn = r->next;
+        if (r->result_cleanup) {
+            r->result_cleanup(r->data);
+            r->data = NULL;
+        }
+        hts_tpool_delete_result(r, free_results);
+    }
+
+    return 0;
+}
+
+/* Returns the process queue size */
+int hts_tpool_process_qsize(hts_tpool_process *q) {
+    return q->qsize;
+}
+
+/*
+ * Destroys a thread pool.  The threads are joined into the main
+ * thread so they will finish their current work load.
+ */
+void hts_tpool_destroy(hts_tpool *p) {
+    int i;
+
+    DBG_OUT(stderr, "Destroying pool %p\n", p);
+
+    /* Send shutdown message to worker threads */
+    pthread_mutex_lock(&p->pool_m);
+    p->shutdown = 1;
+
+    DBG_OUT(stderr, "Sending shutdown request\n");
+
+    for (i = 0; i < p->tsize; i++)
+        pthread_cond_signal(&p->t[i].pending_c);
+
+    pthread_mutex_unlock(&p->pool_m);
+
+    DBG_OUT(stderr, "Shutdown complete\n");
+
+    for (i = 0; i < p->tsize; i++)
+        pthread_join(p->t[i].tid, NULL);
+
+    pthread_mutex_destroy(&p->pool_m);
+    for (i = 0; i < p->tsize; i++)
+        pthread_cond_destroy(&p->t[i].pending_c);
+
+    if (p->t_stack)
+        free(p->t_stack);
+
+    free(p->t);
+    free(p);
+
+    DBG_OUT(stderr, "Destroyed pool %p\n", p);
+}
+
+
+/*
+ * Destroys a thread pool without waiting on jobs to complete.
+ * Use hts_tpool_kill(p) to quickly exit after a fatal error.
+ */
+void hts_tpool_kill(hts_tpool *p) {
+    int i;
+
+    DBG_OUT(stderr, "Destroying pool %p, kill=%d\n", p, kill);
+
+    for (i = 0; i < p->tsize; i++)
+        pthread_kill(p->t[i].tid, SIGINT);
+
+    pthread_mutex_destroy(&p->pool_m);
+    for (i = 0; i < p->tsize; i++)
+        pthread_cond_destroy(&p->t[i].pending_c);
+
+    if (p->t_stack)
+        free(p->t_stack);
+
+    free(p->t);
+    free(p);
+
+    DBG_OUT(stderr, "Destroyed pool %p\n", p);
+}
+
+
+/*=============================================================================
+ * Test app.
+ *
+ * This can be considered both as a basic test and as a worked example for
+ * various usage patterns.
+ *=============================================================================
+ */
+
+#ifdef TEST_MAIN
+
+#include <stdio.h>
+
+#ifndef TASK_SIZE
+#define TASK_SIZE 1000
+#endif
+
+/*-----------------------------------------------------------------------------
+ * Unordered x -> x*x test.
+ * Results arrive in order of completion.
+ */
+void *doit_square_u(void *arg) {
+    int job = *(int *)arg;
+
+    usleep(random() % 100000); // to coerce job completion out of order
+
+    printf("RESULT: %d\n", job*job);
+
+    free(arg);
+    return NULL;
+}
+
+int test_square_u(int n) {
+    hts_tpool *p = hts_tpool_init(n);
+    hts_tpool_process *q = hts_tpool_process_init(p, n*2, 1);
+    int i;
+
+    // Dispatch jobs
+    for (i = 0; i < TASK_SIZE; i++) {
+        int *ip = malloc(sizeof(*ip));
+        *ip = i;
+        hts_tpool_dispatch(p, q, doit_square_u, ip);
+    }
+
+    hts_tpool_process_flush(q);
+    hts_tpool_process_destroy(q);
+    hts_tpool_destroy(p);
+
+    return 0;
+}
+
+
+/*-----------------------------------------------------------------------------
+ * Ordered x -> x*x test.
+ * Results arrive in numerical order.
+ *
+ * This implementation uses a non-blocking dispatch to avoid dead-locks
+ * where one job takes too long to complete.
+ */
+void *doit_square(void *arg) {
+    int job = *(int *)arg;
+    int *res;
+
+    // One excessively slow, to stress test output queue filling and
+    // excessive out of order scenarios.
+    usleep(500000 * ((job&31)==31) + random() % 10000);
+
+    res = malloc(sizeof(*res));
+    *res = (job<0) ? -job*job : job*job;
+
+    free(arg);
+    return res;
+}
+
+int test_square(int n) {
+    hts_tpool *p = hts_tpool_init(n);
+    hts_tpool_process *q = hts_tpool_process_init(p, n*2, 0);
+    int i;
+    hts_tpool_result *r;
+
+    // Dispatch jobs
+    for (i = 0; i < TASK_SIZE; i++) {
+        int *ip = malloc(sizeof(*ip));
+        *ip = i;
+        int blk;
+
+        do {
+            // In the situation where some jobs take much longer than
+            // others, we could end up blocking here as we haven't got
+            // any room in the output queue to place it. (We don't launch a
+            // job if the output queue is full.)
+
+            // This happens when the next serial number to fetch is, eg, 50
+            // but jobs 51-100 have all executed really fast and appeared in
+            // the output queue before 50.  A dispatch & check-results
+            // alternating loop can fail to find job 50 many times over until
+            // eventually the dispatch blocks before it arrives.
+
+            // Our solution is to dispatch in non-blocking mode so we are
+            // always to either dispatch or consume a result.
+            blk = hts_tpool_dispatch2(p, q, doit_square, ip, 1);
+
+            // Check for results.
+            if ((r = hts_tpool_next_result(q))) {
+                printf("RESULT: %d\n", *(int *)hts_tpool_result_data(r));
+                hts_tpool_delete_result(r, 1);
+            }
+            if (blk == -1) {
+                // The alternative is a separate thread for dispatching and/or
+                // consumption of results. See test_squareB.
+                putchar('.'); fflush(stdout);
+                usleep(10000);
+            }
+        } while (blk == -1);
+    }
+
+    // Wait for any input-queued up jobs or in-progress jobs to complete.
+    hts_tpool_process_flush(q);
+
+    while ((r = hts_tpool_next_result(q))) {
+        printf("RESULT: %d\n", *(int *)hts_tpool_result_data(r));
+        hts_tpool_delete_result(r, 1);
+    }
+
+    hts_tpool_process_destroy(q);
+    hts_tpool_destroy(p);
+
+    return 0;
+}
+
+/*-----------------------------------------------------------------------------
+ * Ordered x -> x*x test.
+ * Results arrive in numerical order.
+ *
+ * This implementation uses separate dispatching threads and job consumption
+ * threads (main thread).  This means it can use a blocking calls for
+ * simplicity elsewhere.
+ */
+struct squareB_opt {
+    hts_tpool *p;
+    hts_tpool_process *q;
+    int n;
+};
+static void *test_squareB_dispatcher(void *arg) {
+    struct squareB_opt *o = (struct squareB_opt *)arg;
+    int i, *ip;
+
+    for (i = 0; i < o->n; i++) {
+        ip = malloc(sizeof(*ip));
+        *ip = i;
+
+        hts_tpool_dispatch(o->p, o->q, doit_square, ip);
+    }
+
+    // Dispatch an sentinel job to mark the end
+    *(ip = malloc(sizeof(*ip))) = -1;
+    hts_tpool_dispatch(o->p, o->q, doit_square, ip);
+    pthread_exit(NULL);
+}
+
+int test_squareB(int n) {
+    hts_tpool *p = hts_tpool_init(n);
+    hts_tpool_process *q = hts_tpool_process_init(p, n*2, 0);
+    struct squareB_opt o = {p, q, TASK_SIZE};
+    pthread_t tid;
+
+    // Launch our job creation thread.
+    pthread_create(&tid, NULL, test_squareB_dispatcher, &o);
+
+    // Consume all results until we find the end-of-job marker.
+    for(;;) {
+        hts_tpool_result *r = hts_tpool_next_result_wait(q);
+        int x = *(int *)hts_tpool_result_data(r);
+        hts_tpool_delete_result(r, 1);
+        if (x == -1)
+            break;
+        printf("RESULT: %d\n", x);
+    }
+
+    // Wait for any input-queued up jobs or in-progress jobs to complete.
+    // This should do nothing as we've been executing until the termination
+    // marker of -1.
+    hts_tpool_process_flush(q);
+    assert(hts_tpool_next_result(q) == NULL);
+
+    hts_tpool_process_destroy(q);
+    hts_tpool_destroy(p);
+    pthread_join(tid, NULL);
+
+    return 0;
+}
+
+
+/*-----------------------------------------------------------------------------
+ * A simple pipeline test.
+ * We use a dedicated input thread that does the initial generation of job
+ * and dispatch, several execution steps running in a shared pool, and a
+ * dedicated output thread that prints up the final result.  It's key that our
+ * pipeline execution stages can run independently and don't themselves have
+ * any waits.  To achieve this we therefore also use some dedicated threads
+ * that take the output from one queue and resubmits the job as the input to
+ * the next queue.
+ *
+ * More generally this could perhaps be a single pipeline thread that
+ * marshalls multiple queues and their interactions, but this is simply a
+ * demonstration of a single pipeline.
+ *
+ * Our process fills out the bottom byte of a 32-bit int and then shifts it
+ * left one byte at a time.  Only the final stage needs to be ordered.  Each
+ * stage uses its own queue.
+ *
+ * Possible improvement: we only need the last stage to be ordered.  By
+ * allocating our own serial numbers for the first job and manually setting
+ * these serials in the last job, perhaps we can permit out of order execution
+ * of all the in-between stages.  (I doubt it'll affect speed much though.)
+ */
+
+static void *pipe_input_thread(void *arg);
+static void *pipe_stage1(void *arg);
+static void *pipe_stage2(void *arg);
+static void *pipe_stage3(void *arg);
+static void *pipe_output_thread(void *arg);
+
+typedef struct {
+    hts_tpool *p;
+    hts_tpool_process *q1;
+    hts_tpool_process *q2;
+    hts_tpool_process *q3;
+    int n;
+} pipe_opt;
+
+typedef struct {
+    pipe_opt *o;
+    unsigned int x;
+    int eof; // set with last job.
+} pipe_job;
+
+static void *pipe_input_thread(void *arg) {
+    pipe_opt *o = (pipe_opt *)arg;
+
+    int i;
+    for (i = 1; i <= o->n; i++) {
+        pipe_job *j = malloc(sizeof(*j));
+        j->o = o;
+        j->x = i;
+        j->eof = (i == o->n);
+
+        printf("I  %08x\n", j->x);
+
+        if (hts_tpool_dispatch(o->p, o->q1, pipe_stage1, j) != 0) {
+            free(j);
+            pthread_exit((void *)1);
+        }
+    }
+
+    pthread_exit(NULL);
+}
+
+static void *pipe_stage1(void *arg) {
+    pipe_job *j = (pipe_job *)arg;
+
+    j->x <<= 8;
+    usleep(random() % 10000); // fast job
+    printf("1  %08x\n", j->x);
+
+    return j;
+}
+
+static void *pipe_stage1to2(void *arg) {
+    pipe_opt *o = (pipe_opt *)arg;
+    hts_tpool_result *r;
+
+    while ((r = hts_tpool_next_result_wait(o->q1))) {
+        pipe_job *j = (pipe_job *)hts_tpool_result_data(r);
+        hts_tpool_delete_result(r, 0);
+        if (hts_tpool_dispatch(j->o->p, j->o->q2, pipe_stage2, j) != 0)
+            pthread_exit((void *)1);
+        if (j->eof)
+            break;
+    }
+
+    pthread_exit(NULL);
+}
+
+static void *pipe_stage2(void *arg) {
+    pipe_job *j = (pipe_job *)arg;
+
+    j->x <<= 8;
+    usleep(random() % 100000); // slow job
+    printf("2  %08x\n", j->x);
+
+    return j;
+}
+
+static void *pipe_stage2to3(void *arg) {
+    pipe_opt *o = (pipe_opt *)arg;
+    hts_tpool_result *r;
+
+    while ((r = hts_tpool_next_result_wait(o->q2))) {
+        pipe_job *j = (pipe_job *)hts_tpool_result_data(r);
+        hts_tpool_delete_result(r, 0);
+        if (hts_tpool_dispatch(j->o->p, j->o->q3, pipe_stage3, j) != 0)
+            pthread_exit((void *)1);
+        if (j->eof)
+            break;
+    }
+
+    pthread_exit(NULL);
+}
+
+static void *pipe_stage3(void *arg) {
+    pipe_job *j = (pipe_job *)arg;
+
+    usleep(random() % 10000); // fast job
+    j->x <<= 8;
+    return j;
+}
+
+static void *pipe_output_thread(void *arg) {
+    pipe_opt *o = (pipe_opt *)arg;
+    hts_tpool_result *r;
+
+    while ((r = hts_tpool_next_result_wait(o->q3))) {
+        pipe_job *j = (pipe_job *)hts_tpool_result_data(r);
+        int eof = j->eof;
+        printf("O  %08x\n", j->x);
+        hts_tpool_delete_result(r, 1);
+        if (eof)
+            break;
+    }
+
+    pthread_exit(NULL);
+}
+
+int test_pipe(int n) {
+    hts_tpool *p = hts_tpool_init(n);
+    hts_tpool_process *q1 = hts_tpool_process_init(p, n*2, 0);
+    hts_tpool_process *q2 = hts_tpool_process_init(p, n*2, 0);
+    hts_tpool_process *q3 = hts_tpool_process_init(p, n*2, 0);
+    pipe_opt o = {p, q1, q2, q3, TASK_SIZE};
+    pthread_t tidIto1, tid1to2, tid2to3, tid3toO;
+    void *retv;
+    int ret;
+
+    // Launch our data source and sink threads.
+    pthread_create(&tidIto1, NULL, pipe_input_thread,  &o);
+    pthread_create(&tid1to2, NULL, pipe_stage1to2,     &o);
+    pthread_create(&tid2to3, NULL, pipe_stage2to3,     &o);
+    pthread_create(&tid3toO, NULL, pipe_output_thread, &o);
+
+    // Wait for tasks to finish.
+    ret = 0;
+    pthread_join(tidIto1, &retv); ret |= (retv != NULL);
+    pthread_join(tid1to2, &retv); ret |= (retv != NULL);
+    pthread_join(tid2to3, &retv); ret |= (retv != NULL);
+    pthread_join(tid3toO, &retv); ret |= (retv != NULL);
+    printf("Return value %d\n", ret);
+
+    hts_tpool_process_destroy(q1);
+    hts_tpool_process_destroy(q2);
+    hts_tpool_process_destroy(q3);
+    hts_tpool_destroy(p);
+
+    return 0;
+}
+
+/*-----------------------------------------------------------------------------*/
+int main(int argc, char **argv) {
+    int n;
+    srandom(0);
+
+    if (argc < 3) {
+        fprintf(stderr, "Usage: %s command n_threads\n", argv[0]);
+        fprintf(stderr, "Where commands are:\n\n");
+        fprintf(stderr, "unordered       # Unordered output\n");
+        fprintf(stderr, "ordered1        # Main thread with non-block API\n");
+        fprintf(stderr, "ordered2        # Dispatch thread, blocking API\n");
+        fprintf(stderr, "pipe            # Multi-stage pipeline, several queues\n");
+        exit(1);
+    }
+
+    n = atoi(argv[2]);
+    if (strcmp(argv[1], "unordered") == 0) return test_square_u(n);
+    if (strcmp(argv[1], "ordered1") == 0)  return test_square(n);
+    if (strcmp(argv[1], "ordered2") == 0)  return test_squareB(n);
+    if (strcmp(argv[1], "pipe") == 0)      return test_pipe(n);
+
+    fprintf(stderr, "Unknown sub-command\n");
+    exit(1);
+}
+#endif
diff --git a/ext/htslib/thread_pool_internal.h b/ext/htslib/thread_pool_internal.h
new file mode 100644
index 0000000..c560614
--- /dev/null
+++ b/ext/htslib/thread_pool_internal.h
@@ -0,0 +1,169 @@
+/*  thread_pool_internal.h -- Internal API for the thread pool.
+
+    Copyright (c) 2013-2016 Genome Research Ltd.
+
+    Author: James Bonfield <jkb@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+/*
+ * This file implements a thread pool for multi-threading applications.
+ * It consists of two distinct interfaces: thread pools an thread job queues.
+ *
+ * The pool of threads is given a function pointer and void* data to pass in.
+ * This means the pool can run jobs of multiple types, albeit first come
+ * first served with no job scheduling except to pick tasks from
+ * queues that have room to store the result.
+ *
+ * Upon completion, the return value from the function pointer is
+ * added to back to the queue if the result is required.  We may have
+ * multiple queues in use for the one pool.
+ *
+ * To see example usage, please look at the #ifdef TEST_MAIN code in
+ * thread_pool.c.
+ */
+
+#ifndef THREAD_POOL_INTERNAL_H
+#define THREAD_POOL_INTERNAL_H
+
+#include <pthread.h>
+#include <stdint.h>
+#include "htslib/thread_pool.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * An input job, before execution.
+ */
+typedef struct hts_tpool_job {
+    void *(*func)(void *arg);
+    void *arg;
+    void (*job_cleanup)(void *arg);
+    void (*result_cleanup)(void *data);
+    struct hts_tpool_job *next;
+
+    struct hts_tpool *p;
+    struct hts_tpool_process *q;
+    uint64_t serial;
+} hts_tpool_job;
+
+/*
+ * An output, after job has executed.
+ */
+struct hts_tpool_result {
+    struct hts_tpool_result *next;
+    void (*result_cleanup)(void *data);
+    uint64_t serial; // sequential number for ordering
+    void *data;      // result itself
+};
+
+/*
+ * A per-thread worker struct.
+ */
+typedef struct {
+    struct hts_tpool *p;
+    int idx;
+    pthread_t tid;
+    pthread_cond_t  pending_c; // when waiting for a job
+} hts_tpool_worker;
+
+/*
+ * An IO queue consists of a queue of jobs to execute
+ * (the "input" side) and a queue of job results post-
+ * execution (the "output" side).
+ *
+ * We have size limits to prevent either queue from
+ * growing too large and serial numbers to ensure
+ * sequential consumption of the output.
+ *
+ * The thread pool may have many hetergeneous tasks, each
+ * using its own io_queue mixed into the same thread pool.
+ */
+struct hts_tpool_process {
+    struct hts_tpool *p;             // thread pool
+    hts_tpool_job    *input_head;    // input list
+    hts_tpool_job    *input_tail;
+    hts_tpool_result *output_head;   // output list
+    hts_tpool_result *output_tail;
+    int qsize;                       // max size of i/o queues
+    uint64_t next_serial;            // next serial for output
+    uint64_t curr_serial;            // current serial (next input)
+
+    int no_more_input;               // disable dispatching of more jobs
+    int n_input;                     // no. items in input queue; was njobs
+    int n_output;                    // no. items in output queue
+    int n_processing;                // no. items being processed (executing)
+
+    int shutdown;                    // true if pool is being destroyed
+    int in_only;                     // if true, don't queue result up.
+    int wake_dispatch;               // unblocks waiting dispatchers
+
+    int ref_count;                   // used to track safe destruction
+
+    pthread_cond_t output_avail_c;   // Signalled on each new output
+    pthread_cond_t input_not_full_c; // Input queue is no longer full
+    pthread_cond_t input_empty_c;    // Input queue has become empty
+    pthread_cond_t none_processing_c;// n_processing has hit zero
+
+    struct hts_tpool_process *next, *prev;// to form circular linked list.
+};
+
+/*
+ * The single pool structure itself.
+ *
+ * This knows nothing about the nature of the jobs or where their
+ * output is going, but it maintains a list of queues associated with
+ * this pool from which the jobs are taken.
+ */
+struct hts_tpool {
+    int nwaiting; // how many workers waiting for new jobs
+    int njobs;    // how many total jobs are waiting in all queues
+    int shutdown; // true if pool is being destroyed
+
+    // I/O queues to check for jobs in and to put results.
+    // Forms a circular linked list.  (q_head may be amended
+    // to point to the most recently updated.)
+    hts_tpool_process *q_head;
+
+    // threads
+    int tsize;    // maximum number of jobs
+    hts_tpool_worker *t;
+    // array of worker IDs free
+    int *t_stack, t_stack_top;
+
+    // A single mutex used when updating this and any associated structure.
+    pthread_mutex_t pool_m;
+
+    // Tracking of average number of running jobs.
+    // This can be used to dampen any hysteresis caused by bursty
+    // input availability.
+    int n_count, n_running;
+
+    // Debugging to check wait time.
+    // FIXME: should we just delete these and cull the associated code?
+    long long total_time, wait_time;
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/ext/htslib/vcf.5 b/ext/htslib/vcf.5
new file mode 100644
index 0000000..35d60c1
--- /dev/null
+++ b/ext/htslib/vcf.5
@@ -0,0 +1,120 @@
+'\" t
+.TH vcf 5 "August 2013" "htslib" "Bioinformatics formats"
+.SH NAME
+vcf \- Variant Call Format
+.\"
+.\" Copyright (C) 2011 Broad Institute.
+.\" Copyright (C) 2013-2014 Genome Research Ltd.
+.\"
+.\" Author: Heng Li <lh3@sanger.ac.uk>
+.\"
+.\" Permission is hereby granted, free of charge, to any person obtaining a
+.\" copy of this software and associated documentation files (the "Software"),
+.\" to deal in the Software without restriction, including without limitation
+.\" the rights to use, copy, modify, merge, publish, distribute, sublicense,
+.\" and/or sell copies of the Software, and to permit persons to whom the
+.\" Software is furnished to do so, subject to the following conditions:
+.\"
+.\" The above copyright notice and this permission notice shall be included in
+.\" all copies or substantial portions of the Software.
+.\"
+.\" THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+.\" IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+.\" FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+.\" THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+.\" LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+.\" FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+.\" DEALINGS IN THE SOFTWARE.
+.\"
+.SH DESCRIPTION
+The Variant Call Format (VCF) is a TAB-delimited format with each data line
+consisting of the following fields:
+.TS
+nlbl.
+1	CHROM	CHROMosome name
+2	POS	the left-most POSition of the variant
+3	ID	unique variant IDentifier
+4	REF	the REFerence allele
+5	ALT	the ALTernate allele(s) (comma-separated)
+6	QUAL	variant/reference QUALity
+7	FILTER	FILTERs applied
+8	INFO	INFOrmation related to the variant (semicolon-separated)
+9	FORMAT	FORMAT of the genotype fields (optional; colon-separated)
+10+	SAMPLE	SAMPLE genotypes and per-sample information (optional)
+.TE
+.P
+The following table gives the \fBINFO\fP tags used by samtools and bcftools.
+.TP
+.B AF1
+Max-likelihood estimate of the site allele frequency (AF) of the first ALT allele
+(double)
+.TP
+.B DP
+Raw read depth (without quality filtering)
+(int)
+.TP
+.B DP4
+# high-quality reference forward bases, ref reverse, alternate for and alt rev bases
+(int[4])
+.TP
+.B FQ
+Consensus quality. Positive: sample genotypes different; negative: otherwise
+(int)
+.TP
+.B MQ
+Root-Mean-Square mapping quality of covering reads
+(int)
+.TP
+.B PC2
+Phred probability of AF in group1 samples being larger (,smaller) than in group2
+(int[2])
+.TP
+.B PCHI2
+Posterior weighted chi^2 P-value between group1 and group2 samples
+(double)
+.TP
+.B PV4
+P-value for strand bias, baseQ bias, mapQ bias and tail distance bias
+(double[4])
+.TP
+.B QCHI2
+Phred-scaled PCHI2
+(int)
+.TP
+.B RP
+# permutations yielding a smaller PCHI2
+(int)
+.TP
+.B CLR
+Phred log ratio of genotype likelihoods with and without the trio/pair constraint
+(int)
+.TP
+.B UGT
+Most probable genotype configuration without the trio constraint
+(string)
+.TP
+.B CGT
+Most probable configuration with the trio constraint
+(string)
+.TP
+.B VDB
+Tests variant positions within reads. Intended for filtering RNA-seq artifacts around splice sites
+(float)
+.TP
+.B RPB
+Mann-Whitney rank-sum test for tail distance bias
+(float)
+.TP
+.B HWE
+Hardy-Weinberg equilibrium test (Wigginton et al)
+(float)
+.P
+.SH SEE ALSO
+.TP
+https://github.com/samtools/hts-specs
+The full VCF/BCF file format specification
+.TP
+.I A note on exact tests of Hardy-Weinberg equilibrium
+Wigginton JE et al
+PMID:15789306
+.\" (http://www.ncbi.nlm.nih.gov/pubmed/15789306)
diff --git a/ext/htslib/vcf.c b/ext/htslib/vcf.c
new file mode 100644
index 0000000..105c753
--- /dev/null
+++ b/ext/htslib/vcf.c
@@ -0,0 +1,5952 @@
+/*  vcf.c -- VCF/BCF API functions.
+
+    Copyright (C) 2012, 2013 Broad Institute.
+    Copyright (C) 2012-2024 Genome Research Ltd.
+    Portions copyright (C) 2014 Intel Corporation.
+
+    Author: Heng Li <lh3@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+
+#include <stdio.h>
+#include <assert.h>
+#include <string.h>
+#include <strings.h>
+#include <stdlib.h>
+#include <limits.h>
+#include <stdint.h>
+#include <inttypes.h>
+#include <errno.h>
+
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+#include "fuzz_settings.h"
+#endif
+
+#include "htslib/vcf.h"
+#include "htslib/bgzf.h"
+#include "htslib/tbx.h"
+#include "htslib/hfile.h"
+#include "hts_internal.h"
+#include "htslib/hts_endian.h"
+#include "htslib/khash_str2int.h"
+#include "htslib/kstring.h"
+#include "htslib/sam.h"
+#include "htslib/khash.h"
+
+#if 0
+// This helps on Intel a bit, often 6-7% faster VCF parsing.
+// Conversely sometimes harms AMD Zen4 as ~9% slower.
+// Possibly related to IPC differences.  However for now it's just a
+// curiousity we ignore and stick with the simpler code.
+//
+// Left here as a hint for future explorers.
+static inline int xstreq(const char *a, const char *b) {
+    while (*a && *a == *b)
+        a++, b++;
+    return *a == *b;
+}
+
+#define KHASH_MAP_INIT_XSTR(name, khval_t) \
+  KHASH_INIT(name, kh_cstr_t, khval_t, 1, kh_str_hash_func, xstreq)
+
+KHASH_MAP_INIT_XSTR(vdict, bcf_idinfo_t)
+#else
+KHASH_MAP_INIT_STR(vdict, bcf_idinfo_t)
+#endif
+
+typedef khash_t(vdict) vdict_t;
+
+KHASH_MAP_INIT_STR(hdict, bcf_hrec_t*)
+typedef khash_t(hdict) hdict_t;
+
+
+#include "htslib/kseq.h"
+HTSLIB_EXPORT
+uint32_t bcf_float_missing    = 0x7F800001;
+
+HTSLIB_EXPORT
+uint32_t bcf_float_vector_end = 0x7F800002;
+
+HTSLIB_EXPORT
+uint8_t bcf_type_shift[] = { 0, 0, 1, 2, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
+
+static bcf_idinfo_t bcf_idinfo_def = { .info = { 15, 15, 15 }, .hrec = { NULL, NULL, NULL}, .id = -1 };
+
+/*
+    Partial support for 64-bit POS and Number=1 INFO tags.
+    Notes:
+     - the support for 64-bit values is motivated by POS and INFO/END for large genomes
+     - the use of 64-bit values does not conform to the specification
+     - cannot output 64-bit BCF and if it does, it is not compatible with anything
+     - experimental, use at your risk
+*/
+#ifdef VCF_ALLOW_INT64
+    #define BCF_MAX_BT_INT64 (0x7fffffffffffffff)       /* INT64_MAX, for internal use only */
+    #define BCF_MIN_BT_INT64 -9223372036854775800LL     /* INT64_MIN + 8, for internal use only */
+#endif
+
+#define BCF_IS_64BIT (1<<30)
+
+
+// Opaque structure with auxilary data which allows to extend bcf_hdr_t without breaking ABI.
+// Note that this preserving API and ABI requires that the first element is vdict_t struct
+// rather than a pointer, as user programs may (and in some cases do) access the dictionary
+// directly as (vdict_t*)hdr->dict.
+typedef struct
+{
+    vdict_t dict;   // bcf_hdr_t.dict[0] vdict_t dictionary which keeps bcf_idinfo_t for BCF_HL_FLT,BCF_HL_INFO,BCF_HL_FMT
+    hdict_t *gen;   // hdict_t dictionary which keeps bcf_hrec_t* pointers for generic and structured fields
+    size_t *key_len;// length of h->id[BCF_DT_ID] strings
+}
+bcf_hdr_aux_t;
+
+static inline bcf_hdr_aux_t *get_hdr_aux(const bcf_hdr_t *hdr)
+{
+    return (bcf_hdr_aux_t *)hdr->dict[0];
+}
+
+static char *find_chrom_header_line(char *s)
+{
+    char *nl;
+    if (strncmp(s, "#CHROM\t", 7) == 0) return s;
+    else if ((nl = strstr(s, "\n#CHROM\t")) != NULL) return nl+1;
+    else return NULL;
+}
+
+/*************************
+ *** VCF header parser ***
+ *************************/
+
+static int bcf_hdr_add_sample_len(bcf_hdr_t *h, const char *s, size_t len)
+{
+    const char *ss = s;
+    while ( *ss && isspace_c(*ss) && ss - s < len) ss++;
+    if ( !*ss || ss - s == len)
+    {
+        hts_log_error("Empty sample name: trailing spaces/tabs in the header line?");
+        return -1;
+    }
+
+    vdict_t *d = (vdict_t*)h->dict[BCF_DT_SAMPLE];
+    int ret;
+    char *sdup = malloc(len + 1);
+    if (!sdup) return -1;
+    memcpy(sdup, s, len);
+    sdup[len] = 0;
+
+    // Ensure space is available in h->samples
+    size_t n = kh_size(d);
+    char **new_samples = realloc(h->samples, sizeof(char*) * (n + 1));
+    if (!new_samples) {
+        free(sdup);
+        return -1;
+    }
+    h->samples = new_samples;
+
+    int k = kh_put(vdict, d, sdup, &ret);
+    if (ret < 0) {
+        free(sdup);
+        return -1;
+    }
+    if (ret) { // absent
+        kh_val(d, k) = bcf_idinfo_def;
+        kh_val(d, k).id = n;
+    } else {
+        hts_log_error("Duplicated sample name '%s'", sdup);
+        free(sdup);
+        return -1;
+    }
+    h->samples[n] = sdup;
+    h->dirty = 1;
+    return 0;
+}
+
+int bcf_hdr_add_sample(bcf_hdr_t *h, const char *s)
+{
+    if (!s) {
+        // Allowed for backwards-compatibility, calling with s == NULL
+        // used to trigger bcf_hdr_sync(h);
+        return 0;
+    }
+    return bcf_hdr_add_sample_len(h, s, strlen(s));
+}
+
+int HTS_RESULT_USED bcf_hdr_parse_sample_line(bcf_hdr_t *hdr, const char *str)
+{
+    const char *mandatory = "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO";
+    if ( strncmp(str,mandatory,strlen(mandatory)) )
+    {
+        hts_log_error("Could not parse the \"#CHROM..\" line, either the fields are incorrect or spaces are present instead of tabs:\n\t%s",str);
+        return -1;
+    }
+
+    const char *beg = str + strlen(mandatory), *end;
+    if ( !*beg || *beg=='\n' ) return 0;
+    if ( strncmp(beg,"\tFORMAT\t",8) )
+    {
+        hts_log_error("Could not parse the \"#CHROM..\" line, either FORMAT is missing or spaces are present instead of tabs:\n\t%s",str);
+        return -1;
+    }
+    beg += 8;
+
+    int ret = 0;
+    while ( *beg )
+    {
+        end = beg;
+        while ( *end && *end!='\t' && *end!='\n' ) end++;
+        if ( bcf_hdr_add_sample_len(hdr, beg, end-beg) < 0 ) ret = -1;
+        if ( !*end || *end=='\n' || ret<0 ) break;
+        beg = end + 1;
+    }
+    return ret;
+}
+
+int bcf_hdr_sync(bcf_hdr_t *h)
+{
+    int i;
+    for (i = 0; i < 3; i++)
+    {
+        vdict_t *d = (vdict_t*)h->dict[i];
+        khint_t k;
+        if ( h->n[i] < kh_size(d) )
+        {
+            bcf_idpair_t *new_idpair;
+            // this should be true only for i=2, BCF_DT_SAMPLE
+            new_idpair = (bcf_idpair_t*) realloc(h->id[i], kh_size(d)*sizeof(bcf_idpair_t));
+            if (!new_idpair) return -1;
+            h->n[i] = kh_size(d);
+            h->id[i] = new_idpair;
+        }
+        for (k=kh_begin(d); k<kh_end(d); k++)
+        {
+            if (!kh_exist(d,k)) continue;
+            h->id[i][kh_val(d,k).id].key = kh_key(d,k);
+            h->id[i][kh_val(d,k).id].val = &kh_val(d,k);
+        }
+    }
+
+    // Invalidate key length cache
+    bcf_hdr_aux_t *aux = get_hdr_aux(h);
+    if (aux && aux->key_len) {
+        free(aux->key_len);
+        aux->key_len = NULL;
+    }
+
+    h->dirty = 0;
+    return 0;
+}
+
+void bcf_hrec_destroy(bcf_hrec_t *hrec)
+{
+    if (!hrec) return;
+    free(hrec->key);
+    if ( hrec->value ) free(hrec->value);
+    int i;
+    for (i=0; i<hrec->nkeys; i++)
+    {
+        free(hrec->keys[i]);
+        free(hrec->vals[i]);
+    }
+    free(hrec->keys);
+    free(hrec->vals);
+    free(hrec);
+}
+
+// Copies all fields except IDX.
+bcf_hrec_t *bcf_hrec_dup(bcf_hrec_t *hrec)
+{
+    int save_errno;
+    bcf_hrec_t *out = (bcf_hrec_t*) calloc(1,sizeof(bcf_hrec_t));
+    if (!out) return NULL;
+
+    out->type = hrec->type;
+    if ( hrec->key ) {
+        out->key = strdup(hrec->key);
+        if (!out->key) goto fail;
+    }
+    if ( hrec->value ) {
+        out->value = strdup(hrec->value);
+        if (!out->value) goto fail;
+    }
+    out->nkeys = hrec->nkeys;
+    out->keys = (char**) malloc(sizeof(char*)*hrec->nkeys);
+    if (!out->keys) goto fail;
+    out->vals = (char**) malloc(sizeof(char*)*hrec->nkeys);
+    if (!out->vals) goto fail;
+    int i, j = 0;
+    for (i=0; i<hrec->nkeys; i++)
+    {
+        if ( hrec->keys[i] && !strcmp("IDX",hrec->keys[i]) ) continue;
+        if ( hrec->keys[i] ) {
+            out->keys[j] = strdup(hrec->keys[i]);
+            if (!out->keys[j]) goto fail;
+        }
+        if ( hrec->vals[i] ) {
+            out->vals[j] = strdup(hrec->vals[i]);
+            if (!out->vals[j]) goto fail;
+        }
+        j++;
+    }
+    if ( i!=j ) out->nkeys -= i-j;   // IDX was omitted
+    return out;
+
+ fail:
+    save_errno = errno;
+    hts_log_error("%s", strerror(errno));
+    bcf_hrec_destroy(out);
+    errno = save_errno;
+    return NULL;
+}
+
+void bcf_hrec_debug(FILE *fp, bcf_hrec_t *hrec)
+{
+    fprintf(fp, "key=[%s] value=[%s]", hrec->key, hrec->value?hrec->value:"");
+    int i;
+    for (i=0; i<hrec->nkeys; i++)
+        fprintf(fp, "\t[%s]=[%s]", hrec->keys[i],hrec->vals[i]);
+    fprintf(fp, "\n");
+}
+
+void bcf_header_debug(bcf_hdr_t *hdr)
+{
+    int i, j;
+    for (i=0; i<hdr->nhrec; i++)
+    {
+        if ( !hdr->hrec[i]->value )
+        {
+            fprintf(stderr, "##%s=<", hdr->hrec[i]->key);
+            fprintf(stderr,"%s=%s", hdr->hrec[i]->keys[0], hdr->hrec[i]->vals[0]);
+            for (j=1; j<hdr->hrec[i]->nkeys; j++)
+                fprintf(stderr,",%s=%s", hdr->hrec[i]->keys[j], hdr->hrec[i]->vals[j]);
+            fprintf(stderr,">\n");
+        }
+        else
+            fprintf(stderr,"##%s=%s\n", hdr->hrec[i]->key,hdr->hrec[i]->value);
+    }
+}
+
+int bcf_hrec_add_key(bcf_hrec_t *hrec, const char *str, size_t len)
+{
+    char **tmp;
+    size_t n = hrec->nkeys + 1;
+    assert(len > 0 && len < SIZE_MAX);
+    tmp = realloc(hrec->keys, sizeof(char*)*n);
+    if (!tmp) return -1;
+    hrec->keys = tmp;
+    tmp = realloc(hrec->vals, sizeof(char*)*n);
+    if (!tmp) return -1;
+    hrec->vals = tmp;
+
+    hrec->keys[hrec->nkeys] = (char*) malloc((len+1)*sizeof(char));
+    if (!hrec->keys[hrec->nkeys]) return -1;
+    memcpy(hrec->keys[hrec->nkeys],str,len);
+    hrec->keys[hrec->nkeys][len] = 0;
+    hrec->vals[hrec->nkeys] = NULL;
+    hrec->nkeys = n;
+    return 0;
+}
+
+int bcf_hrec_set_val(bcf_hrec_t *hrec, int i, const char *str, size_t len, int is_quoted)
+{
+    if ( hrec->vals[i] ) {
+        free(hrec->vals[i]);
+        hrec->vals[i] = NULL;
+    }
+    if ( !str ) return 0;
+    if ( is_quoted )
+    {
+        if (len >= SIZE_MAX - 3) {
+            errno = ENOMEM;
+            return -1;
+        }
+        hrec->vals[i] = (char*) malloc((len+3)*sizeof(char));
+        if (!hrec->vals[i]) return -1;
+        hrec->vals[i][0] = '"';
+        memcpy(&hrec->vals[i][1],str,len);
+        hrec->vals[i][len+1] = '"';
+        hrec->vals[i][len+2] = 0;
+    }
+    else
+    {
+        if (len == SIZE_MAX) {
+            errno = ENOMEM;
+            return -1;
+        }
+        hrec->vals[i] = (char*) malloc((len+1)*sizeof(char));
+        if (!hrec->vals[i]) return -1;
+        memcpy(hrec->vals[i],str,len);
+        hrec->vals[i][len] = 0;
+    }
+    return 0;
+}
+
+int hrec_add_idx(bcf_hrec_t *hrec, int idx)
+{
+    int n = hrec->nkeys + 1;
+    char **tmp = (char**) realloc(hrec->keys, sizeof(char*)*n);
+    if (!tmp) return -1;
+    hrec->keys = tmp;
+
+    tmp = (char**) realloc(hrec->vals, sizeof(char*)*n);
+    if (!tmp) return -1;
+    hrec->vals = tmp;
+
+    hrec->keys[hrec->nkeys] = strdup("IDX");
+    if (!hrec->keys[hrec->nkeys]) return -1;
+
+    kstring_t str = {0,0,0};
+    if (kputw(idx, &str) < 0) {
+        free(hrec->keys[hrec->nkeys]);
+        return -1;
+    }
+    hrec->vals[hrec->nkeys] = str.s;
+    hrec->nkeys = n;
+    return 0;
+}
+
+int bcf_hrec_find_key(bcf_hrec_t *hrec, const char *key)
+{
+    int i;
+    for (i=0; i<hrec->nkeys; i++)
+        if ( !strcasecmp(key,hrec->keys[i]) ) return i;
+    return -1;
+}
+
+static void bcf_hrec_set_type(bcf_hrec_t *hrec)
+{
+    if ( !strcmp(hrec->key, "contig") ) hrec->type = BCF_HL_CTG;
+    else if ( !strcmp(hrec->key, "INFO") ) hrec->type = BCF_HL_INFO;
+    else if ( !strcmp(hrec->key, "FILTER") ) hrec->type = BCF_HL_FLT;
+    else if ( !strcmp(hrec->key, "FORMAT") ) hrec->type = BCF_HL_FMT;
+    else if ( hrec->nkeys>0 ) hrec->type = BCF_HL_STR;
+    else hrec->type = BCF_HL_GEN;
+}
+
+
+/**
+    The arrays were generated with
+
+    valid_ctg:
+        perl -le '@v = (split(//,q[!#$%&*+./:;=?@^_|~-]),"a"..."z","A"..."Z","0"..."9"); @a = (0) x 256; foreach $c (@v) { $a[ord($c)] = 1; } print join(", ",@a)' | fold -w 48
+
+    valid_tag:
+        perl -le '@v = (split(//,q[_.]),"a"..."z","A"..."Z","0"..."9"); @a = (0) x 256; foreach $c (@v) { $a[ord($c)] = 1; } print join(", ",@a)' | fold -w 48
+*/
+static const uint8_t valid_ctg[256] =
+{
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 1, 0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1,
+    0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+};
+static const uint8_t valid_tag[256] =
+{
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,
+    0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1,
+    0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+};
+
+/**
+    bcf_hrec_check() - check the validity of structured header lines
+
+    Returns 0 on success or negative value on error.
+
+    Currently the return status is not checked by the caller
+    and only a warning is printed on stderr. This should be improved
+    to propagate the error all the way up to the caller and let it
+    decide what to do: throw an error or proceed anyway.
+ */
+static int bcf_hrec_check(bcf_hrec_t *hrec)
+{
+    int i;
+    bcf_hrec_set_type(hrec);
+
+    if ( hrec->type==BCF_HL_CTG )
+    {
+        i = bcf_hrec_find_key(hrec,"ID");
+        if ( i<0 ) goto err_missing_id;
+        char *val = hrec->vals[i];
+        if ( val[0]=='*' || val[0]=='=' || !valid_ctg[(uint8_t)val[0]] ) goto err_invalid_ctg;
+        while ( *(++val) )
+            if ( !valid_ctg[(uint8_t)*val] ) goto err_invalid_ctg;
+        return 0;
+    }
+    if ( hrec->type==BCF_HL_INFO )
+    {
+        i = bcf_hrec_find_key(hrec,"ID");
+        if ( i<0 ) goto err_missing_id;
+        char *val = hrec->vals[i];
+        if ( !strcmp(val,"1000G") ) return 0;
+        if ( val[0]=='.' || (val[0]>='0' && val[0]<='9') || !valid_tag[(uint8_t)val[0]] ) goto err_invalid_tag;
+        while ( *(++val) )
+            if ( !valid_tag[(uint8_t)*val] ) goto err_invalid_tag;
+        return 0;
+    }
+    if ( hrec->type==BCF_HL_FMT )
+    {
+        i = bcf_hrec_find_key(hrec,"ID");
+        if ( i<0 ) goto err_missing_id;
+        char *val = hrec->vals[i];
+        if ( val[0]=='.' || (val[0]>='0' && val[0]<='9') || !valid_tag[(uint8_t)val[0]] ) goto err_invalid_tag;
+        while ( *(++val) )
+            if ( !valid_tag[(uint8_t)*val] ) goto err_invalid_tag;
+        return 0;
+    }
+    return 0;
+
+  err_missing_id:
+    hts_log_warning("Missing ID attribute in one or more header lines");
+    return -1;
+
+  err_invalid_ctg:
+    hts_log_warning("Invalid contig name: \"%s\"", hrec->vals[i]);
+    return -1;
+
+  err_invalid_tag:
+    hts_log_warning("Invalid tag name: \"%s\"", hrec->vals[i]);
+    return -1;
+}
+
+static inline int is_escaped(const char *min, const char *str)
+{
+    int n = 0;
+    while ( --str>=min && *str=='\\' ) n++;
+    return n%2;
+}
+
+bcf_hrec_t *bcf_hdr_parse_line(const bcf_hdr_t *h, const char *line, int *len)
+{
+    bcf_hrec_t *hrec = NULL;
+    const char *p = line;
+    if (p[0] != '#' || p[1] != '#') { *len = 0; return NULL; }
+    p += 2;
+
+    const char *q = p;
+    while ( *q && *q!='=' && *q != '\n' ) q++;
+    ptrdiff_t n = q-p;
+    if ( *q!='=' || !n ) // wrong format
+        goto malformed_line;
+
+    hrec = (bcf_hrec_t*) calloc(1,sizeof(bcf_hrec_t));
+    if (!hrec) { *len = -1; return NULL; }
+    hrec->key = (char*) malloc(sizeof(char)*(n+1));
+    if (!hrec->key) goto fail;
+    memcpy(hrec->key,p,n);
+    hrec->key[n] = 0;
+    hrec->type = -1;
+
+    p = ++q;
+    if ( *p!='<' ) // generic field, e.g. ##samtoolsVersion=0.1.18-r579
+    {
+        while ( *q && *q!='\n' ) q++;
+        hrec->value = (char*) malloc((q-p+1)*sizeof(char));
+        if (!hrec->value) goto fail;
+        memcpy(hrec->value, p, q-p);
+        hrec->value[q-p] = 0;
+        *len = q - line + (*q ? 1 : 0); // Skip \n but not \0
+        return hrec;
+    }
+
+    // structured line, e.g.
+    // ##INFO=<ID=PV1,Number=1,Type=Float,Description="P-value for baseQ bias">
+    // ##PEDIGREE=<Name_0=G0-ID,Name_1=G1-ID,Name_3=GN-ID>
+    int nopen = 1;
+    while ( *q && *q!='\n' && nopen>0 )
+    {
+        p = ++q;
+        while ( *q && *q==' ' ) { p++; q++; }
+        // ^[A-Za-z_][0-9A-Za-z_.]*$
+        if (p==q && *q && (isalpha_c(*q) || *q=='_'))
+        {
+            q++;
+            while ( *q && (isalnum_c(*q) || *q=='_' || *q=='.') ) q++;
+        }
+        n = q-p;
+        int m = 0;
+        while ( *q && *q==' ' ) { q++; m++; }
+        if ( *q!='=' || !n )
+            goto malformed_line;
+
+        if (bcf_hrec_add_key(hrec, p, q-p-m) < 0) goto fail;
+        p = ++q;
+        while ( *q && *q==' ' ) { p++; q++; }
+
+        int quoted = 0;
+        char ending = '\0';
+        switch (*p) {
+        case '"':
+            quoted = 1;
+            ending = '"';
+            p++;
+            break;
+        case '[':
+            quoted = 1;
+            ending = ']';
+            break;
+        }
+        if ( quoted ) q++;
+        while ( *q && *q != '\n' )
+        {
+            if ( quoted ) { if ( *q==ending && !is_escaped(p,q) ) break; }
+            else
+            {
+                if ( *q=='<' ) nopen++;
+                if ( *q=='>' ) nopen--;
+                if ( !nopen ) break;
+                if ( *q==',' && nopen==1 ) break;
+            }
+            q++;
+        }
+        const char *r = q;
+        if (quoted && ending == ']') {
+            if (*q == ending) {
+                r++;
+                q++;
+                quoted = 0;
+            } else {
+                char buffer[320];
+                hts_log_error("Missing ']' in header line %s",
+                              hts_strprint(buffer, sizeof(buffer), '"',
+                                           line, q-line));
+                goto fail;
+            }
+        }
+        while ( r > p && r[-1] == ' ' ) r--;
+        if (bcf_hrec_set_val(hrec, hrec->nkeys-1, p, r-p, quoted) < 0)
+            goto fail;
+        if ( quoted && *q==ending ) q++;
+        if ( *q=='>' )
+        {
+            if (nopen) nopen--;     // this can happen with nested angle brackets <>
+            q++;
+        }
+    }
+    if ( nopen )
+        hts_log_warning("Incomplete header line, trying to proceed anyway:\n\t[%s]\n\t[%d]",line,q[0]);
+
+    // Skip to end of line
+    int nonspace = 0;
+    p = q;
+    while ( *q && *q!='\n' ) { nonspace |= !isspace_c(*q); q++; }
+    if (nonspace) {
+        char buffer[320];
+        hts_log_warning("Dropped trailing junk from header line '%s'",
+                        hts_strprint(buffer, sizeof(buffer),
+                                     '"', line, q - line));
+    }
+
+    *len = q - line + (*q ? 1 : 0);
+    return hrec;
+
+ fail:
+    *len = -1;
+    bcf_hrec_destroy(hrec);
+    return NULL;
+
+ malformed_line:
+    {
+        char buffer[320];
+        while ( *q && *q!='\n' ) q++;  // Ensure *len includes full line
+        hts_log_error("Could not parse the header line: %s",
+                      hts_strprint(buffer, sizeof(buffer),
+                                   '"', line, q - line));
+        *len = q - line + (*q ? 1 : 0);
+        bcf_hrec_destroy(hrec);
+        return NULL;
+    }
+}
+
+static int bcf_hdr_set_idx(bcf_hdr_t *hdr, int dict_type, const char *tag, bcf_idinfo_t *idinfo)
+{
+    size_t new_n;
+
+    // If available, preserve existing IDX
+    if ( idinfo->id==-1 )
+        idinfo->id = hdr->n[dict_type];
+    else if ( idinfo->id < hdr->n[dict_type] && hdr->id[dict_type][idinfo->id].key )
+    {
+        hts_log_error("Conflicting IDX=%d lines in the header dictionary, the new tag is %s",
+            idinfo->id, tag);
+        errno = EINVAL;
+        return -1;
+    }
+
+    new_n = idinfo->id >= hdr->n[dict_type] ? idinfo->id+1 : hdr->n[dict_type];
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    // hts_resize() can attempt to allocate up to 2 * requested items
+    if (new_n > FUZZ_ALLOC_LIMIT/(2 * sizeof(bcf_idpair_t)))
+        return -1;
+#endif
+    if (hts_resize(bcf_idpair_t, new_n, &hdr->m[dict_type],
+                   &hdr->id[dict_type], HTS_RESIZE_CLEAR)) {
+        return -1;
+    }
+    hdr->n[dict_type] = new_n;
+
+    // NB: the next kh_put call can invalidate the idinfo pointer, therefore
+    // we leave it unassigned here. It must be set explicitly in bcf_hdr_sync.
+    hdr->id[dict_type][idinfo->id].key = tag;
+
+    return 0;
+}
+
+// returns: 1 when hdr needs to be synced, -1 on error, 0 otherwise
+static int bcf_hdr_register_hrec(bcf_hdr_t *hdr, bcf_hrec_t *hrec)
+{
+    // contig
+    int i, ret, replacing = 0;
+    khint_t k;
+    char *str = NULL;
+
+    bcf_hrec_set_type(hrec);
+
+    if ( hrec->type==BCF_HL_CTG )
+    {
+        hts_pos_t len = 0;
+
+        // Get the contig ID ($str) and length ($j)
+        i = bcf_hrec_find_key(hrec,"length");
+        if ( i<0 ) len = 0;
+        else {
+            char *end = hrec->vals[i];
+            len = strtoll(hrec->vals[i], &end, 10);
+            if (end == hrec->vals[i] || len < 0) return 0;
+        }
+
+        i = bcf_hrec_find_key(hrec,"ID");
+        if ( i<0 ) return 0;
+        str = strdup(hrec->vals[i]);
+        if (!str) return -1;
+
+        // Register in the dictionary
+        vdict_t *d = (vdict_t*)hdr->dict[BCF_DT_CTG];
+        khint_t k = kh_get(vdict, d, str);
+        if ( k != kh_end(d) ) { // already present
+            free(str); str=NULL;
+            if (kh_val(d, k).hrec[0] != NULL) // and not removed
+                return 0;
+            replacing = 1;
+        } else {
+            k = kh_put(vdict, d, str, &ret);
+            if (ret < 0) { free(str); return -1; }
+        }
+
+        int idx = bcf_hrec_find_key(hrec,"IDX");
+        if ( idx!=-1 )
+        {
+            char *tmp = hrec->vals[idx];
+            idx = strtol(hrec->vals[idx], &tmp, 10);
+            if ( *tmp || idx < 0 || idx >= INT_MAX - 1)
+            {
+                if (!replacing) {
+                    kh_del(vdict, d, k);
+                    free(str);
+                }
+                hts_log_warning("Error parsing the IDX tag, skipping");
+                return 0;
+            }
+        }
+
+        kh_val(d, k) = bcf_idinfo_def;
+        kh_val(d, k).id = idx;
+        kh_val(d, k).info[0] = len;
+        kh_val(d, k).hrec[0] = hrec;
+        if (bcf_hdr_set_idx(hdr, BCF_DT_CTG, kh_key(d,k), &kh_val(d,k)) < 0) {
+            if (!replacing) {
+                kh_del(vdict, d, k);
+                free(str);
+            }
+            return -1;
+        }
+        if ( idx==-1 ) {
+            if (hrec_add_idx(hrec, kh_val(d,k).id) < 0) {
+               return -1;
+            }
+        }
+
+        return 1;
+    }
+
+    if ( hrec->type==BCF_HL_STR ) return 1;
+    if ( hrec->type!=BCF_HL_INFO && hrec->type!=BCF_HL_FLT && hrec->type!=BCF_HL_FMT ) return 0;
+
+    // INFO/FILTER/FORMAT
+    char *id = NULL;
+    uint32_t type = UINT32_MAX, var = UINT32_MAX;
+    int num = -1, idx = -1;
+    for (i=0; i<hrec->nkeys; i++)
+    {
+        if ( !strcmp(hrec->keys[i], "ID") ) id = hrec->vals[i];
+        else if ( !strcmp(hrec->keys[i], "IDX") )
+        {
+            char *tmp = hrec->vals[i];
+            idx = strtol(hrec->vals[i], &tmp, 10);
+            if ( *tmp || idx < 0 || idx >= INT_MAX - 1)
+            {
+                hts_log_warning("Error parsing the IDX tag, skipping");
+                return 0;
+            }
+        }
+        else if ( !strcmp(hrec->keys[i], "Type") )
+        {
+            if ( !strcmp(hrec->vals[i], "Integer") ) type = BCF_HT_INT;
+            else if ( !strcmp(hrec->vals[i], "Float") ) type = BCF_HT_REAL;
+            else if ( !strcmp(hrec->vals[i], "String") ) type = BCF_HT_STR;
+            else if ( !strcmp(hrec->vals[i], "Character") ) type = BCF_HT_STR;
+            else if ( !strcmp(hrec->vals[i], "Flag") ) type = BCF_HT_FLAG;
+            else
+            {
+                hts_log_warning("The type \"%s\" is not supported, assuming \"String\"", hrec->vals[i]);
+                type = BCF_HT_STR;
+            }
+        }
+        else if ( !strcmp(hrec->keys[i], "Number") )
+        {
+            if ( !strcmp(hrec->vals[i],"A") ) var = BCF_VL_A;
+            else if ( !strcmp(hrec->vals[i],"R") ) var = BCF_VL_R;
+            else if ( !strcmp(hrec->vals[i],"G") ) var = BCF_VL_G;
+            else if ( !strcmp(hrec->vals[i],".") ) var = BCF_VL_VAR;
+            else
+            {
+                sscanf(hrec->vals[i],"%d",&num);
+                var = BCF_VL_FIXED;
+            }
+            if (var != BCF_VL_FIXED) num = 0xfffff;
+        }
+    }
+    if (hrec->type == BCF_HL_INFO || hrec->type == BCF_HL_FMT) {
+        if (type == -1) {
+            hts_log_warning("%s %s field has no Type defined. Assuming String",
+                *hrec->key == 'I' ? "An" : "A", hrec->key);
+            type = BCF_HT_STR;
+        }
+        if (var == -1) {
+            hts_log_warning("%s %s field has no Number defined. Assuming '.'",
+                *hrec->key == 'I' ? "An" : "A", hrec->key);
+            var = BCF_VL_VAR;
+        }
+        if ( type==BCF_HT_FLAG && (var!=BCF_VL_FIXED || num!=0) )
+        {
+            hts_log_warning("The definition of Flag \"%s/%s\" is invalid, forcing Number=0", hrec->key,id);
+            var = BCF_VL_FIXED;
+            num = 0;
+        }
+    }
+    uint32_t info = ((((uint32_t)num) & 0xfffff)<<12 |
+                     (var & 0xf) << 8 |
+                     (type & 0xf) << 4 |
+                     (((uint32_t) hrec->type) & 0xf));
+
+    if ( !id ) return 0;
+    str = strdup(id);
+    if (!str) return -1;
+
+    vdict_t *d = (vdict_t*)hdr->dict[BCF_DT_ID];
+    k = kh_get(vdict, d, str);
+    if ( k != kh_end(d) )
+    {
+        // already present
+        free(str);
+        if ( kh_val(d, k).hrec[info&0xf] ) return 0;
+        kh_val(d, k).info[info&0xf] = info;
+        kh_val(d, k).hrec[info&0xf] = hrec;
+        if ( idx==-1 ) {
+            if (hrec_add_idx(hrec, kh_val(d, k).id) < 0) {
+                return -1;
+            }
+        }
+        return 1;
+    }
+    k = kh_put(vdict, d, str, &ret);
+    if (ret < 0) {
+        free(str);
+        return -1;
+    }
+    kh_val(d, k) = bcf_idinfo_def;
+    kh_val(d, k).info[info&0xf] = info;
+    kh_val(d, k).hrec[info&0xf] = hrec;
+    kh_val(d, k).id = idx;
+    if (bcf_hdr_set_idx(hdr, BCF_DT_ID, kh_key(d,k), &kh_val(d,k)) < 0) {
+        kh_del(vdict, d, k);
+        free(str);
+        return -1;
+    }
+    if ( idx==-1 ) {
+        if (hrec_add_idx(hrec, kh_val(d,k).id) < 0) {
+            return -1;
+        }
+    }
+
+    return 1;
+}
+
+static void bcf_hdr_unregister_hrec(bcf_hdr_t *hdr, bcf_hrec_t *hrec)
+{
+    if (hrec->type == BCF_HL_FLT ||
+        hrec->type == BCF_HL_INFO ||
+        hrec->type == BCF_HL_FMT ||
+        hrec->type == BCF_HL_CTG) {
+        int id = bcf_hrec_find_key(hrec, "ID");
+        if (id < 0 || !hrec->vals[id])
+            return;
+        vdict_t *dict = (hrec->type == BCF_HL_CTG
+                         ? (vdict_t*)hdr->dict[BCF_DT_CTG]
+                         : (vdict_t*)hdr->dict[BCF_DT_ID]);
+        khint_t k = kh_get(vdict, dict, hrec->vals[id]);
+        if (k != kh_end(dict))
+            kh_val(dict, k).hrec[hrec->type==BCF_HL_CTG ? 0 : hrec->type] = NULL;
+    }
+}
+
+static void bcf_hdr_remove_from_hdict(bcf_hdr_t *hdr, bcf_hrec_t *hrec)
+{
+    kstring_t str = KS_INITIALIZE;
+    bcf_hdr_aux_t *aux = get_hdr_aux(hdr);
+    khint_t k;
+    int id;
+
+    switch (hrec->type) {
+    case BCF_HL_GEN:
+        if (ksprintf(&str, "##%s=%s", hrec->key,hrec->value) < 0)
+            str.l = 0;
+        break;
+    case BCF_HL_STR:
+        id = bcf_hrec_find_key(hrec, "ID");
+        if (id < 0)
+            return;
+        if (!hrec->vals[id] ||
+            ksprintf(&str, "##%s=<ID=%s>", hrec->key, hrec->vals[id]) < 0)
+            str.l = 0;
+        break;
+    default:
+        return;
+    }
+    if (str.l) {
+        k = kh_get(hdict, aux->gen, str.s);
+    } else {
+        // Couldn't get a string for some reason, so try the hard way...
+        for (k = kh_begin(aux->gen); k < kh_end(aux->gen); k++) {
+            if (kh_exist(aux->gen, k) && kh_val(aux->gen, k) == hrec)
+                break;
+        }
+    }
+    if (k != kh_end(aux->gen) && kh_val(aux->gen, k) == hrec) {
+        kh_val(aux->gen, k) = NULL;
+        free((char *) kh_key(aux->gen, k));
+        kh_key(aux->gen, k) = NULL;
+        kh_del(hdict, aux->gen, k);
+    }
+    free(str.s);
+}
+
+int bcf_hdr_update_hrec(bcf_hdr_t *hdr, bcf_hrec_t *hrec, const bcf_hrec_t *tmp)
+{
+    // currently only for bcf_hdr_set_version
+    assert( hrec->type==BCF_HL_GEN );
+    int ret;
+    khint_t k;
+    bcf_hdr_aux_t *aux = get_hdr_aux(hdr);
+    for (k=kh_begin(aux->gen); k<kh_end(aux->gen); k++)
+    {
+        if ( !kh_exist(aux->gen,k) ) continue;
+        if ( hrec!=(bcf_hrec_t*)kh_val(aux->gen,k) ) continue;
+        break;
+    }
+    assert( k<kh_end(aux->gen) );   // something went wrong, should never happen
+    free((char*)kh_key(aux->gen,k));
+    kh_del(hdict,aux->gen,k);
+    kstring_t str = {0,0,0};
+    if ( ksprintf(&str, "##%s=%s", tmp->key,tmp->value) < 0 )
+    {
+        free(str.s);
+        return -1;
+    }
+    k = kh_put(hdict, aux->gen, str.s, &ret);
+    if ( ret<0 )
+    {
+        free(str.s);
+        return -1;
+    }
+    free(hrec->value);
+    hrec->value = strdup(tmp->value);
+    if ( !hrec->value ) return -1;
+    return 0;
+}
+
+int bcf_hdr_add_hrec(bcf_hdr_t *hdr, bcf_hrec_t *hrec)
+{
+    kstring_t str = {0,0,0};
+    bcf_hdr_aux_t *aux = get_hdr_aux(hdr);
+
+    int res;
+    if ( !hrec ) return 0;
+
+    bcf_hrec_check(hrec);   // todo: check return status and propagate errors up
+
+    res = bcf_hdr_register_hrec(hdr,hrec);
+    if (res < 0) return -1;
+    if ( !res )
+    {
+        // If one of the hashed field, then it is already present
+        if ( hrec->type != BCF_HL_GEN )
+        {
+            bcf_hrec_destroy(hrec);
+            return 0;
+        }
+
+        // Is one of the generic fields and already present?
+        if ( ksprintf(&str, "##%s=%s", hrec->key,hrec->value) < 0 )
+        {
+            free(str.s);
+            return -1;
+        }
+        khint_t k = kh_get(hdict, aux->gen, str.s);
+        if ( k != kh_end(aux->gen) )
+        {
+            // duplicate record
+            bcf_hrec_destroy(hrec);
+            free(str.s);
+            return 0;
+        }
+    }
+
+    int i;
+    if ( hrec->type==BCF_HL_STR && (i=bcf_hrec_find_key(hrec,"ID"))>=0 )
+    {
+        if ( ksprintf(&str, "##%s=<ID=%s>", hrec->key,hrec->vals[i]) < 0 )
+        {
+            free(str.s);
+            return -1;
+        }
+        khint_t k = kh_get(hdict, aux->gen, str.s);
+        if ( k != kh_end(aux->gen) )
+        {
+            // duplicate record
+            bcf_hrec_destroy(hrec);
+            free(str.s);
+            return 0;
+        }
+    }
+
+    // New record, needs to be added
+    int n = hdr->nhrec + 1;
+    bcf_hrec_t **new_hrec = realloc(hdr->hrec, n*sizeof(bcf_hrec_t*));
+    if (!new_hrec) {
+        free(str.s);
+        bcf_hdr_unregister_hrec(hdr, hrec);
+        return -1;
+    }
+    hdr->hrec = new_hrec;
+
+    if ( str.s )
+    {
+        khint_t k = kh_put(hdict, aux->gen, str.s, &res);
+        if ( res<0 )
+        {
+            free(str.s);
+            return -1;
+        }
+        kh_val(aux->gen,k) = hrec;
+    }
+
+    hdr->hrec[hdr->nhrec] = hrec;
+    hdr->dirty = 1;
+    hdr->nhrec = n;
+
+    return hrec->type==BCF_HL_GEN ? 0 : 1;
+}
+
+bcf_hrec_t *bcf_hdr_get_hrec(const bcf_hdr_t *hdr, int type, const char *key, const char *value, const char *str_class)
+{
+    int i;
+    if ( type==BCF_HL_GEN )
+    {
+        // e.g. ##fileformat=VCFv4.2
+        //      ##source=GenomicsDBImport
+        //      ##bcftools_viewVersion=1.16-80-gdfdb0923+htslib-1.16-34-g215d364
+        if ( value )
+        {
+            kstring_t str = {0,0,0};
+            ksprintf(&str, "##%s=%s", key,value);
+            bcf_hdr_aux_t *aux = get_hdr_aux(hdr);
+            khint_t k = kh_get(hdict, aux->gen, str.s);
+            free(str.s);
+            if ( k == kh_end(aux->gen) ) return NULL;
+            return kh_val(aux->gen, k);
+        }
+        for (i=0; i<hdr->nhrec; i++)
+        {
+            if ( hdr->hrec[i]->type!=type ) continue;
+            if ( strcmp(hdr->hrec[i]->key,key) ) continue;
+            return hdr->hrec[i];
+        }
+        return NULL;
+    }
+    else if ( type==BCF_HL_STR )
+    {
+        // e.g. ##GATKCommandLine=<ID=GenomicsDBImport,CommandLine="GenomicsDBImport....">
+        //      ##ALT=<ID=NON_REF,Description="Represents any possible alternative allele not already represented at this location by REF and ALT">
+        if (!str_class) return NULL;
+        if ( !strcmp("ID",key) )
+        {
+            kstring_t str = {0,0,0};
+            ksprintf(&str, "##%s=<%s=%s>",str_class,key,value);
+            bcf_hdr_aux_t *aux = get_hdr_aux(hdr);
+            khint_t k = kh_get(hdict, aux->gen, str.s);
+            free(str.s);
+            if ( k == kh_end(aux->gen) ) return NULL;
+            return kh_val(aux->gen, k);
+        }
+        for (i=0; i<hdr->nhrec; i++)
+        {
+            if ( hdr->hrec[i]->type!=type ) continue;
+            if ( strcmp(hdr->hrec[i]->key,str_class) ) continue;
+            int j = bcf_hrec_find_key(hdr->hrec[i],key);
+            if ( j>=0 && !strcmp(hdr->hrec[i]->vals[j],value) ) return hdr->hrec[i];
+        }
+        return NULL;
+    }
+    vdict_t *d = type==BCF_HL_CTG ? (vdict_t*)hdr->dict[BCF_DT_CTG] : (vdict_t*)hdr->dict[BCF_DT_ID];
+    khint_t k = kh_get(vdict, d, value);
+    if ( k == kh_end(d) ) return NULL;
+    return kh_val(d, k).hrec[type==BCF_HL_CTG?0:type];
+}
+
+void bcf_hdr_check_sanity(bcf_hdr_t *hdr)
+{
+    static int PL_warned = 0, GL_warned = 0;
+
+    if ( !PL_warned )
+    {
+        int id = bcf_hdr_id2int(hdr, BCF_DT_ID, "PL");
+        if ( bcf_hdr_idinfo_exists(hdr,BCF_HL_FMT,id) && bcf_hdr_id2length(hdr,BCF_HL_FMT,id)!=BCF_VL_G )
+        {
+            hts_log_warning("PL should be declared as Number=G");
+            PL_warned = 1;
+        }
+    }
+    if ( !GL_warned )
+    {
+        int id = bcf_hdr_id2int(hdr, BCF_DT_ID, "GL");
+        if ( bcf_hdr_idinfo_exists(hdr,BCF_HL_FMT,id) && bcf_hdr_id2length(hdr,BCF_HL_FMT,id)!=BCF_VL_G )
+        {
+            hts_log_warning("GL should be declared as Number=G");
+            GL_warned = 1;
+        }
+    }
+}
+
+int bcf_hdr_parse(bcf_hdr_t *hdr, char *htxt)
+{
+    int len, done = 0;
+    char *p = htxt;
+
+    // Check sanity: "fileformat" string must come as first
+    bcf_hrec_t *hrec = bcf_hdr_parse_line(hdr,p,&len);
+    if ( !hrec || !hrec->key || strcasecmp(hrec->key,"fileformat") )
+        hts_log_warning("The first line should be ##fileformat; is the VCF/BCF header broken?");
+    if (bcf_hdr_add_hrec(hdr, hrec) < 0) {
+        bcf_hrec_destroy(hrec);
+        return -1;
+    }
+
+    // The filter PASS must appear first in the dictionary
+    hrec = bcf_hdr_parse_line(hdr,"##FILTER=<ID=PASS,Description=\"All filters passed\">",&len);
+    if (!hrec || bcf_hdr_add_hrec(hdr, hrec) < 0) {
+        bcf_hrec_destroy(hrec);
+        return -1;
+    }
+
+    // Parse the whole header
+    do {
+        while (NULL != (hrec = bcf_hdr_parse_line(hdr, p, &len))) {
+            if (bcf_hdr_add_hrec(hdr, hrec) < 0) {
+                bcf_hrec_destroy(hrec);
+                return -1;
+            }
+            p += len;
+        }
+        assert(hrec == NULL);
+        if (len < 0) {
+            // len < 0 indicates out-of-memory, or similar error
+            hts_log_error("Could not parse header line: %s", strerror(errno));
+            return -1;
+        } else if (len > 0) {
+            // Bad header line.  bcf_hdr_parse_line() will have logged it.
+            // Skip and try again on the next line (p + len will be the start
+            // of the next one).
+            p += len;
+            continue;
+        }
+
+        // Next should be the sample line.  If not, it was a malformed
+        // header, in which case print a warning and skip (many VCF
+        // operations do not really care about a few malformed lines).
+        // In the future we may want to add a strict mode that errors in
+        // this case.
+        if ( strncmp("#CHROM\t",p,7) && strncmp("#CHROM ",p,7) ) {
+            char *eol = strchr(p, '\n');
+            if (*p != '\0') {
+                char buffer[320];
+                hts_log_warning("Could not parse header line: %s",
+                                hts_strprint(buffer, sizeof(buffer),
+                                               '"', p,
+                                               eol ? (eol - p) : SIZE_MAX));
+            }
+            if (eol) {
+                p = eol + 1; // Try from the next line.
+            } else {
+                done = -1; // No more lines left, give up.
+            }
+        } else {
+            done = 1; // Sample line found
+        }
+    } while (!done);
+
+    if (done < 0) {
+        // No sample line is fatal.
+        hts_log_error("Could not parse the header, sample line not found");
+        return -1;
+    }
+
+    if (bcf_hdr_parse_sample_line(hdr,p) < 0)
+        return -1;
+    if (bcf_hdr_sync(hdr) < 0)
+        return -1;
+    bcf_hdr_check_sanity(hdr);
+    return 0;
+}
+
+int bcf_hdr_append(bcf_hdr_t *hdr, const char *line)
+{
+    int len;
+    bcf_hrec_t *hrec = bcf_hdr_parse_line(hdr, (char*) line, &len);
+    if ( !hrec ) return -1;
+    if (bcf_hdr_add_hrec(hdr, hrec) < 0)
+        return -1;
+    return 0;
+}
+
+void bcf_hdr_remove(bcf_hdr_t *hdr, int type, const char *key)
+{
+    int i = 0;
+    bcf_hrec_t *hrec;
+    if ( !key )
+    {
+        // no key, remove all entries of this type
+        while ( i<hdr->nhrec )
+        {
+            if ( hdr->hrec[i]->type!=type ) { i++; continue; }
+            hrec = hdr->hrec[i];
+            bcf_hdr_unregister_hrec(hdr, hrec);
+            bcf_hdr_remove_from_hdict(hdr, hrec);
+            hdr->dirty = 1;
+            hdr->nhrec--;
+            if ( i < hdr->nhrec )
+                memmove(&hdr->hrec[i],&hdr->hrec[i+1],(hdr->nhrec-i)*sizeof(bcf_hrec_t*));
+            bcf_hrec_destroy(hrec);
+        }
+        return;
+    }
+    while (1)
+    {
+        if ( type==BCF_HL_FLT || type==BCF_HL_INFO || type==BCF_HL_FMT || type== BCF_HL_CTG )
+        {
+            hrec = bcf_hdr_get_hrec(hdr, type, "ID", key, NULL);
+            if ( !hrec ) return;
+
+            for (i=0; i<hdr->nhrec; i++)
+                if ( hdr->hrec[i]==hrec ) break;
+            assert( i<hdr->nhrec );
+
+            vdict_t *d = type==BCF_HL_CTG ? (vdict_t*)hdr->dict[BCF_DT_CTG] : (vdict_t*)hdr->dict[BCF_DT_ID];
+            khint_t k = kh_get(vdict, d, key);
+            kh_val(d, k).hrec[type==BCF_HL_CTG?0:type] = NULL;
+        }
+        else
+        {
+            for (i=0; i<hdr->nhrec; i++)
+            {
+                if ( hdr->hrec[i]->type!=type ) continue;
+                if ( type==BCF_HL_GEN )
+                {
+                    if ( !strcmp(hdr->hrec[i]->key,key) ) break;
+                }
+                else
+                {
+                    // not all structured lines have ID, we could be more sophisticated as in bcf_hdr_get_hrec()
+                    int j = bcf_hrec_find_key(hdr->hrec[i], "ID");
+                    if ( j>=0 && !strcmp(hdr->hrec[i]->vals[j],key) ) break;
+                }
+            }
+            if ( i==hdr->nhrec ) return;
+            hrec = hdr->hrec[i];
+            bcf_hdr_remove_from_hdict(hdr, hrec);
+        }
+
+        hdr->nhrec--;
+        if ( i < hdr->nhrec )
+            memmove(&hdr->hrec[i],&hdr->hrec[i+1],(hdr->nhrec-i)*sizeof(bcf_hrec_t*));
+        bcf_hrec_destroy(hrec);
+        hdr->dirty = 1;
+    }
+}
+
+int bcf_hdr_printf(bcf_hdr_t *hdr, const char *fmt, ...)
+{
+    char tmp[256], *line = tmp;
+    va_list ap;
+    va_start(ap, fmt);
+    int n = vsnprintf(line, sizeof(tmp), fmt, ap);
+    va_end(ap);
+
+    if (n >= sizeof(tmp)) {
+        n++; // For trailing NUL
+        line = (char*)malloc(n);
+        if (!line)
+            return -1;
+
+        va_start(ap, fmt);
+        vsnprintf(line, n, fmt, ap);
+        va_end(ap);
+    }
+
+    int ret = bcf_hdr_append(hdr, line);
+
+    if (line != tmp) free(line);
+    return ret;
+}
+
+
+/**********************
+ *** BCF header I/O ***
+ **********************/
+
+const char *bcf_hdr_get_version(const bcf_hdr_t *hdr)
+{
+    bcf_hrec_t *hrec = bcf_hdr_get_hrec(hdr, BCF_HL_GEN, "fileformat", NULL, NULL);
+    if ( !hrec )
+    {
+        hts_log_warning("No version string found, assuming VCFv4.2");
+        return "VCFv4.2";
+    }
+    return hrec->value;
+}
+
+int bcf_hdr_set_version(bcf_hdr_t *hdr, const char *version)
+{
+    bcf_hrec_t *hrec = bcf_hdr_get_hrec(hdr, BCF_HL_GEN, "fileformat", NULL, NULL);
+    if ( !hrec )
+    {
+        int len;
+        kstring_t str = {0,0,0};
+        if ( ksprintf(&str,"##fileformat=%s", version) < 0 ) return -1;
+        hrec = bcf_hdr_parse_line(hdr, str.s, &len);
+        free(str.s);
+    }
+    else
+    {
+        bcf_hrec_t *tmp = bcf_hrec_dup(hrec);
+        if ( !tmp ) return -1;
+        free(tmp->value);
+        tmp->value = strdup(version);
+        if ( !tmp->value ) return -1;
+        bcf_hdr_update_hrec(hdr, hrec, tmp);
+        bcf_hrec_destroy(tmp);
+    }
+    hdr->dirty = 1;
+    return 0; // FIXME: check for errs in this function (return < 0 if so)
+}
+
+bcf_hdr_t *bcf_hdr_init(const char *mode)
+{
+    int i;
+    bcf_hdr_t *h;
+    h = (bcf_hdr_t*)calloc(1, sizeof(bcf_hdr_t));
+    if (!h) return NULL;
+    for (i = 0; i < 3; ++i) {
+        if ((h->dict[i] = kh_init(vdict)) == NULL) goto fail;
+        // Supersize the hash to make collisions very unlikely
+        static int dsize[3] = {16384,16384,2048}; // info, contig, format
+        if (kh_resize(vdict, h->dict[i], dsize[i]) < 0) goto fail;
+    }
+
+    bcf_hdr_aux_t *aux = (bcf_hdr_aux_t*)calloc(1,sizeof(bcf_hdr_aux_t));
+    if ( !aux ) goto fail;
+    if ( (aux->gen = kh_init(hdict))==NULL ) { free(aux); goto fail; }
+    aux->key_len = NULL;
+    aux->dict = *((vdict_t*)h->dict[0]);
+    free(h->dict[0]);
+    h->dict[0] = aux;
+
+    if ( strchr(mode,'w') )
+    {
+        bcf_hdr_append(h, "##fileformat=VCFv4.2");
+        // The filter PASS must appear first in the dictionary
+        bcf_hdr_append(h, "##FILTER=<ID=PASS,Description=\"All filters passed\">");
+    }
+    return h;
+
+ fail:
+    for (i = 0; i < 3; ++i)
+        kh_destroy(vdict, h->dict[i]);
+    free(h);
+    return NULL;
+}
+
+void bcf_hdr_destroy(bcf_hdr_t *h)
+{
+    int i;
+    khint_t k;
+    if (!h) return;
+    for (i = 0; i < 3; ++i) {
+        vdict_t *d = (vdict_t*)h->dict[i];
+        if (d == 0) continue;
+        for (k = kh_begin(d); k != kh_end(d); ++k)
+            if (kh_exist(d, k)) free((char*)kh_key(d, k));
+        if ( i==0 )
+        {
+            bcf_hdr_aux_t *aux = get_hdr_aux(h);
+            for (k=kh_begin(aux->gen); k<kh_end(aux->gen); k++)
+                if ( kh_exist(aux->gen,k) ) free((char*)kh_key(aux->gen,k));
+            kh_destroy(hdict, aux->gen);
+            free(aux->key_len); // may exist for dict[0] only
+        }
+        kh_destroy(vdict, d);
+        free(h->id[i]);
+    }
+    for (i=0; i<h->nhrec; i++)
+        bcf_hrec_destroy(h->hrec[i]);
+    if (h->nhrec) free(h->hrec);
+    if (h->samples) free(h->samples);
+    free(h->keep_samples);
+    free(h->transl[0]); free(h->transl[1]);
+    free(h->mem.s);
+    free(h);
+}
+
+bcf_hdr_t *bcf_hdr_read(htsFile *hfp)
+{
+    if (hfp->format.format == vcf)
+        return vcf_hdr_read(hfp);
+    if (hfp->format.format != bcf) {
+        hts_log_error("Input is not detected as bcf or vcf format");
+        return NULL;
+    }
+
+    assert(hfp->is_bgzf);
+
+    BGZF *fp = hfp->fp.bgzf;
+    uint8_t magic[5];
+    bcf_hdr_t *h;
+    h = bcf_hdr_init("r");
+    if (!h) {
+        hts_log_error("Failed to allocate bcf header");
+        return NULL;
+    }
+    if (bgzf_read(fp, magic, 5) != 5)
+    {
+        hts_log_error("Failed to read the header (reading BCF in text mode?)");
+        bcf_hdr_destroy(h);
+        return NULL;
+    }
+    if (strncmp((char*)magic, "BCF\2\2", 5) != 0)
+    {
+        if (!strncmp((char*)magic, "BCF", 3))
+            hts_log_error("Invalid BCF2 magic string: only BCFv2.2 is supported");
+        else
+            hts_log_error("Invalid BCF2 magic string");
+        bcf_hdr_destroy(h);
+        return NULL;
+    }
+    uint8_t buf[4];
+    size_t hlen;
+    char *htxt = NULL;
+    if (bgzf_read(fp, buf, 4) != 4) goto fail;
+    hlen = buf[0] | (buf[1] << 8) | (buf[2] << 16) | ((size_t) buf[3] << 24);
+    if (hlen >= SIZE_MAX) { errno = ENOMEM; goto fail; }
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    if (hlen > FUZZ_ALLOC_LIMIT/2) { errno = ENOMEM; goto fail; }
+#endif
+    htxt = (char*)malloc(hlen + 1);
+    if (!htxt) goto fail;
+    if (bgzf_read(fp, htxt, hlen) != hlen) goto fail;
+    htxt[hlen] = '\0'; // Ensure htxt is terminated
+    if ( bcf_hdr_parse(h, htxt) < 0 ) goto fail;
+    free(htxt);
+    return h;
+ fail:
+    hts_log_error("Failed to read BCF header");
+    free(htxt);
+    bcf_hdr_destroy(h);
+    return NULL;
+}
+
+int bcf_hdr_write(htsFile *hfp, bcf_hdr_t *h)
+{
+    if (!h) {
+        errno = EINVAL;
+        return -1;
+    }
+    if ( h->dirty ) {
+        if (bcf_hdr_sync(h) < 0) return -1;
+    }
+    hfp->format.category = variant_data;
+    if (hfp->format.format == vcf || hfp->format.format == text_format) {
+        hfp->format.format = vcf;
+        return vcf_hdr_write(hfp, h);
+    }
+
+    if (hfp->format.format == binary_format)
+        hfp->format.format = bcf;
+
+    kstring_t htxt = {0,0,0};
+    if (bcf_hdr_format(h, 1, &htxt) < 0) {
+        free(htxt.s);
+        return -1;
+    }
+    kputc('\0', &htxt); // include the \0 byte
+
+    BGZF *fp = hfp->fp.bgzf;
+    if ( bgzf_write(fp, "BCF\2\2", 5) !=5 ) return -1;
+    uint8_t hlen[4];
+    u32_to_le(htxt.l, hlen);
+    if ( bgzf_write(fp, hlen, 4) !=4 ) return -1;
+    if ( bgzf_write(fp, htxt.s, htxt.l) != htxt.l ) return -1;
+    if ( bgzf_flush(fp) < 0) return -1;
+
+    free(htxt.s);
+    return 0;
+}
+
+/********************
+ *** BCF site I/O ***
+ ********************/
+
+bcf1_t *bcf_init(void)
+{
+    bcf1_t *v;
+    v = (bcf1_t*)calloc(1, sizeof(bcf1_t));
+    return v;
+}
+
+void bcf_clear(bcf1_t *v)
+{
+    int i;
+    for (i=0; i<v->d.m_info; i++)
+    {
+        if ( v->d.info[i].vptr_free )
+        {
+            free(v->d.info[i].vptr - v->d.info[i].vptr_off);
+            v->d.info[i].vptr_free = 0;
+        }
+    }
+    for (i=0; i<v->d.m_fmt; i++)
+    {
+        if ( v->d.fmt[i].p_free )
+        {
+            free(v->d.fmt[i].p - v->d.fmt[i].p_off);
+            v->d.fmt[i].p_free = 0;
+        }
+    }
+    v->rid = v->pos = v->rlen = v->unpacked = 0;
+    bcf_float_set_missing(v->qual);
+    v->n_info = v->n_allele = v->n_fmt = v->n_sample = 0;
+    v->shared.l = v->indiv.l = 0;
+    v->d.var_type = -1;
+    v->d.shared_dirty = 0;
+    v->d.indiv_dirty  = 0;
+    v->d.n_flt = 0;
+    v->errcode = 0;
+    if (v->d.m_als) v->d.als[0] = 0;
+    if (v->d.m_id) v->d.id[0] = 0;
+}
+
+void bcf_empty(bcf1_t *v)
+{
+    bcf_clear1(v);
+    free(v->d.id);
+    free(v->d.als);
+    free(v->d.allele); free(v->d.flt); free(v->d.info); free(v->d.fmt);
+    if (v->d.var ) free(v->d.var);
+    free(v->shared.s); free(v->indiv.s);
+    memset(&v->d,0,sizeof(v->d));
+    memset(&v->shared,0,sizeof(v->shared));
+    memset(&v->indiv,0,sizeof(v->indiv));
+}
+
+void bcf_destroy(bcf1_t *v)
+{
+    if (!v) return;
+    bcf_empty1(v);
+    free(v);
+}
+
+static inline int bcf_read1_core(BGZF *fp, bcf1_t *v)
+{
+    uint8_t x[32];
+    ssize_t ret;
+    uint32_t shared_len, indiv_len;
+    if ((ret = bgzf_read(fp, x, 32)) != 32) {
+        if (ret == 0) return -1;
+        return -2;
+    }
+    bcf_clear1(v);
+    shared_len = le_to_u32(x);
+    if (shared_len < 24) return -2;
+    shared_len -= 24; // to exclude six 32-bit integers
+    indiv_len = le_to_u32(x + 4);
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    // ks_resize() normally allocates 1.5 * requested size to allow for growth
+    if ((uint64_t) shared_len + indiv_len > FUZZ_ALLOC_LIMIT / 3 * 2) return -2;
+#endif
+    if (ks_resize(&v->shared, shared_len ? shared_len : 1) != 0) return -2;
+    if (ks_resize(&v->indiv, indiv_len ? indiv_len : 1) != 0) return -2;
+    v->rid  = le_to_i32(x + 8);
+    v->pos  = le_to_u32(x + 12);
+    if ( v->pos==UINT32_MAX ) v->pos = -1;  // this is for telomere coordinate, e.g. MT:0
+    v->rlen = le_to_i32(x + 16);
+    v->qual = le_to_float(x + 20);
+    v->n_info = le_to_u16(x + 24);
+    v->n_allele = le_to_u16(x + 26);
+    v->n_sample = le_to_u32(x + 28) & 0xffffff;
+    v->n_fmt = x[31];
+    v->shared.l = shared_len;
+    v->indiv.l = indiv_len;
+    // silent fix of broken BCFs produced by earlier versions of bcf_subset, prior to and including bd6ed8b4
+    if ( (!v->indiv.l || !v->n_sample) && v->n_fmt ) v->n_fmt = 0;
+
+    if (bgzf_read(fp, v->shared.s, v->shared.l) != v->shared.l) return -2;
+    if (bgzf_read(fp, v->indiv.s, v->indiv.l) != v->indiv.l) return -2;
+    return 0;
+}
+
+#define bit_array_size(n) ((n)/8+1)
+#define bit_array_set(a,i)   ((a)[(i)/8] |=   1 << ((i)%8))
+#define bit_array_clear(a,i) ((a)[(i)/8] &= ~(1 << ((i)%8)))
+#define bit_array_test(a,i)  ((a)[(i)/8] &   (1 << ((i)%8)))
+
+static int bcf_dec_typed_int1_safe(uint8_t *p, uint8_t *end, uint8_t **q,
+                                   int32_t *val) {
+    uint32_t t;
+    if (end - p < 2) return -1;
+    t = *p++ & 0xf;
+    /* Use if .. else if ... else instead of switch to force order.  Assumption
+       is that small integers are more frequent than big ones. */
+    if (t == BCF_BT_INT8) {
+        *val = *(int8_t *) p++;
+    } else {
+        if (end - p < (1<<bcf_type_shift[t])) return -1;
+        if (t == BCF_BT_INT16) {
+            *val = le_to_i16(p);
+            p += 2;
+        } else if (t == BCF_BT_INT32) {
+            *val = le_to_i32(p);
+            p += 4;
+#ifdef VCF_ALLOW_INT64
+        } else if (t == BCF_BT_INT64) {
+            // This case should never happen because there should be no
+            // 64-bit BCFs at all, definitely not coming from htslib
+            *val = le_to_i64(p);
+            p += 8;
+#endif
+        } else {
+            return -1;
+        }
+    }
+    *q = p;
+    return 0;
+}
+
+static int bcf_dec_size_safe(uint8_t *p, uint8_t *end, uint8_t **q,
+                             int *num, int *type) {
+    int r;
+    if (p >= end) return -1;
+    *type = *p & 0xf;
+    if (*p>>4 != 15) {
+        *q = p + 1;
+        *num = *p >> 4;
+        return 0;
+    }
+    r = bcf_dec_typed_int1_safe(p + 1, end, q, num);
+    if (r) return r;
+    return *num >= 0 ? 0 : -1;
+}
+
+static const char *get_type_name(int type) {
+    const char *types[9] = {
+        "null", "int (8-bit)", "int (16 bit)", "int (32 bit)",
+        "unknown", "float", "unknown", "char", "unknown"
+    };
+    int t = (type >= 0 && type < 8) ? type : 8;
+    return types[t];
+}
+
+static void bcf_record_check_err(const bcf_hdr_t *hdr, bcf1_t *rec,
+                                 char *type, uint32_t *reports, int i) {
+    if (*reports == 0 || hts_verbose >= HTS_LOG_DEBUG)
+        hts_log_warning("Bad BCF record at %s:%"PRIhts_pos
+                        ": Invalid FORMAT %s %d",
+                        bcf_seqname_safe(hdr,rec), rec->pos+1, type, i);
+    (*reports)++;
+}
+
+static int bcf_record_check(const bcf_hdr_t *hdr, bcf1_t *rec) {
+    uint8_t *ptr, *end;
+    size_t bytes;
+    uint32_t err = 0;
+    int type = 0;
+    int num  = 0;
+    int reflen = 0;
+    uint32_t i, reports;
+    const uint32_t is_integer = ((1 << BCF_BT_INT8)  |
+                                 (1 << BCF_BT_INT16) |
+#ifdef VCF_ALLOW_INT64
+                                 (1 << BCF_BT_INT64) |
+#endif
+                                 (1 << BCF_BT_INT32));
+    const uint32_t is_valid_type = (is_integer          |
+                                    (1 << BCF_BT_NULL)  |
+                                    (1 << BCF_BT_FLOAT) |
+                                    (1 << BCF_BT_CHAR));
+    int32_t max_id = hdr ? hdr->n[BCF_DT_ID] : 0;
+
+    // Check for valid contig ID
+    if (rec->rid < 0
+        || (hdr && (rec->rid >= hdr->n[BCF_DT_CTG]
+                    || hdr->id[BCF_DT_CTG][rec->rid].key == NULL))) {
+        hts_log_warning("Bad BCF record at %"PRIhts_pos": Invalid %s id %d", rec->pos+1, "CONTIG", rec->rid);
+        err |= BCF_ERR_CTG_INVALID;
+    }
+
+    // Check ID
+    ptr = (uint8_t *) rec->shared.s;
+    end = ptr + rec->shared.l;
+    if (bcf_dec_size_safe(ptr, end, &ptr, &num, &type) != 0) goto bad_shared;
+    if (type != BCF_BT_CHAR) {
+        hts_log_warning("Bad BCF record at %s:%"PRIhts_pos": Invalid %s type %d (%s)", bcf_seqname_safe(hdr,rec), rec->pos+1, "ID", type, get_type_name(type));
+        err |= BCF_ERR_TAG_INVALID;
+    }
+    bytes = (size_t) num << bcf_type_shift[type];
+    if (end - ptr < bytes) goto bad_shared;
+    ptr += bytes;
+
+    // Check REF and ALT
+    if (rec->n_allele < 1) {
+        hts_log_warning("Bad BCF record at %s:%"PRIhts_pos": No REF allele",
+                        bcf_seqname_safe(hdr,rec), rec->pos+1);
+        err |= BCF_ERR_TAG_UNDEF;
+    }
+
+    reports = 0;
+    for (i = 0; i < rec->n_allele; i++) {
+        if (bcf_dec_size_safe(ptr, end, &ptr, &num, &type) != 0) goto bad_shared;
+        if (type != BCF_BT_CHAR) {
+            if (!reports++ || hts_verbose >= HTS_LOG_DEBUG)
+                hts_log_warning("Bad BCF record at %s:%"PRIhts_pos": Invalid %s type %d (%s)", bcf_seqname_safe(hdr,rec), rec->pos+1, "REF/ALT", type, get_type_name(type));
+            err |= BCF_ERR_CHAR;
+        }
+        if (i == 0) reflen = num;
+        bytes = (size_t) num << bcf_type_shift[type];
+        if (end - ptr < bytes) goto bad_shared;
+        ptr += bytes;
+    }
+
+    // Check FILTER
+    reports = 0;
+    if (bcf_dec_size_safe(ptr, end, &ptr, &num, &type) != 0) goto bad_shared;
+    if (num > 0) {
+        bytes = (size_t) num << bcf_type_shift[type];
+        if (((1 << type) & is_integer) == 0) {
+            hts_log_warning("Bad BCF record at %s:%"PRIhts_pos": Invalid %s type %d (%s)", bcf_seqname_safe(hdr,rec), rec->pos+1, "FILTER", type, get_type_name(type));
+            err |= BCF_ERR_TAG_INVALID;
+            if (end - ptr < bytes) goto bad_shared;
+            ptr += bytes;
+        } else {
+            if (end - ptr < bytes) goto bad_shared;
+            for (i = 0; i < num; i++) {
+                int32_t key = bcf_dec_int1(ptr, type, &ptr);
+                if (key < 0
+                    || (hdr && (key >= max_id
+                                || hdr->id[BCF_DT_ID][key].key == NULL))) {
+                    if (!reports++ || hts_verbose >= HTS_LOG_DEBUG)
+                        hts_log_warning("Bad BCF record at %s:%"PRIhts_pos": Invalid %s id %d", bcf_seqname_safe(hdr,rec), rec->pos+1, "FILTER", key);
+                    err |= BCF_ERR_TAG_UNDEF;
+                }
+            }
+        }
+    }
+
+    // Check INFO
+    reports = 0;
+    bcf_idpair_t *id_tmp = hdr ? hdr->id[BCF_DT_ID] : NULL;
+    for (i = 0; i < rec->n_info; i++) {
+        int32_t key = -1;
+        if (bcf_dec_typed_int1_safe(ptr, end, &ptr, &key) != 0) goto bad_shared;
+        if (key < 0 || (hdr && (key >= max_id
+                                || id_tmp[key].key == NULL))) {
+            if (!reports++ || hts_verbose >= HTS_LOG_DEBUG)
+                hts_log_warning("Bad BCF record at %s:%"PRIhts_pos": Invalid %s id %d", bcf_seqname_safe(hdr,rec), rec->pos+1, "INFO", key);
+            err |= BCF_ERR_TAG_UNDEF;
+        }
+        if (bcf_dec_size_safe(ptr, end, &ptr, &num, &type) != 0) goto bad_shared;
+        if (((1 << type) & is_valid_type) == 0
+            || (type == BCF_BT_NULL && num > 0)) {
+            if (!reports++ || hts_verbose >= HTS_LOG_DEBUG)
+                hts_log_warning("Bad BCF record at %s:%"PRIhts_pos": Invalid %s type %d (%s)", bcf_seqname_safe(hdr,rec), rec->pos+1, "INFO", type, get_type_name(type));
+            err |= BCF_ERR_TAG_INVALID;
+        }
+        bytes = (size_t) num << bcf_type_shift[type];
+        if (end - ptr < bytes) goto bad_shared;
+        ptr += bytes;
+    }
+
+    // Check FORMAT and individual information
+    ptr = (uint8_t *) rec->indiv.s;
+    end = ptr + rec->indiv.l;
+    reports = 0;
+    for (i = 0; i < rec->n_fmt; i++) {
+        int32_t key = -1;
+        if (bcf_dec_typed_int1_safe(ptr, end, &ptr, &key) != 0) goto bad_indiv;
+        if (key < 0
+            || (hdr && (key >= max_id
+                        || id_tmp[key].key == NULL))) {
+            bcf_record_check_err(hdr, rec, "id", &reports, key);
+            err |= BCF_ERR_TAG_UNDEF;
+        }
+        if (bcf_dec_size_safe(ptr, end, &ptr, &num, &type) != 0) goto bad_indiv;
+        if (((1 << type) & is_valid_type) == 0
+            || (type == BCF_BT_NULL && num > 0)) {
+            bcf_record_check_err(hdr, rec, "type", &reports, type);
+            err |= BCF_ERR_TAG_INVALID;
+        }
+        bytes = ((size_t) num << bcf_type_shift[type]) * rec->n_sample;
+        if (end - ptr < bytes) goto bad_indiv;
+        ptr += bytes;
+    }
+
+    if (!err && rec->rlen < 0) {
+        // Treat bad rlen as a warning instead of an error, and try to
+        // fix up by using the length of the stored REF allele.
+        static int warned = 0;
+        if (!warned) {
+            hts_log_warning("BCF record at %s:%"PRIhts_pos" has invalid RLEN (%"PRIhts_pos"). "
+                            "Only one invalid RLEN will be reported.",
+                            bcf_seqname_safe(hdr,rec), rec->pos+1, rec->rlen);
+            warned = 1;
+        }
+        rec->rlen = reflen >= 0 ? reflen : 0;
+    }
+
+    rec->errcode |= err;
+
+    return err ? -2 : 0; // Return -2 so bcf_read() reports an error
+
+ bad_shared:
+    hts_log_error("Bad BCF record at %s:%"PRIhts_pos" - shared section malformed or too short", bcf_seqname_safe(hdr,rec), rec->pos+1);
+    return -2;
+
+ bad_indiv:
+    hts_log_error("Bad BCF record at %s:%"PRIhts_pos" - individuals section malformed or too short", bcf_seqname_safe(hdr,rec), rec->pos+1);
+    return -2;
+}
+
+static inline uint8_t *bcf_unpack_fmt_core1(uint8_t *ptr, int n_sample, bcf_fmt_t *fmt);
+int bcf_subset_format(const bcf_hdr_t *hdr, bcf1_t *rec)
+{
+    if ( !hdr->keep_samples ) return 0;
+    if ( !bcf_hdr_nsamples(hdr) )
+    {
+        rec->indiv.l = rec->n_sample = 0;
+        return 0;
+    }
+
+    int i, j;
+    uint8_t *ptr = (uint8_t*)rec->indiv.s, *dst = NULL, *src;
+    bcf_dec_t *dec = &rec->d;
+    hts_expand(bcf_fmt_t, rec->n_fmt, dec->m_fmt, dec->fmt);
+    for (i=0; i<dec->m_fmt; ++i) dec->fmt[i].p_free = 0;
+
+    for (i=0; i<rec->n_fmt; i++)
+    {
+        ptr = bcf_unpack_fmt_core1(ptr, rec->n_sample, &dec->fmt[i]);
+        src = dec->fmt[i].p - dec->fmt[i].size;
+        if ( dst )
+        {
+            memmove(dec->fmt[i-1].p + dec->fmt[i-1].p_len, dec->fmt[i].p - dec->fmt[i].p_off, dec->fmt[i].p_off);
+            dec->fmt[i].p = dec->fmt[i-1].p + dec->fmt[i-1].p_len + dec->fmt[i].p_off;
+        }
+        dst = dec->fmt[i].p;
+        for (j=0; j<hdr->nsamples_ori; j++)
+        {
+            src += dec->fmt[i].size;
+            if ( !bit_array_test(hdr->keep_samples,j) ) continue;
+            memmove(dst, src, dec->fmt[i].size);
+            dst += dec->fmt[i].size;
+        }
+        rec->indiv.l -= dec->fmt[i].p_len - (dst - dec->fmt[i].p);
+        dec->fmt[i].p_len = dst - dec->fmt[i].p;
+    }
+    rec->unpacked |= BCF_UN_FMT;
+
+    rec->n_sample = bcf_hdr_nsamples(hdr);
+    return 0;
+}
+
+int bcf_read(htsFile *fp, const bcf_hdr_t *h, bcf1_t *v)
+{
+    if (fp->format.format == vcf) return vcf_read(fp,h,v);
+    int ret = bcf_read1_core(fp->fp.bgzf, v);
+    if (ret == 0) ret = bcf_record_check(h, v);
+    if ( ret!=0 || !h->keep_samples ) return ret;
+    return bcf_subset_format(h,v);
+}
+
+int bcf_readrec(BGZF *fp, void *null, void *vv, int *tid, hts_pos_t *beg, hts_pos_t *end)
+{
+    bcf1_t *v = (bcf1_t *) vv;
+    int ret = bcf_read1_core(fp, v);
+    if (ret == 0) ret = bcf_record_check(NULL, v);
+    if (ret  >= 0)
+        *tid = v->rid, *beg = v->pos, *end = v->pos + v->rlen;
+    return ret;
+}
+
+static inline int bcf1_sync_id(bcf1_t *line, kstring_t *str)
+{
+    // single typed string
+    if ( line->d.id && strcmp(line->d.id, ".") ) {
+        return bcf_enc_vchar(str, strlen(line->d.id), line->d.id);
+    } else {
+        return bcf_enc_size(str, 0, BCF_BT_CHAR);
+    }
+}
+static inline int bcf1_sync_alleles(bcf1_t *line, kstring_t *str)
+{
+    // list of typed strings
+    int i;
+    for (i=0; i<line->n_allele; i++) {
+        if (bcf_enc_vchar(str, strlen(line->d.allele[i]), line->d.allele[i]) < 0)
+            return -1;
+    }
+    if ( !line->rlen && line->n_allele ) line->rlen = strlen(line->d.allele[0]);
+    return 0;
+}
+static inline int bcf1_sync_filter(bcf1_t *line, kstring_t *str)
+{
+    // typed vector of integers
+    if ( line->d.n_flt ) {
+        return bcf_enc_vint(str, line->d.n_flt, line->d.flt, -1);
+    } else {
+        return bcf_enc_vint(str, 0, 0, -1);
+    }
+}
+
+static inline int bcf1_sync_info(bcf1_t *line, kstring_t *str)
+{
+    // pairs of typed vectors
+    int i, irm = -1, e = 0;
+    for (i=0; i<line->n_info; i++)
+    {
+        bcf_info_t *info = &line->d.info[i];
+        if ( !info->vptr )
+        {
+            // marked for removal
+            if ( irm < 0 ) irm = i;
+            continue;
+        }
+        e |= kputsn_(info->vptr - info->vptr_off, info->vptr_len + info->vptr_off, str) < 0;
+        if ( irm >=0 )
+        {
+            bcf_info_t tmp = line->d.info[irm]; line->d.info[irm] = line->d.info[i]; line->d.info[i] = tmp;
+            while ( irm<=i && line->d.info[irm].vptr ) irm++;
+        }
+    }
+    if ( irm>=0 ) line->n_info = irm;
+    return e == 0 ? 0 : -1;
+}
+
+static int bcf1_sync(bcf1_t *line)
+{
+    char *shared_ori = line->shared.s;
+    size_t prev_len;
+
+    kstring_t tmp = {0,0,0};
+    if ( !line->shared.l )
+    {
+        // New line created via API, BCF data blocks do not exist. Get it ready for BCF output
+        tmp = line->shared;
+        bcf1_sync_id(line, &tmp);
+        line->unpack_size[0] = tmp.l; prev_len = tmp.l;
+
+        bcf1_sync_alleles(line, &tmp);
+        line->unpack_size[1] = tmp.l - prev_len; prev_len = tmp.l;
+
+        bcf1_sync_filter(line, &tmp);
+        line->unpack_size[2] = tmp.l - prev_len;
+
+        bcf1_sync_info(line, &tmp);
+        line->shared = tmp;
+    }
+    else if ( line->d.shared_dirty )
+    {
+        // The line was edited, update the BCF data block.
+
+        if ( !(line->unpacked & BCF_UN_STR) ) bcf_unpack(line,BCF_UN_STR);
+
+        // ptr_ori points to the original unchanged BCF data.
+        uint8_t *ptr_ori = (uint8_t *) line->shared.s;
+
+        // ID: single typed string
+        if ( line->d.shared_dirty & BCF1_DIRTY_ID )
+            bcf1_sync_id(line, &tmp);
+        else
+            kputsn_(ptr_ori, line->unpack_size[0], &tmp);
+        ptr_ori += line->unpack_size[0];
+        line->unpack_size[0] = tmp.l; prev_len = tmp.l;
+
+        // REF+ALT: list of typed strings
+        if ( line->d.shared_dirty & BCF1_DIRTY_ALS )
+            bcf1_sync_alleles(line, &tmp);
+        else
+        {
+            kputsn_(ptr_ori, line->unpack_size[1], &tmp);
+            if ( !line->rlen && line->n_allele ) line->rlen = strlen(line->d.allele[0]);
+        }
+        ptr_ori += line->unpack_size[1];
+        line->unpack_size[1] = tmp.l - prev_len; prev_len = tmp.l;
+
+        if ( line->unpacked & BCF_UN_FLT )
+        {
+            // FILTER: typed vector of integers
+            if ( line->d.shared_dirty & BCF1_DIRTY_FLT )
+                bcf1_sync_filter(line, &tmp);
+            else if ( line->d.n_flt )
+                kputsn_(ptr_ori, line->unpack_size[2], &tmp);
+            else
+                bcf_enc_vint(&tmp, 0, 0, -1);
+            ptr_ori += line->unpack_size[2];
+            line->unpack_size[2] = tmp.l - prev_len;
+
+            if ( line->unpacked & BCF_UN_INFO )
+            {
+                // INFO: pairs of typed vectors
+                if ( line->d.shared_dirty & BCF1_DIRTY_INF )
+                {
+                    bcf1_sync_info(line, &tmp);
+                    ptr_ori = (uint8_t*)line->shared.s + line->shared.l;
+                }
+            }
+        }
+
+        int size = line->shared.l - (size_t)ptr_ori + (size_t)line->shared.s;
+        if ( size ) kputsn_(ptr_ori, size, &tmp);
+
+        free(line->shared.s);
+        line->shared = tmp;
+    }
+    if ( line->shared.s != shared_ori && line->unpacked & BCF_UN_INFO )
+    {
+        // Reallocated line->shared.s block invalidated line->d.info[].vptr pointers
+        size_t off_new = line->unpack_size[0] + line->unpack_size[1] + line->unpack_size[2];
+        int i;
+        for (i=0; i<line->n_info; i++)
+        {
+            uint8_t *vptr_free = line->d.info[i].vptr_free ? line->d.info[i].vptr - line->d.info[i].vptr_off : NULL;
+            line->d.info[i].vptr = (uint8_t*) line->shared.s + off_new + line->d.info[i].vptr_off;
+            off_new += line->d.info[i].vptr_len + line->d.info[i].vptr_off;
+            if ( vptr_free )
+            {
+                free(vptr_free);
+                line->d.info[i].vptr_free = 0;
+            }
+        }
+    }
+
+    if ( line->n_sample && line->n_fmt && (!line->indiv.l || line->d.indiv_dirty) )
+    {
+        // The genotype fields changed or are not present
+        tmp.l = tmp.m = 0; tmp.s = NULL;
+        int i, irm = -1;
+        for (i=0; i<line->n_fmt; i++)
+        {
+            bcf_fmt_t *fmt = &line->d.fmt[i];
+            if ( !fmt->p )
+            {
+                // marked for removal
+                if ( irm < 0 ) irm = i;
+                continue;
+            }
+            kputsn_(fmt->p - fmt->p_off, fmt->p_len + fmt->p_off, &tmp);
+            if ( irm >=0 )
+            {
+                bcf_fmt_t tfmt = line->d.fmt[irm]; line->d.fmt[irm] = line->d.fmt[i]; line->d.fmt[i] = tfmt;
+                while ( irm<=i && line->d.fmt[irm].p ) irm++;
+            }
+
+        }
+        if ( irm>=0 ) line->n_fmt = irm;
+        free(line->indiv.s);
+        line->indiv = tmp;
+
+        // Reallocated line->indiv.s block invalidated line->d.fmt[].p pointers
+        size_t off_new = 0;
+        for (i=0; i<line->n_fmt; i++)
+        {
+            uint8_t *p_free = line->d.fmt[i].p_free ? line->d.fmt[i].p - line->d.fmt[i].p_off : NULL;
+            line->d.fmt[i].p = (uint8_t*) line->indiv.s + off_new + line->d.fmt[i].p_off;
+            off_new += line->d.fmt[i].p_len + line->d.fmt[i].p_off;
+            if ( p_free )
+            {
+                free(p_free);
+                line->d.fmt[i].p_free = 0;
+            }
+        }
+    }
+    if ( !line->n_sample ) line->n_fmt = 0;
+    line->d.shared_dirty = line->d.indiv_dirty = 0;
+    return 0;
+}
+
+bcf1_t *bcf_copy(bcf1_t *dst, bcf1_t *src)
+{
+    bcf1_sync(src);
+
+    bcf_clear(dst);
+    dst->rid  = src->rid;
+    dst->pos  = src->pos;
+    dst->rlen = src->rlen;
+    dst->qual = src->qual;
+    dst->n_info = src->n_info; dst->n_allele = src->n_allele;
+    dst->n_fmt = src->n_fmt; dst->n_sample = src->n_sample;
+
+    if ( dst->shared.m < src->shared.l )
+    {
+        dst->shared.s = (char*) realloc(dst->shared.s, src->shared.l);
+        dst->shared.m = src->shared.l;
+    }
+    dst->shared.l = src->shared.l;
+    memcpy(dst->shared.s,src->shared.s,dst->shared.l);
+
+    if ( dst->indiv.m < src->indiv.l )
+    {
+        dst->indiv.s = (char*) realloc(dst->indiv.s, src->indiv.l);
+        dst->indiv.m = src->indiv.l;
+    }
+    dst->indiv.l = src->indiv.l;
+    memcpy(dst->indiv.s,src->indiv.s,dst->indiv.l);
+
+    return dst;
+}
+bcf1_t *bcf_dup(bcf1_t *src)
+{
+    bcf1_t *out = bcf_init1();
+    return bcf_copy(out, src);
+}
+
+int bcf_write(htsFile *hfp, bcf_hdr_t *h, bcf1_t *v)
+{
+    if ( h->dirty ) {
+        if (bcf_hdr_sync(h) < 0) return -1;
+    }
+    if ( bcf_hdr_nsamples(h)!=v->n_sample )
+    {
+        hts_log_error("Broken VCF record, the number of columns at %s:%"PRIhts_pos" does not match the number of samples (%d vs %d)",
+            bcf_seqname_safe(h,v), v->pos+1, v->n_sample, bcf_hdr_nsamples(h));
+        return -1;
+    }
+
+    if ( hfp->format.format == vcf || hfp->format.format == text_format )
+        return vcf_write(hfp,h,v);
+
+    if ( v->errcode & ~BCF_ERR_LIMITS ) // todo: unsure about the other BCF_ERR_LIMITS branches in vcf_parse_format_alloc4()
+    {
+        // vcf_parse1() encountered a new contig or tag, undeclared in the
+        // header.  At this point, the header must have been printed,
+        // proceeding would lead to a broken BCF file. Errors must be checked
+        // and cleared by the caller before we can proceed.
+        char errdescription[1024] = "";
+        hts_log_error("Unchecked error (%d %s) at %s:%"PRIhts_pos, v->errcode, bcf_strerror(v->errcode, errdescription, sizeof(errdescription)), bcf_seqname_safe(h,v), v->pos+1);
+        return -1;
+    }
+    bcf1_sync(v);   // check if the BCF record was modified
+
+    if ( v->unpacked & BCF_IS_64BIT )
+    {
+        hts_log_error("Data at %s:%"PRIhts_pos" contains 64-bit values not representable in BCF. Please use VCF instead", bcf_seqname_safe(h,v), v->pos+1);
+        return -1;
+    }
+
+    BGZF *fp = hfp->fp.bgzf;
+    uint8_t x[32];
+    u32_to_le(v->shared.l + 24, x); // to include six 32-bit integers
+    u32_to_le(v->indiv.l, x + 4);
+    i32_to_le(v->rid, x + 8);
+    u32_to_le(v->pos, x + 12);
+    u32_to_le(v->rlen, x + 16);
+    float_to_le(v->qual, x + 20);
+    u16_to_le(v->n_info, x + 24);
+    u16_to_le(v->n_allele, x + 26);
+    u32_to_le((uint32_t)v->n_fmt<<24 | (v->n_sample & 0xffffff), x + 28);
+    if ( bgzf_write(fp, x, 32) != 32 ) return -1;
+    if ( bgzf_write(fp, v->shared.s, v->shared.l) != v->shared.l ) return -1;
+    if ( bgzf_write(fp, v->indiv.s, v->indiv.l) != v->indiv.l ) return -1;
+
+    if (hfp->idx) {
+        if (bgzf_idx_push(fp, hfp->idx, v->rid, v->pos, v->pos + v->rlen,
+                          bgzf_tell(fp), 1) < 0)
+            return -1;
+    }
+
+    return 0;
+}
+
+/**********************
+ *** VCF header I/O ***
+ **********************/
+
+static int add_missing_contig_hrec(bcf_hdr_t *h, const char *name) {
+    bcf_hrec_t *hrec = calloc(1, sizeof(bcf_hrec_t));
+    int save_errno;
+    if (!hrec) goto fail;
+
+    hrec->key = strdup("contig");
+    if (!hrec->key) goto fail;
+
+    if (bcf_hrec_add_key(hrec, "ID", strlen("ID")) < 0) goto fail;
+    if (bcf_hrec_set_val(hrec, hrec->nkeys-1, name, strlen(name), 0) < 0)
+        goto fail;
+    if (bcf_hdr_add_hrec(h, hrec) < 0)
+        goto fail;
+    return 0;
+
+ fail:
+    save_errno = errno;
+    hts_log_error("%s", strerror(errno));
+    if (hrec) bcf_hrec_destroy(hrec);
+    errno = save_errno;
+    return -1;
+}
+
+bcf_hdr_t *vcf_hdr_read(htsFile *fp)
+{
+    kstring_t txt, *s = &fp->line;
+    int ret;
+    bcf_hdr_t *h;
+    tbx_t *idx = NULL;
+    const char **names = NULL;
+    h = bcf_hdr_init("r");
+    if (!h) {
+        hts_log_error("Failed to allocate bcf header");
+        return NULL;
+    }
+    txt.l = txt.m = 0; txt.s = 0;
+    while ((ret = hts_getline(fp, KS_SEP_LINE, s)) >= 0) {
+        int e = 0;
+        if (s->l == 0) continue;
+        if (s->s[0] != '#') {
+            hts_log_error("No sample line");
+            goto error;
+        }
+        if (s->s[1] != '#' && fp->fn_aux) { // insert contigs here
+            kstring_t tmp = { 0, 0, NULL };
+            hFILE *f = hopen(fp->fn_aux, "r");
+            if (f == NULL) {
+                hts_log_error("Couldn't open \"%s\"", fp->fn_aux);
+                goto error;
+            }
+            while (tmp.l = 0, kgetline(&tmp, (kgets_func *) hgets, f) >= 0) {
+                char *tab = strchr(tmp.s, '\t');
+                if (tab == NULL) continue;
+                e |= (kputs("##contig=<ID=", &txt) < 0);
+                e |= (kputsn(tmp.s, tab - tmp.s, &txt) < 0);
+                e |= (kputs(",length=", &txt) < 0);
+                e |= (kputl(atol(tab), &txt) < 0);
+                e |= (kputsn(">\n", 2, &txt) < 0);
+            }
+            free(tmp.s);
+            if (hclose(f) != 0) {
+                hts_log_error("Error on closing %s", fp->fn_aux);
+                goto error;
+            }
+            if (e) goto error;
+        }
+        if (kputsn(s->s, s->l, &txt) < 0) goto error;
+        if (kputc('\n', &txt) < 0) goto error;
+        if (s->s[1] != '#') break;
+    }
+    if ( ret < -1 ) goto error;
+    if ( !txt.s )
+    {
+        hts_log_error("Could not read the header");
+        goto error;
+    }
+    if ( bcf_hdr_parse(h, txt.s) < 0 ) goto error;
+
+    // check tabix index, are all contigs listed in the header? add the missing ones
+    idx = tbx_index_load3(fp->fn, NULL, HTS_IDX_SILENT_FAIL);
+    if ( idx )
+    {
+        int i, n, need_sync = 0;
+        names = tbx_seqnames(idx, &n);
+        if (!names) goto error;
+        for (i=0; i<n; i++)
+        {
+            bcf_hrec_t *hrec = bcf_hdr_get_hrec(h, BCF_HL_CTG, "ID", (char*) names[i], NULL);
+            if ( hrec ) continue;
+            if (add_missing_contig_hrec(h, names[i]) < 0) goto error;
+            need_sync = 1;
+        }
+        if ( need_sync ) {
+            if (bcf_hdr_sync(h) < 0) goto error;
+        }
+        free(names);
+        tbx_destroy(idx);
+    }
+    free(txt.s);
+    return h;
+
+ error:
+    if (idx) tbx_destroy(idx);
+    free(names);
+    free(txt.s);
+    if (h) bcf_hdr_destroy(h);
+    return NULL;
+}
+
+int bcf_hdr_set(bcf_hdr_t *hdr, const char *fname)
+{
+    int i = 0, n = 0, save_errno;
+    char **lines = hts_readlines(fname, &n);
+    if ( !lines ) return 1;
+    for (i=0; i<n-1; i++)
+    {
+        int k;
+        bcf_hrec_t *hrec = bcf_hdr_parse_line(hdr,lines[i],&k);
+        if (!hrec) goto fail;
+        if (bcf_hdr_add_hrec(hdr, hrec) < 0) {
+            bcf_hrec_destroy(hrec);
+            goto fail;
+        }
+        free(lines[i]);
+        lines[i] = NULL;
+    }
+    if (bcf_hdr_parse_sample_line(hdr, lines[n-1]) < 0) goto fail;
+    if (bcf_hdr_sync(hdr) < 0) goto fail;
+    free(lines[n-1]);
+    free(lines);
+    return 0;
+
+ fail:
+    save_errno = errno;
+    for (; i < n; i++)
+        free(lines[i]);
+    free(lines);
+    errno = save_errno;
+    return 1;
+}
+
+static int _bcf_hrec_format(const bcf_hrec_t *hrec, int is_bcf, kstring_t *str)
+{
+    uint32_t e = 0;
+    if ( !hrec->value )
+    {
+        int j, nout = 0;
+        e |= ksprintf(str, "##%s=<", hrec->key) < 0;
+        for (j=0; j<hrec->nkeys; j++)
+        {
+            // do not output IDX if output is VCF
+            if ( !is_bcf && !strcmp("IDX",hrec->keys[j]) ) continue;
+            if ( nout ) e |= kputc(',',str) < 0;
+            e |= ksprintf(str,"%s=%s", hrec->keys[j], hrec->vals[j]) < 0;
+            nout++;
+        }
+        e |= ksprintf(str,">\n") < 0;
+    }
+    else
+        e |= ksprintf(str,"##%s=%s\n", hrec->key,hrec->value) < 0;
+
+    return e == 0 ? 0 : -1;
+}
+
+int bcf_hrec_format(const bcf_hrec_t *hrec, kstring_t *str)
+{
+    return _bcf_hrec_format(hrec,0,str);
+}
+
+int bcf_hdr_format(const bcf_hdr_t *hdr, int is_bcf, kstring_t *str)
+{
+    int i, r = 0;
+    for (i=0; i<hdr->nhrec; i++)
+        r |= _bcf_hrec_format(hdr->hrec[i], is_bcf, str) < 0;
+
+    r |= ksprintf(str, "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO") < 0;
+    if ( bcf_hdr_nsamples(hdr) )
+    {
+        r |= ksprintf(str, "\tFORMAT") < 0;
+        for (i=0; i<bcf_hdr_nsamples(hdr); i++)
+            r |= ksprintf(str, "\t%s", hdr->samples[i]) < 0;
+    }
+    r |= ksprintf(str, "\n") < 0;
+
+    return r ? -1 : 0;
+}
+
+char *bcf_hdr_fmt_text(const bcf_hdr_t *hdr, int is_bcf, int *len)
+{
+    kstring_t txt = {0,0,0};
+    if (bcf_hdr_format(hdr, is_bcf, &txt) < 0)
+        return NULL;
+    if ( len ) *len = txt.l;
+    return txt.s;
+}
+
+const char **bcf_hdr_seqnames(const bcf_hdr_t *h, int *n)
+{
+    vdict_t *d = (vdict_t*)h->dict[BCF_DT_CTG];
+    int i, tid, m = kh_size(d);
+    const char **names = (const char**) calloc(m,sizeof(const char*));
+    if ( !names )
+    {
+        hts_log_error("Failed to allocate memory");
+        *n = 0;
+        return NULL;
+    }
+    khint_t k;
+    for (k=kh_begin(d); k<kh_end(d); k++)
+    {
+        if ( !kh_exist(d,k) ) continue;
+        if ( !kh_val(d, k).hrec[0] ) continue;  // removed via bcf_hdr_remove
+        tid = kh_val(d,k).id;
+        if ( tid >= m )
+        {
+            // This can happen after a contig has been removed from BCF header via bcf_hdr_remove()
+            if ( hts_resize(const char*, tid + 1, &m, &names, HTS_RESIZE_CLEAR)<0 )
+            {
+                hts_log_error("Failed to allocate memory");
+                *n = 0;
+                free(names);
+                return NULL;
+            }
+            m = tid + 1;
+        }
+        names[tid] = kh_key(d,k);
+    }
+    // ensure there are no gaps
+    for (i=0,tid=0; tid<m; i++,tid++)
+    {
+        while ( tid<m && !names[tid] ) tid++;
+        if ( tid==m ) break;
+        if ( i==tid ) continue;
+        names[i] = names[tid];
+        names[tid] = 0;
+    }
+    *n = i;
+    return names;
+}
+
+int vcf_hdr_write(htsFile *fp, const bcf_hdr_t *h)
+{
+    kstring_t htxt = {0,0,0};
+    if (bcf_hdr_format(h, 0, &htxt) < 0) {
+        free(htxt.s);
+        return -1;
+    }
+    while (htxt.l && htxt.s[htxt.l-1] == '\0') --htxt.l; // kill trailing zeros
+    int ret;
+    if ( fp->format.compression!=no_compression ) {
+        ret = bgzf_write(fp->fp.bgzf, htxt.s, htxt.l);
+        if (bgzf_flush(fp->fp.bgzf) != 0) return -1;
+    } else {
+        ret = hwrite(fp->fp.hfile, htxt.s, htxt.l);
+    }
+    free(htxt.s);
+    return ret<0 ? -1 : 0;
+}
+
+/***********************
+ *** Typed value I/O ***
+ ***********************/
+
+int bcf_enc_vint(kstring_t *s, int n, int32_t *a, int wsize)
+{
+    int32_t max = INT32_MIN, min = INT32_MAX;
+    int i;
+    if (n <= 0) {
+        return bcf_enc_size(s, 0, BCF_BT_NULL);
+    } else if (n == 1) {
+        return bcf_enc_int1(s, a[0]);
+    } else {
+        if (wsize <= 0) wsize = n;
+
+        // Equivalent to:
+        // for (i = 0; i < n; ++i) {
+        //     if (a[i] == bcf_int32_missing || a[i] == bcf_int32_vector_end )
+        //         continue;
+        //     if (max < a[i]) max = a[i];
+        //     if (min > a[i]) min = a[i];
+        // }
+        int max4[4] = {INT32_MIN, INT32_MIN, INT32_MIN, INT32_MIN};
+        int min4[4] = {INT32_MAX, INT32_MAX, INT32_MAX, INT32_MAX};
+        for (i = 0; i < (n&~3); i+=4) {
+            // bcf_int32_missing    == INT32_MIN and
+            // bcf_int32_vector_end == INT32_MIN+1.
+            // We skip these, but can mostly avoid explicit checking
+            if (max4[0] < a[i+0]) max4[0] = a[i+0];
+            if (max4[1] < a[i+1]) max4[1] = a[i+1];
+            if (max4[2] < a[i+2]) max4[2] = a[i+2];
+            if (max4[3] < a[i+3]) max4[3] = a[i+3];
+            if (min4[0] > a[i+0] && a[i+0] > INT32_MIN+1) min4[0] = a[i+0];
+            if (min4[1] > a[i+1] && a[i+1] > INT32_MIN+1) min4[1] = a[i+1];
+            if (min4[2] > a[i+2] && a[i+2] > INT32_MIN+1) min4[2] = a[i+2];
+            if (min4[3] > a[i+3] && a[i+3] > INT32_MIN+1) min4[3] = a[i+3];
+        }
+        min = min4[0];
+        if (min > min4[1]) min = min4[1];
+        if (min > min4[2]) min = min4[2];
+        if (min > min4[3]) min = min4[3];
+        max = max4[0];
+        if (max < max4[1]) max = max4[1];
+        if (max < max4[2]) max = max4[2];
+        if (max < max4[3]) max = max4[3];
+        for (; i < n; ++i) {
+            if (max < a[i]) max = a[i];
+            if (min > a[i] && a[i] > INT32_MIN+1) min = a[i];
+        }
+
+        if (max <= BCF_MAX_BT_INT8 && min >= BCF_MIN_BT_INT8) {
+            if (bcf_enc_size(s, wsize, BCF_BT_INT8) < 0 ||
+                ks_resize(s, s->l + n) < 0)
+                return -1;
+            uint8_t *p = (uint8_t *) s->s + s->l;
+            for (i = 0; i < n; ++i, p++) {
+                if ( a[i]==bcf_int32_vector_end )   *p = bcf_int8_vector_end;
+                else if ( a[i]==bcf_int32_missing ) *p = bcf_int8_missing;
+                else *p = a[i];
+            }
+            s->l += n;
+        } else if (max <= BCF_MAX_BT_INT16 && min >= BCF_MIN_BT_INT16) {
+            uint8_t *p;
+            if (bcf_enc_size(s, wsize, BCF_BT_INT16) < 0 ||
+                ks_resize(s, s->l + n * sizeof(int16_t)) < 0)
+                return -1;
+            p = (uint8_t *) s->s + s->l;
+            for (i = 0; i < n; ++i)
+            {
+                int16_t x;
+                if ( a[i]==bcf_int32_vector_end ) x = bcf_int16_vector_end;
+                else if ( a[i]==bcf_int32_missing ) x = bcf_int16_missing;
+                else x = a[i];
+                i16_to_le(x, p);
+                p += sizeof(int16_t);
+            }
+            s->l += n * sizeof(int16_t);
+        } else {
+            uint8_t *p;
+            if (bcf_enc_size(s, wsize, BCF_BT_INT32) < 0 ||
+                ks_resize(s, s->l + n * sizeof(int32_t)) < 0)
+                return -1;
+            p = (uint8_t *) s->s + s->l;
+            for (i = 0; i < n; ++i) {
+                i32_to_le(a[i], p);
+                p += sizeof(int32_t);
+            }
+            s->l += n * sizeof(int32_t);
+        }
+    }
+
+    return 0;
+}
+
+#ifdef VCF_ALLOW_INT64
+static int bcf_enc_long1(kstring_t *s, int64_t x) {
+    uint32_t e = 0;
+    if (x <= BCF_MAX_BT_INT32 && x >= BCF_MIN_BT_INT32)
+        return bcf_enc_int1(s, x);
+    if (x == bcf_int64_vector_end) {
+        e |= bcf_enc_size(s, 1, BCF_BT_INT8);
+        e |= kputc(bcf_int8_vector_end, s) < 0;
+    } else if (x == bcf_int64_missing) {
+        e |= bcf_enc_size(s, 1, BCF_BT_INT8);
+        e |= kputc(bcf_int8_missing, s) < 0;
+    } else {
+        e |= bcf_enc_size(s, 1, BCF_BT_INT64);
+        e |= ks_expand(s, 8);
+        if (e == 0) { u64_to_le(x, (uint8_t *) s->s + s->l); s->l += 8; }
+    }
+    return e == 0 ? 0 : -1;
+}
+#endif
+
+static inline int serialize_float_array(kstring_t *s, size_t n, const float *a) {
+    uint8_t *p;
+    size_t i;
+    size_t bytes = n * sizeof(float);
+
+    if (bytes / sizeof(float) != n) return -1;
+    if (ks_resize(s, s->l + bytes) < 0) return -1;
+
+    p = (uint8_t *) s->s + s->l;
+    for (i = 0; i < n; i++) {
+        float_to_le(a[i], p);
+        p += sizeof(float);
+    }
+    s->l += bytes;
+
+    return 0;
+}
+
+int bcf_enc_vfloat(kstring_t *s, int n, float *a)
+{
+    assert(n >= 0);
+    bcf_enc_size(s, n, BCF_BT_FLOAT);
+    serialize_float_array(s, n, a);
+    return 0; // FIXME: check for errs in this function
+}
+
+int bcf_enc_vchar(kstring_t *s, int l, const char *a)
+{
+    bcf_enc_size(s, l, BCF_BT_CHAR);
+    kputsn(a, l, s);
+    return 0; // FIXME: check for errs in this function
+}
+
+// Special case of n==1 as it also occurs quite often in FORMAT data.
+// This version is also small enough to get inlined.
+static inline int bcf_fmt_array1(kstring_t *s, int type, void *data) {
+    uint32_t e = 0;
+    uint8_t *p = (uint8_t *)data;
+    int32_t v;
+
+    // helps gcc more than clang here. In billions of cycles:
+    //          bcf_fmt_array1  bcf_fmt_array
+    // gcc7:    23.2            24.3
+    // gcc13:   21.6            23.0
+    // clang13: 27.1            27.8
+    switch (type) {
+    case BCF_BT_CHAR:
+        e |= kputc_(*p == bcf_str_missing ? '.' : *p, s) < 0;
+        break;
+
+    case BCF_BT_INT8:
+        if (*(int8_t *)p != bcf_int8_vector_end) {
+            e |= ((*(int8_t *)p == bcf_int8_missing)
+                  ? kputc_('.', s)
+                  : kputw(*(int8_t *)p, s)) < 0;
+        }
+        break;
+    case BCF_BT_INT16:
+        v = le_to_i16(p);
+        if (v != bcf_int16_vector_end) {
+            e |= (v == bcf_int16_missing
+                  ? kputc_('.', s)
+                  : kputw(v, s)) < 0;
+        }
+        break;
+
+    case BCF_BT_INT32:
+        v = le_to_i32(p);
+        if (v != bcf_int32_vector_end) {
+            e |= (v == bcf_int32_missing
+                  ? kputc_('.', s)
+                  : kputw(v, s)) < 0;
+        }
+        break;
+
+    case BCF_BT_FLOAT:
+        v = le_to_u32(p);
+        if (v != bcf_float_vector_end) {
+            e |= (v == bcf_float_missing
+                  ? kputc_('.', s)
+                  : kputd(le_to_float(p), s)) < 0;
+        }
+        break;
+
+    default:
+        hts_log_error("Unexpected type %d", type);
+        return -1;
+    }
+
+    return e == 0 ? 0 : -1;
+}
+
+int bcf_fmt_array(kstring_t *s, int n, int type, void *data)
+{
+    int j = 0;
+    uint32_t e = 0;
+    if (n == 0) {
+        return kputc_('.', s) >= 0 ? 0 : -1;
+    }
+
+    if (type == BCF_BT_CHAR)
+    {
+        char *p = (char *)data;
+
+        // Note bcf_str_missing is already accounted for in n==0 above.
+        if (n >= 8) {
+            char *p_end = memchr(p, 0, n);
+            e |= kputsn(p, p_end ? p_end-p : n, s) < 0;
+        } else {
+            for (j = 0; j < n && *p; ++j, ++p)
+               e |= kputc(*p, s) < 0;
+        }
+    }
+    else
+    {
+        #define BRANCH(type_t, convert, is_missing, is_vector_end, kprint) { \
+            uint8_t *p = (uint8_t *) data; \
+            for (j=0; j<n; j++, p += sizeof(type_t))    \
+            { \
+                type_t v = convert(p); \
+                if ( is_vector_end ) break; \
+                if ( j ) e |= kputc_(',', s) < 0; \
+                e |= (is_missing ? kputc('.', s) : kprint) < 0; \
+            } \
+        }
+        switch (type) {
+            case BCF_BT_INT8:  BRANCH(int8_t,  le_to_i8, v==bcf_int8_missing,  v==bcf_int8_vector_end,  kputw(v, s)); break;
+            case BCF_BT_INT16: BRANCH(int16_t, le_to_i16, v==bcf_int16_missing, v==bcf_int16_vector_end, kputw(v, s)); break;
+            case BCF_BT_INT32: BRANCH(int32_t, le_to_i32, v==bcf_int32_missing, v==bcf_int32_vector_end, kputw(v, s)); break;
+            case BCF_BT_FLOAT: BRANCH(uint32_t, le_to_u32, v==bcf_float_missing, v==bcf_float_vector_end, kputd(le_to_float(p), s)); break;
+            default: hts_log_error("Unexpected type %d", type); exit(1); break;
+        }
+        #undef BRANCH
+    }
+    return e == 0 ? 0 : -1;
+}
+
+uint8_t *bcf_fmt_sized_array(kstring_t *s, uint8_t *ptr)
+{
+    int x, type;
+    x = bcf_dec_size(ptr, &ptr, &type);
+    bcf_fmt_array(s, x, type, ptr);
+    return ptr + (x << bcf_type_shift[type]);
+}
+
+/********************
+ *** VCF site I/O ***
+ ********************/
+
+typedef struct {
+    int key;            // Key for h->id[BCF_DT_ID][key] vdict
+    int max_m;          // number of elements in field array (ie commas)
+    int size;           // field size (max_l or max_g*4 if is_gt)
+    int offset;         // offset of buf into h->mem
+    uint32_t is_gt:1,   // is genotype
+             max_g:31;  // maximum number of genotypes
+    uint32_t max_l;     // length of field
+    uint32_t y;         // h->id[0][fmt[j].key].val->info[BCF_HL_FMT]
+    uint8_t *buf;       // Pointer into h->mem
+} fmt_aux_t;
+
+// fmt_aux_t field notes:
+// max_* are biggest sizes of the various FORMAT fields across all samples.
+// We use these after pivoting the data to ensure easy random access
+// of a specific sample.
+//
+// max_m is only used for type BCF_HT_REAL or BCF_HT_INT
+// max_g is only used for is_gt == 1 (will be BCF_HT_STR)
+// max_l is only used for is_gt == 0 (will be BCF_HT_STR)
+//
+// These are computed in vcf_parse_format_max3 and used in
+// vcf_parse_format_alloc4 to get the size.
+//
+// size is computed from max_g, max_l, max_m and is_gt.  Once computed
+// the max values are never accessed again.
+//
+// In theory all 4 vars could be coalesced into a single variable, but this
+// significantly harms speed (even if done via a union).  It's about 25-30%
+// slower.
+
+static inline int align_mem(kstring_t *s)
+{
+    int e = 0;
+    if (s->l&7) {
+        uint64_t zero = 0;
+        e = kputsn((char*)&zero, 8 - (s->l&7), s) < 0;
+    }
+    return e == 0 ? 0 : -1;
+}
+
+#define MAX_N_FMT 255   /* Limited by size of bcf1_t n_fmt field */
+
+// detect FORMAT "."
+static int vcf_parse_format_empty1(kstring_t *s, const bcf_hdr_t *h, bcf1_t *v,
+                                   const char *p, const char *q) {
+    const char *end = s->s + s->l;
+    if ( q>=end )
+    {
+        hts_log_error("FORMAT column with no sample columns starting at %s:%"PRIhts_pos"", bcf_seqname_safe(h,v), v->pos+1);
+        v->errcode |= BCF_ERR_NCOLS;
+        return -1;
+    }
+
+    v->n_fmt = 0;
+    if ( p[0]=='.' && p[1]==0 ) // FORMAT field is empty "."
+    {
+        v->n_sample = bcf_hdr_nsamples(h);
+        return 1;
+    }
+
+    return 0;
+}
+
+// get format information from the dictionary
+static int vcf_parse_format_dict2(kstring_t *s, const bcf_hdr_t *h, bcf1_t *v,
+                                  const char *p, const char *q, fmt_aux_t *fmt) {
+    const vdict_t *d = (vdict_t*)h->dict[BCF_DT_ID];
+    char *t;
+    int j;
+    ks_tokaux_t aux1;
+
+    for (j = 0, t = kstrtok(p, ":", &aux1); t; t = kstrtok(0, 0, &aux1), ++j) {
+        if (j >= MAX_N_FMT) {
+            v->errcode |= BCF_ERR_LIMITS;
+            hts_log_error("FORMAT column at %s:%"PRIhts_pos" lists more identifiers than htslib can handle",
+                bcf_seqname_safe(h,v), v->pos+1);
+            return -1;
+        }
+
+        *(char*)aux1.p = 0;
+        khint_t k = kh_get(vdict, d, t);
+        if (k == kh_end(d) || kh_val(d, k).info[BCF_HL_FMT] == 15) {
+            if ( t[0]=='.' && t[1]==0 )
+            {
+                hts_log_error("Invalid FORMAT tag name '.' at %s:%"PRIhts_pos, bcf_seqname_safe(h,v), v->pos+1);
+                v->errcode |= BCF_ERR_TAG_INVALID;
+                return -1;
+            }
+            hts_log_warning("FORMAT '%s' at %s:%"PRIhts_pos" is not defined in the header, assuming Type=String", t, bcf_seqname_safe(h,v), v->pos+1);
+            kstring_t tmp = {0,0,0};
+            int l;
+            ksprintf(&tmp, "##FORMAT=<ID=%s,Number=1,Type=String,Description=\"Dummy\">", t);
+            bcf_hrec_t *hrec = bcf_hdr_parse_line(h,tmp.s,&l);
+            free(tmp.s);
+            int res = hrec ? bcf_hdr_add_hrec((bcf_hdr_t*)h, hrec) : -1;
+            if (res < 0) bcf_hrec_destroy(hrec);
+            if (res > 0) res = bcf_hdr_sync((bcf_hdr_t*)h);
+
+            k = kh_get(vdict, d, t);
+            v->errcode |= BCF_ERR_TAG_UNDEF;
+            if (res || k == kh_end(d)) {
+                hts_log_error("Could not add dummy header for FORMAT '%s' at %s:%"PRIhts_pos, t, bcf_seqname_safe(h,v), v->pos+1);
+                v->errcode |= BCF_ERR_TAG_INVALID;
+                return -1;
+            }
+        }
+        fmt[j].max_l = fmt[j].max_m = fmt[j].max_g = 0;
+        fmt[j].key = kh_val(d, k).id;
+        fmt[j].is_gt = (t[0] == 'G' && t[1] == 'T' && !t[2]);
+        fmt[j].y = h->id[0][fmt[j].key].val->info[BCF_HL_FMT];
+        v->n_fmt++;
+    }
+    return 0;
+}
+
+// compute max
+static int vcf_parse_format_max3(kstring_t *s, const bcf_hdr_t *h, bcf1_t *v,
+                                 char *p, char *q, fmt_aux_t *fmt) {
+    int n_sample_ori = -1;
+    char *r = q + 1;  // r: position in the format string
+    int l = 0, m = 1, g = 1, j;
+    v->n_sample = 0;  // m: max vector size, l: max field len, g: max number of alleles
+    const char *end = s->s + s->l;
+
+    while ( r<end )
+    {
+        // can we skip some samples?
+        if ( h->keep_samples )
+        {
+            n_sample_ori++;
+            if ( !bit_array_test(h->keep_samples,n_sample_ori) )
+            {
+                while ( *r!='\t' && r<end ) r++;
+                if ( *r=='\t' ) { *r = 0; r++; }
+                continue;
+            }
+        }
+
+        // collect fmt stats: max vector size, length, number of alleles
+        j = 0;  // j-th format field
+        fmt_aux_t *f = fmt;
+        static char meta[256] = {
+            // \0 \t , / : |
+            1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+            0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1, 0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,
+            0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+            0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,
+            0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+            0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+            0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+            0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+        };
+
+        char *r_start = r;
+        for (;;) {
+            // Quickly skip ahead to an appropriate meta-character
+            while (!meta[(unsigned char)*r]) r++;
+
+            switch (*r) {
+            case ',':
+                m++;
+                break;
+
+            case '|':
+            case '/':
+                if (f->is_gt) g++;
+                break;
+
+            case '\t':
+                *r = 0; // fall through
+
+            default: // valid due to while loop above.
+            case '\0':
+            case ':':
+                l = r - r_start; r_start = r;
+                if (f->max_m < m) f->max_m = m;
+                if (f->max_l < l) f->max_l = l;
+                if (f->is_gt && f->max_g < g) f->max_g = g;
+                l = 0, m = g = 1;
+                if ( *r==':' ) {
+                    j++; f++;
+                    if ( j>=v->n_fmt ) {
+                        hts_log_error("Incorrect number of FORMAT fields at %s:%"PRIhts_pos"",
+                                      h->id[BCF_DT_CTG][v->rid].key, v->pos+1);
+                        v->errcode |= BCF_ERR_NCOLS;
+                        return -1;
+                    }
+                } else goto end_for;
+                break;
+            }
+            if ( r>=end ) break;
+            r++;
+        }
+    end_for:
+        v->n_sample++;
+        if ( v->n_sample == bcf_hdr_nsamples(h) ) break;
+        r++;
+    }
+
+    return 0;
+}
+
+// allocate memory for arrays
+static int vcf_parse_format_alloc4(kstring_t *s, const bcf_hdr_t *h, bcf1_t *v,
+                                   const char *p, const char *q,
+                                   fmt_aux_t *fmt) {
+    kstring_t *mem = (kstring_t*)&h->mem;
+
+    int j;
+    for (j = 0; j < v->n_fmt; ++j) {
+        fmt_aux_t *f = &fmt[j];
+        if ( !f->max_m ) f->max_m = 1;  // omitted trailing format field
+
+        if ((f->y>>4&0xf) == BCF_HT_STR) {
+            f->size = f->is_gt? f->max_g << 2 : f->max_l;
+        } else if ((f->y>>4&0xf) == BCF_HT_REAL || (f->y>>4&0xf) == BCF_HT_INT) {
+            f->size = f->max_m << 2;
+        } else {
+            hts_log_error("The format type %d at %s:%"PRIhts_pos" is currently not supported", f->y>>4&0xf, bcf_seqname_safe(h,v), v->pos+1);
+            v->errcode |= BCF_ERR_TAG_INVALID;
+            return -1;
+        }
+
+        if (align_mem(mem) < 0) {
+            hts_log_error("Memory allocation failure at %s:%"PRIhts_pos, bcf_seqname_safe(h,v), v->pos+1);
+            v->errcode |= BCF_ERR_LIMITS;
+            return -1;
+        }
+
+        // Limit the total memory to ~2Gb per VCF row.  This should mean
+        // malformed VCF data is less likely to take excessive memory and/or
+        // time.
+        if ((uint64_t) mem->l + v->n_sample * (uint64_t)f->size > INT_MAX) {
+            static int warned = 0;
+            if ( !warned ) hts_log_warning("Excessive memory required by FORMAT fields at %s:%"PRIhts_pos, bcf_seqname_safe(h,v), v->pos+1);
+            warned = 1;
+            v->errcode |= BCF_ERR_LIMITS;
+            f->size = -1;
+            f->offset = 0;
+            continue;
+        }
+
+        f->offset = mem->l;
+        if (ks_resize(mem, mem->l + v->n_sample * (size_t)f->size) < 0) {
+            hts_log_error("Memory allocation failure at %s:%"PRIhts_pos, bcf_seqname_safe(h,v), v->pos+1);
+            v->errcode |= BCF_ERR_LIMITS;
+            return -1;
+        }
+        mem->l += v->n_sample * f->size;
+    }
+
+    {
+        int j;
+        for (j = 0; j < v->n_fmt; ++j)
+            fmt[j].buf = (uint8_t*)mem->s + fmt[j].offset;
+    }
+
+    // check for duplicate tags
+    int i;
+    for (i=1; i<v->n_fmt; i++)
+    {
+        fmt_aux_t *ifmt = &fmt[i];
+        if ( ifmt->size==-1 ) continue; // already marked for removal
+        for (j=0; j<i; j++)
+        {
+            fmt_aux_t *jfmt = &fmt[j];
+            if ( jfmt->size==-1 ) continue; // already marked for removal
+            if ( ifmt->key!=jfmt->key ) continue;
+            static int warned = 0;
+            if ( !warned ) hts_log_warning("Duplicate FORMAT tag %s at %s:%"PRIhts_pos, bcf_hdr_int2id(h,BCF_DT_ID,ifmt->key), bcf_seqname_safe(h,v), v->pos+1);
+            warned = 1;
+            v->errcode |= BCF_ERR_TAG_INVALID;
+            ifmt->size = -1;
+            ifmt->offset = 0;
+            break;
+        }
+    }
+    return 0;
+}
+
+// Fill the sample fields
+static int vcf_parse_format_fill5(kstring_t *s, const bcf_hdr_t *h, bcf1_t *v,
+                                  const char *p, const char *q, fmt_aux_t *fmt) {
+    static int extreme_val_warned = 0;
+    int n_sample_ori = -1;
+    // At beginning of the loop t points to the first char of a format
+    const char *t = q + 1;
+    int m = 0;   // m: sample id
+    const int nsamples = bcf_hdr_nsamples(h);
+
+    const char *end = s->s + s->l;
+    while ( t<end )
+    {
+        // can we skip some samples?
+        if ( h->keep_samples )
+        {
+            n_sample_ori++;
+            if ( !bit_array_test(h->keep_samples,n_sample_ori) )
+            {
+                while ( *t && t<end ) t++;
+                t++;
+                continue;
+            }
+        }
+        if ( m == nsamples ) break;
+
+        int j = 0; // j-th format field, m-th sample
+        while ( t < end )
+        {
+            fmt_aux_t *z = &fmt[j++];
+            const int htype = z->y>>4&0xf;
+            if (!z->buf) {
+                hts_log_error("Memory allocation failure for FORMAT field type %d at %s:%"PRIhts_pos,
+                              z->y>>4&0xf, bcf_seqname_safe(h,v), v->pos+1);
+                v->errcode |= BCF_ERR_LIMITS;
+                return -1;
+            }
+
+            if ( z->size==-1 )
+            {
+                // this field is to be ignored, it's either too big or a duplicate
+                while ( *t != ':' && *t ) t++;
+            }
+            else if (htype == BCF_HT_STR) {
+                int l;
+                if (z->is_gt) {
+                    // Genotypes.
+                    // <val>([|/]<val>)+... where <val> is [0-9]+ or ".".
+                    int32_t is_phased = 0;
+                    uint32_t *x = (uint32_t*)(z->buf + z->size * (size_t)m);
+                    uint32_t unreadable = 0;
+                    uint32_t max = 0;
+                    int overflow = 0;
+                    for (l = 0;; ++t) {
+                        if (*t == '.') {
+                            ++t, x[l++] = is_phased;
+                        } else {
+                            const char *tt = t;
+                            uint32_t val;
+                            // Or "v->n_allele < 10", but it doesn't
+                            // seem to be any faster and this feels safer.
+                            if (*t >= '0' && *t <= '9' &&
+                                !(t[1] >= '0' && t[1] <= '9')) {
+                                val = *t++ - '0';
+                            } else {
+                                val = hts_str2uint(t, (char **)&t,
+                                                   sizeof(val) * CHAR_MAX - 2,
+                                                   &overflow);
+                                unreadable |= tt == t;
+                            }
+                            if (max < val) max = val;
+                            x[l++] = (val + 1) << 1 | is_phased;
+                        }
+                        is_phased = (*t == '|');
+                        if (*t != '|' && *t != '/') break;
+                    }
+                    // Possibly check max against v->n_allele instead?
+                    if (overflow || max > (INT32_MAX >> 1) - 1) {
+                        hts_log_error("Couldn't read GT data: value too large at %s:%"PRIhts_pos, bcf_seqname_safe(h,v), v->pos+1);
+                        return -1;
+                    }
+                    if (unreadable) {
+                        hts_log_error("Couldn't read GT data: value not a number or '.' at %s:%"PRIhts_pos, bcf_seqname_safe(h,v), v->pos+1);
+                        return -1;
+                    }
+                    if ( !l ) x[l++] = 0;   // An empty field, insert missing value
+                    for (; l < z->size>>2; ++l)
+                        x[l] = bcf_int32_vector_end;
+
+                } else {
+                    // Otherwise arbitrary strings
+                    char *x = (char*)z->buf + z->size * (size_t)m;
+                    for (l = 0; *t != ':' && *t; ++t)
+                        x[l++] = *t;
+                    if (z->size > l)
+                        memset(&x[l], 0, (z->size-l) * sizeof(*x));
+                }
+
+            } else if (htype == BCF_HT_INT) {
+                // One or more integers in an array
+                int32_t *x = (int32_t*)(z->buf + z->size * (size_t)m);
+                int l;
+                for (l = 0;; ++t) {
+                    if (*t == '.') {
+                        x[l++] = bcf_int32_missing, ++t; // ++t to skip "."
+                    } else {
+                        int overflow = 0;
+                        char *te;
+                        long int tmp_val = hts_str2int(t, &te, sizeof(tmp_val)*CHAR_BIT, &overflow);
+                        if ( te==t || overflow || tmp_val<BCF_MIN_BT_INT32 || tmp_val>BCF_MAX_BT_INT32 )
+                        {
+                            if ( !extreme_val_warned )
+                            {
+                                hts_log_warning("Extreme FORMAT/%s value encountered and set to missing at %s:%"PRIhts_pos,
+                                                h->id[BCF_DT_ID][fmt[j-1].key].key, bcf_seqname_safe(h,v), v->pos+1);
+                                extreme_val_warned = 1;
+                            }
+                            tmp_val = bcf_int32_missing;
+                        }
+                        x[l++] = tmp_val;
+                        t = te;
+                    }
+                    if (*t != ',') break;
+                }
+                if ( !l )
+                    x[l++] = bcf_int32_missing;
+                for (; l < z->size>>2; ++l)
+                    x[l] = bcf_int32_vector_end;
+
+            } else if (htype == BCF_HT_REAL) {
+                // One of more floating point values in an array
+                float *x = (float*)(z->buf + z->size * (size_t)m);
+                int l;
+                for (l = 0;; ++t) {
+                    if (*t == '.' && !isdigit_c(t[1])) {
+                        bcf_float_set_missing(x[l++]), ++t; // ++t to skip "."
+                    } else {
+                        int overflow = 0;
+                        char *te;
+                        float tmp_val = hts_str2dbl(t, &te, &overflow);
+                        if ( (te==t || overflow) && !extreme_val_warned )
+                        {
+                            hts_log_warning("Extreme FORMAT/%s value encountered at %s:%"PRIhts_pos, h->id[BCF_DT_ID][fmt[j-1].key].key, bcf_seqname(h,v), v->pos+1);
+                            extreme_val_warned = 1;
+                        }
+                        x[l++] = tmp_val;
+                        t = te;
+                    }
+                    if (*t != ',') break;
+                }
+                if ( !l )
+                    // An empty field, insert missing value
+                    bcf_float_set_missing(x[l++]);
+                for (; l < z->size>>2; ++l)
+                    bcf_float_set_vector_end(x[l]);
+            } else {
+                hts_log_error("Unknown FORMAT field type %d at %s:%"PRIhts_pos, htype, bcf_seqname_safe(h,v), v->pos+1);
+                v->errcode |= BCF_ERR_TAG_INVALID;
+                return -1;
+            }
+
+            if (*t == '\0') {
+                break;
+            }
+            else if (*t == ':') {
+                t++;
+            }
+            else {
+                char buffer[8];
+                hts_log_error("Invalid character %s in '%s' FORMAT field at %s:%"PRIhts_pos"",
+                    hts_strprint(buffer, sizeof buffer, '\'', t, 1),
+                    h->id[BCF_DT_ID][z->key].key, bcf_seqname_safe(h,v), v->pos+1);
+                v->errcode |= BCF_ERR_CHAR;
+                return -1;
+            }
+        }
+
+        // fill end-of-vector values
+        for (; j < v->n_fmt; ++j) {
+            fmt_aux_t *z = &fmt[j];
+            const int htype = z->y>>4&0xf;
+            int l;
+
+            if (z->size == -1) // this field is to be ignored
+                continue;
+
+            if (htype == BCF_HT_STR) {
+                if (z->is_gt) {
+                    int32_t *x = (int32_t*)(z->buf + z->size * (size_t)m);
+                    if (z->size) x[0] = bcf_int32_missing;
+                    for (l = 1; l < z->size>>2; ++l) x[l] = bcf_int32_vector_end;
+                } else {
+                    char *x = (char*)z->buf + z->size * (size_t)m;
+                    if ( z->size ) {
+                        x[0] = '.';
+                        memset(&x[1], 0, (z->size-1) * sizeof(*x));
+                    }
+                }
+            } else if (htype == BCF_HT_INT) {
+                int32_t *x = (int32_t*)(z->buf + z->size * (size_t)m);
+                x[0] = bcf_int32_missing;
+                for (l = 1; l < z->size>>2; ++l) x[l] = bcf_int32_vector_end;
+            } else if (htype == BCF_HT_REAL) {
+                float *x = (float*)(z->buf + z->size * (size_t)m);
+                bcf_float_set_missing(x[0]);
+                for (l = 1; l < z->size>>2; ++l) bcf_float_set_vector_end(x[l]);
+            }
+        }
+
+        m++; t++;
+    }
+
+    return 0;
+}
+
+// write individual genotype information
+static int vcf_parse_format_gt6(kstring_t *s, const bcf_hdr_t *h, bcf1_t *v,
+                                const char *p, const char *q, fmt_aux_t *fmt) {
+    kstring_t *str = &v->indiv;
+    int i, need_downsize = 0;
+    if (v->n_sample > 0) {
+        for (i = 0; i < v->n_fmt; ++i) {
+            fmt_aux_t *z = &fmt[i];
+            if ( z->size==-1 ) {
+                need_downsize = 1;
+                continue;
+            }
+            bcf_enc_int1(str, z->key);
+            if ((z->y>>4&0xf) == BCF_HT_STR && !z->is_gt) {
+                bcf_enc_size(str, z->size, BCF_BT_CHAR);
+                kputsn((char*)z->buf, z->size * (size_t)v->n_sample, str);
+            } else if ((z->y>>4&0xf) == BCF_HT_INT || z->is_gt) {
+                bcf_enc_vint(str, (z->size>>2) * v->n_sample, (int32_t*)z->buf, z->size>>2);
+            } else {
+                bcf_enc_size(str, z->size>>2, BCF_BT_FLOAT);
+                if (serialize_float_array(str, (z->size>>2) * (size_t)v->n_sample,
+                                          (float *) z->buf) != 0) {
+                    v->errcode |= BCF_ERR_LIMITS;
+                    hts_log_error("Out of memory at %s:%"PRIhts_pos, bcf_seqname_safe(h,v), v->pos+1);
+                    return -1;
+                }
+            }
+        }
+
+    }
+    if ( need_downsize ) {
+        i = 0;
+        while ( i < v->n_fmt ) {
+            if ( fmt[i].size==-1 )
+            {
+                v->n_fmt--;
+                if ( i < v->n_fmt ) memmove(&fmt[i],&fmt[i+1],sizeof(*fmt)*(v->n_fmt-i));
+            }
+            else
+                i++;
+        }
+    }
+    return 0;
+}
+
+// validity checking
+static int vcf_parse_format_check7(const bcf_hdr_t *h, bcf1_t *v) {
+    if ( v->n_sample!=bcf_hdr_nsamples(h) )
+    {
+        hts_log_error("Number of columns at %s:%"PRIhts_pos" does not match the number of samples (%d vs %d)",
+            bcf_seqname_safe(h,v), v->pos+1, v->n_sample, bcf_hdr_nsamples(h));
+        v->errcode |= BCF_ERR_NCOLS;
+        return -1;
+    }
+    if ( v->indiv.l > 0xffffffff )
+    {
+        hts_log_error("The FORMAT at %s:%"PRIhts_pos" is too long", bcf_seqname_safe(h,v), v->pos+1);
+        v->errcode |= BCF_ERR_LIMITS;
+
+        // Error recovery: return -1 if this is a critical error or 0 if we want to ignore the FORMAT and proceed
+        v->n_fmt = 0;
+        return -1;
+    }
+
+    return 0;
+}
+
+// p,q is the start and the end of the FORMAT field
+static int vcf_parse_format(kstring_t *s, const bcf_hdr_t *h, bcf1_t *v,
+                            char *p, char *q)
+{
+    if ( !bcf_hdr_nsamples(h) ) return 0;
+    kstring_t *mem = (kstring_t*)&h->mem;
+    mem->l = 0;
+
+    fmt_aux_t fmt[MAX_N_FMT];
+
+    // detect FORMAT "."
+    int ret; // +ve = ok, -ve = err
+    if ((ret = vcf_parse_format_empty1(s, h, v, p, q)))
+        return ret ? 0 : -1;
+
+    // get format information from the dictionary
+    if (vcf_parse_format_dict2(s, h, v, p, q, fmt) < 0)
+        return -1;
+
+    // FORMAT data is per-sample A:B:C A:B:C A:B:C ... but in memory it is
+    // stored as per-type arrays AAA... BBB... CCC...  This is basically
+    // a data rotation or pivot.
+
+    // The size of elements in the array grow to their maximum needed,
+    // permitting fast random access.  This means however we have to first
+    // scan the whole FORMAT line to find the maximum of each type, and
+    // then scan it again to find the store the data.
+    // We break this down into compute-max, allocate, fill-out-buffers
+
+    // TODO: ?
+    // The alternative would be to pivot on the first pass, with fixed
+    // size entries for numerics and concatenated strings otherwise, also
+    // tracking maximum sizes.  Then on a second pass we reallocate and
+    // copy the data again to a uniformly sized array.  Two passes through
+    // memory, but without doubling string parsing.
+
+    // compute max
+    if (vcf_parse_format_max3(s, h, v, p, q, fmt) < 0)
+        return -1;
+
+    // allocate memory for arrays
+    if (vcf_parse_format_alloc4(s, h, v, p, q, fmt) < 0)
+        return -1;
+
+    // fill the sample fields; at beginning of the loop
+    if (vcf_parse_format_fill5(s, h, v, p, q, fmt) < 0)
+        return -1;
+
+    // write individual genotype information
+    if (vcf_parse_format_gt6(s, h, v, p, q, fmt) < 0)
+        return -1;
+
+    // validity checking
+    if (vcf_parse_format_check7(h, v) < 0)
+        return -1;
+
+    return 0;
+}
+
+static khint_t fix_chromosome(const bcf_hdr_t *h, vdict_t *d, const char *p) {
+    // Simple error recovery for chromosomes not defined in the header. It will not help when VCF header has
+    // been already printed, but will enable tools like vcfcheck to proceed.
+
+    kstring_t tmp = {0,0,0};
+    khint_t k;
+    int l;
+    if (ksprintf(&tmp, "##contig=<ID=%s>", p) < 0)
+        return kh_end(d);
+    bcf_hrec_t *hrec = bcf_hdr_parse_line(h,tmp.s,&l);
+    free(tmp.s);
+    int res = hrec ? bcf_hdr_add_hrec((bcf_hdr_t*)h, hrec) : -1;
+    if (res < 0) bcf_hrec_destroy(hrec);
+    if (res > 0) res = bcf_hdr_sync((bcf_hdr_t*)h);
+    k = kh_get(vdict, d, p);
+
+    return k;
+}
+
+static int vcf_parse_filter(kstring_t *str, const bcf_hdr_t *h, bcf1_t *v, char *p, char *q) {
+    int i, n_flt = 1, max_n_flt = 0;
+    char *r, *t;
+    int32_t *a_flt = NULL;
+    ks_tokaux_t aux1;
+    khint_t k;
+    vdict_t *d = (vdict_t*)h->dict[BCF_DT_ID];
+    // count the number of filters
+    if (*(q-1) == ';') *(q-1) = 0;
+    for (r = p; *r; ++r)
+        if (*r == ';') ++n_flt;
+    if (n_flt > max_n_flt) {
+        a_flt = malloc(n_flt * sizeof(*a_flt));
+        if (!a_flt) {
+            hts_log_error("Could not allocate memory at %s:%"PRIhts_pos, bcf_seqname_safe(h,v), v->pos+1);
+            v->errcode |= BCF_ERR_LIMITS; // No appropriate code?
+            return -1;
+        }
+        max_n_flt = n_flt;
+    }
+    // add filters
+    for (t = kstrtok(p, ";", &aux1), i = 0; t; t = kstrtok(0, 0, &aux1)) {
+        *(char*)aux1.p = 0;
+        k = kh_get(vdict, d, t);
+        if (k == kh_end(d))
+        {
+            // Simple error recovery for FILTERs not defined in the header. It will not help when VCF header has
+            // been already printed, but will enable tools like vcfcheck to proceed.
+            hts_log_warning("FILTER '%s' is not defined in the header", t);
+            kstring_t tmp = {0,0,0};
+            int l;
+            ksprintf(&tmp, "##FILTER=<ID=%s,Description=\"Dummy\">", t);
+            bcf_hrec_t *hrec = bcf_hdr_parse_line(h,tmp.s,&l);
+            free(tmp.s);
+            int res = hrec ? bcf_hdr_add_hrec((bcf_hdr_t*)h, hrec) : -1;
+            if (res < 0) bcf_hrec_destroy(hrec);
+            if (res > 0) res = bcf_hdr_sync((bcf_hdr_t*)h);
+            k = kh_get(vdict, d, t);
+            v->errcode |= BCF_ERR_TAG_UNDEF;
+            if (res || k == kh_end(d)) {
+                hts_log_error("Could not add dummy header for FILTER '%s' at %s:%"PRIhts_pos, t, bcf_seqname_safe(h,v), v->pos+1);
+                v->errcode |= BCF_ERR_TAG_INVALID;
+                free(a_flt);
+                return -1;
+            }
+        }
+        a_flt[i++] = kh_val(d, k).id;
+    }
+
+    bcf_enc_vint(str, n_flt, a_flt, -1);
+    free(a_flt);
+
+    return 0;
+}
+
+static int vcf_parse_info(kstring_t *str, const bcf_hdr_t *h, bcf1_t *v, char *p, char *q) {
+    static int extreme_int_warned = 0, negative_rlen_warned = 0;
+    int max_n_val = 0, overflow = 0;
+    char *r, *key;
+    khint_t k;
+    vdict_t *d = (vdict_t*)h->dict[BCF_DT_ID];
+    int32_t *a_val = NULL;
+
+    v->n_info = 0;
+    if (*(q-1) == ';') *(q-1) = 0;
+    for (r = key = p;; ++r) {
+        int c;
+        char *val, *end;
+        while (*r > '=' || (*r != ';' && *r != '=' && *r != 0)) r++;
+        if (v->n_info == UINT16_MAX) {
+            hts_log_error("Too many INFO entries at %s:%"PRIhts_pos,
+                          bcf_seqname_safe(h,v), v->pos+1);
+            v->errcode |= BCF_ERR_LIMITS;
+            goto fail;
+        }
+        val = end = NULL;
+        c = *r; *r = 0;
+        if (c == '=') {
+            val = r + 1;
+
+            for (end = val; *end != ';' && *end != 0; ++end);
+            c = *end; *end = 0;
+        } else end = r;
+        if ( !*key ) { if (c==0) break; r = end; key = r + 1; continue; }  // faulty VCF, ";;" in the INFO
+        k = kh_get(vdict, d, key);
+        if (k == kh_end(d) || kh_val(d, k).info[BCF_HL_INFO] == 15)
+        {
+            hts_log_warning("INFO '%s' is not defined in the header, assuming Type=String", key);
+            kstring_t tmp = {0,0,0};
+            int l;
+            ksprintf(&tmp, "##INFO=<ID=%s,Number=1,Type=String,Description=\"Dummy\">", key);
+            bcf_hrec_t *hrec = bcf_hdr_parse_line(h,tmp.s,&l);
+            free(tmp.s);
+            int res = hrec ? bcf_hdr_add_hrec((bcf_hdr_t*)h, hrec) : -1;
+            if (res < 0) bcf_hrec_destroy(hrec);
+            if (res > 0) res = bcf_hdr_sync((bcf_hdr_t*)h);
+            k = kh_get(vdict, d, key);
+            v->errcode |= BCF_ERR_TAG_UNDEF;
+            if (res || k == kh_end(d)) {
+                hts_log_error("Could not add dummy header for INFO '%s' at %s:%"PRIhts_pos, key, bcf_seqname_safe(h,v), v->pos+1);
+                v->errcode |= BCF_ERR_TAG_INVALID;
+                goto fail;
+            }
+        }
+        uint32_t y = kh_val(d, k).info[BCF_HL_INFO];
+        ++v->n_info;
+        bcf_enc_int1(str, kh_val(d, k).id);
+        if (val == 0) {
+            bcf_enc_size(str, 0, BCF_BT_NULL);
+        } else if ((y>>4&0xf) == BCF_HT_FLAG || (y>>4&0xf) == BCF_HT_STR) { // if Flag has a value, treat it as a string
+            bcf_enc_vchar(str, end - val, val);
+        } else { // int/float value/array
+            int i, n_val;
+            char *t, *te;
+            for (t = val, n_val = 1; *t; ++t) // count the number of values
+                if (*t == ',') ++n_val;
+            // Check both int and float size in one step for simplicity
+            if (n_val > max_n_val) {
+                int32_t *a_tmp = (int32_t *)realloc(a_val, n_val * sizeof(*a_val));
+                if (!a_tmp) {
+                    hts_log_error("Could not allocate memory at %s:%"PRIhts_pos, bcf_seqname_safe(h,v), v->pos+1);
+                    v->errcode |= BCF_ERR_LIMITS; // No appropriate code?
+                    goto fail;
+                }
+                a_val = a_tmp;
+                max_n_val = n_val;
+            }
+            if ((y>>4&0xf) == BCF_HT_INT) {
+                i = 0, t = val;
+                int64_t val1;
+                int is_int64 = 0;
+#ifdef VCF_ALLOW_INT64
+                if ( n_val==1 )
+                {
+                    overflow = 0;
+                    long long int tmp_val = hts_str2int(val, &te, sizeof(tmp_val)*CHAR_BIT, &overflow);
+                    if ( te==val ) tmp_val = bcf_int32_missing;
+                    else if ( overflow || tmp_val<BCF_MIN_BT_INT64 || tmp_val>BCF_MAX_BT_INT64 )
+                    {
+                        if ( !extreme_int_warned )
+                        {
+                            hts_log_warning("Extreme INFO/%s value encountered and set to missing at %s:%"PRIhts_pos,key,bcf_seqname_safe(h,v), v->pos+1);
+                            extreme_int_warned = 1;
+                        }
+                        tmp_val = bcf_int32_missing;
+                    }
+                    else
+                        is_int64 = 1;
+                    val1 = tmp_val;
+                    t = te;
+                    i = 1;  // this is just to avoid adding another nested block...
+                }
+#endif
+                for (; i < n_val; ++i, ++t)
+                {
+                    overflow = 0;
+                    long int tmp_val = hts_str2int(t, &te, sizeof(tmp_val)*CHAR_BIT, &overflow);
+                    if ( te==t ) tmp_val = bcf_int32_missing;
+                    else if ( overflow || tmp_val<BCF_MIN_BT_INT32 || tmp_val>BCF_MAX_BT_INT32 )
+                    {
+                        if ( !extreme_int_warned )
+                        {
+                            hts_log_warning("Extreme INFO/%s value encountered and set to missing at %s:%"PRIhts_pos,key,bcf_seqname_safe(h,v), v->pos+1);
+                            extreme_int_warned = 1;
+                        }
+                        tmp_val = bcf_int32_missing;
+                    }
+                    a_val[i] = tmp_val;
+                    for (t = te; *t && *t != ','; t++);
+                }
+                if (n_val == 1) {
+#ifdef VCF_ALLOW_INT64
+                    if ( is_int64 )
+                    {
+                        v->unpacked |= BCF_IS_64BIT;
+                        bcf_enc_long1(str, val1);
+                    }
+                    else
+                        bcf_enc_int1(str, (int32_t)val1);
+#else
+                    val1 = a_val[0];
+                    bcf_enc_int1(str, (int32_t)val1);
+#endif
+                } else {
+                    bcf_enc_vint(str, n_val, a_val, -1);
+                }
+                if (n_val==1 && (val1!=bcf_int32_missing || is_int64)
+                    && memcmp(key, "END", 4) == 0)
+                {
+                    if ( val1 <= v->pos )
+                    {
+                        if ( !negative_rlen_warned )
+                        {
+                            hts_log_warning("INFO/END=%"PRIhts_pos" is smaller than POS at %s:%"PRIhts_pos,val1,bcf_seqname_safe(h,v),v->pos+1);
+                            negative_rlen_warned = 1;
+                        }
+                    }
+                    else
+                        v->rlen = val1 - v->pos;
+                }
+            } else if ((y>>4&0xf) == BCF_HT_REAL) {
+                float *val_f = (float *)a_val;
+                for (i = 0, t = val; i < n_val; ++i, ++t)
+                {
+                    overflow = 0;
+                    val_f[i] = hts_str2dbl(t, &te, &overflow);
+                    if ( te==t || overflow ) // conversion failed
+                        bcf_float_set_missing(val_f[i]);
+                    for (t = te; *t && *t != ','; t++);
+                }
+                bcf_enc_vfloat(str, n_val, val_f);
+            }
+        }
+        if (c == 0) break;
+        r = end;
+        key = r + 1;
+    }
+
+    free(a_val);
+    return 0;
+
+ fail:
+    free(a_val);
+    return -1;
+}
+
+int vcf_parse(kstring_t *s, const bcf_hdr_t *h, bcf1_t *v)
+{
+    int ret = -2, overflow = 0;
+    char *p, *q, *r, *t;
+    kstring_t *str;
+    khint_t k;
+    ks_tokaux_t aux;
+
+//#define NOT_DOT(p) strcmp((p), ".")
+//#define NOT_DOT(p) (!(*p == '.' && !p[1]))
+//#define NOT_DOT(p) ((*p) != '.' || (p)[1])
+//#define NOT_DOT(p) (q-p != 1 || memcmp(p, ".\0", 2))
+#define NOT_DOT(p) (memcmp(p, ".\0", 2))
+
+    if (!s || !h || !v || !(s->s))
+        return ret;
+
+    // Assumed in lots of places, but we may as well spot this early
+    assert(sizeof(float) == sizeof(int32_t));
+
+    // Ensure string we parse has space to permit some over-flow when during
+    // parsing.  Eg to do memcmp(key, "END", 4) in vcf_parse_info over
+    // the more straight forward looking strcmp, giving a speed advantage.
+    if (ks_resize(s, s->l+4) < 0)
+        return -1;
+
+    // Force our memory to be initialised so we avoid the technicality of
+    // undefined behaviour in using a 4-byte memcmp.  (The reality is this
+    // almost certainly is never detected by the compiler so has no impact,
+    // but equally so this code has minimal (often beneficial) impact on
+    // performance too.)
+    s->s[s->l+0] = 0;
+    s->s[s->l+1] = 0;
+    s->s[s->l+2] = 0;
+    s->s[s->l+3] = 0;
+
+    bcf_clear1(v);
+    str = &v->shared;
+    memset(&aux, 0, sizeof(ks_tokaux_t));
+
+    // CHROM
+    if (!(p = kstrtok(s->s, "\t", &aux)))
+        goto err;
+    *(q = (char*)aux.p) = 0;
+
+    vdict_t *d = (vdict_t*)h->dict[BCF_DT_CTG];
+    k = kh_get(vdict, d, p);
+    if (k == kh_end(d)) {
+        hts_log_warning("Contig '%s' is not defined in the header. (Quick workaround: index the file with tabix.)", p);
+        v->errcode = BCF_ERR_CTG_UNDEF;
+        if ((k = fix_chromosome(h, d, p)) == kh_end(d)) {
+            hts_log_error("Could not add dummy header for contig '%s'", p);
+            v->errcode |= BCF_ERR_CTG_INVALID;
+            goto err;
+        }
+    }
+    v->rid = kh_val(d, k).id;
+
+    // POS
+    if (!(p = kstrtok(0, 0, &aux)))
+        goto err;
+    *(q = (char*)aux.p) = 0;
+
+    overflow = 0;
+    char *tmp = p;
+    v->pos = hts_str2uint(p, &p, 62, &overflow);
+    if (overflow) {
+        hts_log_error("Position value '%s' is too large", tmp);
+        goto err;
+    } else if ( *p ) {
+        hts_log_error("Could not parse the position '%s'", tmp);
+        goto err;
+    } else {
+        v->pos -= 1;
+    }
+    if (v->pos >= INT32_MAX)
+        v->unpacked |= BCF_IS_64BIT;
+
+    // ID
+    if (!(p = kstrtok(0, 0, &aux)))
+        goto err;
+    *(q = (char*)aux.p) = 0;
+
+    if (NOT_DOT(p)) bcf_enc_vchar(str, q - p, p);
+    else bcf_enc_size(str, 0, BCF_BT_CHAR);
+
+    // REF
+    if (!(p = kstrtok(0, 0, &aux)))
+        goto err;
+    *(q = (char*)aux.p) = 0;
+
+    bcf_enc_vchar(str, q - p, p);
+    v->n_allele = 1, v->rlen = q - p;
+
+    // ALT
+    if (!(p = kstrtok(0, 0, &aux)))
+        goto err;
+    *(q = (char*)aux.p) = 0;
+
+    if (NOT_DOT(p)) {
+        for (r = t = p;; ++r) {
+            if (*r == ',' || *r == 0) {
+                if (v->n_allele == UINT16_MAX) {
+                    hts_log_error("Too many ALT alleles at %s:%"PRIhts_pos,
+                                  bcf_seqname_safe(h,v), v->pos+1);
+                    v->errcode |= BCF_ERR_LIMITS;
+                    goto err;
+                }
+                bcf_enc_vchar(str, r - t, t);
+                t = r + 1;
+                ++v->n_allele;
+            }
+            if (r == q) break;
+        }
+    }
+
+    // QUAL
+    if (!(p = kstrtok(0, 0, &aux)))
+        goto err;
+    *(q = (char*)aux.p) = 0;
+
+    if (NOT_DOT(p)) v->qual = atof(p);
+    else bcf_float_set_missing(v->qual);
+    if ( v->max_unpack && !(v->max_unpack>>1) ) goto end; // BCF_UN_STR
+
+    // FILTER
+    if (!(p = kstrtok(0, 0, &aux)))
+        goto err;
+    *(q = (char*)aux.p) = 0;
+
+    if (NOT_DOT(p)) {
+        if (vcf_parse_filter(str, h, v, p, q)) {
+            goto err;
+        }
+    } else bcf_enc_vint(str, 0, 0, -1);
+    if ( v->max_unpack && !(v->max_unpack>>2) ) goto end; // BCF_UN_FLT
+
+    // INFO
+    if (!(p = kstrtok(0, 0, &aux)))
+        goto err;
+    *(q = (char*)aux.p) = 0;
+
+    if (NOT_DOT(p)) {
+        if (vcf_parse_info(str, h, v, p, q)) {
+            goto err;
+        }
+    }
+    if ( v->max_unpack && !(v->max_unpack>>3) ) goto end;
+
+    // FORMAT; optional
+    p = kstrtok(0, 0, &aux);
+    if (p) {
+        *(q = (char*)aux.p) = 0;
+
+        return vcf_parse_format(s, h, v, p, q) == 0 ? 0 : -2;
+    } else {
+        return 0;
+    }
+
+ end:
+    ret = 0;
+
+ err:
+    return ret;
+}
+
+int vcf_open_mode(char *mode, const char *fn, const char *format)
+{
+    if (format == NULL) {
+        // Try to pick a format based on the filename extension
+        char extension[HTS_MAX_EXT_LEN];
+        if (find_file_extension(fn, extension) < 0) return -1;
+        return vcf_open_mode(mode, fn, extension);
+    }
+    else if (strcasecmp(format, "bcf") == 0) strcpy(mode, "b");
+    else if (strcasecmp(format, "vcf") == 0) strcpy(mode, "");
+    else if (strcasecmp(format, "vcf.gz") == 0 || strcasecmp(format, "vcf.bgz") == 0) strcpy(mode, "z");
+    else return -1;
+
+    return 0;
+}
+
+int vcf_read(htsFile *fp, const bcf_hdr_t *h, bcf1_t *v)
+{
+    int ret;
+    ret = hts_getline(fp, KS_SEP_LINE, &fp->line);
+    if (ret < 0) return ret;
+    return vcf_parse1(&fp->line, h, v);
+}
+
+static inline uint8_t *bcf_unpack_fmt_core1(uint8_t *ptr, int n_sample, bcf_fmt_t *fmt)
+{
+    uint8_t *ptr_start = ptr;
+    fmt->id = bcf_dec_typed_int1(ptr, &ptr);
+    fmt->n = bcf_dec_size(ptr, &ptr, &fmt->type);
+    fmt->size = fmt->n << bcf_type_shift[fmt->type];
+    fmt->p = ptr;
+    fmt->p_off  = ptr - ptr_start;
+    fmt->p_free = 0;
+    ptr += n_sample * fmt->size;
+    fmt->p_len = ptr - fmt->p;
+    return ptr;
+}
+
+static inline uint8_t *bcf_unpack_info_core1(uint8_t *ptr, bcf_info_t *info)
+{
+    uint8_t *ptr_start = ptr;
+    int64_t len = 0;
+    info->key = bcf_dec_typed_int1(ptr, &ptr);
+    len = info->len = bcf_dec_size(ptr, &ptr, &info->type);
+    info->vptr = ptr;
+    info->vptr_off  = ptr - ptr_start;
+    info->vptr_free = 0;
+    info->v1.i = 0;
+    if (info->len == 1) {
+        switch(info->type) {
+        case BCF_BT_INT8:
+        case BCF_BT_CHAR:
+            info->v1.i = *(int8_t*)ptr;
+            break;
+        case BCF_BT_INT16:
+            info->v1.i = le_to_i16(ptr);
+            len <<= 1;
+            break;
+        case BCF_BT_INT32:
+            info->v1.i = le_to_i32(ptr);
+            len <<= 2;
+            break;
+        case BCF_BT_FLOAT:
+            info->v1.f = le_to_float(ptr);
+            len <<= 2;
+            break;
+        case BCF_BT_INT64:
+            info->v1.i = le_to_i64(ptr);
+            len <<= 3;
+            break;
+        }
+    } else {
+        len <<= bcf_type_shift[info->type];
+    }
+    ptr += len;
+
+    info->vptr_len = ptr - info->vptr;
+    return ptr;
+}
+
+int bcf_unpack(bcf1_t *b, int which)
+{
+    if ( !b->shared.l ) return 0; // Building a new BCF record from scratch
+    uint8_t *ptr = (uint8_t*)b->shared.s, *ptr_ori;
+    int i;
+    bcf_dec_t *d = &b->d;
+    if (which & BCF_UN_FLT) which |= BCF_UN_STR;
+    if (which & BCF_UN_INFO) which |= BCF_UN_SHR;
+    if ((which&BCF_UN_STR) && !(b->unpacked&BCF_UN_STR))
+    {
+        kstring_t tmp;
+
+        // ID
+        tmp.l = 0; tmp.s = d->id; tmp.m = d->m_id;
+        ptr_ori = ptr;
+        ptr = bcf_fmt_sized_array(&tmp, ptr);
+        b->unpack_size[0] = ptr - ptr_ori;
+        kputc_('\0', &tmp);
+        d->id = tmp.s; d->m_id = tmp.m;
+
+        // REF and ALT are in a single block (d->als) and d->alleles are pointers into this block
+        hts_expand(char*, b->n_allele, d->m_allele, d->allele); // NM: hts_expand() is a macro
+        tmp.l = 0; tmp.s = d->als; tmp.m = d->m_als;
+        ptr_ori = ptr;
+        for (i = 0; i < b->n_allele; ++i) {
+            // Use offset within tmp.s as realloc may change pointer
+            d->allele[i] = (char *)(intptr_t)tmp.l;
+            ptr = bcf_fmt_sized_array(&tmp, ptr);
+            kputc_('\0', &tmp);
+        }
+        b->unpack_size[1] = ptr - ptr_ori;
+        d->als = tmp.s; d->m_als = tmp.m;
+
+        // Convert our offsets within tmp.s back to pointers again
+        for (i = 0; i < b->n_allele; ++i)
+            d->allele[i] = d->als + (ptrdiff_t)d->allele[i];
+        b->unpacked |= BCF_UN_STR;
+    }
+    if ((which&BCF_UN_FLT) && !(b->unpacked&BCF_UN_FLT)) { // FILTER
+        ptr = (uint8_t*)b->shared.s + b->unpack_size[0] + b->unpack_size[1];
+        ptr_ori = ptr;
+        if (*ptr>>4) {
+            int type;
+            d->n_flt = bcf_dec_size(ptr, &ptr, &type);
+            hts_expand(int, d->n_flt, d->m_flt, d->flt);
+            for (i = 0; i < d->n_flt; ++i)
+                d->flt[i] = bcf_dec_int1(ptr, type, &ptr);
+        } else ++ptr, d->n_flt = 0;
+        b->unpack_size[2] = ptr - ptr_ori;
+        b->unpacked |= BCF_UN_FLT;
+    }
+    if ((which&BCF_UN_INFO) && !(b->unpacked&BCF_UN_INFO)) { // INFO
+        ptr = (uint8_t*)b->shared.s + b->unpack_size[0] + b->unpack_size[1] + b->unpack_size[2];
+        hts_expand(bcf_info_t, b->n_info, d->m_info, d->info);
+        for (i = 0; i < d->m_info; ++i) d->info[i].vptr_free = 0;
+        for (i = 0; i < b->n_info; ++i)
+            ptr = bcf_unpack_info_core1(ptr, &d->info[i]);
+        b->unpacked |= BCF_UN_INFO;
+    }
+    if ((which&BCF_UN_FMT) && b->n_sample && !(b->unpacked&BCF_UN_FMT)) { // FORMAT
+        ptr = (uint8_t*)b->indiv.s;
+        hts_expand(bcf_fmt_t, b->n_fmt, d->m_fmt, d->fmt);
+        for (i = 0; i < d->m_fmt; ++i) d->fmt[i].p_free = 0;
+        for (i = 0; i < b->n_fmt; ++i)
+            ptr = bcf_unpack_fmt_core1(ptr, b->n_sample, &d->fmt[i]);
+        b->unpacked |= BCF_UN_FMT;
+    }
+    return 0;
+}
+
+int vcf_format(const bcf_hdr_t *h, const bcf1_t *v, kstring_t *s)
+{
+    int i;
+    int32_t max_dt_id = h->n[BCF_DT_ID];
+    const char *chrom = bcf_seqname(h, v);
+    if (!chrom) {
+        hts_log_error("Invalid BCF, CONTIG id=%d not present in the header",
+                      v->rid);
+        errno = EINVAL;
+        return -1;
+    }
+
+    bcf_unpack((bcf1_t*)v, BCF_UN_ALL & ~(BCF_UN_INFO|BCF_UN_FMT));
+
+    // Cache of key lengths so we don't keep repeatedly using them.
+    // This assumes we're not modifying the header between successive calls
+    // to vcf_format, but that would lead to many other forms of breakage
+    // so it feels like a valid assumption to make.
+    //
+    // We cannot just do this in bcf_hdr_sync as some code (eg bcftools
+    // annotate) manipulates the headers directly without calling sync to
+    // refresh the data structures.  So we must do just-in-time length
+    // calculation during writes instead.
+    bcf_hdr_aux_t *aux = get_hdr_aux(h);
+    if (!aux->key_len) {
+        if (!(aux->key_len = calloc(h->n[BCF_DT_ID]+1, sizeof(*aux->key_len))))
+            return -1;
+    }
+    size_t *key_len = aux->key_len;
+
+    kputs(chrom, s); // CHROM
+    kputc_('\t', s); kputll(v->pos + 1, s); // POS
+    kputc_('\t', s); kputs(v->d.id ? v->d.id : ".", s); // ID
+    kputc_('\t', s); // REF
+    if (v->n_allele > 0) kputs(v->d.allele[0], s);
+    else kputc_('.', s);
+    kputc_('\t', s); // ALT
+    if (v->n_allele > 1) {
+        for (i = 1; i < v->n_allele; ++i) {
+            if (i > 1) kputc_(',', s);
+            kputs(v->d.allele[i], s);
+        }
+    } else kputc_('.', s);
+    kputc_('\t', s); // QUAL
+    if ( bcf_float_is_missing(v->qual) ) kputc_('.', s); // QUAL
+    else kputd(v->qual, s);
+    kputc_('\t', s); // FILTER
+    if (v->d.n_flt) {
+        for (i = 0; i < v->d.n_flt; ++i) {
+            int32_t idx = v->d.flt[i];
+            if (idx < 0 || idx >= max_dt_id
+                || h->id[BCF_DT_ID][idx].key == NULL) {
+                hts_log_error("Invalid BCF, the FILTER tag id=%d at %s:%"PRIhts_pos" not present in the header",
+                              idx, bcf_seqname_safe(h, v), v->pos + 1);
+                errno = EINVAL;
+                return -1;
+            }
+            if (i) kputc_(';', s);
+            if (!key_len[idx])
+                key_len[idx] = strlen(h->id[BCF_DT_ID][idx].key);
+            kputsn(h->id[BCF_DT_ID][idx].key, key_len[idx], s);
+        }
+    } else kputc_('.', s);
+
+    kputc_('\t', s); // INFO
+    if (v->n_info) {
+        uint8_t *ptr = v->shared.s
+            ? (uint8_t *)v->shared.s + v->unpack_size[0] +
+               v->unpack_size[1] + v->unpack_size[2]
+            : NULL;
+        int first = 1;
+        bcf_info_t *info = v->d.info;
+
+        // Note if we duplicate this code into custom packed and unpacked
+        // implementations then we gain a bit more speed, particularly with
+        // clang 13 (up to 5%).  Not sure why this is, but code duplication
+        // isn't pleasant and it's still faster adding packed support than
+        // not so it's a win, just not as good as it should be.
+        const int info_packed = !(v->unpacked & BCF_UN_INFO) && v->shared.l;
+        for (i = 0; i < v->n_info; ++i) {
+            bcf_info_t in, *z;
+            if (info_packed) {
+                // Use a local bcf_info_t when data is packed
+                z = &in;
+                z->key  = bcf_dec_typed_int1(ptr, &ptr);
+                z->len  = bcf_dec_size(ptr, &ptr, &z->type);
+                z->vptr = ptr;
+                ptr += z->len << bcf_type_shift[z->type];
+            } else {
+                // Else previously unpacked INFO struct
+                z = &info[i];
+
+                // Also potentially since deleted
+                if ( !z->vptr ) continue;
+            }
+
+            bcf_idpair_t *id = z->key >= 0 && z->key < max_dt_id
+                ? &h->id[BCF_DT_ID][z->key]
+                : NULL;
+
+            if (!id || !id->key) {
+                hts_log_error("Invalid BCF, the INFO tag id=%d is %s at %s:%"PRIhts_pos,
+                              z->key,
+                              z->key < 0 ? "negative"
+                              : (z->key >= max_dt_id ? "too large" : "not present in the header"),
+                              bcf_seqname_safe(h, v), v->pos+1);
+                errno = EINVAL;
+                return -1;
+            }
+
+            // KEY
+            if (!key_len[z->key])
+                key_len[z->key] = strlen(id->key);
+            size_t id_len = key_len[z->key];
+            if (ks_resize(s, s->l + 3 + id_len) < 0)
+                return -1;
+            char *sptr = s->s + s->l;
+            if ( !first ) {
+                *sptr++ = ';';
+                s->l++;
+            }
+            first = 0;
+            memcpy(sptr, id->key, id_len);
+            s->l += id_len;
+
+            // VALUE
+            if (z->len <= 0) continue;
+            sptr[id_len] = '=';
+            s->l++;
+
+            if (z->len != 1 || info_packed) {
+                bcf_fmt_array(s, z->len, z->type, z->vptr);
+            } else {
+                // Single length vectors are unpacked into their
+                // own info.v1 union and handled separately.
+                if (z->type == BCF_BT_FLOAT) {
+                    if ( bcf_float_is_missing(z->v1.f) )
+                        kputc_('.', s);
+                    else
+                        kputd(z->v1.f, s);
+                } else if (z->type == BCF_BT_CHAR) {
+                    kputc_(z->v1.i, s);
+                } else if (z->type < BCF_BT_INT64) {
+                    int64_t missing[] = {
+                        0, // BCF_BT_NULL
+                        bcf_int8_missing,
+                        bcf_int16_missing,
+                        bcf_int32_missing,
+                    };
+                    if (z->v1.i == missing[z->type])
+                        kputc_('.', s);
+                    else
+                        kputw(z->v1.i, s);
+                } else if (z->type == BCF_BT_INT64) {
+                    if (z->v1.i == bcf_int64_missing)
+                        kputc_('.', s);
+                    else
+                        kputll(z->v1.i, s);
+                } else {
+                    hts_log_error("Unexpected type %d at %s:%"PRIhts_pos, z->type, bcf_seqname_safe(h, v), v->pos+1);
+                    errno = EINVAL;
+                    return -1;
+                }
+            }
+        }
+        if ( first ) kputc_('.', s);
+    } else kputc_('.', s);
+
+    // FORMAT and individual information
+    if (v->n_sample) {
+        int i,j;
+        if ( v->n_fmt) {
+            uint8_t *ptr = (uint8_t *)v->indiv.s;
+            int gt_i = -1;
+            bcf_fmt_t *fmt = v->d.fmt;
+            int first = 1;
+            int fmt_packed = !(v->unpacked & BCF_UN_FMT);
+
+            if (fmt_packed) {
+                // Local fmt as we have an array of num FORMAT keys,
+                // each of which points to N.Sample values.
+
+                // No real gain to be had in handling unpacked data here,
+                // but it doesn't cost us much in complexity either and
+                // it gives us flexibility.
+                fmt = malloc(v->n_fmt * sizeof(*fmt));
+                if (!fmt)
+                    return -1;
+            }
+
+            // KEYS
+            for (i = 0; i < (int)v->n_fmt; ++i) {
+                bcf_fmt_t *z;
+                z = &fmt[i];
+                if (fmt_packed) {
+                    z->id   = bcf_dec_typed_int1(ptr, &ptr);
+                    z->n    = bcf_dec_size(ptr, &ptr, &z->type);
+                    z->p    = ptr;
+                    z->size = z->n << bcf_type_shift[z->type];
+                    ptr += v->n_sample * z->size;
+                }
+                if ( !z->p ) continue;
+                kputc_(!first ? ':' : '\t', s); first = 0;
+
+                bcf_idpair_t *id = z->id >= 0 && z->id < max_dt_id
+                    ? &h->id[BCF_DT_ID][z->id]
+                    : NULL;
+
+                if (!id || !id->key) {
+                    hts_log_error("Invalid BCF, the FORMAT tag id=%d at %s:%"PRIhts_pos" not present in the header", z->id, bcf_seqname_safe(h, v), v->pos+1);
+                    errno = EINVAL;
+                    return -1;
+                }
+
+                if (!key_len[z->id])
+                    key_len[z->id] = strlen(id->key);
+                size_t id_len = key_len[z->id];
+                kputsn(id->key, id_len, s);
+                if (id_len == 2 && id->key[0] == 'G' && id->key[1] == 'T')
+                    gt_i = i;
+            }
+            if ( first ) kputsn("\t.", 2, s);
+
+            // VALUES per sample
+            for (j = 0; j < v->n_sample; ++j) {
+                kputc_('\t', s);
+                first = 1;
+                bcf_fmt_t *f = fmt;
+                for (i = 0; i < (int)v->n_fmt; i++, f++) {
+                    if ( !f->p ) continue;
+                    if (!first) kputc_(':', s);
+                    first = 0;
+                    if (gt_i == i) {
+                        bcf_format_gt(f,j,s);
+                        break;
+                    }
+                    else if (f->n == 1)
+                        bcf_fmt_array1(s, f->type, f->p + j * (size_t)f->size);
+                    else
+                        bcf_fmt_array(s, f->n, f->type, f->p + j * (size_t)f->size);
+                }
+
+                // Simpler loop post GT and at least 1 iteration
+                for (i++, f++; i < (int)v->n_fmt; i++, f++) {
+                    if ( !f->p ) continue;
+                    kputc_(':', s);
+                    if (f->n == 1)
+                        bcf_fmt_array1(s, f->type, f->p + j * (size_t)f->size);
+                    else
+                        bcf_fmt_array(s, f->n, f->type, f->p + j * (size_t)f->size);
+                }
+                if ( first ) kputc_('.', s);
+            }
+            if (fmt_packed)
+                free(fmt);
+        }
+        else
+            for (j=0; j<=v->n_sample; j++)
+                kputsn("\t.", 2, s);
+    }
+    kputc('\n', s);
+    return 0;
+}
+
+int vcf_write_line(htsFile *fp, kstring_t *line)
+{
+    int ret;
+    if ( line->s[line->l-1]!='\n' ) kputc('\n',line);
+    if ( fp->format.compression!=no_compression )
+        ret = bgzf_write(fp->fp.bgzf, line->s, line->l);
+    else
+        ret = hwrite(fp->fp.hfile, line->s, line->l);
+    return ret==line->l ? 0 : -1;
+}
+
+int vcf_write(htsFile *fp, const bcf_hdr_t *h, bcf1_t *v)
+{
+    ssize_t ret;
+    fp->line.l = 0;
+    if (vcf_format1(h, v, &fp->line) != 0)
+        return -1;
+    if ( fp->format.compression!=no_compression ) {
+        if (bgzf_flush_try(fp->fp.bgzf, fp->line.l) < 0)
+            return -1;
+        if (fp->idx && !fp->fp.bgzf->mt)
+            hts_idx_amend_last(fp->idx, bgzf_tell(fp->fp.bgzf));
+        ret = bgzf_write(fp->fp.bgzf, fp->line.s, fp->line.l);
+    } else {
+        ret = hwrite(fp->fp.hfile, fp->line.s, fp->line.l);
+    }
+
+    if (fp->idx && fp->format.compression == bgzf) {
+        int tid;
+        if ((tid = hts_idx_tbi_name(fp->idx, v->rid, bcf_seqname_safe(h, v))) < 0)
+            return -1;
+
+        if (bgzf_idx_push(fp->fp.bgzf, fp->idx,
+                          tid, v->pos, v->pos + v->rlen,
+                          bgzf_tell(fp->fp.bgzf), 1) < 0)
+            return -1;
+    }
+
+    return ret==fp->line.l ? 0 : -1;
+}
+
+/************************
+ * Data access routines *
+ ************************/
+
+int bcf_hdr_id2int(const bcf_hdr_t *h, int which, const char *id)
+{
+    khint_t k;
+    vdict_t *d = (vdict_t*)h->dict[which];
+    k = kh_get(vdict, d, id);
+    return k == kh_end(d)? -1 : kh_val(d, k).id;
+}
+
+
+/********************
+ *** BCF indexing ***
+ ********************/
+
+// Calculate number of index levels given min_shift and the header contig
+// list.  Also returns number of contigs in *nids_out.
+static int idx_calc_n_lvls_ids(const bcf_hdr_t *h, int min_shift,
+                               int starting_n_lvls, int *nids_out)
+{
+    int n_lvls, i, nids = 0;
+    int64_t max_len = 0, s;
+
+    for (i = 0; i < h->n[BCF_DT_CTG]; ++i)
+    {
+        if ( !h->id[BCF_DT_CTG][i].val ) continue;
+        if ( max_len < h->id[BCF_DT_CTG][i].val->info[0] )
+            max_len = h->id[BCF_DT_CTG][i].val->info[0];
+        nids++;
+    }
+    if ( !max_len ) max_len = (1LL<<31) - 1;  // In case contig line is broken.
+    max_len += 256;
+    s = hts_bin_maxpos(min_shift, starting_n_lvls);
+    for (n_lvls = starting_n_lvls; max_len > s; ++n_lvls, s <<= 3);
+
+    if (nids_out) *nids_out = nids;
+    return n_lvls;
+}
+
+hts_idx_t *bcf_index(htsFile *fp, int min_shift)
+{
+    int n_lvls;
+    bcf1_t *b = NULL;
+    hts_idx_t *idx = NULL;
+    bcf_hdr_t *h;
+    int r;
+    h = bcf_hdr_read(fp);
+    if ( !h ) return NULL;
+    int nids = 0;
+    n_lvls = idx_calc_n_lvls_ids(h, min_shift, 0, &nids);
+    idx = hts_idx_init(nids, HTS_FMT_CSI, bgzf_tell(fp->fp.bgzf), min_shift, n_lvls);
+    if (!idx) goto fail;
+    b = bcf_init1();
+    if (!b) goto fail;
+    while ((r = bcf_read1(fp,h, b)) >= 0) {
+        int ret;
+        ret = hts_idx_push(idx, b->rid, b->pos, b->pos + b->rlen, bgzf_tell(fp->fp.bgzf), 1);
+        if (ret < 0) goto fail;
+    }
+    if (r < -1) goto fail;
+    hts_idx_finish(idx, bgzf_tell(fp->fp.bgzf));
+    bcf_destroy1(b);
+    bcf_hdr_destroy(h);
+    return idx;
+
+ fail:
+    hts_idx_destroy(idx);
+    bcf_destroy1(b);
+    bcf_hdr_destroy(h);
+    return NULL;
+}
+
+hts_idx_t *bcf_index_load2(const char *fn, const char *fnidx)
+{
+    return fnidx? hts_idx_load2(fn, fnidx) : bcf_index_load(fn);
+}
+
+hts_idx_t *bcf_index_load3(const char *fn, const char *fnidx, int flags)
+{
+    return hts_idx_load3(fn, fnidx, HTS_FMT_CSI, flags);
+}
+
+int bcf_index_build3(const char *fn, const char *fnidx, int min_shift, int n_threads)
+{
+    htsFile *fp;
+    hts_idx_t *idx;
+    tbx_t *tbx;
+    int ret;
+    if ((fp = hts_open(fn, "rb")) == 0) return -2;
+    if (n_threads)
+        hts_set_threads(fp, n_threads);
+    if ( fp->format.compression!=bgzf ) { hts_close(fp); return -3; }
+    switch (fp->format.format) {
+        case bcf:
+            if (!min_shift) {
+                hts_log_error("TBI indices for BCF files are not supported");
+                ret = -1;
+            } else {
+                idx = bcf_index(fp, min_shift);
+                if (idx) {
+                    ret = hts_idx_save_as(idx, fn, fnidx, HTS_FMT_CSI);
+                    if (ret < 0) ret = -4;
+                    hts_idx_destroy(idx);
+                }
+                else ret = -1;
+            }
+            break;
+
+        case vcf:
+            tbx = tbx_index(hts_get_bgzfp(fp), min_shift, &tbx_conf_vcf);
+            if (tbx) {
+                ret = hts_idx_save_as(tbx->idx, fn, fnidx, min_shift > 0 ? HTS_FMT_CSI : HTS_FMT_TBI);
+                if (ret < 0) ret = -4;
+                tbx_destroy(tbx);
+            }
+            else ret = -1;
+            break;
+
+        default:
+            ret = -3;
+            break;
+    }
+    hts_close(fp);
+    return ret;
+}
+
+int bcf_index_build2(const char *fn, const char *fnidx, int min_shift)
+{
+    return bcf_index_build3(fn, fnidx, min_shift, 0);
+}
+
+int bcf_index_build(const char *fn, int min_shift)
+{
+    return bcf_index_build3(fn, NULL, min_shift, 0);
+}
+
+// Initialise fp->idx for the current format type.
+// This must be called after the header has been written but no other data.
+static int vcf_idx_init(htsFile *fp, bcf_hdr_t *h, int min_shift, const char *fnidx) {
+    int n_lvls, fmt;
+
+    if (min_shift == 0) {
+        min_shift = 14;
+        n_lvls = 5;
+        fmt = HTS_FMT_TBI;
+    } else {
+        // Set initial n_lvls to match tbx_index()
+        int starting_n_lvls = (TBX_MAX_SHIFT - min_shift + 2) / 3;
+        // Increase if necessary
+        n_lvls = idx_calc_n_lvls_ids(h, min_shift, starting_n_lvls, NULL);
+        fmt = HTS_FMT_CSI;
+    }
+
+    fp->idx = hts_idx_init(0, fmt, bgzf_tell(fp->fp.bgzf), min_shift, n_lvls);
+    if (!fp->idx) return -1;
+
+    // Tabix meta data, added even in CSI for VCF
+    uint8_t conf[4*7];
+    u32_to_le(TBX_VCF, conf+0);  // fmt
+    u32_to_le(1,       conf+4);  // name col
+    u32_to_le(2,       conf+8);  // beg col
+    u32_to_le(0,       conf+12); // end col
+    u32_to_le('#',     conf+16); // comment
+    u32_to_le(0,       conf+20); // n.skip
+    u32_to_le(0,       conf+24); // ref name len
+    if (hts_idx_set_meta(fp->idx, sizeof(conf)*sizeof(*conf), (uint8_t *)conf, 1) < 0) {
+        hts_idx_destroy(fp->idx);
+        fp->idx = NULL;
+        return -1;
+    }
+    fp->fnidx = fnidx;
+
+    return 0;
+}
+
+// Initialise fp->idx for the current format type.
+// This must be called after the header has been written but no other data.
+int bcf_idx_init(htsFile *fp, bcf_hdr_t *h, int min_shift, const char *fnidx) {
+    int n_lvls, nids = 0;
+
+    if (fp->format.compression != bgzf) {
+        hts_log_error("Indexing is only supported on BGZF-compressed files");
+        return -3; // Matches no-compression return for bcf_index_build3()
+    }
+
+    if (fp->format.format == vcf)
+        return vcf_idx_init(fp, h, min_shift, fnidx);
+
+    if (!min_shift)
+        min_shift = 14;
+
+    n_lvls = idx_calc_n_lvls_ids(h, min_shift, 0, &nids);
+
+    fp->idx = hts_idx_init(nids, HTS_FMT_CSI, bgzf_tell(fp->fp.bgzf), min_shift, n_lvls);
+    if (!fp->idx) return -1;
+    fp->fnidx = fnidx;
+
+    return 0;
+}
+
+// Finishes an index. Call after the last record has been written.
+// Returns 0 on success, <0 on failure.
+//
+// NB: same format as SAM/BAM as it uses bgzf.
+int bcf_idx_save(htsFile *fp) {
+    return sam_idx_save(fp);
+}
+
+/*****************
+ *** Utilities ***
+ *****************/
+
+int bcf_hdr_combine(bcf_hdr_t *dst, const bcf_hdr_t *src)
+{
+    int i, ndst_ori = dst->nhrec, need_sync = 0, ret = 0, res;
+    for (i=0; i<src->nhrec; i++)
+    {
+        if ( src->hrec[i]->type==BCF_HL_GEN && src->hrec[i]->value )
+        {
+            int j;
+            for (j=0; j<ndst_ori; j++)
+            {
+                if ( dst->hrec[j]->type!=BCF_HL_GEN ) continue;
+
+                // Checking only the key part of generic lines, otherwise
+                // the VCFs are too verbose. Should we perhaps add a flag
+                // to bcf_hdr_combine() and make this optional?
+                if ( !strcmp(src->hrec[i]->key,dst->hrec[j]->key) ) break;
+            }
+            if ( j>=ndst_ori ) {
+                res = bcf_hdr_add_hrec(dst, bcf_hrec_dup(src->hrec[i]));
+                if (res < 0) return -1;
+                need_sync += res;
+            }
+        }
+        else if ( src->hrec[i]->type==BCF_HL_STR )
+        {
+            // NB: we are ignoring fields without ID
+            int j = bcf_hrec_find_key(src->hrec[i],"ID");
+            if ( j>=0 )
+            {
+                bcf_hrec_t *rec = bcf_hdr_get_hrec(dst, src->hrec[i]->type, "ID", src->hrec[i]->vals[j], src->hrec[i]->key);
+                if ( !rec ) {
+                    res = bcf_hdr_add_hrec(dst, bcf_hrec_dup(src->hrec[i]));
+                    if (res < 0) return -1;
+                    need_sync += res;
+                }
+            }
+        }
+        else
+        {
+            int j = bcf_hrec_find_key(src->hrec[i],"ID");
+            assert( j>=0 ); // this should always be true for valid VCFs
+
+            bcf_hrec_t *rec = bcf_hdr_get_hrec(dst, src->hrec[i]->type, "ID", src->hrec[i]->vals[j], NULL);
+            if ( !rec ) {
+                res = bcf_hdr_add_hrec(dst, bcf_hrec_dup(src->hrec[i]));
+                if (res < 0) return -1;
+                need_sync += res;
+            } else if ( src->hrec[i]->type==BCF_HL_INFO || src->hrec[i]->type==BCF_HL_FMT )
+            {
+                // Check that both records are of the same type. The bcf_hdr_id2length
+                // macro cannot be used here because dst header is not synced yet.
+                vdict_t *d_src = (vdict_t*)src->dict[BCF_DT_ID];
+                vdict_t *d_dst = (vdict_t*)dst->dict[BCF_DT_ID];
+                khint_t k_src  = kh_get(vdict, d_src, src->hrec[i]->vals[0]);
+                khint_t k_dst  = kh_get(vdict, d_dst, src->hrec[i]->vals[0]);
+                if ( (kh_val(d_src,k_src).info[rec->type]>>8 & 0xf) != (kh_val(d_dst,k_dst).info[rec->type]>>8 & 0xf) )
+                {
+                    hts_log_warning("Trying to combine \"%s\" tag definitions of different lengths",
+                        src->hrec[i]->vals[0]);
+                    ret |= 1;
+                }
+                if ( (kh_val(d_src,k_src).info[rec->type]>>4 & 0xf) != (kh_val(d_dst,k_dst).info[rec->type]>>4 & 0xf) )
+                {
+                    hts_log_warning("Trying to combine \"%s\" tag definitions of different types",
+                        src->hrec[i]->vals[0]);
+                    ret |= 1;
+                }
+            }
+        }
+    }
+    if ( need_sync ) {
+        if (bcf_hdr_sync(dst) < 0) return -1;
+    }
+    return ret;
+}
+
+bcf_hdr_t *bcf_hdr_merge(bcf_hdr_t *dst, const bcf_hdr_t *src)
+{
+    if ( !dst )
+    {
+        // this will effectively strip existing IDX attributes from src to become dst
+        dst = bcf_hdr_init("r");
+        kstring_t htxt = {0,0,0};
+        if (bcf_hdr_format(src, 0, &htxt) < 0) {
+            free(htxt.s);
+            return NULL;
+        }
+        if ( bcf_hdr_parse(dst, htxt.s) < 0 ) {
+            bcf_hdr_destroy(dst);
+            dst = NULL;
+        }
+        free(htxt.s);
+        return dst;
+    }
+
+    int i, ndst_ori = dst->nhrec, need_sync = 0, res;
+    for (i=0; i<src->nhrec; i++)
+    {
+        if ( src->hrec[i]->type==BCF_HL_GEN && src->hrec[i]->value )
+        {
+            int j;
+            for (j=0; j<ndst_ori; j++)
+            {
+                if ( dst->hrec[j]->type!=BCF_HL_GEN ) continue;
+
+                // Checking only the key part of generic lines, otherwise
+                // the VCFs are too verbose. Should we perhaps add a flag
+                // to bcf_hdr_combine() and make this optional?
+                if ( !strcmp(src->hrec[i]->key,dst->hrec[j]->key) ) break;
+            }
+            if ( j>=ndst_ori ) {
+                res = bcf_hdr_add_hrec(dst, bcf_hrec_dup(src->hrec[i]));
+                if (res < 0) return NULL;
+                need_sync += res;
+            }
+        }
+        else if ( src->hrec[i]->type==BCF_HL_STR )
+        {
+            // NB: we are ignoring fields without ID
+            int j = bcf_hrec_find_key(src->hrec[i],"ID");
+            if ( j>=0 )
+            {
+                bcf_hrec_t *rec = bcf_hdr_get_hrec(dst, src->hrec[i]->type, "ID", src->hrec[i]->vals[j], src->hrec[i]->key);
+                if ( !rec ) {
+                    res = bcf_hdr_add_hrec(dst, bcf_hrec_dup(src->hrec[i]));
+                    if (res < 0) return NULL;
+                    need_sync += res;
+                }
+            }
+        }
+        else
+        {
+            int j = bcf_hrec_find_key(src->hrec[i],"ID");
+            assert( j>=0 ); // this should always be true for valid VCFs
+
+            bcf_hrec_t *rec = bcf_hdr_get_hrec(dst, src->hrec[i]->type, "ID", src->hrec[i]->vals[j], NULL);
+            if ( !rec ) {
+                res = bcf_hdr_add_hrec(dst, bcf_hrec_dup(src->hrec[i]));
+                if (res < 0) return NULL;
+                need_sync += res;
+            } else if ( src->hrec[i]->type==BCF_HL_INFO || src->hrec[i]->type==BCF_HL_FMT )
+            {
+                // Check that both records are of the same type. The bcf_hdr_id2length
+                // macro cannot be used here because dst header is not synced yet.
+                vdict_t *d_src = (vdict_t*)src->dict[BCF_DT_ID];
+                vdict_t *d_dst = (vdict_t*)dst->dict[BCF_DT_ID];
+                khint_t k_src  = kh_get(vdict, d_src, src->hrec[i]->vals[0]);
+                khint_t k_dst  = kh_get(vdict, d_dst, src->hrec[i]->vals[0]);
+                if ( (kh_val(d_src,k_src).info[rec->type]>>8 & 0xf) != (kh_val(d_dst,k_dst).info[rec->type]>>8 & 0xf) )
+                {
+                    hts_log_warning("Trying to combine \"%s\" tag definitions of different lengths",
+                        src->hrec[i]->vals[0]);
+                }
+                if ( (kh_val(d_src,k_src).info[rec->type]>>4 & 0xf) != (kh_val(d_dst,k_dst).info[rec->type]>>4 & 0xf) )
+                {
+                    hts_log_warning("Trying to combine \"%s\" tag definitions of different types",
+                        src->hrec[i]->vals[0]);
+                }
+            }
+        }
+    }
+    if ( need_sync ) {
+        if (bcf_hdr_sync(dst) < 0) return NULL;
+    }
+    return dst;
+}
+
+int bcf_translate(const bcf_hdr_t *dst_hdr, bcf_hdr_t *src_hdr, bcf1_t *line)
+{
+    int i;
+    if ( line->errcode )
+    {
+        char errordescription[1024] = "";
+        hts_log_error("Unchecked error (%d %s) at %s:%"PRIhts_pos", exiting", line->errcode, bcf_strerror(line->errcode, errordescription, sizeof(errordescription)),  bcf_seqname_safe(src_hdr,line), line->pos+1);
+        exit(1);
+    }
+    if ( src_hdr->ntransl==-1 ) return 0;    // no need to translate, all tags have the same id
+    if ( !src_hdr->ntransl )  // called for the first time, see what needs translating
+    {
+        int dict;
+        for (dict=0; dict<2; dict++)    // BCF_DT_ID and BCF_DT_CTG
+        {
+            src_hdr->transl[dict] = (int*) malloc(src_hdr->n[dict]*sizeof(int));
+            for (i=0; i<src_hdr->n[dict]; i++)
+            {
+                if ( !src_hdr->id[dict][i].key ) // gap left after removed BCF header lines
+                {
+                    src_hdr->transl[dict][i] = -1;
+                    continue;
+                }
+                src_hdr->transl[dict][i] = bcf_hdr_id2int(dst_hdr,dict,src_hdr->id[dict][i].key);
+                if ( src_hdr->transl[dict][i]!=-1 && i!=src_hdr->transl[dict][i] ) src_hdr->ntransl++;
+            }
+        }
+        if ( !src_hdr->ntransl )
+        {
+            free(src_hdr->transl[0]); src_hdr->transl[0] = NULL;
+            free(src_hdr->transl[1]); src_hdr->transl[1] = NULL;
+            src_hdr->ntransl = -1;
+        }
+        if ( src_hdr->ntransl==-1 ) return 0;
+    }
+    bcf_unpack(line,BCF_UN_ALL);
+
+    // CHROM
+    if ( src_hdr->transl[BCF_DT_CTG][line->rid] >=0 ) line->rid = src_hdr->transl[BCF_DT_CTG][line->rid];
+
+    // FILTER
+    for (i=0; i<line->d.n_flt; i++)
+    {
+        int src_id = line->d.flt[i];
+        if ( src_hdr->transl[BCF_DT_ID][src_id] >=0 )
+            line->d.flt[i] = src_hdr->transl[BCF_DT_ID][src_id];
+        line->d.shared_dirty |= BCF1_DIRTY_FLT;
+    }
+
+    // INFO
+    for (i=0; i<line->n_info; i++)
+    {
+        int src_id = line->d.info[i].key;
+        int dst_id = src_hdr->transl[BCF_DT_ID][src_id];
+        if ( dst_id<0 ) continue;
+        line->d.info[i].key = dst_id;
+        if ( !line->d.info[i].vptr ) continue;  // skip deleted
+        int src_size = src_id>>7 ? ( src_id>>15 ? BCF_BT_INT32 : BCF_BT_INT16) : BCF_BT_INT8;
+        int dst_size = dst_id>>7 ? ( dst_id>>15 ? BCF_BT_INT32 : BCF_BT_INT16) : BCF_BT_INT8;
+        if ( src_size==dst_size )   // can overwrite
+        {
+            uint8_t *vptr = line->d.info[i].vptr - line->d.info[i].vptr_off;
+            if ( dst_size==BCF_BT_INT8 ) { vptr[1] = (uint8_t)dst_id; }
+            else if ( dst_size==BCF_BT_INT16 ) { *(uint16_t*)vptr = (uint16_t)dst_id; }
+            else { *(uint32_t*)vptr = (uint32_t)dst_id; }
+        }
+        else    // must realloc
+        {
+            bcf_info_t *info = &line->d.info[i];
+            kstring_t str = {0,0,0};
+            bcf_enc_int1(&str, dst_id);
+            bcf_enc_size(&str, info->len,info->type);
+            uint32_t vptr_off = str.l;
+            kputsn((char*)info->vptr, info->vptr_len, &str);
+            if( info->vptr_free ) free(info->vptr - info->vptr_off);
+            info->vptr_off = vptr_off;
+            info->vptr = (uint8_t*)str.s + info->vptr_off;
+            info->vptr_free = 1;
+            line->d.shared_dirty |= BCF1_DIRTY_INF;
+        }
+    }
+
+    // FORMAT
+    for (i=0; i<line->n_fmt; i++)
+    {
+        int src_id = line->d.fmt[i].id;
+        int dst_id = src_hdr->transl[BCF_DT_ID][src_id];
+        if ( dst_id<0 ) continue;
+        line->d.fmt[i].id = dst_id;
+        if( !line->d.fmt[i].p ) continue;  // skip deleted
+        int src_size = src_id>>7 ? ( src_id>>15 ? BCF_BT_INT32 : BCF_BT_INT16) : BCF_BT_INT8;
+        int dst_size = dst_id>>7 ? ( dst_id>>15 ? BCF_BT_INT32 : BCF_BT_INT16) : BCF_BT_INT8;
+        if ( src_size==dst_size )   // can overwrite
+        {
+            uint8_t *p = line->d.fmt[i].p - line->d.fmt[i].p_off;    // pointer to the vector size (4bits) and BT type (4bits)
+            if ( dst_size==BCF_BT_INT8 ) { p[1] = dst_id; }
+            else if ( dst_size==BCF_BT_INT16 ) { i16_to_le(dst_id, p + 1); }
+            else { i32_to_le(dst_id, p + 1); }
+        }
+        else    // must realloc
+        {
+            bcf_fmt_t *fmt = &line->d.fmt[i];
+            kstring_t str = {0,0,0};
+            bcf_enc_int1(&str, dst_id);
+            bcf_enc_size(&str, fmt->n, fmt->type);
+            uint32_t p_off = str.l;
+            kputsn((char*)fmt->p, fmt->p_len, &str);
+            if( fmt->p_free ) free(fmt->p - fmt->p_off);
+            fmt->p_off = p_off;
+            fmt->p = (uint8_t*)str.s + fmt->p_off;
+            fmt->p_free = 1;
+            line->d.indiv_dirty = 1;
+        }
+    }
+    return 0;
+}
+
+bcf_hdr_t *bcf_hdr_dup(const bcf_hdr_t *hdr)
+{
+    bcf_hdr_t *hout = bcf_hdr_init("r");
+    if (!hout) {
+        hts_log_error("Failed to allocate bcf header");
+        return NULL;
+    }
+    kstring_t htxt = {0,0,0};
+    if (bcf_hdr_format(hdr, 1, &htxt) < 0) {
+        free(htxt.s);
+        return NULL;
+    }
+    if ( bcf_hdr_parse(hout, htxt.s) < 0 ) {
+        bcf_hdr_destroy(hout);
+        hout = NULL;
+    }
+    free(htxt.s);
+    return hout;
+}
+
+bcf_hdr_t *bcf_hdr_subset(const bcf_hdr_t *h0, int n, char *const* samples, int *imap)
+{
+    void *names_hash = khash_str2int_init();
+    kstring_t htxt = {0,0,0};
+    kstring_t str = {0,0,0};
+    bcf_hdr_t *h = bcf_hdr_init("w");
+    int r = 0;
+    if (!h || !names_hash) {
+        hts_log_error("Failed to allocate bcf header");
+        goto err;
+    }
+    if (bcf_hdr_format(h0, 1, &htxt) < 0) {
+        hts_log_error("Failed to get header text");
+        goto err;
+    }
+    bcf_hdr_set_version(h,bcf_hdr_get_version(h0));
+    int j;
+    for (j=0; j<n; j++) imap[j] = -1;
+    if ( bcf_hdr_nsamples(h0) > 0) {
+        char *p = find_chrom_header_line(htxt.s);
+        int i = 0, end = n? 8 : 7;
+        while ((p = strchr(p, '\t')) != 0 && i < end) ++i, ++p;
+        if (i != end) {
+            hts_log_error("Wrong number of columns in header #CHROM line");
+            goto err;
+        }
+        r |= kputsn(htxt.s, p - htxt.s, &str) < 0;
+        for (i = 0; i < n; ++i) {
+            if ( khash_str2int_has_key(names_hash,samples[i]) )
+            {
+                hts_log_error("Duplicate sample name \"%s\"", samples[i]);
+                goto err;
+            }
+            imap[i] = bcf_hdr_id2int(h0, BCF_DT_SAMPLE, samples[i]);
+            if (imap[i] < 0) continue;
+            r |= kputc('\t', &str) < 0;
+            r |= kputs(samples[i], &str) < 0;
+            r |= khash_str2int_inc(names_hash,samples[i]) < 0;
+        }
+    } else r |= kputsn(htxt.s, htxt.l, &str) < 0;
+    while (str.l && (!str.s[str.l-1] || str.s[str.l-1]=='\n') ) str.l--; // kill trailing zeros and newlines
+    r |= kputc('\n',&str) < 0;
+    if (r) {
+        hts_log_error("%s", strerror(errno));
+        goto err;
+    }
+    if ( bcf_hdr_parse(h, str.s) < 0 ) {
+        bcf_hdr_destroy(h);
+        h = NULL;
+    }
+    free(str.s);
+    free(htxt.s);
+    khash_str2int_destroy(names_hash);
+    return h;
+
+ err:
+    ks_free(&str);
+    ks_free(&htxt);
+    khash_str2int_destroy(names_hash);
+    bcf_hdr_destroy(h);
+    return NULL;
+}
+
+int bcf_hdr_set_samples(bcf_hdr_t *hdr, const char *samples, int is_file)
+{
+    if ( samples && !strcmp("-",samples) ) return 0;            // keep all samples
+
+    int i, narr = bit_array_size(bcf_hdr_nsamples(hdr));
+    hdr->keep_samples = (uint8_t*) calloc(narr,1);
+    if (!hdr->keep_samples) return -1;
+
+    hdr->nsamples_ori = bcf_hdr_nsamples(hdr);
+    if ( !samples )
+    {
+        // exclude all samples
+        khint_t k;
+        vdict_t *d = (vdict_t*)hdr->dict[BCF_DT_SAMPLE], *new_dict;
+        new_dict = kh_init(vdict);
+        if (!new_dict) return -1;
+
+        bcf_hdr_nsamples(hdr) = 0;
+
+        for (k = kh_begin(d); k != kh_end(d); ++k)
+            if (kh_exist(d, k)) free((char*)kh_key(d, k));
+        kh_destroy(vdict, d);
+        hdr->dict[BCF_DT_SAMPLE] = new_dict;
+        if (bcf_hdr_sync(hdr) < 0) return -1;
+
+        return 0;
+    }
+
+    if ( samples[0]=='^' )
+        for (i=0; i<bcf_hdr_nsamples(hdr); i++) bit_array_set(hdr->keep_samples,i);
+
+    int idx, n, ret = 0;
+    char **smpls = hts_readlist(samples[0]=='^'?samples+1:samples, is_file, &n);
+    if ( !smpls ) return -1;
+    for (i=0; i<n; i++)
+    {
+        idx = bcf_hdr_id2int(hdr,BCF_DT_SAMPLE,smpls[i]);
+        if ( idx<0 )
+        {
+            if ( !ret ) ret = i+1;
+            continue;
+        }
+        assert( idx<bcf_hdr_nsamples(hdr) );
+        if (  samples[0]=='^' )
+            bit_array_clear(hdr->keep_samples, idx);
+        else
+            bit_array_set(hdr->keep_samples, idx);
+    }
+    for (i=0; i<n; i++) free(smpls[i]);
+    free(smpls);
+
+    bcf_hdr_nsamples(hdr) = 0;
+    for (i=0; i<hdr->nsamples_ori; i++)
+        if ( bit_array_test(hdr->keep_samples,i) ) bcf_hdr_nsamples(hdr)++;
+
+    if ( !bcf_hdr_nsamples(hdr) ) { free(hdr->keep_samples); hdr->keep_samples=NULL; }
+    else
+    {
+        // Make new list and dictionary with desired samples
+        char **samples = (char**) malloc(sizeof(char*)*bcf_hdr_nsamples(hdr));
+        vdict_t *new_dict, *d;
+        int k, res;
+        if (!samples) return -1;
+
+        new_dict = kh_init(vdict);
+        if (!new_dict) {
+            free(samples);
+            return -1;
+        }
+        idx = 0;
+        for (i=0; i<hdr->nsamples_ori; i++) {
+            if ( bit_array_test(hdr->keep_samples,i) ) {
+                samples[idx] = hdr->samples[i];
+                k = kh_put(vdict, new_dict, hdr->samples[i], &res);
+                if (res < 0) {
+                    free(samples);
+                    kh_destroy(vdict, new_dict);
+                    return -1;
+                }
+                kh_val(new_dict, k) = bcf_idinfo_def;
+                kh_val(new_dict, k).id = idx;
+                idx++;
+            }
+        }
+
+        // Delete desired samples from old dictionary, so we don't free them
+        d = (vdict_t*)hdr->dict[BCF_DT_SAMPLE];
+        for (i=0; i < idx; i++) {
+            int k = kh_get(vdict, d, samples[i]);
+            if (k < kh_end(d)) kh_del(vdict, d, k);
+        }
+
+        // Free everything else
+        for (k = kh_begin(d); k != kh_end(d); ++k)
+            if (kh_exist(d, k)) free((char*)kh_key(d, k));
+        kh_destroy(vdict, d);
+        hdr->dict[BCF_DT_SAMPLE] = new_dict;
+
+        free(hdr->samples);
+        hdr->samples = samples;
+
+        if (bcf_hdr_sync(hdr) < 0)
+            return -1;
+    }
+
+    return ret;
+}
+
+int bcf_subset(const bcf_hdr_t *h, bcf1_t *v, int n, int *imap)
+{
+    kstring_t ind;
+    ind.s = 0; ind.l = ind.m = 0;
+    if (n) {
+        bcf_fmt_t fmt[MAX_N_FMT];
+        int i, j;
+        uint8_t *ptr = (uint8_t*)v->indiv.s;
+        for (i = 0; i < v->n_fmt; ++i)
+            ptr = bcf_unpack_fmt_core1(ptr, v->n_sample, &fmt[i]);
+        for (i = 0; i < (int)v->n_fmt; ++i) {
+            bcf_fmt_t *f = &fmt[i];
+            bcf_enc_int1(&ind, f->id);
+            bcf_enc_size(&ind, f->n, f->type);
+            for (j = 0; j < n; ++j)
+                if (imap[j] >= 0) kputsn((char*)(f->p + imap[j] * f->size), f->size, &ind);
+        }
+        for (i = j = 0; j < n; ++j) if (imap[j] >= 0) ++i;
+        v->n_sample = i;
+    } else v->n_sample = 0;
+    if ( !v->n_sample ) v->n_fmt = 0;
+    free(v->indiv.s);
+    v->indiv = ind;
+    v->unpacked &= ~BCF_UN_FMT;    // only BCF is ready for output, VCF will need to unpack again
+    return 0;
+}
+
+int bcf_is_snp(bcf1_t *v)
+{
+    int i;
+    bcf_unpack(v, BCF_UN_STR);
+    for (i = 0; i < v->n_allele; ++i)
+    {
+        if ( v->d.allele[i][1]==0 && v->d.allele[i][0]!='*' ) continue;
+
+        // mpileup's <X> allele, see also below. This is not completely satisfactory,
+        // a general library is here narrowly tailored to fit samtools.
+        if ( v->d.allele[i][0]=='<' && v->d.allele[i][1]=='X' && v->d.allele[i][2]=='>' ) continue;
+        if ( v->d.allele[i][0]=='<' && v->d.allele[i][1]=='*' && v->d.allele[i][2]=='>' ) continue;
+
+        break;
+    }
+    return i == v->n_allele;
+}
+
+static void bcf_set_variant_type(const char *ref, const char *alt, bcf_variant_t *var)
+{
+    if ( *alt == '*' && !alt[1] ) { var->n = 0; var->type = VCF_OVERLAP; return; }  // overlapping variant
+
+    // The most frequent case
+    if ( !ref[1] && !alt[1] )
+    {
+        if ( *alt == '.' || *ref==*alt ) { var->n = 0; var->type = VCF_REF; return; }
+        if ( *alt == 'X' ) { var->n = 0; var->type = VCF_REF; return; }  // mpileup's X allele shouldn't be treated as variant
+        var->n = 1; var->type = VCF_SNP; return;
+    }
+    if ( alt[0]=='<' )
+    {
+        if ( alt[1]=='X' && alt[2]=='>' ) { var->n = 0; var->type = VCF_REF; return; }  // mpileup's X allele shouldn't be treated as variant
+        if ( alt[1]=='*' && alt[2]=='>' ) { var->n = 0; var->type = VCF_REF; return; }
+        if ( !strcmp("NON_REF>",alt+1) ) { var->n = 0; var->type = VCF_REF; return; }
+        var->type = VCF_OTHER;
+        return;
+    }
+
+    // Catch "joined before" breakend case
+    if ( alt[0]==']' || alt[0] == '[' )
+    {
+        var->type = VCF_BND; return;
+    }
+
+    // Iterate through alt characters that match the reference
+    const char *r = ref, *a = alt;
+    while (*r && *a && toupper_c(*r)==toupper_c(*a) ) { r++; a++; }     // unfortunately, matching REF,ALT case is not guaranteed
+
+    if ( *a && !*r )
+    {
+        if ( *a==']' || *a=='[' ) { var->type = VCF_BND; return; } // "joined after" breakend
+        while ( *a ) a++;
+        var->n = (a-alt)-(r-ref); var->type = VCF_INDEL | VCF_INS; return;
+    }
+    else if ( *r && !*a )
+    {
+        while ( *r ) r++;
+        var->n = (a-alt)-(r-ref); var->type = VCF_INDEL | VCF_DEL; return;
+    }
+    else if ( !*r && !*a )
+    {
+        var->n = 0; var->type = VCF_REF; return;
+    }
+
+    const char *re = r, *ae = a;
+    while ( re[1] ) re++;
+    while ( ae[1] ) ae++;
+    while ( re>r && ae>a && toupper_c(*re)==toupper_c(*ae) ) { re--; ae--; }
+    if ( ae==a )
+    {
+        if ( re==r ) { var->n = 1; var->type = VCF_SNP; return; }
+        var->n = -(re-r);
+        if ( toupper_c(*re)==toupper_c(*ae) ) { var->type = VCF_INDEL | VCF_DEL; return; }
+        var->type = VCF_OTHER; return;
+    }
+    else if ( re==r )
+    {
+        var->n = ae-a;
+        if ( toupper_c(*re)==toupper_c(*ae) ) { var->type = VCF_INDEL | VCF_INS; return; }
+        var->type = VCF_OTHER; return;
+    }
+
+    var->type = ( re-r == ae-a ) ? VCF_MNP : VCF_OTHER;
+    var->n = ( re-r > ae-a ) ? -(re-r+1) : ae-a+1;
+
+    // should do also complex events, SVs, etc...
+}
+
+static int bcf_set_variant_types(bcf1_t *b)
+{
+    if ( !(b->unpacked & BCF_UN_STR) ) bcf_unpack(b, BCF_UN_STR);
+    bcf_dec_t *d = &b->d;
+    if ( d->n_var < b->n_allele )
+    {
+        bcf_variant_t *new_var = realloc(d->var, sizeof(bcf_variant_t)*b->n_allele);
+        if (!new_var)
+            return -1;
+        d->var = new_var;
+        d->n_var = b->n_allele;
+    }
+    int i;
+    b->d.var_type = 0;
+    d->var[0].type = VCF_REF;
+    d->var[0].n    = 0;
+    for (i=1; i<b->n_allele; i++)
+    {
+        bcf_set_variant_type(d->allele[0],d->allele[i], &d->var[i]);
+        b->d.var_type |= d->var[i].type;
+        //fprintf(stderr,"[set_variant_type] %d   %s %s -> %d %d .. %d\n", b->pos+1,d->allele[0],d->allele[i],d->var[i].type,d->var[i].n, b->d.var_type);
+    }
+    return 0;
+}
+
+// bcf_get_variant_type/bcf_get_variant_types should only return the following,
+// to be compatible with callers that are not expecting newer values
+// like VCF_INS, VCF_DEL.  The full set is available from the newer
+// vcf_has_variant_type* interfaces.
+#define ORIG_VAR_TYPES (VCF_SNP|VCF_MNP|VCF_INDEL|VCF_OTHER|VCF_BND|VCF_OVERLAP)
+int bcf_get_variant_types(bcf1_t *rec)
+{
+    if ( rec->d.var_type==-1 ) {
+        if (bcf_set_variant_types(rec) != 0) {
+            hts_log_error("Couldn't get variant types: %s", strerror(errno));
+            exit(1); // Due to legacy API having no way to report failures
+        }
+    }
+    return rec->d.var_type & ORIG_VAR_TYPES;
+}
+
+int bcf_get_variant_type(bcf1_t *rec, int ith_allele)
+{
+    if ( rec->d.var_type==-1 ) {
+        if (bcf_set_variant_types(rec) != 0) {
+            hts_log_error("Couldn't get variant types: %s", strerror(errno));
+            exit(1); // Due to legacy API having no way to report failures
+        }
+    }
+    if (ith_allele < 0 || ith_allele >= rec->n_allele) {
+        hts_log_error("Requested allele outside valid range");
+        exit(1);
+    }
+    return rec->d.var[ith_allele].type & ORIG_VAR_TYPES;
+}
+#undef ORIG_VAR_TYPES
+
+int bcf_has_variant_type(bcf1_t *rec, int ith_allele, uint32_t bitmask)
+{
+    if ( rec->d.var_type==-1 ) {
+        if (bcf_set_variant_types(rec) != 0) return -1;
+    }
+    if (ith_allele < 0 || ith_allele >= rec->n_allele) return -1;
+    if (bitmask == VCF_REF) {  // VCF_REF is 0, so handled as a special case
+        return rec->d.var[ith_allele].type == VCF_REF;
+    }
+    return bitmask & rec->d.var[ith_allele].type;
+}
+
+int bcf_variant_length(bcf1_t *rec, int ith_allele)
+{
+    if ( rec->d.var_type==-1 ) {
+        if (bcf_set_variant_types(rec) != 0) return bcf_int32_missing;
+    }
+    if (ith_allele < 0 || ith_allele >= rec->n_allele) return bcf_int32_missing;
+    return rec->d.var[ith_allele].n;
+}
+
+int bcf_has_variant_types(bcf1_t *rec, uint32_t bitmask,
+                          enum bcf_variant_match mode)
+{
+    if ( rec->d.var_type==-1 ) {
+        if (bcf_set_variant_types(rec) != 0) return -1;
+    }
+    uint32_t type = rec->d.var_type;
+    if ( mode==bcf_match_overlap ) return bitmask & type;
+
+    // VCF_INDEL is always set with VCF_INS and VCF_DEL by bcf_set_variant_type[s], but the bitmask may
+    // ask for say `VCF_INS` or `VCF_INDEL` only
+    if ( bitmask&(VCF_INS|VCF_DEL) && !(bitmask&VCF_INDEL) ) type &= ~VCF_INDEL;
+    else if ( bitmask&VCF_INDEL && !(bitmask&(VCF_INS|VCF_DEL)) ) type &= ~(VCF_INS|VCF_DEL);
+
+    if ( mode==bcf_match_subset )
+    {
+        if ( ~bitmask & type ) return 0;
+        else return bitmask & type;
+    }
+    // mode == bcf_match_exact
+    return type==bitmask ? type : 0;
+}
+
+int bcf_update_info(const bcf_hdr_t *hdr, bcf1_t *line, const char *key, const void *values, int n, int type)
+{
+    static int negative_rlen_warned = 0;
+    int is_end_tag;
+
+    // Is the field already present?
+    int i, inf_id = bcf_hdr_id2int(hdr,BCF_DT_ID,key);
+    if ( !bcf_hdr_idinfo_exists(hdr,BCF_HL_INFO,inf_id) ) return -1;    // No such INFO field in the header
+    if ( !(line->unpacked & BCF_UN_INFO) ) bcf_unpack(line, BCF_UN_INFO);
+
+    is_end_tag = strcmp(key, "END") == 0;
+
+    for (i=0; i<line->n_info; i++)
+        if ( inf_id==line->d.info[i].key ) break;
+    bcf_info_t *inf = i==line->n_info ? NULL : &line->d.info[i];
+
+    if ( !n || (type==BCF_HT_STR && !values) )
+    {
+        if ( n==0 && is_end_tag )
+            line->rlen = line->n_allele ? strlen(line->d.allele[0]) : 0;
+        if ( inf )
+        {
+            // Mark the tag for removal, free existing memory if necessary
+            if ( inf->vptr_free )
+            {
+                free(inf->vptr - inf->vptr_off);
+                inf->vptr_free = 0;
+            }
+            line->d.shared_dirty |= BCF1_DIRTY_INF;
+            inf->vptr = NULL;
+            inf->vptr_off = inf->vptr_len = 0;
+        }
+        return 0;
+    }
+
+    if (is_end_tag)
+    {
+        if (n != 1)
+        {
+            hts_log_error("END info tag should only have one value at %s:%"PRIhts_pos, bcf_seqname_safe(hdr,line), line->pos+1);
+            line->errcode |= BCF_ERR_TAG_INVALID;
+            return -1;
+        }
+        if (type != BCF_HT_INT && type != BCF_HT_LONG)
+        {
+            hts_log_error("Wrong type (%d) for END info tag at %s:%"PRIhts_pos, type, bcf_seqname_safe(hdr,line), line->pos+1);
+            line->errcode |= BCF_ERR_TAG_INVALID;
+            return -1;
+        }
+    }
+
+    // Encode the values and determine the size required to accommodate the values
+    kstring_t str = {0,0,0};
+    bcf_enc_int1(&str, inf_id);
+    if ( type==BCF_HT_INT )
+        bcf_enc_vint(&str, n, (int32_t*)values, -1);
+    else if ( type==BCF_HT_REAL )
+        bcf_enc_vfloat(&str, n, (float*)values);
+    else if ( type==BCF_HT_FLAG || type==BCF_HT_STR )
+    {
+        if ( values==NULL )
+            bcf_enc_size(&str, 0, BCF_BT_NULL);
+        else
+            bcf_enc_vchar(&str, strlen((char*)values), (char*)values);
+    }
+#ifdef VCF_ALLOW_INT64
+    else if ( type==BCF_HT_LONG )
+    {
+        if (n != 1) {
+            hts_log_error("Only storing a single BCF_HT_LONG value is supported at %s:%"PRIhts_pos, bcf_seqname_safe(hdr,line), line->pos+1);
+            abort();
+        }
+        bcf_enc_long1(&str, *(int64_t *) values);
+    }
+#endif
+    else
+    {
+        hts_log_error("The type %d not implemented yet at %s:%"PRIhts_pos, type, bcf_seqname_safe(hdr,line), line->pos+1);
+        abort();
+    }
+
+    // Is the INFO tag already present
+    if ( inf )
+    {
+        // Is it big enough to accommodate new block?
+        if ( inf->vptr && str.l <= inf->vptr_len + inf->vptr_off )
+        {
+            if ( str.l != inf->vptr_len + inf->vptr_off ) line->d.shared_dirty |= BCF1_DIRTY_INF;
+            uint8_t *ptr = inf->vptr - inf->vptr_off;
+            memcpy(ptr, str.s, str.l);
+            free(str.s);
+            int vptr_free = inf->vptr_free;
+            bcf_unpack_info_core1(ptr, inf);
+            inf->vptr_free = vptr_free;
+        }
+        else
+        {
+            if ( inf->vptr_free )
+                free(inf->vptr - inf->vptr_off);
+            bcf_unpack_info_core1((uint8_t*)str.s, inf);
+            inf->vptr_free = 1;
+            line->d.shared_dirty |= BCF1_DIRTY_INF;
+        }
+    }
+    else
+    {
+        // The tag is not present, create new one
+        line->n_info++;
+        hts_expand0(bcf_info_t, line->n_info, line->d.m_info , line->d.info);
+        inf = &line->d.info[line->n_info-1];
+        bcf_unpack_info_core1((uint8_t*)str.s, inf);
+        inf->vptr_free = 1;
+        line->d.shared_dirty |= BCF1_DIRTY_INF;
+    }
+    line->unpacked |= BCF_UN_INFO;
+
+   if ( n==1 && is_end_tag) {
+        hts_pos_t end = type == BCF_HT_INT ? *(int32_t *) values : *(int64_t *) values;
+        if ( (type == BCF_HT_INT && end!=bcf_int32_missing) || (type == BCF_HT_LONG && end!=bcf_int64_missing) )
+        {
+            if ( end <= line->pos )
+            {
+                if ( !negative_rlen_warned )
+                {
+                    hts_log_warning("INFO/END=%"PRIhts_pos" is smaller than POS at %s:%"PRIhts_pos,end,bcf_seqname_safe(hdr,line),line->pos+1);
+                    negative_rlen_warned = 1;
+                }
+                line->rlen = line->n_allele ? strlen(line->d.allele[0]) : 0;
+            }
+            else
+                line->rlen = end - line->pos;
+        }
+    }
+    return 0;
+}
+
+int bcf_update_format_string(const bcf_hdr_t *hdr, bcf1_t *line, const char *key, const char **values, int n)
+{
+    if ( !n )
+        return bcf_update_format(hdr,line,key,NULL,0,BCF_HT_STR);
+
+    int i, max_len = 0;
+    for (i=0; i<n; i++)
+    {
+        int len = strlen(values[i]);
+        if ( len > max_len ) max_len = len;
+    }
+    char *out = (char*) malloc(max_len*n);
+    if ( !out ) return -2;
+    for (i=0; i<n; i++)
+    {
+        char *dst = out+i*max_len;
+        const char *src = values[i];
+        int j = 0;
+        while ( src[j] ) { dst[j] = src[j]; j++; }
+        for (; j<max_len; j++) dst[j] = 0;
+    }
+    int ret = bcf_update_format(hdr,line,key,out,max_len*n,BCF_HT_STR);
+    free(out);
+    return ret;
+}
+
+int bcf_update_format(const bcf_hdr_t *hdr, bcf1_t *line, const char *key, const void *values, int n, int type)
+{
+    // Is the field already present?
+    int i, fmt_id = bcf_hdr_id2int(hdr,BCF_DT_ID,key);
+    if ( !bcf_hdr_idinfo_exists(hdr,BCF_HL_FMT,fmt_id) )
+    {
+        if ( !n ) return 0;
+        return -1;  // the key not present in the header
+    }
+
+    if ( !(line->unpacked & BCF_UN_FMT) ) bcf_unpack(line, BCF_UN_FMT);
+
+    for (i=0; i<line->n_fmt; i++)
+        if ( line->d.fmt[i].id==fmt_id ) break;
+    bcf_fmt_t *fmt = i==line->n_fmt ? NULL : &line->d.fmt[i];
+
+    if ( !n )
+    {
+        if ( fmt )
+        {
+            // Mark the tag for removal, free existing memory if necessary
+            if ( fmt->p_free )
+            {
+                free(fmt->p - fmt->p_off);
+                fmt->p_free = 0;
+            }
+            line->d.indiv_dirty = 1;
+            fmt->p = NULL;
+        }
+        return 0;
+    }
+
+    line->n_sample = bcf_hdr_nsamples(hdr);
+    int nps = n / line->n_sample;  // number of values per sample
+    assert( nps && nps*line->n_sample==n );     // must be divisible by n_sample
+
+    // Encode the values and determine the size required to accommodate the values
+    kstring_t str = {0,0,0};
+    bcf_enc_int1(&str, fmt_id);
+    assert(values != NULL);
+    if ( type==BCF_HT_INT )
+        bcf_enc_vint(&str, n, (int32_t*)values, nps);
+    else if ( type==BCF_HT_REAL )
+    {
+        bcf_enc_size(&str, nps, BCF_BT_FLOAT);
+        serialize_float_array(&str, nps*line->n_sample, (float *) values);
+    }
+    else if ( type==BCF_HT_STR )
+    {
+        bcf_enc_size(&str, nps, BCF_BT_CHAR);
+        kputsn((char*)values, nps*line->n_sample, &str);
+    }
+    else
+    {
+        hts_log_error("The type %d not implemented yet at %s:%"PRIhts_pos, type, bcf_seqname_safe(hdr,line), line->pos+1);
+        abort();
+    }
+
+    if ( !fmt )
+    {
+        // Not present, new format field
+        line->n_fmt++;
+        hts_expand0(bcf_fmt_t, line->n_fmt, line->d.m_fmt, line->d.fmt);
+
+        // Special case: VCF specification requires that GT is always first
+        if ( line->n_fmt > 1 && key[0]=='G' && key[1]=='T' && !key[2] )
+        {
+            for (i=line->n_fmt-1; i>0; i--)
+                line->d.fmt[i] = line->d.fmt[i-1];
+            fmt = &line->d.fmt[0];
+        }
+        else
+            fmt = &line->d.fmt[line->n_fmt-1];
+        bcf_unpack_fmt_core1((uint8_t*)str.s, line->n_sample, fmt);
+        line->d.indiv_dirty = 1;
+        fmt->p_free = 1;
+    }
+    else
+    {
+        // The tag is already present, check if it is big enough to accommodate the new block
+        if ( fmt->p && str.l <= fmt->p_len + fmt->p_off )
+        {
+            // good, the block is big enough
+            if ( str.l != fmt->p_len + fmt->p_off ) line->d.indiv_dirty = 1;
+            uint8_t *ptr = fmt->p - fmt->p_off;
+            memcpy(ptr, str.s, str.l);
+            free(str.s);
+            int p_free = fmt->p_free;
+            bcf_unpack_fmt_core1(ptr, line->n_sample, fmt);
+            fmt->p_free = p_free;
+        }
+        else
+        {
+            if ( fmt->p_free )
+                free(fmt->p - fmt->p_off);
+            bcf_unpack_fmt_core1((uint8_t*)str.s, line->n_sample, fmt);
+            fmt->p_free = 1;
+            line->d.indiv_dirty = 1;
+        }
+    }
+    line->unpacked |= BCF_UN_FMT;
+    return 0;
+}
+
+
+int bcf_update_filter(const bcf_hdr_t *hdr, bcf1_t *line, int *flt_ids, int n)
+{
+    if ( !(line->unpacked & BCF_UN_FLT) ) bcf_unpack(line, BCF_UN_FLT);
+    line->d.shared_dirty |= BCF1_DIRTY_FLT;
+    line->d.n_flt = n;
+    if ( !n ) return 0;
+    hts_expand(int, line->d.n_flt, line->d.m_flt, line->d.flt);
+    int i;
+    for (i=0; i<n; i++)
+        line->d.flt[i] = flt_ids[i];
+    return 0;
+}
+
+int bcf_add_filter(const bcf_hdr_t *hdr, bcf1_t *line, int flt_id)
+{
+    if ( !(line->unpacked & BCF_UN_FLT) ) bcf_unpack(line, BCF_UN_FLT);
+    int i;
+    for (i=0; i<line->d.n_flt; i++)
+        if ( flt_id==line->d.flt[i] ) break;
+    if ( i<line->d.n_flt ) return 0;    // this filter is already set
+    line->d.shared_dirty |= BCF1_DIRTY_FLT;
+    if ( flt_id==0 )    // set to PASS
+        line->d.n_flt = 1;
+    else if ( line->d.n_flt==1 && line->d.flt[0]==0 )
+        line->d.n_flt = 1;
+    else
+        line->d.n_flt++;
+    hts_expand(int, line->d.n_flt, line->d.m_flt, line->d.flt);
+    line->d.flt[line->d.n_flt-1] = flt_id;
+    return 1;
+}
+int bcf_remove_filter(const bcf_hdr_t *hdr, bcf1_t *line, int flt_id, int pass)
+{
+    if ( !(line->unpacked & BCF_UN_FLT) ) bcf_unpack(line, BCF_UN_FLT);
+    int i;
+    for (i=0; i<line->d.n_flt; i++)
+        if ( flt_id==line->d.flt[i] ) break;
+    if ( i==line->d.n_flt ) return 0;   // the filter is not present
+    line->d.shared_dirty |= BCF1_DIRTY_FLT;
+    if ( i!=line->d.n_flt-1 ) memmove(line->d.flt+i,line->d.flt+i+1,(line->d.n_flt-i-1)*sizeof(*line->d.flt));
+    line->d.n_flt--;
+    if ( !line->d.n_flt && pass ) bcf_add_filter(hdr,line,0);
+    return 0;
+}
+
+int bcf_has_filter(const bcf_hdr_t *hdr, bcf1_t *line, char *filter)
+{
+    if ( filter[0]=='.' && !filter[1] ) filter = "PASS";
+    int id = bcf_hdr_id2int(hdr, BCF_DT_ID, filter);
+    if ( !bcf_hdr_idinfo_exists(hdr,BCF_HL_FLT,id) ) return -1;  // not defined in the header
+
+    if ( !(line->unpacked & BCF_UN_FLT) ) bcf_unpack(line, BCF_UN_FLT);
+    if ( id==0 && !line->d.n_flt) return 1; // PASS
+
+    int i;
+    for (i=0; i<line->d.n_flt; i++)
+        if ( line->d.flt[i]==id ) return 1;
+    return 0;
+}
+
+static inline int _bcf1_sync_alleles(const bcf_hdr_t *hdr, bcf1_t *line, int nals)
+{
+    line->d.shared_dirty |= BCF1_DIRTY_ALS;
+
+    line->n_allele = nals;
+    hts_expand(char*, line->n_allele, line->d.m_allele, line->d.allele);
+
+    char *als = line->d.als;
+    int n = 0;
+    while (n<nals)
+    {
+        line->d.allele[n] = als;
+        while ( *als ) als++;
+        als++;
+        n++;
+    }
+
+    // Update REF length. Note that END is 1-based while line->pos 0-based
+    bcf_info_t *end_info = bcf_get_info(hdr,line,"END");
+    if ( end_info )
+    {
+        if ( end_info->type==BCF_HT_INT && end_info->v1.i==bcf_int32_missing ) end_info = NULL;
+        else if ( end_info->type==BCF_HT_LONG && end_info->v1.i==bcf_int64_missing ) end_info = NULL;
+    }
+    if ( end_info && end_info->v1.i > line->pos )
+        line->rlen = end_info->v1.i - line->pos;
+    else if ( nals > 0 )
+        line->rlen = strlen(line->d.allele[0]);
+    else
+        line->rlen = 0;
+
+    return 0;
+}
+int bcf_update_alleles(const bcf_hdr_t *hdr, bcf1_t *line, const char **alleles, int nals)
+{
+    if ( !(line->unpacked & BCF_UN_STR) ) bcf_unpack(line, BCF_UN_STR);
+    char *free_old = NULL;
+    char buffer[256];
+    size_t used = 0;
+
+    // The pointers in alleles may point into the existing line->d.als memory,
+    // so care needs to be taken not to clobber them while updating.  Usually
+    // they will be short so we can copy through an intermediate buffer.
+    // If they're longer, or won't fit in the existing allocation we
+    // can allocate a new buffer to write into.  Note that in either case
+    // pointers to line->d.als memory in alleles may not be valid when we've
+    // finished.
+    int i;
+    size_t avail = line->d.m_als < sizeof(buffer) ? line->d.m_als : sizeof(buffer);
+    for (i=0; i<nals; i++) {
+        size_t sz = strlen(alleles[i]) + 1;
+        if (avail - used < sz)
+            break;
+        memcpy(buffer + used, alleles[i], sz);
+        used += sz;
+    }
+
+    // Did we miss anything?
+    if (i < nals) {
+        int j;
+        size_t needed = used;
+        char *new_als;
+        for (j = i; j < nals; j++)
+            needed += strlen(alleles[j]) + 1;
+        if (needed < line->d.m_als) // Don't shrink the buffer
+            needed = line->d.m_als;
+        if (needed > INT_MAX) {
+            hts_log_error("REF + alleles too long to fit in a BCF record");
+            return -1;
+        }
+        new_als = malloc(needed);
+        if (!new_als)
+            return -1;
+        free_old = line->d.als;
+        line->d.als = new_als;
+        line->d.m_als = needed;
+    }
+
+    // Copy from the temp buffer to the destination
+    if (used) {
+        assert(used <= line->d.m_als);
+        memcpy(line->d.als, buffer, used);
+    }
+
+    // Add in any remaining entries - if this happens we will always be
+    // writing to a newly-allocated buffer.
+    for (; i < nals; i++) {
+        size_t sz = strlen(alleles[i]) + 1;
+        memcpy(line->d.als + used, alleles[i], sz);
+        used += sz;
+    }
+
+    if (free_old)
+        free(free_old);
+    return _bcf1_sync_alleles(hdr,line,nals);
+}
+
+int bcf_update_alleles_str(const bcf_hdr_t *hdr, bcf1_t *line, const char *alleles_string)
+{
+    if ( !(line->unpacked & BCF_UN_STR) ) bcf_unpack(line, BCF_UN_STR);
+    kstring_t tmp;
+    tmp.l = 0; tmp.s = line->d.als; tmp.m = line->d.m_als;
+    kputs(alleles_string, &tmp);
+    line->d.als = tmp.s; line->d.m_als = tmp.m;
+
+    int nals = 1;
+    char *t = line->d.als;
+    while (*t)
+    {
+        if ( *t==',' ) { *t = 0; nals++; }
+        t++;
+    }
+    return _bcf1_sync_alleles(hdr, line, nals);
+}
+
+int bcf_update_id(const bcf_hdr_t *hdr, bcf1_t *line, const char *id)
+{
+    if ( !(line->unpacked & BCF_UN_STR) ) bcf_unpack(line, BCF_UN_STR);
+    kstring_t tmp;
+    tmp.l = 0; tmp.s = line->d.id; tmp.m = line->d.m_id;
+    if ( id )
+        kputs(id, &tmp);
+    else
+        kputs(".", &tmp);
+    line->d.id = tmp.s; line->d.m_id = tmp.m;
+    line->d.shared_dirty |= BCF1_DIRTY_ID;
+    return 0;
+}
+
+int bcf_add_id(const bcf_hdr_t *hdr, bcf1_t *line, const char *id)
+{
+    if ( !id ) return 0;
+    if ( !(line->unpacked & BCF_UN_STR) ) bcf_unpack(line, BCF_UN_STR);
+
+    kstring_t tmp;
+    tmp.l = 0; tmp.s = line->d.id; tmp.m = line->d.m_id;
+
+    int len = strlen(id);
+    char *dst = line->d.id;
+    while ( *dst && (dst=strstr(dst,id)) )
+    {
+        if ( dst[len]!=0 && dst[len]!=';' ) dst++;              // a prefix, not a match
+        else if ( dst==line->d.id || dst[-1]==';' ) return 0;   // already present
+        dst++;  // a suffix, not a match
+    }
+    if ( line->d.id && (line->d.id[0]!='.' || line->d.id[1]) )
+    {
+        tmp.l = strlen(line->d.id);
+        kputc(';',&tmp);
+    }
+    kputs(id,&tmp);
+
+    line->d.id = tmp.s; line->d.m_id = tmp.m;
+    line->d.shared_dirty |= BCF1_DIRTY_ID;
+    return 0;
+
+}
+
+bcf_fmt_t *bcf_get_fmt(const bcf_hdr_t *hdr, bcf1_t *line, const char *key)
+{
+    int id = bcf_hdr_id2int(hdr, BCF_DT_ID, key);
+    if ( !bcf_hdr_idinfo_exists(hdr,BCF_HL_FMT,id) ) return NULL;   // no such FMT field in the header
+    return bcf_get_fmt_id(line, id);
+}
+
+bcf_info_t *bcf_get_info(const bcf_hdr_t *hdr, bcf1_t *line, const char *key)
+{
+    int id = bcf_hdr_id2int(hdr, BCF_DT_ID, key);
+    if ( !bcf_hdr_idinfo_exists(hdr,BCF_HL_INFO,id) ) return NULL;   // no such INFO field in the header
+    return bcf_get_info_id(line, id);
+}
+
+bcf_fmt_t *bcf_get_fmt_id(bcf1_t *line, const int id)
+{
+    int i;
+    if ( !(line->unpacked & BCF_UN_FMT) ) bcf_unpack(line, BCF_UN_FMT);
+    for (i=0; i<line->n_fmt; i++)
+    {
+        if ( line->d.fmt[i].id==id ) return &line->d.fmt[i];
+    }
+    return NULL;
+}
+
+bcf_info_t *bcf_get_info_id(bcf1_t *line, const int id)
+{
+    int i;
+    if ( !(line->unpacked & BCF_UN_INFO) ) bcf_unpack(line, BCF_UN_INFO);
+    for (i=0; i<line->n_info; i++)
+    {
+        if ( line->d.info[i].key==id ) return &line->d.info[i];
+    }
+    return NULL;
+}
+
+
+int bcf_get_info_values(const bcf_hdr_t *hdr, bcf1_t *line, const char *tag, void **dst, int *ndst, int type)
+{
+    int i, ret = -4, tag_id = bcf_hdr_id2int(hdr, BCF_DT_ID, tag);
+    if ( !bcf_hdr_idinfo_exists(hdr,BCF_HL_INFO,tag_id) ) return -1;    // no such INFO field in the header
+    if ( bcf_hdr_id2type(hdr,BCF_HL_INFO,tag_id)!=(type & 0xff) ) return -2;     // expected different type
+
+    if ( !(line->unpacked & BCF_UN_INFO) ) bcf_unpack(line, BCF_UN_INFO);
+
+    for (i=0; i<line->n_info; i++)
+        if ( line->d.info[i].key==tag_id ) break;
+    if ( i==line->n_info ) return ( type==BCF_HT_FLAG ) ? 0 : -3;       // the tag is not present in this record
+    if ( type==BCF_HT_FLAG ) return 1;
+
+    bcf_info_t *info = &line->d.info[i];
+    if ( !info->vptr ) return -3;           // the tag was marked for removal
+    if ( type==BCF_HT_STR )
+    {
+        if ( *ndst < info->len+1 )
+        {
+            *ndst = info->len + 1;
+            *dst  = realloc(*dst, *ndst);
+        }
+        memcpy(*dst,info->vptr,info->len);
+        ((uint8_t*)*dst)[info->len] = 0;
+        return info->len;
+    }
+
+    // Make sure the buffer is big enough
+    int size1;
+    switch (type) {
+        case BCF_HT_INT:  size1 = sizeof(int32_t); break;
+        case BCF_HT_LONG: size1 = sizeof(int64_t); break;
+        case BCF_HT_REAL: size1 = sizeof(float); break;
+        default:
+            hts_log_error("Unexpected output type %d at %s:%"PRIhts_pos, type, bcf_seqname_safe(hdr,line), line->pos+1);
+            return -2;
+    }
+    if ( *ndst < info->len )
+    {
+        *ndst = info->len;
+        *dst  = realloc(*dst, *ndst * size1);
+    }
+
+    #define BRANCH(type_t, convert, is_missing, is_vector_end, set_missing, set_regular, out_type_t) do { \
+        out_type_t *tmp = (out_type_t *) *dst; \
+        int j; \
+        for (j=0; j<info->len; j++) \
+        { \
+            type_t p = convert(info->vptr + j * sizeof(type_t)); \
+            if ( is_vector_end ) break; \
+            if ( is_missing ) set_missing; \
+            else set_regular; \
+            tmp++; \
+        } \
+        ret = j; \
+    } while (0)
+    switch (info->type) {
+        case BCF_BT_INT8:
+            if (type == BCF_HT_LONG) {
+                BRANCH(int8_t,  le_to_i8,  p==bcf_int8_missing,  p==bcf_int8_vector_end,  *tmp=bcf_int64_missing, *tmp=p, int64_t);
+            } else {
+                BRANCH(int8_t,  le_to_i8,  p==bcf_int8_missing,  p==bcf_int8_vector_end,  *tmp=bcf_int32_missing, *tmp=p, int32_t);
+            }
+            break;
+        case BCF_BT_INT16:
+            if (type == BCF_HT_LONG) {
+                BRANCH(int16_t, le_to_i16, p==bcf_int16_missing, p==bcf_int16_vector_end, *tmp=bcf_int64_missing, *tmp=p, int64_t);
+            } else {
+                BRANCH(int16_t, le_to_i16, p==bcf_int16_missing, p==bcf_int16_vector_end, *tmp=bcf_int32_missing, *tmp=p, int32_t);
+            }
+            break;
+        case BCF_BT_INT32:
+            if (type == BCF_HT_LONG) {
+                BRANCH(int32_t, le_to_i32, p==bcf_int32_missing, p==bcf_int32_vector_end, *tmp=bcf_int64_missing, *tmp=p, int64_t); break;
+            } else {
+                BRANCH(int32_t, le_to_i32, p==bcf_int32_missing, p==bcf_int32_vector_end, *tmp=bcf_int32_missing, *tmp=p, int32_t); break;
+            }
+        case BCF_BT_FLOAT: BRANCH(uint32_t, le_to_u32, p==bcf_float_missing, p==bcf_float_vector_end, bcf_float_set_missing(*tmp), bcf_float_set(tmp, p), float); break;
+        default: hts_log_error("Unexpected type %d at %s:%"PRIhts_pos, info->type, bcf_seqname_safe(hdr,line), line->pos+1); return -2;
+    }
+    #undef BRANCH
+    return ret;  // set by BRANCH
+}
+
+int bcf_get_format_string(const bcf_hdr_t *hdr, bcf1_t *line, const char *tag, char ***dst, int *ndst)
+{
+    int i,tag_id = bcf_hdr_id2int(hdr, BCF_DT_ID, tag);
+    if ( !bcf_hdr_idinfo_exists(hdr,BCF_HL_FMT,tag_id) ) return -1;    // no such FORMAT field in the header
+    if ( bcf_hdr_id2type(hdr,BCF_HL_FMT,tag_id)!=BCF_HT_STR ) return -2;     // expected different type
+
+    if ( !(line->unpacked & BCF_UN_FMT) ) bcf_unpack(line, BCF_UN_FMT);
+
+    for (i=0; i<line->n_fmt; i++)
+        if ( line->d.fmt[i].id==tag_id ) break;
+    if ( i==line->n_fmt ) return -3;                               // the tag is not present in this record
+    bcf_fmt_t *fmt = &line->d.fmt[i];
+    if ( !fmt->p ) return -3;                                      // the tag was marked for removal
+
+    int nsmpl = bcf_hdr_nsamples(hdr);
+    if ( !*dst )
+    {
+        *dst = (char**) malloc(sizeof(char*)*nsmpl);
+        if ( !*dst ) return -4;     // could not alloc
+        (*dst)[0] = NULL;
+    }
+    int n = (fmt->n+1)*nsmpl;
+    if ( *ndst < n )
+    {
+        (*dst)[0] = realloc((*dst)[0], n);
+        if ( !(*dst)[0] ) return -4;    // could not alloc
+        *ndst = n;
+    }
+    for (i=0; i<nsmpl; i++)
+    {
+        uint8_t *src = fmt->p + i*fmt->n;
+        uint8_t *tmp = (uint8_t*)(*dst)[0] + i*(fmt->n+1);
+        memcpy(tmp,src,fmt->n);
+        tmp[fmt->n] = 0;
+        (*dst)[i] = (char*) tmp;
+    }
+    return n;
+}
+
+int bcf_get_format_values(const bcf_hdr_t *hdr, bcf1_t *line, const char *tag, void **dst, int *ndst, int type)
+{
+    int i,j, tag_id = bcf_hdr_id2int(hdr, BCF_DT_ID, tag);
+    if ( !bcf_hdr_idinfo_exists(hdr,BCF_HL_FMT,tag_id) ) return -1;    // no such FORMAT field in the header
+    if ( tag[0]=='G' && tag[1]=='T' && tag[2]==0 )
+    {
+        // Ugly: GT field is considered to be a string by the VCF header but BCF represents it as INT.
+        if ( bcf_hdr_id2type(hdr,BCF_HL_FMT,tag_id)!=BCF_HT_STR ) return -2;
+    }
+    else if ( bcf_hdr_id2type(hdr,BCF_HL_FMT,tag_id)!=type ) return -2;     // expected different type
+
+    if ( !(line->unpacked & BCF_UN_FMT) ) bcf_unpack(line, BCF_UN_FMT);
+
+    for (i=0; i<line->n_fmt; i++)
+        if ( line->d.fmt[i].id==tag_id ) break;
+    if ( i==line->n_fmt ) return -3;                               // the tag is not present in this record
+    bcf_fmt_t *fmt = &line->d.fmt[i];
+    if ( !fmt->p ) return -3;                                      // the tag was marked for removal
+
+    if ( type==BCF_HT_STR )
+    {
+        int n = fmt->n*bcf_hdr_nsamples(hdr);
+        if ( *ndst < n )
+        {
+            *dst  = realloc(*dst, n);
+            if ( !*dst ) return -4;     // could not alloc
+            *ndst = n;
+        }
+        memcpy(*dst,fmt->p,n);
+        return n;
+    }
+
+    // Make sure the buffer is big enough
+    int nsmpl = bcf_hdr_nsamples(hdr);
+    int size1 = type==BCF_HT_INT ? sizeof(int32_t) : sizeof(float);
+    if ( *ndst < fmt->n*nsmpl )
+    {
+        *ndst = fmt->n*nsmpl;
+        *dst  = realloc(*dst, *ndst*size1);
+        if ( !*dst ) return -4;     // could not alloc
+    }
+
+    #define BRANCH(type_t, convert, is_missing, is_vector_end, set_missing, set_vector_end, set_regular, out_type_t) { \
+        out_type_t *tmp = (out_type_t *) *dst; \
+        uint8_t *fmt_p = fmt->p; \
+        for (i=0; i<nsmpl; i++) \
+        { \
+            for (j=0; j<fmt->n; j++) \
+            { \
+                type_t p = convert(fmt_p + j * sizeof(type_t)); \
+                if ( is_missing ) set_missing; \
+                else if ( is_vector_end ) { set_vector_end; break; } \
+                else set_regular; \
+                tmp++; \
+            } \
+            for (; j<fmt->n; j++) { set_vector_end; tmp++; } \
+            fmt_p += fmt->size; \
+        } \
+    }
+    switch (fmt->type) {
+        case BCF_BT_INT8:  BRANCH(int8_t,  le_to_i8, p==bcf_int8_missing,  p==bcf_int8_vector_end,  *tmp=bcf_int32_missing, *tmp=bcf_int32_vector_end, *tmp=p, int32_t); break;
+        case BCF_BT_INT16: BRANCH(int16_t, le_to_i16, p==bcf_int16_missing, p==bcf_int16_vector_end, *tmp=bcf_int32_missing, *tmp=bcf_int32_vector_end, *tmp=p, int32_t); break;
+        case BCF_BT_INT32: BRANCH(int32_t, le_to_i32, p==bcf_int32_missing, p==bcf_int32_vector_end, *tmp=bcf_int32_missing, *tmp=bcf_int32_vector_end, *tmp=p, int32_t); break;
+        case BCF_BT_FLOAT: BRANCH(uint32_t, le_to_u32, p==bcf_float_missing, p==bcf_float_vector_end, bcf_float_set_missing(*tmp), bcf_float_set_vector_end(*tmp), bcf_float_set(tmp, p), float); break;
+        default: hts_log_error("Unexpected type %d at %s:%"PRIhts_pos, fmt->type, bcf_seqname_safe(hdr,line), line->pos+1); exit(1);
+    }
+    #undef BRANCH
+    return nsmpl*fmt->n;
+}
+
+//error description structure definition
+typedef struct err_desc {
+    int  errorcode;
+    const char *description;
+}err_desc;
+
+// error descriptions
+static const err_desc errdesc_bcf[] = {
+    { BCF_ERR_CTG_UNDEF, "Contig not defined in header"},
+    { BCF_ERR_TAG_UNDEF, "Tag not defined in header" },
+    { BCF_ERR_NCOLS, "Incorrect number of columns" },
+    { BCF_ERR_LIMITS, "Limits reached" },
+    { BCF_ERR_CHAR, "Invalid character" },
+    { BCF_ERR_CTG_INVALID, "Invalid contig" },
+    { BCF_ERR_TAG_INVALID, "Invalid tag" },
+};
+
+/// append given description to buffer based on available size and add ... when not enough space
+    /** @param buffer       buffer to which description to be appended
+        @param offset       offset at which to be appended
+        @param maxbuffer    maximum size of the buffer
+        @param description  the description to be appended
+on failure returns -1 - when buffer is not big enough; returns -1 on invalid params and on too small buffer which are improbable due to validation at caller site
+on success returns 0
+    */
+static int add_desc_to_buffer(char *buffer, size_t *offset, size_t maxbuffer, const char *description) {
+
+    if (!description || !buffer || !offset || (maxbuffer < 4))
+        return -1;
+
+    size_t rembuffer = maxbuffer - *offset;
+    if (rembuffer > (strlen(description) + (rembuffer == maxbuffer ? 0 : 1))) {    //add description with optionally required ','
+        *offset += snprintf(buffer + *offset, rembuffer, "%s%s", (rembuffer == maxbuffer)? "": ",", description);
+    } else {    //not enough space for description, put ...
+        size_t tmppos = (rembuffer <= 4) ? maxbuffer - 4 : *offset;
+        snprintf(buffer + tmppos, 4, "...");    //ignore offset update
+        return -1;
+    }
+    return 0;
+}
+
+//get description for given error code. return NULL on error
+const char *bcf_strerror(int errorcode, char *buffer, size_t maxbuffer) {
+    size_t usedup = 0;
+    int ret = 0;
+    int idx;
+
+    if (!buffer || maxbuffer < 4)
+        return NULL;           //invalid / insufficient buffer
+
+    if (!errorcode) {
+        buffer[0] = '\0';      //no error, set null
+        return buffer;
+    }
+
+    for (idx = 0; idx < sizeof(errdesc_bcf) / sizeof(err_desc); ++idx) {
+        if (errorcode & errdesc_bcf[idx].errorcode) {    //error is set, add description
+            ret = add_desc_to_buffer(buffer, &usedup, maxbuffer, errdesc_bcf[idx].description);
+            if (ret < 0)
+                break;         //not enough space, ... added, no need to continue
+
+            errorcode &= ~errdesc_bcf[idx].errorcode;    //reset the error
+        }
+    }
+
+    if (errorcode && (ret >= 0))  {     //undescribed error is present in error code and had enough buffer, try to add unkonwn error as well§
+        add_desc_to_buffer(buffer, &usedup, maxbuffer, "Unknown error");
+    }
+    return buffer;
+}
+
diff --git a/ext/htslib/vcf_sweep.c b/ext/htslib/vcf_sweep.c
new file mode 100644
index 0000000..f3fb5fa
--- /dev/null
+++ b/ext/htslib/vcf_sweep.c
@@ -0,0 +1,190 @@
+/*  vcf_sweep.c -- forward/reverse sweep API.
+
+    Copyright (C) 2013-2014, 2019 Genome Research Ltd.
+
+    Author: Petr Danecek <pd3@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+
+#include <assert.h>
+
+#include "htslib/vcf_sweep.h"
+#include "htslib/bgzf.h"
+
+#define SW_FWD 0
+#define SW_BWD 1
+
+struct bcf_sweep_t
+{
+    htsFile *file;
+    bcf_hdr_t *hdr;
+    BGZF *fp;
+
+    int direction;          // to tell if the direction has changed
+    int block_size;         // the size of uncompressed data to hold in memory
+    bcf1_t *rec;            // bcf buffer
+    int nrec, mrec;         // number of used records; total size of the buffer
+    int lrid, lpos, lnals, lals_len, mlals;   // to check uniqueness of a record
+    char *lals;
+
+    uint64_t *idx;          // uncompressed offsets of VCF/BCF records
+    int iidx, nidx, midx;   // i: current offset; n: used; m: allocated
+    int idx_done;           // the index is built during the first pass
+};
+
+BGZF *hts_get_bgzfp(htsFile *fp);
+int hts_useek(htsFile *file, off_t uoffset, int where);
+off_t hts_utell(htsFile *file);
+
+static inline int sw_rec_equal(bcf_sweep_t *sw, bcf1_t *rec)
+{
+    if ( sw->lrid!=rec->rid ) return 0;
+    if ( sw->lpos!=rec->pos ) return 0;
+    if ( sw->lnals!=rec->n_allele ) return 0;
+
+    char *t = rec->d.allele[sw->lnals-1];
+    int len = t - rec->d.allele[0] + 1;
+    while ( *t ) { t++; len++; }
+    if ( sw->lals_len!=len ) return 0;
+    if ( memcmp(sw->lals,rec->d.allele[0],len) ) return 0;
+    return 1;
+}
+
+static int sw_rec_save(bcf_sweep_t *sw, bcf1_t *rec)
+{
+    sw->lrid  = rec->rid;
+    sw->lpos  = rec->pos;
+    sw->lnals = rec->n_allele;
+
+    char *t = rec->d.allele[sw->lnals-1];
+    int len = t - rec->d.allele[0] + 1;
+    while ( *t ) { t++; len++; }
+    sw->lals_len = len;
+    hts_expand(char, len, sw->mlals, sw->lals);
+    memcpy(sw->lals, rec->d.allele[0], len);
+
+    return 0; // FIXME: check for errs in this function
+}
+
+static int sw_fill_buffer(bcf_sweep_t *sw)
+{
+    if ( !sw->iidx ) return 0;
+    sw->iidx--;
+
+    int ret = hts_useek(sw->file, sw->idx[sw->iidx], 0);
+    assert( ret==0 );
+
+    sw->nrec = 0;
+    bcf1_t *rec = &sw->rec[sw->nrec];
+    while ( (ret=bcf_read1(sw->file, sw->hdr, rec))==0 )
+    {
+        bcf_unpack(rec, BCF_UN_STR);
+
+        // if not in the last block, stop at the saved record
+        if ( sw->iidx+1 < sw->nidx && sw_rec_equal(sw,rec) ) break;
+
+        sw->nrec++;
+        hts_expand0(bcf1_t, sw->nrec+1, sw->mrec, sw->rec);
+        rec = &sw->rec[sw->nrec];
+    }
+    sw_rec_save(sw, &sw->rec[0]);
+
+    return 0; // FIXME: check for errs in this function
+}
+
+bcf_sweep_t *bcf_sweep_init(const char *fname)
+{
+    bcf_sweep_t *sw = (bcf_sweep_t*) calloc(1,sizeof(bcf_sweep_t));
+    sw->file = hts_open(fname, "r");
+    sw->fp   = hts_get_bgzfp(sw->file);
+    if (sw->fp) bgzf_index_build_init(sw->fp);
+    sw->hdr  = bcf_hdr_read(sw->file);
+    sw->mrec = 1;
+    sw->rec  = (bcf1_t*) calloc(sw->mrec,(sizeof(bcf1_t)));
+    sw->block_size = 1024*1024*3;
+    sw->direction = SW_FWD;
+    return sw;
+}
+
+void bcf_sweep_destroy(bcf_sweep_t *sw)
+{
+    int i;
+    for (i=0; i<sw->mrec; i++) bcf_empty1(&sw->rec[i]);
+    free(sw->idx);
+    free(sw->rec);
+    free(sw->lals);
+    bcf_hdr_destroy(sw->hdr);
+    hts_close(sw->file);
+    free(sw);
+}
+
+static void sw_seek(bcf_sweep_t *sw, int direction)
+{
+    sw->direction = direction;
+    if ( direction==SW_FWD )
+        hts_useek(sw->file, sw->idx[0], 0);
+    else
+    {
+        sw->iidx = sw->nidx;
+        sw->nrec = 0;
+    }
+}
+
+bcf1_t *bcf_sweep_fwd(bcf_sweep_t *sw)
+{
+    if ( sw->direction==SW_BWD ) sw_seek(sw, SW_FWD);
+
+    off_t pos = hts_utell(sw->file);
+
+    bcf1_t *rec = &sw->rec[0];
+    int ret = bcf_read1(sw->file, sw->hdr, rec);
+
+    if ( ret!=0 )   // last record, get ready for sweeping backwards
+    {
+        sw->idx_done = 1;
+        if (sw->fp) sw->fp->idx_build_otf = 0;
+        sw_seek(sw, SW_BWD);
+        return NULL;
+    }
+
+    if ( !sw->idx_done )
+    {
+        if ( !sw->nidx || pos - sw->idx[sw->nidx-1] > sw->block_size )
+        {
+            sw->nidx++;
+            hts_expand(uint64_t, sw->nidx, sw->midx, sw->idx);
+            sw->idx[sw->nidx-1] = pos;
+        }
+    }
+    return rec;
+}
+
+bcf1_t *bcf_sweep_bwd(bcf_sweep_t *sw)
+{
+    if ( sw->direction==SW_FWD ) sw_seek(sw, SW_BWD);
+    if ( !sw->nrec ) sw_fill_buffer(sw);
+    if ( !sw->nrec ) return NULL;
+    return &sw->rec[ --sw->nrec ];
+}
+
+bcf_hdr_t *bcf_sweep_hdr(bcf_sweep_t *sw) { return sw->hdr; }
+
diff --git a/ext/htslib/vcfutils.c b/ext/htslib/vcfutils.c
new file mode 100644
index 0000000..890c50a
--- /dev/null
+++ b/ext/htslib/vcfutils.c
@@ -0,0 +1,854 @@
+/*  vcfutils.c -- allele-related utility functions.
+
+    Copyright (C) 2012-2018, 2020-2022 Genome Research Ltd.
+
+    Author: Petr Danecek <pd3@sanger.ac.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.  */
+
+#define HTS_BUILDING_LIBRARY // Enables HTSLIB_EXPORT, see htslib/hts_defs.h
+#include <config.h>
+#include <inttypes.h>
+
+#include "htslib/vcfutils.h"
+#include "htslib/kbitset.h"
+
+int bcf_calc_ac(const bcf_hdr_t *header, bcf1_t *line, int *ac, int which)
+{
+    int i;
+    for (i=0; i<line->n_allele; i++) ac[i]=0;
+
+    // Use INFO/AC,AN field only when asked
+    if ( which&BCF_UN_INFO )
+    {
+        bcf_unpack(line, BCF_UN_INFO);
+        int an_id = bcf_hdr_id2int(header, BCF_DT_ID, "AN");
+        int ac_id = bcf_hdr_id2int(header, BCF_DT_ID, "AC");
+        int i, an=-1, ac_len=0, ac_type=0;
+        uint8_t *ac_ptr=NULL;
+        if ( an_id>=0 && ac_id>=0 )
+        {
+            for (i=0; i<line->n_info; i++)
+            {
+                bcf_info_t *z = &line->d.info[i];
+                if ( z->key == an_id ) an = z->v1.i;
+                else if ( z->key == ac_id ) { ac_ptr = z->vptr; ac_len = z->len; ac_type = z->type; }
+            }
+        }
+        if ( an>=0 && ac_ptr )
+        {
+            if ( ac_len != line->n_allele - 1 )
+            {
+                static int warned = 0;
+                if ( !warned )
+                {
+                    hts_log_warning("Incorrect number of AC fields at %s:%"PRIhts_pos". (This message is printed only once.)\n",
+                            header->id[BCF_DT_CTG][line->rid].key, line->pos+1);
+                    warned = 1;
+                }
+                return 0;
+            }
+            int nac = 0;
+            #define BRANCH_INT(type_t, convert) {        \
+                for (i=0; i<ac_len; i++)        \
+                {                               \
+                    type_t val = convert(ac_ptr + i * sizeof(type_t)); \
+                    ac[i+1] = val;             \
+                    nac += val;                \
+                }                               \
+            }
+            switch (ac_type) {
+                case BCF_BT_INT8:  BRANCH_INT(int8_t,  le_to_i8); break;
+                case BCF_BT_INT16: BRANCH_INT(int16_t, le_to_i16); break;
+                case BCF_BT_INT32: BRANCH_INT(int32_t, le_to_i32); break;
+                default: hts_log_error("Unexpected type %d at %s:%"PRIhts_pos, ac_type, header->id[BCF_DT_CTG][line->rid].key, line->pos+1); exit(1); break;
+            }
+            #undef BRANCH_INT
+            if ( an<nac )
+            {
+                hts_log_error("Incorrect AN/AC counts at %s:%"PRIhts_pos, header->id[BCF_DT_CTG][line->rid].key, line->pos+1);
+                exit(1);
+            }
+            ac[0] = an - nac;
+            return 1;
+        }
+    }
+
+    // Split genotype fields only when asked
+    if ( which&BCF_UN_FMT )
+    {
+        int i, gt_id = bcf_hdr_id2int(header,BCF_DT_ID,"GT");
+        if ( gt_id<0 ) return 0;
+        bcf_unpack(line, BCF_UN_FMT);
+        bcf_fmt_t *fmt_gt = NULL;
+        for (i=0; i<(int)line->n_fmt; i++)
+            if ( line->d.fmt[i].id==gt_id ) { fmt_gt = &line->d.fmt[i]; break; }
+        if ( !fmt_gt ) return 0;
+        #define BRANCH_INT(type_t, convert, vector_end) { \
+            for (i=0; i<line->n_sample; i++) \
+            { \
+                uint8_t *p = (fmt_gt->p + i*fmt_gt->size); \
+                int ial; \
+                for (ial=0; ial<fmt_gt->n; ial++) \
+                { \
+                    int32_t val = convert(&p[ial * sizeof(type_t)]); \
+                    if ( val==vector_end ) break; /* smaller ploidy */ \
+                    if ( bcf_gt_is_missing(val) ) continue; /* missing allele */ \
+                    if ( val>>1 > line->n_allele ) \
+                    { \
+                        hts_log_error("Incorrect allele (\"%d\") in %s at %s:%"PRIhts_pos, (val>>1)-1, header->samples[i], header->id[BCF_DT_CTG][line->rid].key, line->pos+1); \
+                        exit(1); \
+                    } \
+                    ac[(val>>1)-1]++; \
+                } \
+            } \
+        }
+        switch (fmt_gt->type) {
+            case BCF_BT_INT8:  BRANCH_INT(int8_t,  le_to_i8,  bcf_int8_vector_end); break;
+            case BCF_BT_INT16: BRANCH_INT(int16_t, le_to_i16, bcf_int16_vector_end); break;
+            case BCF_BT_INT32: BRANCH_INT(int32_t, le_to_i32, bcf_int32_vector_end); break;
+            default: hts_log_error("Unexpected type %d at %s:%"PRIhts_pos, fmt_gt->type, header->id[BCF_DT_CTG][line->rid].key, line->pos+1); exit(1); break;
+        }
+        #undef BRANCH_INT
+        return 1;
+    }
+    return 0;
+}
+
+int bcf_gt_type(bcf_fmt_t *fmt_ptr, int isample, int *_ial, int *_jal)
+{
+    int i, nals = 0, has_ref = 0, has_alt = 0, ial = 0, jal = 0;
+    #define BRANCH_INT(type_t, convert, vector_end) { \
+        uint8_t *p = fmt_ptr->p + isample*fmt_ptr->size; \
+        for (i=0; i<fmt_ptr->n; i++) \
+        { \
+            int32_t val = convert(&p[i * sizeof(type_t)]); \
+            if ( val == vector_end ) break; /* smaller ploidy */ \
+            if ( bcf_gt_is_missing(val) ) return GT_UNKN; /* missing allele */ \
+            int tmp = val>>1; \
+            if ( tmp>1 ) \
+            { \
+                if ( !ial ) { ial = tmp; has_alt = 1; } \
+                else if ( tmp!=ial ) \
+                { \
+                    if ( tmp<ial ) \
+                    { \
+                        jal = ial; \
+                        ial = tmp; \
+                    } \
+                    else \
+                    { \
+                        jal = tmp; \
+                    } \
+                    has_alt = 2; \
+                } \
+            } \
+            else has_ref = 1; \
+            nals++; \
+        } \
+    }
+    switch (fmt_ptr->type) {
+        case BCF_BT_INT8:  BRANCH_INT(int8_t,  le_to_i8,  bcf_int8_vector_end); break;
+        case BCF_BT_INT16: BRANCH_INT(int16_t, le_to_i16, bcf_int16_vector_end); break;
+        case BCF_BT_INT32: BRANCH_INT(int32_t, le_to_i32, bcf_int32_vector_end); break;
+        default: hts_log_error("Unexpected type %d", fmt_ptr->type); exit(1); break;
+    }
+    #undef BRANCH_INT
+
+    if ( _ial ) *_ial = ial>0 ? ial-1 : ial;
+    if ( _jal ) *_jal = jal>0 ? jal-1 : jal;
+    if ( !nals ) return GT_UNKN;
+    if ( nals==1 )
+        return has_ref ? GT_HAPL_R : GT_HAPL_A;
+    if ( !has_ref )
+        return has_alt==1 ? GT_HOM_AA : GT_HET_AA;
+    if ( !has_alt )
+        return GT_HOM_RR;
+    return GT_HET_RA;
+}
+
+int bcf_trim_alleles(const bcf_hdr_t *header, bcf1_t *line)
+{
+    int i, ret = 0, nrm = 0;
+    kbitset_t *rm_set = NULL;
+    bcf_fmt_t *gt = bcf_get_fmt(header, line, "GT");
+    if ( !gt ) return 0;
+
+    int *ac = (int*) calloc(line->n_allele,sizeof(int));
+
+    // check if all alleles are populated
+    #define BRANCH(type_t, convert, vector_end) { \
+        for (i=0; i<line->n_sample; i++) \
+        { \
+            uint8_t *p = gt->p + i*gt->size; \
+            int ial; \
+            for (ial=0; ial<gt->n; ial++) \
+            { \
+                int32_t val = convert(&p[ial * sizeof(type_t)]); \
+                if ( val==vector_end ) break; /* smaller ploidy */ \
+                if ( bcf_gt_is_missing(val) ) continue; /* missing allele */ \
+                if ( (val>>1)-1 >= line->n_allele ) { \
+                    hts_log_error("Allele index is out of bounds at %s:%"PRIhts_pos, header->id[BCF_DT_CTG][line->rid].key, line->pos+1); \
+                    ret = -1; \
+                    goto clean; \
+                } \
+                ac[(val>>1)-1]++; \
+            } \
+        } \
+    }
+    switch (gt->type) {
+        case BCF_BT_INT8:  BRANCH(int8_t,  le_to_i8,  bcf_int8_vector_end); break;
+        case BCF_BT_INT16: BRANCH(int16_t, le_to_i16, bcf_int16_vector_end); break;
+        case BCF_BT_INT32: BRANCH(int32_t, le_to_i32, bcf_int32_vector_end); break;
+        default: hts_log_error("Unexpected GT %d at %s:%"PRIhts_pos,
+            gt->type, header->id[BCF_DT_CTG][line->rid].key, line->pos + 1);
+            goto clean;
+    }
+    #undef BRANCH
+
+    rm_set = kbs_init(line->n_allele);
+    for (i=1; i<line->n_allele; i++) {
+        if ( !ac[i] ) { kbs_insert(rm_set, i); nrm++; }
+    }
+
+    if (nrm) {
+        if (bcf_remove_allele_set(header, line, rm_set))
+            ret = -2;
+    }
+
+clean:
+    free(ac);
+    if (rm_set) kbs_destroy(rm_set);
+    return ret ? ret : nrm;
+}
+
+int bcf_remove_alleles(const bcf_hdr_t *header, bcf1_t *line, int rm_mask)
+{
+    int i;
+    kbitset_t *rm_set = kbs_init(line->n_allele);
+    for (i=1; i<line->n_allele; i++)
+        if ( rm_mask & 1<<i ) kbs_insert(rm_set, i);
+
+    bcf_remove_allele_set(header, line, rm_set);
+    kbs_destroy(rm_set);
+
+    return 0; // FIXME: check for errs in this function
+}
+
+int bcf_remove_allele_set(const bcf_hdr_t *header, bcf1_t *line, const struct kbitset_t *rm_set)
+{
+    int *map = (int*) calloc(line->n_allele, sizeof(int));
+    uint8_t *dat = NULL;
+
+    bcf_unpack(line, BCF_UN_ALL);
+
+    // create map of indexes from old to new ALT numbering and modify ALT
+    kstring_t str = {0,0,0};
+    kputs(line->d.allele[0], &str);
+
+    int nrm = 0, i,j;  // i: ori alleles, j: new alleles
+    for (i=1, j=1; i<line->n_allele; i++)
+    {
+        if ( kbs_exists(rm_set, i) )
+        {
+            // remove this allele
+            line->d.allele[i] = NULL;
+            nrm++;
+            continue;
+        }
+        kputc(',', &str);
+        kputs(line->d.allele[i], &str);
+        map[i] = j;
+        j++;
+    }
+    if ( !nrm ) goto clean;
+
+    int nR_ori = line->n_allele;
+    int nR_new = line->n_allele-nrm;
+    if ( nR_new<=0 ) // should not be able to remove reference allele
+    {
+        hts_log_error("Cannot remove reference allele at %s:%"PRIhts_pos" [%d]",
+            bcf_seqname_safe(header,line), line->pos+1, nR_new);
+        goto err;
+    }
+    int nA_ori = nR_ori-1;
+    int nA_new = nR_new-1;
+
+    int nG_ori = nR_ori*(nR_ori + 1)/2;
+    int nG_new = nR_new*(nR_new + 1)/2;
+
+    bcf_update_alleles_str(header, line, str.s);
+
+    // remove from Number=G, Number=R and Number=A INFO fields.
+    int mdat = 0, ndat = 0, mdat_bytes = 0, nret;
+    for (i=0; i<line->n_info; i++)
+    {
+        bcf_info_t *info = &line->d.info[i];
+        int vlen = bcf_hdr_id2length(header,BCF_HL_INFO,info->key);
+
+        if ( vlen!=BCF_VL_A && vlen!=BCF_VL_G && vlen!=BCF_VL_R ) continue; // no need to change
+
+        int type = bcf_hdr_id2type(header,BCF_HL_INFO,info->key);
+        if ( type==BCF_HT_FLAG ) continue;
+        int size = 1;
+        if ( type==BCF_HT_REAL || type==BCF_HT_INT ) size = 4;
+
+        mdat = mdat_bytes / size;
+        nret = bcf_get_info_values(header, line, bcf_hdr_int2id(header,BCF_DT_ID,info->key), (void**)&dat, &mdat, type);
+        mdat_bytes = mdat * size;
+        if ( nret<0 )
+        {
+            hts_log_error("Could not access INFO/%s at %s:%"PRIhts_pos" [%d]",
+                bcf_hdr_int2id(header,BCF_DT_ID,info->key), bcf_seqname_safe(header,line), line->pos+1, nret);
+            goto err;
+        }
+        if ( nret==0 ) continue; // no data for this tag
+
+        if ( type==BCF_HT_STR )
+        {
+            str.l = 0;
+            char *ss = (char*) dat, *se = (char*) dat, s = ss[0];
+            if ( vlen==BCF_VL_A || vlen==BCF_VL_R )
+            {
+                int nexp, inc = 0;
+                if ( vlen==BCF_VL_A )
+                {
+                    nexp = nA_ori;
+                    inc  = 1;
+                }
+                else
+                    nexp = nR_ori;
+                for (j=0; j<nexp; j++)
+                {
+                    if ( !*se ) break;
+                    while ( *se && *se!=',' ) se++;
+                    if ( kbs_exists(rm_set, j+inc) )
+                    {
+                        if ( *se ) se++;
+                        ss = se;
+                        continue;
+                    }
+                    if ( str.l ) kputc(',',&str);
+                    kputsn(ss,se-ss,&str);
+                    if ( *se ) se++;
+                    ss = se;
+                }
+                if ( j==1 && s == '.' ) continue; // missing
+                if ( j!=nexp )
+                {
+                    hts_log_error("Unexpected number of values in INFO/%s at %s:%"PRIhts_pos"; expected Number=%c=%d, but found %d",
+                        bcf_hdr_int2id(header,BCF_DT_ID,info->key), bcf_seqname_safe(header,line), line->pos+1, vlen==BCF_VL_A ? 'A' : 'R', nexp, j);
+                    goto err;
+                }
+            }
+            else    // Number=G, assuming diploid genotype
+            {
+                int k = 0, n = 0;
+                for (j=0; j<nR_ori; j++)
+                {
+                    for (k=0; k<=j; k++)
+                    {
+                        if ( !*se ) break;
+                        while ( *se && *se!=',' ) se++;
+                        n++;
+                        if ( kbs_exists(rm_set, j) || kbs_exists(rm_set, k) )
+                        {
+                            if ( *se ) se++;
+                            ss = se;
+                            continue;
+                        }
+                        if ( str.l ) kputc(',',&str);
+                        kputsn(ss,se-ss,&str);
+                        if ( *se ) se++;
+                        ss = se;
+                    }
+                    if ( !*se ) break;
+                }
+                if ( n==1 && s == '.' ) continue; // missing
+                if ( n!=nG_ori )
+                {
+                    hts_log_error("Unexpected number of values in INFO/%s at %s:%"PRIhts_pos"; expected Number=G=%d, but found %d",
+                        bcf_hdr_int2id(header,BCF_DT_ID,info->key), bcf_seqname_safe(header,line), line->pos+1, nG_ori, n);
+                    goto err;
+                }
+            }
+
+            nret = bcf_update_info(header, line, bcf_hdr_int2id(header,BCF_DT_ID,info->key), (void*)str.s, str.l, type);
+            if ( nret<0 )
+            {
+                hts_log_error("Could not update INFO/%s at %s:%"PRIhts_pos" [%d]",
+                    bcf_hdr_int2id(header,BCF_DT_ID,info->key), bcf_seqname_safe(header,line), line->pos+1, nret);
+                goto err;
+            }
+            continue;
+        }
+
+        if (nret==1) // could be missing - check
+        {
+            int missing = 0;
+            #define BRANCH(type_t, convert, is_missing) { \
+                type_t val = convert(info->vptr); \
+                if ( is_missing ) missing = 1; \
+            }
+            switch (info->type) {
+                case BCF_BT_INT8:  BRANCH(int8_t, le_to_i8,  val==bcf_int8_missing); break;
+                case BCF_BT_INT16: BRANCH(int16_t, le_to_i16, val==bcf_int16_missing); break;
+                case BCF_BT_INT32: BRANCH(int32_t, le_to_i32, val==bcf_int32_missing); break;
+                case BCF_BT_FLOAT: BRANCH(float,   le_to_float, bcf_float_is_missing(val)); break;
+                default: hts_log_error("Unexpected type %d", info->type); goto err;
+            }
+            #undef BRANCH
+            if (missing) continue; // could remove this INFO tag?
+        }
+
+        if ( vlen==BCF_VL_A || vlen==BCF_VL_R )
+        {
+            int inc = 0, ntop;
+            if ( vlen==BCF_VL_A )
+            {
+                if ( nret!=nA_ori )
+                {
+                    hts_log_error("Unexpected number of values in INFO/%s at %s:%"PRIhts_pos"; expected Number=A=%d, but found %d",
+                        bcf_hdr_int2id(header,BCF_DT_ID,info->key), bcf_seqname_safe(header,line), line->pos+1, nA_ori, nret);
+                    goto err;
+                }
+                ntop = nA_ori;
+                ndat = nA_new;
+                inc  = 1;
+            }
+            else
+            {
+                if ( nret!=nR_ori )
+                {
+                    hts_log_error("Unexpected number of values in INFO/%s at %s:%"PRIhts_pos"; expected Number=R=%d, but found %d",
+                        bcf_hdr_int2id(header,BCF_DT_ID,info->key), bcf_seqname_safe(header,line), line->pos+1, nR_ori, nret);
+                    goto err;
+                }
+                ntop = nR_ori;
+                ndat = nR_new;
+            }
+            int k = 0;
+
+            #define BRANCH(type_t,is_vector_end) \
+            { \
+                type_t *ptr = (type_t*) dat; \
+                int size = sizeof(type_t); \
+                for (j=0; j<ntop; j++) /* j:ori, k:new */ \
+                { \
+                    if ( is_vector_end ) { memcpy(dat+k*size, dat+j*size, size); break; } \
+                    if ( kbs_exists(rm_set, j+inc) ) continue; \
+                    if ( j!=k ) memcpy(dat+k*size, dat+j*size, size); \
+                    k++; \
+                } \
+            }
+            switch (type)
+            {
+                case BCF_HT_INT:  BRANCH(int32_t,ptr[j]==bcf_int32_vector_end); break;
+                case BCF_HT_REAL: BRANCH(float,bcf_float_is_vector_end(ptr[j])); break;
+            }
+            #undef BRANCH
+        }
+        else    // Number=G
+        {
+            if ( nret!=nG_ori )
+            {
+                hts_log_error("Unexpected number of values in INFO/%s at %s:%"PRIhts_pos"; expected Number=G=%d, but found %d",
+                    bcf_hdr_int2id(header,BCF_DT_ID,info->key), bcf_seqname_safe(header,line), line->pos+1, nG_ori, nret);
+                goto err;
+            }
+            int k, l_ori = -1, l_new = 0;
+            ndat = nG_new;
+
+            #define BRANCH(type_t,is_vector_end) \
+            { \
+                type_t *ptr = (type_t*) dat; \
+                int size = sizeof(type_t); \
+                for (j=0; j<nR_ori; j++) \
+                { \
+                    for (k=0; k<=j; k++) \
+                    { \
+                        l_ori++; \
+                        if ( is_vector_end ) { memcpy(dat+l_new*size, dat+l_ori*size, size); break; } \
+                        if ( kbs_exists(rm_set, j) || kbs_exists(rm_set, k) ) continue; \
+                        if ( l_ori!=l_new ) memcpy(dat+l_new*size, dat+l_ori*size, size); \
+                        l_new++; \
+                    } \
+                } \
+            }
+            switch (type)
+            {
+                case BCF_HT_INT:  BRANCH(int32_t,ptr[l_ori]==bcf_int32_vector_end); break;
+                case BCF_HT_REAL: BRANCH(float,bcf_float_is_vector_end(ptr[l_ori])); break;
+            }
+            #undef BRANCH
+        }
+
+        nret = bcf_update_info(header, line, bcf_hdr_int2id(header,BCF_DT_ID,info->key), (void*)dat, ndat, type);
+        if ( nret<0 )
+        {
+            hts_log_error("Could not update INFO/%s at %s:%"PRIhts_pos" [%d]",
+                bcf_hdr_int2id(header,BCF_DT_ID,info->key), bcf_seqname_safe(header,line), line->pos+1, nret);
+            goto err;
+        }
+    }
+
+    // Update GT fields, the allele indexes might have changed
+    for (i=1; i<nR_ori; i++) if ( map[i]!=i ) break;
+    if ( i<nR_ori )
+    {
+        mdat = mdat_bytes / 4;  // sizeof(int32_t)
+        nret = bcf_get_genotypes(header,line,(void**)&dat,&mdat);
+        mdat_bytes = mdat * 4;
+        if ( nret>0 )
+        {
+            nret /= line->n_sample;
+            int32_t *ptr = (int32_t*) dat;
+            for (i=0; i<line->n_sample; i++)
+            {
+                for (j=0; j<nret; j++)
+                {
+                    if ( bcf_gt_is_missing(ptr[j]) ) continue;
+                    if ( ptr[j]==bcf_int32_vector_end ) break;
+                    int al = bcf_gt_allele(ptr[j]);
+                    if ( !( al<nR_ori && map[al]>=0 ) )
+                    {
+                        hts_log_error("Problem updating genotypes at %s:%"PRIhts_pos" [ al<nR_ori && map[al]>=0 :: al=%d,nR_ori=%d,map[al]=%d ]",
+                            bcf_seqname_safe(header,line), line->pos+1, al, nR_ori, map[al]);
+                        goto err;
+                    }
+                    // if an allele other than the reference is mapped to 0, it has been removed,
+                    // so translate it to 'missing', while preserving the phasing bit
+                    ptr[j] = ((al>0 && !map[al]) ? bcf_gt_missing : (map[al]+1)<<1) | (ptr[j]&1);
+                }
+                ptr += nret;
+            }
+            nret = bcf_update_genotypes(header, line, (void*)dat, nret*line->n_sample);
+            if ( nret<0 )
+            {
+                hts_log_error("Could not update FORMAT/GT at %s:%"PRIhts_pos" [%d]",
+                    bcf_seqname_safe(header,line), line->pos+1, nret);
+                goto err;
+            }
+        }
+    }
+
+    // Remove from Number=G, Number=R and Number=A FORMAT fields.
+    // Assuming haploid or diploid GTs
+    for (i=0; i<line->n_fmt; i++)
+    {
+        bcf_fmt_t *fmt = &line->d.fmt[i];
+        int vlen = bcf_hdr_id2length(header,BCF_HL_FMT,fmt->id);
+
+        if ( vlen!=BCF_VL_A && vlen!=BCF_VL_G && vlen!=BCF_VL_R ) continue; // no need to change
+
+        int type = bcf_hdr_id2type(header,BCF_HL_FMT,fmt->id);
+        if ( type==BCF_HT_FLAG ) continue;
+
+        int size = 1;
+        if ( type==BCF_HT_REAL || type==BCF_HT_INT ) size = 4;
+
+        mdat = mdat_bytes / size;
+        nret = bcf_get_format_values(header, line, bcf_hdr_int2id(header,BCF_DT_ID,fmt->id), (void**)&dat, &mdat, type);
+        mdat_bytes = mdat * size;
+        if ( nret<0 )
+        {
+            hts_log_error("Could not access FORMAT/%s at %s:%"PRIhts_pos" [%d]",
+                bcf_hdr_int2id(header,BCF_DT_ID,fmt->id), bcf_seqname_safe(header,line), line->pos+1, nret);
+            goto err;
+        }
+        if ( nret == 0 ) continue; // no data for this tag
+
+        if ( type==BCF_HT_STR )
+        {
+            int size = nret/line->n_sample;     // number of bytes per sample
+            str.l = 0;
+            if ( vlen==BCF_VL_A || vlen==BCF_VL_R )
+            {
+                int nexp, inc = 0;
+                if ( vlen==BCF_VL_A )
+                {
+                    nexp = nA_ori;
+                    inc  = 1;
+                }
+                else
+                    nexp = nR_ori;
+                for (j=0; j<line->n_sample; j++)
+                {
+                    char *ss = ((char*)dat) + j*size, *se = ss + size, *ptr = ss, s = ss[0];
+                    int k_src = 0, k_dst = 0, l = str.l;
+                    for (k_src=0; k_src<nexp; k_src++)
+                    {
+                        if ( ptr>=se || !*ptr) break;
+                        while ( ptr<se && *ptr && *ptr!=',' ) ptr++;
+                        if ( kbs_exists(rm_set, k_src+inc) )
+                        {
+                            ss = ++ptr;
+                            continue;
+                        }
+                        if ( k_dst ) kputc(',',&str);
+                        kputsn(ss,ptr-ss,&str);
+                        ss = ++ptr;
+                        k_dst++;
+                    }
+                    if ( k_src==1 && s == '.' ) continue; // missing
+                    if ( k_src!=nexp )
+                    {
+                        hts_log_error("Unexpected number of values in FORMAT/%s at %s:%"PRIhts_pos"; expected Number=%c=%d, but found %d",
+                            bcf_hdr_int2id(header,BCF_DT_ID,fmt->id), bcf_seqname_safe(header,line), line->pos+1, vlen==BCF_VL_A ? 'A' : 'R', nexp, k_src);
+                        goto err;
+                    }
+                    l = str.l - l;
+                    for (; l<size; l++) kputc(0, &str);
+                }
+            }
+            else    // Number=G, diploid or haploid
+            {
+                for (j=0; j<line->n_sample; j++)
+                {
+                    char *ss = ((char*)dat) + j*size, *se = ss + size, *ptr = ss, s = ss[0];
+                    int k_src = 0, k_dst = 0, l = str.l;
+                    int nexp = 0; // diploid or haploid?
+                    while ( ptr<se )
+                    {
+                        if ( !*ptr ) break;
+                        if ( *ptr==',' ) nexp++;
+                        ptr++;
+                    }
+                    if ( ptr!=ss ) nexp++;
+                    if ( nexp==1 && s == '.' ) continue; // missing
+                    if ( nexp!=nG_ori && nexp!=nR_ori )
+                    {
+                        hts_log_error("Unexpected number of values in FORMAT/%s at %s:%"PRIhts_pos"; expected Number=G=%d(diploid) or %d(haploid), but found %d",
+                            bcf_hdr_int2id(header,BCF_DT_ID,fmt->id), bcf_seqname_safe(header,line), line->pos+1, nG_ori, nR_ori, nexp);
+                        goto err;
+                    }
+                    ptr = ss;
+                    if ( nexp==nG_ori ) // diploid
+                    {
+                        int ia, ib;
+                        for (ia=0; ia<nR_ori; ia++)
+                        {
+                            for (ib=0; ib<=ia; ib++)
+                            {
+                                if ( ptr>=se || !*ptr ) break;
+                                while ( ptr<se && *ptr && *ptr!=',' ) ptr++;
+                                if ( kbs_exists(rm_set, ia) || kbs_exists(rm_set, ib) )
+                                {
+                                    ss = ++ptr;
+                                    continue;
+                                }
+                                if ( k_dst ) kputc(',',&str);
+                                kputsn(ss,ptr-ss,&str);
+                                ss = ++ptr;
+                                k_dst++;
+                            }
+                            if ( ptr>=se || !*ptr ) break;
+                        }
+                    }
+                    else    // haploid
+                    {
+                        for (k_src=0; k_src<nR_ori; k_src++)
+                        {
+                            if ( ptr>=se || !*ptr ) break;
+                            while ( ptr<se && *ptr && *ptr!=',' ) ptr++;
+                            if ( kbs_exists(rm_set, k_src) )
+                            {
+                                ss = ++ptr;
+                                continue;
+                            }
+                            if ( k_dst ) kputc(',',&str);
+                            kputsn(ss,ptr-ss,&str);
+                            ss = ++ptr;
+                            k_dst++;
+                        }
+                        if ( k_src!=nR_ori )
+                        {
+                            hts_log_error("Unexpected number of values in FORMAT/%s at %s:%"PRIhts_pos"; expected Number=G=%d(haploid), but found %d",
+                                bcf_hdr_int2id(header,BCF_DT_ID,fmt->id), bcf_seqname_safe(header,line), line->pos+1, nR_ori, k_src);
+                            goto err;
+                        }
+                        l = str.l - l;
+                        for (; l<size; l++) kputc(0, &str);
+                    }
+                }
+            }
+            nret = bcf_update_format(header, line, bcf_hdr_int2id(header,BCF_DT_ID,fmt->id), (void*)str.s, str.l, type);
+            if ( nret<0 )
+            {
+                hts_log_error("Could not update FORMAT/%s at %s:%"PRIhts_pos" [%d]",
+                    bcf_hdr_int2id(header,BCF_DT_ID,fmt->id), bcf_seqname_safe(header,line), line->pos+1, nret);
+                goto err;
+            }
+            continue;
+        }
+
+        int nori = nret / line->n_sample;
+        if ( nori==1 && !(vlen==BCF_VL_A && nori==nA_ori) ) // all values may be missing - check
+        {
+            int all_missing = 1;
+            #define BRANCH(type_t, convert, is_missing) { \
+                for (j=0; j<line->n_sample; j++) \
+                { \
+                    type_t val = convert(fmt->p + j*fmt->size); \
+                    if ( !(is_missing)) { all_missing = 0; break; } \
+                } \
+            }
+            switch (fmt->type) {
+                case BCF_BT_INT8:  BRANCH(int8_t,  le_to_i8, val==bcf_int8_missing); break;
+                case BCF_BT_INT16: BRANCH(int16_t, le_to_i16, val==bcf_int16_missing); break;
+                case BCF_BT_INT32: BRANCH(int32_t, le_to_i32, val==bcf_int32_missing); break;
+                case BCF_BT_FLOAT: BRANCH(float,   le_to_float, bcf_float_is_missing(val)); break;
+                default: hts_log_error("Unexpected type %d", fmt->type); goto err;
+            }
+            #undef BRANCH
+            if (all_missing) continue; // could remove this FORMAT tag?
+        }
+
+        if ( vlen==BCF_VL_A || vlen==BCF_VL_R || (vlen==BCF_VL_G && nori==nR_ori) ) // Number=A, R or haploid Number=G
+        {
+            int inc = 0, nnew;
+            if ( vlen==BCF_VL_A )
+            {
+                if ( nori!=nA_ori )
+                {
+                    hts_log_error("Unexpected number of values in FORMAT/%s at %s:%"PRIhts_pos"; expected Number=A=%d, but found %d",
+                        bcf_hdr_int2id(header,BCF_DT_ID,fmt->id), bcf_seqname_safe(header,line), line->pos+1, nA_ori, nori);
+                    goto err;
+                }
+                ndat = nA_new*line->n_sample;
+                nnew = nA_new;
+                inc  = 1;
+            }
+            else
+            {
+                if ( nori!=nR_ori )
+                {
+                    hts_log_error("Unexpected number of values in FORMAT/%s at %s:%"PRIhts_pos"; expected Number=R=%d, but found %d",
+                        bcf_hdr_int2id(header,BCF_DT_ID,fmt->id), bcf_seqname_safe(header,line), line->pos+1, nR_ori, nori);
+                    goto err;
+                }
+                ndat = nR_new*line->n_sample;
+                nnew = nR_new;
+            }
+
+            #define BRANCH(type_t,is_vector_end,set_missing) \
+            { \
+                for (j=0; j<line->n_sample; j++) \
+                { \
+                    type_t *ptr_src = ((type_t*)dat) + j*nori; \
+                    type_t *ptr_dst = ((type_t*)dat) + j*nnew; \
+                    int size = sizeof(type_t); \
+                    int k_src, k_dst = 0; \
+                    for (k_src=0; k_src<nori; k_src++) \
+                    { \
+                        if ( is_vector_end ) \
+                        { \
+                            if ( k_dst ) memcpy(ptr_dst+k_dst, ptr_src+k_src, size); \
+                            else set_missing; \
+                            break; \
+                        } \
+                        if ( kbs_exists(rm_set, k_src+inc) ) continue; \
+                        memcpy(ptr_dst+k_dst, ptr_src+k_src, size); \
+                        k_dst++; \
+                    } \
+                } \
+            }
+            switch (type)
+            {
+                case BCF_HT_INT:  BRANCH(int32_t,ptr_src[k_src]==bcf_int32_vector_end,ptr_dst[k_dst]=bcf_int32_missing); break;
+                case BCF_HT_REAL: BRANCH(float,bcf_float_is_vector_end(ptr_src[k_src]),bcf_float_set_missing(ptr_dst[k_dst])); break;
+            }
+            #undef BRANCH
+        }
+        else    // Number=G, diploid or mixture of haploid+diploid
+        {
+            if ( nori!=nG_ori )
+            {
+                hts_log_error("Unexpected number of values in FORMAT/%s at %s:%"PRIhts_pos"; expected Number=G=%d, but found %d",
+                    bcf_hdr_int2id(header,BCF_DT_ID,fmt->id), bcf_seqname_safe(header,line), line->pos+1, nG_ori, nori);
+                goto err;
+            }
+            ndat = nG_new*line->n_sample;
+
+            #define BRANCH(type_t,is_vector_end) \
+            { \
+                for (j=0; j<line->n_sample; j++) \
+                { \
+                    type_t *ptr_src = ((type_t*)dat) + j*nori; \
+                    type_t *ptr_dst = ((type_t*)dat) + j*nG_new; \
+                    int size = sizeof(type_t); \
+                    int ia, ib, k_dst = 0, k_src; \
+                    int nset = 0;   /* haploid or diploid? */ \
+                    for (k_src=0; k_src<nG_ori; k_src++) { if ( is_vector_end ) break; nset++; } \
+                    if ( nset==nR_ori ) /* haploid */ \
+                    { \
+                        for (k_src=0; k_src<nR_ori; k_src++) \
+                        { \
+                            if ( kbs_exists(rm_set, k_src) ) continue; \
+                            memcpy(ptr_dst+k_dst, ptr_src+k_src, size); \
+                            k_dst++; \
+                        } \
+                        memcpy(ptr_dst+k_dst, ptr_src+k_src, size); \
+                    } \
+                    else /* diploid */ \
+                    { \
+                        k_src = -1; \
+                        for (ia=0; ia<nR_ori; ia++) \
+                        { \
+                            for (ib=0; ib<=ia; ib++) \
+                            { \
+                                k_src++; \
+                                if ( is_vector_end ) { memcpy(ptr_dst+k_dst, ptr_src+k_src, size); ia = nR_ori; break; }  \
+                                if ( kbs_exists(rm_set, ia) || kbs_exists(rm_set, ib) ) continue; \
+                                memcpy(ptr_dst+k_dst, ptr_src+k_src, size); \
+                                k_dst++; \
+                            } \
+                        } \
+                    } \
+                } \
+            }
+            switch (type)
+            {
+                case BCF_HT_INT:  BRANCH(int32_t,ptr_src[k_src]==bcf_int32_vector_end); break;
+                case BCF_HT_REAL: BRANCH(float,bcf_float_is_vector_end(ptr_src[k_src])); break;
+            }
+            #undef BRANCH
+        }
+        nret = bcf_update_format(header, line, bcf_hdr_int2id(header,BCF_DT_ID,fmt->id), (void*)dat, ndat, type);
+        if ( nret<0 )
+        {
+            hts_log_error("Could not update FORMAT/%s at %s:%"PRIhts_pos" [%d]",
+                bcf_hdr_int2id(header,BCF_DT_ID,fmt->id), bcf_seqname_safe(header,line), line->pos+1, nret);
+            goto err;
+        }
+    }
+
+clean:
+    free(str.s);
+    free(map);
+    free(dat);
+    return 0;
+
+err:
+    free(str.s);
+    free(map);
+    free(dat);
+    return -1;
+}
+
diff --git a/ext/htslib/version.sh b/ext/htslib/version.sh
new file mode 100755
index 0000000..f35234c
--- /dev/null
+++ b/ext/htslib/version.sh
@@ -0,0 +1,59 @@
+#!/bin/sh
+# version.sh -- Script to build the htslib version string
+#
+#     Author : James Bonfield <jkb@sanger.ac.uk>
+#
+#     Copyright (C) 2017-2018, 2021 Genome Research Ltd.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+# Master version, for use in tarballs or non-git source copies
+VERSION=1.21
+
+# If we have a git clone, then check against the current tag
+srcdir=${0%/version.sh}
+if [ -e $srcdir/.git ]
+then
+    # If we ever get to 10.x this will need to be more liberal
+    v=`cd $srcdir && git describe --always --match '[0-9].[0-9]*' --dirty`
+    case $v in
+        [0-9]*.[0-9]*) VERSION="$v" ;;
+        [0-9a-f][0-9a-f]*) VERSION="$VERSION-1-g$v" ;;
+    esac
+fi
+
+# Numeric version is for use in .dylib or .so libraries
+#
+# Follows the same logic from the Makefile commit c2e93911
+# as non-numeric versions get bumped to patch level 255 to indicate
+# an unknown value.
+if [ "$1" = "numeric" ]
+then
+    v1=`expr "$VERSION" : '\([0-9]*\)'`
+    v2=`expr "$VERSION" : '[0-9]*.\([0-9]*\)'`
+    v3=`expr "$VERSION" : '[0-9]*.[0-9]*.\([0-9]*\)'`
+    if [ -z "`expr "$VERSION" : '\([0-9.]*\)$'`" ]
+    then
+        VERSION="$v1.$v2.255"
+    else
+        VERSION="$v1.$v2${v3:+.}$v3"
+    fi
+fi
+
+echo $VERSION
diff --git a/ext/klib/.gitignore b/ext/klib/.gitignore
new file mode 100644
index 0000000..010a8eb
--- /dev/null
+++ b/ext/klib/.gitignore
@@ -0,0 +1,40 @@
+# General
+*.a
+*.dSYM/
+*.la
+*.lo
+*.o
+*.opensdf
+*.orig
+*.sdf
+*.suo
+*.swp
+*.tests
+*.vcxproj.filters
+*.vcxproj.user
+*~
+.git
+TAGS
+
+# Mac/Xcode-specfic
+xcuserdata
+contents.xcworkspacedata
+.DS_Store
+._*
+
+# Test byproducts
+test/kbtree_test
+test/khash_keith
+test/khash_keith2
+test/khash_test
+test/klist_test
+test/kmin_test
+test/kseq_bench
+test/kseq_bench2
+test/kseq_test
+test/ksort_test
+test/ksort_test-stl
+test/kstring_bench
+test/kstring_bench2
+test/kstring_test
+test/kvec_test
diff --git a/ext/klib/LICENSE.txt b/ext/klib/LICENSE.txt
new file mode 100644
index 0000000..0a996fa
--- /dev/null
+++ b/ext/klib/LICENSE.txt
@@ -0,0 +1,23 @@
+The MIT License
+
+Copyright (c) 2008-     Attractive Chaos <attractor@live.co.uk>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/ext/klib/README.md b/ext/klib/README.md
new file mode 100644
index 0000000..7c7e216
--- /dev/null
+++ b/ext/klib/README.md
@@ -0,0 +1,243 @@
+# Klib: a Generic Library in C
+
+## <a name="overview"></a>Overview
+
+Klib is a standalone and lightweight C library distributed under [MIT/X11
+license][1]. Most components are independent of external libraries, except the
+standard C library, and independent of each other. To use a component of this
+library, you only need to copy a couple of files to your source code tree
+without worrying about library dependencies.
+
+Klib strives for efficiency and a small memory footprint. Some components, such
+as khash.h, kbtree.h, ksort.h and kvec.h, are among the most efficient
+implementations of similar algorithms or data structures in all programming
+languages, in terms of both speed and memory use.
+
+A new documentation is available [here](http://attractivechaos.github.io/klib/)
+which includes most information in this README file.
+
+#### Common components
+
+* [khash.h][khash]: generic [hash table][2] with open addressing.
+* [kbtree.h][kbtree]: generic search tree based on [B-tree][3].
+* [kavl.h][kavl]: generic intrusive [AVL tree][wiki-avl].
+* [ksort.h][ksort]: generic sort, including [introsort][4], [merge sort][5], [heap sort][6], [comb sort][7], [Knuth shuffle][8] and the [k-small][9] algorithm.
+* [kseq.h][kseq]: generic stream buffer and a [FASTA][10]/[FASTQ][11] format parser.
+* kvec.h: generic dynamic array.
+* klist.h: generic single-linked list and [memory pool][12].
+* kstring.{h,c}: basic string library.
+* kmath.{h,c}: numerical routines including [MT19937-64][13] [pseudorandom generator][14], basic [nonlinear programming][15] and a few special math functions.
+* [ketopt.h][ketopt]: portable command-line argument parser with getopt\_long-like API.
+
+#### Components for more specific use cases
+
+* ksa.c: constructing [suffix arrays][16] for strings with multiple sentinels, based on a revised [SAIS algorithm][17].
+* knetfile.{h,c}: random access to remote files on HTTP or FTP.
+* kopen.c: smart stream opening.
+* khmm.{h,c}: basic [HMM][18] library.
+* ksw.(h,c}: Striped [Smith-Waterman algorithm][19].
+* knhx.{h,c}: [Newick tree format][20] parser.
+
+
+## <a name="methodology"></a>Methodology
+
+For the implementation of generic [containers][21], klib extensively uses C
+macros. To use these data structures, we usually need to instantiate methods by
+expanding a long macro. This makes the source code look unusual or even ugly
+and adds difficulty to debugging. Unfortunately, for efficient generic
+programming in C that lacks [template][22], using macros is the only
+solution. Only with macros, we can write a generic container which, once
+instantiated, compete with a type-specific container in efficiency. Some
+generic libraries in C, such as [Glib][23], use the `void*` type to implement
+containers. These implementations are usually slower and use more memory than
+klib (see [this benchmark][31]).
+
+To effectively use klib, it is important to understand how it achieves generic
+programming. We will use the hash table library as an example:
+
+    #include "khash.h"
+    KHASH_MAP_INIT_INT(m32, char)        // instantiate structs and methods
+    int main() {
+        int ret, is_missing;
+        khint_t k;
+        khash_t(m32) *h = kh_init(m32);  // allocate a hash table
+        k = kh_put(m32, h, 5, &ret);     // insert a key to the hash table
+        if (!ret) kh_del(m32, h, k);
+        kh_value(h, k) = 10;             // set the value
+        k = kh_get(m32, h, 10);          // query the hash table
+        is_missing = (k == kh_end(h));   // test if the key is present
+        k = kh_get(m32, h, 5);
+        kh_del(m32, h, k);               // remove a key-value pair
+        for (k = kh_begin(h); k != kh_end(h); ++k)  // traverse
+            if (kh_exist(h, k))          // test if a bucket contains data
+    			kh_value(h, k) = 1;
+        kh_destroy(m32, h);              // deallocate the hash table
+        return 0;
+    }
+
+In this example, the second line instantiates a hash table with `unsigned` as
+the key type and `char` as the value type. `m32` names such a type of hash table.
+All types and functions associated with this name are macros, which will be
+explained later. Macro `kh_init()` initiates a hash table and `kh_destroy()`
+frees it. `kh_put()` inserts a key and returns the iterator (or the position)
+in the hash table. `kh_get()` and `kh_del()` get a key and delete an element,
+respectively. Macro `kh_exist()` tests if an iterator (or a position) is filled
+with data.
+
+An immediate question is this piece of code does not look like a valid C
+program (e.g. lacking semicolon, assignment to an _apparent_ function call and
+_apparent_ undefined `m32` 'variable'). To understand why the code is correct,
+let's go a bit further into the source code of `khash.h`, whose skeleton looks
+like:
+
+    #define KHASH_INIT(name, SCOPE, key_t, val_t, is_map, _hashf, _hasheq) \
+      typedef struct { \
+        int n_buckets, size, n_occupied, upper_bound; \
+        unsigned *flags; \
+        key_t *keys; \
+        val_t *vals; \
+      } kh_##name##_t; \
+      SCOPE inline kh_##name##_t *init_##name() { \
+        return (kh_##name##_t*)calloc(1, sizeof(kh_##name##_t)); \
+      } \
+      SCOPE inline int get_##name(kh_##name##_t *h, key_t k) \
+      ... \
+      SCOPE inline void destroy_##name(kh_##name##_t *h) { \
+        if (h) { \
+          free(h->keys); free(h->flags); free(h->vals); free(h); \
+        } \
+      }
+    
+    #define _int_hf(key) (unsigned)(key)
+    #define _int_heq(a, b) (a == b)
+    #define khash_t(name) kh_##name##_t
+    #define kh_value(h, k) ((h)->vals[k])
+    #define kh_begin(h, k) 0
+    #define kh_end(h) ((h)->n_buckets)
+    #define kh_init(name) init_##name()
+    #define kh_get(name, h, k) get_##name(h, k)
+    #define kh_destroy(name, h) destroy_##name(h)
+    ...
+    #define KHASH_MAP_INIT_INT(name, val_t) \
+    	KHASH_INIT(name, static, unsigned, val_t, is_map, _int_hf, _int_heq)
+
+`KHASH_INIT()` is a huge macro defining all the structs and methods. When this
+macro is called, all the code inside it will be inserted by the [C
+preprocess][37] to the place where it is called. If the macro is called
+multiple times, multiple copies of the code will be inserted. To avoid naming
+conflict of hash tables with different key-value types, the library uses [token
+concatenation][36], which is a preprocessor feature whereby we can substitute
+part of a symbol based on the parameter of the macro. In the end, the C
+preprocessor will generate the following code and feed it to the compiler
+(macro `kh_exist(h,k)` is a little complex and not expanded for simplicity):
+
+    typedef struct {
+      int n_buckets, size, n_occupied, upper_bound;
+      unsigned *flags;
+      unsigned *keys;
+      char *vals;
+    } kh_m32_t;
+    static inline kh_m32_t *init_m32() {
+      return (kh_m32_t*)calloc(1, sizeof(kh_m32_t));
+    }
+    static inline int get_m32(kh_m32_t *h, unsigned k)
+    ...
+    static inline void destroy_m32(kh_m32_t *h) {
+      if (h) {
+        free(h->keys); free(h->flags); free(h->vals); free(h);
+      }
+    }
+
+	int main() {
+		int ret, is_missing;
+		khint_t k;
+		kh_m32_t *h = init_m32();
+		k = put_m32(h, 5, &ret);
+		if (!ret) del_m32(h, k);
+		h->vals[k] = 10;
+		k = get_m32(h, 10);
+		is_missing = (k == h->n_buckets);
+		k = get_m32(h, 5);
+		del_m32(h, k);
+		for (k = 0; k != h->n_buckets; ++k)
+			if (kh_exist(h, k)) h->vals[k] = 1;
+		destroy_m32(h);
+		return 0;
+	}
+
+This is the C program we know.
+
+From this example, we can see that macros and the C preprocessor plays a key
+role in klib. Klib is fast partly because the compiler knows the key-value
+type at the compile time and is able to optimize the code to the same level
+as type-specific code. A generic library written with `void*` will not get such
+performance boost.
+
+Massively inserting code upon instantiation may remind us of C++'s slow
+compiling speed and huge binary size when STL/boost is in use. Klib is much
+better in this respect due to its small code size and component independency.
+Inserting several hundreds lines of code won't make compiling obviously slower.
+
+## <a name="resources"></a>Resources
+
+* Library documentation, if present, is available in the header files. Examples
+can be found in the [test/][24] directory.
+* **Obsolete** documentation of the hash table library can be found at
+[SourceForge][25]. This README is partly adapted from the old documentation.
+* [Blog post][26] describing the hash table library.
+* [Blog post][27] on why using `void*` for generic programming may be inefficient.
+* [Blog post][28] on the generic stream buffer.
+* [Blog post][29] evaluating the performance of `kvec.h`.
+* [Blog post][30] arguing B-tree may be a better data structure than a binary search tree.
+* [Blog post][31] evaluating the performance of `khash.h` and `kbtree.h` among many other implementations.
+[An older version][33] of the benchmark is also available.
+* [Blog post][34] benchmarking internal sorting algorithms and implementations.
+* [Blog post][32] on the k-small algorithm.
+* [Blog post][35] on the Hooke-Jeeve's algorithm for nonlinear programming.
+
+[1]: http://en.wikipedia.org/wiki/MIT_License
+[2]: https://en.wikipedia.org/wiki/Hash_table
+[3]: http://en.wikipedia.org/wiki/B-tree
+[4]: http://en.wikipedia.org/wiki/Introsort
+[5]: http://en.wikipedia.org/wiki/Merge_sort
+[6]: http://en.wikipedia.org/wiki/Heapsort
+[7]: http://en.wikipedia.org/wiki/Comb_sort
+[8]: http://en.wikipedia.org/wiki/Fisher-Yates_shuffle
+[9]: http://en.wikipedia.org/wiki/Selection_algorithm
+[10]: http://en.wikipedia.org/wiki/FASTA_format
+[11]: http://en.wikipedia.org/wiki/FASTQ_format
+[12]: http://en.wikipedia.org/wiki/Memory_pool
+[13]: http://en.wikipedia.org/wiki/Mersenne_twister
+[14]: http://en.wikipedia.org/wiki/Pseudorandom_generator
+[15]: http://en.wikipedia.org/wiki/Nonlinear_programming
+[16]: http://en.wikipedia.org/wiki/Suffix_array
+[17]: https://sites.google.com/site/yuta256/sais
+[18]: http://en.wikipedia.org/wiki/Hidden_Markov_model
+[19]: http://en.wikipedia.org/wiki/Smith-Waterman_algorithm
+[20]: http://en.wikipedia.org/wiki/Newick_format
+[21]: http://en.wikipedia.org/wiki/Container_(abstract_data_type)
+[22]: http://en.wikipedia.org/wiki/Template_(C%2B%2B)
+[23]: http://en.wikipedia.org/wiki/GLib
+[24]: https://github.com/attractivechaos/klib/tree/master/test
+[25]: http://klib.sourceforge.net/
+[26]: http://attractivechaos.wordpress.com/2008/09/02/implementing-generic-hash-library-in-c/
+[27]: http://attractivechaos.wordpress.com/2008/10/02/using-void-in-generic-c-programming-may-be-inefficient/
+[28]: http://attractivechaos.wordpress.com/2008/10/11/a-generic-buffered-stream-wrapper/
+[29]: http://attractivechaos.wordpress.com/2008/09/19/c-array-vs-c-vector/
+[30]: http://attractivechaos.wordpress.com/2008/09/24/b-tree-vs-binary-search-tree/
+[31]: http://attractivechaos.wordpress.com/2008/10/07/another-look-at-my-old-benchmark/
+[32]: http://attractivechaos.wordpress.com/2008/09/13/calculating-median/
+[33]: http://attractivechaos.wordpress.com/2008/08/28/comparison-of-hash-table-libraries/
+[34]: http://attractivechaos.wordpress.com/2008/08/28/comparison-of-internal-sorting-algorithms/
+[35]: http://attractivechaos.wordpress.com/2008/08/24/derivative-free-optimization-dfo/
+[36]: http://en.wikipedia.org/wiki/C_preprocessor#Token_concatenation
+[37]: http://en.wikipedia.org/wiki/C_preprocessor
+
+[wiki-avl]: https://en.wikipedia.org/wiki/AVL_tree
+
+[kbtree]: http://attractivechaos.github.io/klib/#KBtree%3A%20generic%20ordered%20map:%5B%5BKBtree%3A%20generic%20ordered%20map%5D%5D
+[khash]: http://attractivechaos.github.io/klib/#Khash%3A%20generic%20hash%20table:%5B%5BKhash%3A%20generic%20hash%20table%5D%5D
+[kseq]: http://attractivechaos.github.io/klib/#Kseq%3A%20stream%20buffer%20and%20FASTA%2FQ%20parser:%5B%5BKseq%3A%20stream%20buffer%20and%20FASTA%2FQ%20parser%5D%5D
+[ksort]: http://attractivechaos.github.io/klib/#Ksort%3A%20sorting%2C%20shuffling%2C%20heap%20and%20k-small:%5B%5BKsort%3A%20sorting%2C%20shuffling%2C%20heap%20and%20k-small%5D%5D
+[kavl]: http://attractivechaos.github.io/klib/#KAVL%3A%20generic%20intrusive%20AVL%20tree
+[ketopt]: http://attractivechaos.github.io/klib/#Ketopt%3A%20parsing%20command-line%20arguments
diff --git a/ext/klib/bgzf.c b/ext/klib/bgzf.c
new file mode 100644
index 0000000..9833414
--- /dev/null
+++ b/ext/klib/bgzf.c
@@ -0,0 +1,555 @@
+/* The MIT License
+
+   Copyright (c) 2008 Broad Institute / Massachusetts Institute of Technology
+                 2011 Attractive Chaos <attractor@live.co.uk>
+
+   Permission is hereby granted, free of charge, to any person obtaining a copy
+   of this software and associated documentation files (the "Software"), to deal
+   in the Software without restriction, including without limitation the rights
+   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+   copies of the Software, and to permit persons to whom the Software is
+   furnished to do so, subject to the following conditions:
+
+   The above copyright notice and this permission notice shall be included in
+   all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+   THE SOFTWARE.
+*/
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <assert.h>
+#include <sys/types.h>
+#include "bgzf.h"
+
+#ifdef _USE_KNETFILE
+#include "knetfile.h"
+typedef knetFile *_bgzf_file_t;
+#define _bgzf_open(fn, mode) knet_open(fn, mode)
+#define _bgzf_dopen(fp, mode) knet_dopen(fp, mode)
+#define _bgzf_close(fp) knet_close(fp)
+#define _bgzf_fileno(fp) ((fp)->fd)
+#define _bgzf_tell(fp) knet_tell(fp)
+#define _bgzf_seek(fp, offset, whence) knet_seek(fp, offset, whence)
+#define _bgzf_read(fp, buf, len) knet_read(fp, buf, len)
+#define _bgzf_write(fp, buf, len) knet_write(fp, buf, len)
+#else // ~defined(_USE_KNETFILE)
+#if defined(_WIN32) || defined(_MSC_VER)
+#define ftello(fp) ftell(fp)
+#define fseeko(fp, offset, whence) fseek(fp, offset, whence)
+#else // ~defined(_WIN32)
+extern off_t ftello(FILE *stream);
+extern int fseeko(FILE *stream, off_t offset, int whence);
+#endif // ~defined(_WIN32)
+typedef FILE *_bgzf_file_t;
+#define _bgzf_open(fn, mode) fopen(fn, mode)
+#define _bgzf_dopen(fp, mode) fdopen(fp, mode)
+#define _bgzf_close(fp) fclose(fp)
+#define _bgzf_fileno(fp) fileno(fp)
+#define _bgzf_tell(fp) ftello(fp)
+#define _bgzf_seek(fp, offset, whence) fseeko(fp, offset, whence)
+#define _bgzf_read(fp, buf, len) fread(buf, 1, len, fp)
+#define _bgzf_write(fp, buf, len) fwrite(buf, 1, len, fp)
+#endif // ~define(_USE_KNETFILE)
+
+#define BLOCK_HEADER_LENGTH 18
+#define BLOCK_FOOTER_LENGTH 8
+
+/* BGZF/GZIP header (speciallized from RFC 1952; little endian):
+ +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+
+ | 31|139|  8|  4|              0|  0|255|      6| 66| 67|      2|BLK_LEN|
+ +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+
+*/
+static const uint8_t g_magic[19] = "\037\213\010\4\0\0\0\0\0\377\6\0\102\103\2\0\0\0";
+
+#ifdef BGZF_CACHE
+typedef struct {
+	int size;
+	uint8_t *block;
+	int64_t end_offset;
+} cache_t;
+#include "khash.h"
+KHASH_MAP_INIT_INT64(cache, cache_t)
+#endif
+
+static inline void packInt16(uint8_t *buffer, uint16_t value)
+{
+	buffer[0] = value;
+	buffer[1] = value >> 8;
+}
+
+static inline int unpackInt16(const uint8_t *buffer)
+{
+	return buffer[0] | buffer[1] << 8;
+}
+
+static inline void packInt32(uint8_t *buffer, uint32_t value)
+{
+	buffer[0] = value;
+	buffer[1] = value >> 8;
+	buffer[2] = value >> 16;
+	buffer[3] = value >> 24;
+}
+
+static BGZF *bgzf_read_init()
+{
+	BGZF *fp;
+	fp = calloc(1, sizeof(BGZF));
+	fp->open_mode = 'r';
+	fp->uncompressed_block = malloc(BGZF_MAX_BLOCK_SIZE);
+	fp->compressed_block = malloc(BGZF_MAX_BLOCK_SIZE);
+#ifdef BGZF_CACHE
+	fp->cache = kh_init(cache);
+#endif
+	return fp;
+}
+
+static BGZF *bgzf_write_init(int compress_level) // compress_level==-1 for the default level
+{
+	BGZF *fp;
+	fp = calloc(1, sizeof(BGZF));
+	fp->open_mode = 'w';
+	fp->uncompressed_block = malloc(BGZF_MAX_BLOCK_SIZE);
+	fp->compressed_block = malloc(BGZF_MAX_BLOCK_SIZE);
+	fp->compress_level = compress_level < 0? Z_DEFAULT_COMPRESSION : compress_level; // Z_DEFAULT_COMPRESSION==-1
+	if (fp->compress_level > 9) fp->compress_level = Z_DEFAULT_COMPRESSION;
+	return fp;
+}
+// get the compress level from the mode string
+static int mode2level(const char *__restrict mode)
+{
+	int i, compress_level = -1;
+	for (i = 0; mode[i]; ++i)
+		if (mode[i] >= '0' && mode[i] <= '9') break;
+	if (mode[i]) compress_level = (int)mode[i] - '0';
+	if (strchr(mode, 'u')) compress_level = 0;
+	return compress_level;
+}
+
+BGZF *bgzf_open(const char *path, const char *mode)
+{
+	BGZF *fp = 0;
+	if (strchr(mode, 'r') || strchr(mode, 'R')) {
+		_bgzf_file_t fpr;
+		if ((fpr = _bgzf_open(path, "r")) == 0) return 0;
+		fp = bgzf_read_init();
+		fp->fp = fpr;
+	} else if (strchr(mode, 'w') || strchr(mode, 'W')) {
+		FILE *fpw;
+		if ((fpw = fopen(path, "w")) == 0) return 0;
+		fp = bgzf_write_init(mode2level(mode));
+		fp->fp = fpw;
+	}
+	return fp;
+}
+
+BGZF *bgzf_dopen(int fd, const char *mode)
+{
+	BGZF *fp = 0;
+	if (strchr(mode, 'r') || strchr(mode, 'R')) {
+		_bgzf_file_t fpr;
+		if ((fpr = _bgzf_dopen(fd, "r")) == 0) return 0;
+		fp = bgzf_read_init();
+		fp->fp = fpr;
+	} else if (strchr(mode, 'w') || strchr(mode, 'W')) {
+		FILE *fpw;
+		if ((fpw = fdopen(fd, "w")) == 0) return 0;
+		fp = bgzf_write_init(mode2level(mode));
+		fp->fp = fpw;
+	}
+	return fp;
+}
+
+// Deflate the block in fp->uncompressed_block into fp->compressed_block. Also adds an extra field that stores the compressed block length.
+static int deflate_block(BGZF *fp, int block_length)
+{
+	uint8_t *buffer = fp->compressed_block;
+	int buffer_size = BGZF_BLOCK_SIZE;
+	int input_length = block_length;
+	int compressed_length = 0;
+	int remaining;
+	uint32_t crc;
+
+	assert(block_length <= BGZF_BLOCK_SIZE); // guaranteed by the caller
+	memcpy(buffer, g_magic, BLOCK_HEADER_LENGTH); // the last two bytes are a place holder for the length of the block
+	while (1) { // loop to retry for blocks that do not compress enough
+		int status;
+		z_stream zs;
+		zs.zalloc = NULL;
+		zs.zfree = NULL;
+		zs.next_in = fp->uncompressed_block;
+		zs.avail_in = input_length;
+		zs.next_out = (void*)&buffer[BLOCK_HEADER_LENGTH];
+		zs.avail_out = buffer_size - BLOCK_HEADER_LENGTH - BLOCK_FOOTER_LENGTH;
+		status = deflateInit2(&zs, fp->compress_level, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY); // -15 to disable zlib header/footer
+		if (status != Z_OK) {
+			fp->errcode |= BGZF_ERR_ZLIB;
+			return -1;
+		}
+		status = deflate(&zs, Z_FINISH);
+		if (status != Z_STREAM_END) { // not compressed enough
+			deflateEnd(&zs); // reset the stream
+			if (status == Z_OK) { // reduce the size and recompress
+				input_length -= 1024;
+				assert(input_length > 0); // logically, this should not happen
+				continue;
+			}
+			fp->errcode |= BGZF_ERR_ZLIB;
+			return -1;
+		}
+		if (deflateEnd(&zs) != Z_OK) {
+			fp->errcode |= BGZF_ERR_ZLIB;
+			return -1;
+		}
+		compressed_length = zs.total_out;
+		compressed_length += BLOCK_HEADER_LENGTH + BLOCK_FOOTER_LENGTH;
+		assert(compressed_length <= BGZF_BLOCK_SIZE);
+		break;
+	}
+
+	assert(compressed_length > 0);
+	packInt16((uint8_t*)&buffer[16], compressed_length - 1); // write the compressed_length; -1 to fit 2 bytes
+	crc = crc32(0L, NULL, 0L);
+	crc = crc32(crc, fp->uncompressed_block, input_length);
+	packInt32((uint8_t*)&buffer[compressed_length-8], crc);
+	packInt32((uint8_t*)&buffer[compressed_length-4], input_length);
+
+	remaining = block_length - input_length;
+	if (remaining > 0) {
+		assert(remaining <= input_length);
+		memcpy(fp->uncompressed_block, fp->uncompressed_block + input_length, remaining);
+	}
+	fp->block_offset = remaining;
+	return compressed_length;
+}
+
+// Inflate the block in fp->compressed_block into fp->uncompressed_block
+static int inflate_block(BGZF* fp, int block_length)
+{
+	z_stream zs;
+	zs.zalloc = NULL;
+	zs.zfree = NULL;
+	zs.next_in = fp->compressed_block + 18;
+	zs.avail_in = block_length - 16;
+	zs.next_out = fp->uncompressed_block;
+	zs.avail_out = BGZF_BLOCK_SIZE;
+
+	if (inflateInit2(&zs, -15) != Z_OK) {
+		fp->errcode |= BGZF_ERR_ZLIB;
+		return -1;
+	}
+	if (inflate(&zs, Z_FINISH) != Z_STREAM_END) {
+		inflateEnd(&zs);
+		fp->errcode |= BGZF_ERR_ZLIB;
+		return -1;
+	}
+	if (inflateEnd(&zs) != Z_OK) {
+		fp->errcode |= BGZF_ERR_ZLIB;
+		return -1;
+	}
+	return zs.total_out;
+}
+
+static int check_header(const uint8_t *header)
+{
+	return (header[0] == 31 && header[1] == 139 && header[2] == 8 && (header[3] & 4) != 0
+			&& unpackInt16((uint8_t*)&header[10]) == 6
+			&& header[12] == 'B' && header[13] == 'C'
+			&& unpackInt16((uint8_t*)&header[14]) == 2);
+}
+
+#ifdef BGZF_CACHE
+static void free_cache(BGZF *fp)
+{
+	khint_t k;
+	khash_t(cache) *h = (khash_t(cache)*)fp->cache;
+	if (fp->open_mode != 'r') return;
+	for (k = kh_begin(h); k < kh_end(h); ++k)
+		if (kh_exist(h, k)) free(kh_val(h, k).block);
+	kh_destroy(cache, h);
+}
+
+static int load_block_from_cache(BGZF *fp, int64_t block_address)
+{
+	khint_t k;
+	cache_t *p;
+	khash_t(cache) *h = (khash_t(cache)*)fp->cache;
+	k = kh_get(cache, h, block_address);
+	if (k == kh_end(h)) return 0;
+	p = &kh_val(h, k);
+	if (fp->block_length != 0) fp->block_offset = 0;
+	fp->block_address = block_address;
+	fp->block_length = p->size;
+	memcpy(fp->uncompressed_block, p->block, BGZF_BLOCK_SIZE);
+	_bgzf_seek((_bgzf_file_t)fp->fp, p->end_offset, SEEK_SET);
+	return p->size;
+}
+
+static void cache_block(BGZF *fp, int size)
+{
+	int ret;
+	khint_t k;
+	cache_t *p;
+	khash_t(cache) *h = (khash_t(cache)*)fp->cache;
+	if (BGZF_BLOCK_SIZE >= fp->cache_size) return;
+	if ((kh_size(h) + 1) * BGZF_BLOCK_SIZE > fp->cache_size) {
+		/* A better way would be to remove the oldest block in the
+		 * cache, but here we remove a random one for simplicity. This
+		 * should not have a big impact on performance. */
+		for (k = kh_begin(h); k < kh_end(h); ++k)
+			if (kh_exist(h, k)) break;
+		if (k < kh_end(h)) {
+			free(kh_val(h, k).block);
+			kh_del(cache, h, k);
+		}
+	}
+	k = kh_put(cache, h, fp->block_address, &ret);
+	if (ret == 0) return; // if this happens, a bug!
+	p = &kh_val(h, k);
+	p->size = fp->block_length;
+	p->end_offset = fp->block_address + size;
+	p->block = malloc(BGZF_BLOCK_SIZE);
+	memcpy(kh_val(h, k).block, fp->uncompressed_block, BGZF_BLOCK_SIZE);
+}
+#else
+static void free_cache(BGZF *fp) {}
+static int load_block_from_cache(BGZF *fp, int64_t block_address) {return 0;}
+static void cache_block(BGZF *fp, int size) {}
+#endif
+
+int bgzf_read_block(BGZF *fp)
+{
+	uint8_t header[BLOCK_HEADER_LENGTH], *compressed_block;
+	int count, size = 0, block_length, remaining;
+	int64_t block_address;
+	block_address = _bgzf_tell((_bgzf_file_t)fp->fp);
+	if (load_block_from_cache(fp, block_address)) return 0;
+	count = _bgzf_read(fp->fp, header, sizeof(header));
+	if (count == 0) { // no data read
+		fp->block_length = 0;
+		return 0;
+	}
+	if (count != sizeof(header) || !check_header(header)) {
+		fp->errcode |= BGZF_ERR_HEADER;
+		return -1;
+	}
+	size = count;
+	block_length = unpackInt16((uint8_t*)&header[16]) + 1; // +1 because when writing this number, we used "-1"
+	compressed_block = (uint8_t*)fp->compressed_block;
+	memcpy(compressed_block, header, BLOCK_HEADER_LENGTH);
+	remaining = block_length - BLOCK_HEADER_LENGTH;
+	count = _bgzf_read(fp->fp, &compressed_block[BLOCK_HEADER_LENGTH], remaining);
+	if (count != remaining) {
+		fp->errcode |= BGZF_ERR_IO;
+		return -1;
+	}
+	size += count;
+	if ((count = inflate_block(fp, block_length)) < 0) return -1;
+	if (fp->block_length != 0) fp->block_offset = 0; // Do not reset offset if this read follows a seek.
+	fp->block_address = block_address;
+	fp->block_length = count;
+	cache_block(fp, size);
+	return 0;
+}
+
+ssize_t bgzf_read(BGZF *fp, void *data, ssize_t length)
+{
+	ssize_t bytes_read = 0;
+	uint8_t *output = data;
+	if (length <= 0) return 0;
+	assert(fp->open_mode == 'r');
+	while (bytes_read < length) {
+		int copy_length, available = fp->block_length - fp->block_offset;
+		uint8_t *buffer;
+		if (available <= 0) {
+			if (bgzf_read_block(fp) != 0) return -1;
+			available = fp->block_length - fp->block_offset;
+			if (available <= 0) break;
+		}
+		copy_length = length - bytes_read < available? length - bytes_read : available;
+		buffer = fp->uncompressed_block;
+		memcpy(output, buffer + fp->block_offset, copy_length);
+		fp->block_offset += copy_length;
+		output += copy_length;
+		bytes_read += copy_length;
+	}
+	if (fp->block_offset == fp->block_length) {
+		fp->block_address = _bgzf_tell((_bgzf_file_t)fp->fp);
+		fp->block_offset = fp->block_length = 0;
+	}
+	return bytes_read;
+}
+
+int bgzf_flush(BGZF *fp)
+{
+	assert(fp->open_mode == 'w');
+	while (fp->block_offset > 0) {
+		int block_length;
+		block_length = deflate_block(fp, fp->block_offset);
+		if (block_length < 0) return -1;
+		if (fwrite(fp->compressed_block, 1, block_length, fp->fp) != block_length) {
+			fp->errcode |= BGZF_ERR_IO; // possibly truncated file
+			return -1;
+		}
+		fp->block_address += block_length;
+	}
+	return 0;
+}
+
+int bgzf_flush_try(BGZF *fp, ssize_t size)
+{
+	if (fp->block_offset + size > BGZF_BLOCK_SIZE)
+		return bgzf_flush(fp);
+	return -1;
+}
+
+ssize_t bgzf_write(BGZF *fp, const void *data, ssize_t length)
+{
+	const uint8_t *input = data;
+	int block_length = BGZF_BLOCK_SIZE, bytes_written;
+	assert(fp->open_mode == 'w');
+	input = data;
+	bytes_written = 0;
+	while (bytes_written < length) {
+		uint8_t* buffer = fp->uncompressed_block;
+		int copy_length = block_length - fp->block_offset < length - bytes_written? block_length - fp->block_offset : length - bytes_written;
+		memcpy(buffer + fp->block_offset, input, copy_length);
+		fp->block_offset += copy_length;
+		input += copy_length;
+		bytes_written += copy_length;
+		if (fp->block_offset == block_length && bgzf_flush(fp)) break;
+	}
+	return bytes_written;
+}
+
+int bgzf_close(BGZF* fp)
+{
+	int ret, count, block_length;
+	if (fp == 0) return -1;
+	if (fp->open_mode == 'w') {
+		if (bgzf_flush(fp) != 0) return -1;
+		block_length = deflate_block(fp, 0); // write an empty block
+		count = fwrite(fp->compressed_block, 1, block_length, fp->fp);
+		if (fflush(fp->fp) != 0) {
+			fp->errcode |= BGZF_ERR_IO;
+			return -1;
+		}
+	}
+	ret = fp->open_mode == 'w'? fclose(fp->fp) : _bgzf_close(fp->fp);
+	if (ret != 0) return -1;
+	free(fp->uncompressed_block);
+	free(fp->compressed_block);
+	free_cache(fp);
+	free(fp);
+	return 0;
+}
+
+void bgzf_set_cache_size(BGZF *fp, int cache_size)
+{
+	if (fp) fp->cache_size = cache_size;
+}
+
+int bgzf_check_EOF(BGZF *fp)
+{
+	static uint8_t magic[28] = "\037\213\010\4\0\0\0\0\0\377\6\0\102\103\2\0\033\0\3\0\0\0\0\0\0\0\0\0";
+	uint8_t buf[28];
+	off_t offset;
+	offset = _bgzf_tell((_bgzf_file_t)fp->fp);
+	if (_bgzf_seek(fp->fp, -28, SEEK_END) < 0) return 0;
+	_bgzf_read(fp->fp, buf, 28);
+	_bgzf_seek(fp->fp, offset, SEEK_SET);
+	return (memcmp(magic, buf, 28) == 0)? 1 : 0;
+}
+
+int64_t bgzf_seek(BGZF* fp, int64_t pos, int where)
+{
+	int block_offset;
+	int64_t block_address;
+
+	if (fp->open_mode != 'r' || where != SEEK_SET) {
+		fp->errcode |= BGZF_ERR_MISUSE;
+		return -1;
+	}
+	block_offset = pos & 0xFFFF;
+	block_address = pos >> 16;
+	if (_bgzf_seek(fp->fp, block_address, SEEK_SET) < 0) {
+		fp->errcode |= BGZF_ERR_IO;
+		return -1;
+	}
+	fp->block_length = 0;  // indicates current block has not been loaded
+	fp->block_address = block_address;
+	fp->block_offset = block_offset;
+	return 0;
+}
+
+int bgzf_is_bgzf(const char *fn)
+{
+	uint8_t buf[16];
+	int n;
+	_bgzf_file_t fp;
+	if ((fp = _bgzf_open(fn, "r")) == 0) return 0;
+	n = _bgzf_read(fp, buf, 16);
+	_bgzf_close(fp);
+	if (n != 16) return 0;
+	return memcmp(g_magic, buf, 16) == 0? 1 : 0;
+}
+
+int bgzf_getc(BGZF *fp)
+{
+	int c;
+	if (fp->block_offset >= fp->block_length) {
+		if (bgzf_read_block(fp) != 0) return -2; /* error */
+		if (fp->block_length == 0) return -1; /* end-of-file */
+	}
+	c = ((unsigned char*)fp->uncompressed_block)[fp->block_offset++];
+    if (fp->block_offset == fp->block_length) {
+        fp->block_address = _bgzf_tell((_bgzf_file_t)fp->fp);
+        fp->block_offset = 0;
+        fp->block_length = 0;
+    }
+	return c;
+}
+
+#ifndef kroundup32
+#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
+#endif
+
+int bgzf_getline(BGZF *fp, int delim, kstring_t *str)
+{
+	int l, state = 0;
+	unsigned char *buf = (unsigned char*)fp->uncompressed_block;
+	str->l = 0;
+	do {
+		if (fp->block_offset >= fp->block_length) {
+			if (bgzf_read_block(fp) != 0) { state = -2; break; }
+			if (fp->block_length == 0) { state = -1; break; }
+		}
+		for (l = fp->block_offset; l < fp->block_length && buf[l] != delim; ++l);
+		if (l < fp->block_length) state = 1;
+		l -= fp->block_offset;
+		if (str->l + l + 1 >= str->m) {
+			str->m = str->l + l + 2;
+			kroundup32(str->m);
+			str->s = (char*)realloc(str->s, str->m);
+		}
+		memcpy(str->s + str->l, buf + fp->block_offset, l);
+		str->l += l;
+		fp->block_offset += l + 1;
+		if (fp->block_offset >= fp->block_length) {
+			fp->block_address = _bgzf_tell((_bgzf_file_t)fp->fp);
+			fp->block_offset = 0;
+			fp->block_length = 0;
+		} 
+	} while (state == 0);
+	if (str->l == 0 && state < 0) return state;
+	str->s[str->l] = 0;
+	return str->l;
+}
diff --git a/ext/klib/bgzf.h b/ext/klib/bgzf.h
new file mode 100644
index 0000000..29fe0e5
--- /dev/null
+++ b/ext/klib/bgzf.h
@@ -0,0 +1,196 @@
+/* The MIT License
+
+   Copyright (c) 2008 Broad Institute / Massachusetts Institute of Technology
+                 2011 Attractive Chaos <attractor@live.co.uk>
+
+   Permission is hereby granted, free of charge, to any person obtaining a copy
+   of this software and associated documentation files (the "Software"), to deal
+   in the Software without restriction, including without limitation the rights
+   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+   copies of the Software, and to permit persons to whom the Software is
+   furnished to do so, subject to the following conditions:
+
+   The above copyright notice and this permission notice shall be included in
+   all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+   THE SOFTWARE.
+*/
+
+/* The BGZF library was originally written by Bob Handsaker from the Broad
+ * Institute. It was later improved by the SAMtools developers. */
+
+#ifndef __BGZF_H
+#define __BGZF_H
+
+#include <stdint.h>
+#include <stdio.h>
+#include <zlib.h>
+
+#define BGZF_BLOCK_SIZE     0x10000
+#define BGZF_MAX_BLOCK_SIZE 0x10000
+
+#define BGZF_ERR_ZLIB   1
+#define BGZF_ERR_HEADER 2
+#define BGZF_ERR_IO     4
+#define BGZF_ERR_MISUSE 8
+
+typedef struct {
+    int open_mode:8, compress_level:8, errcode:16;
+	int cache_size;
+    int block_length, block_offset;
+    int64_t block_address;
+    void *uncompressed_block, *compressed_block;
+	void *cache; // a pointer to a hash table
+	void *fp; // actual file handler; FILE* on writing; FILE* or knetFile* on reading
+} BGZF;
+
+#ifndef KSTRING_T
+#define KSTRING_T kstring_t
+typedef struct __kstring_t {
+	size_t l, m;
+	char *s;
+} kstring_t;
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+	/******************
+	 * Basic routines *
+	 ******************/
+
+	/**
+	 * Open an existing file descriptor for reading or writing.
+	 *
+	 * @param fd    file descriptor
+	 * @param mode  mode matching /[rwu0-9]+/: 'r' for reading, 'w' for writing and a digit specifies
+	 *              the zlib compression level; if both 'r' and 'w' are present, 'w' is ignored.
+	 * @return      BGZF file handler; 0 on error
+	 */
+	BGZF* bgzf_dopen(int fd, const char *mode);
+
+	#define bgzf_fdopen(fd, mode) bgzf_dopen((fd), (mode)) // for backward compatibility
+
+	/**
+	 * Open the specified file for reading or writing.
+	 */
+	BGZF* bgzf_open(const char* path, const char *mode);
+
+	/**
+	 * Close the BGZF and free all associated resources.
+	 *
+	 * @param fp    BGZF file handler
+	 * @return      0 on success and -1 on error
+	 */
+	int bgzf_close(BGZF *fp);
+
+	/**
+	 * Read up to _length_ bytes from the file storing into _data_.
+	 *
+	 * @param fp     BGZF file handler
+	 * @param data   data array to read into
+	 * @param length size of data to read
+	 * @return       number of bytes actually read; 0 on end-of-file and -1 on error
+	 */
+	ssize_t bgzf_read(BGZF *fp, void *data, ssize_t length);
+
+	/**
+	 * Write _length_ bytes from _data_ to the file.
+	 *
+	 * @param fp     BGZF file handler
+	 * @param data   data array to write
+	 * @param length size of data to write
+	 * @return       number of bytes actually written; -1 on error
+	 */
+	ssize_t bgzf_write(BGZF *fp, const void *data, ssize_t length);
+
+	/**
+	 * Write the data in the buffer to the file.
+	 */
+	int bgzf_flush(BGZF *fp);
+
+	/**
+	 * Return a virtual file pointer to the current location in the file.
+	 * No interpetation of the value should be made, other than a subsequent
+	 * call to bgzf_seek can be used to position the file at the same point.
+	 * Return value is non-negative on success.
+	 */
+	#define bgzf_tell(fp) ((fp->block_address << 16) | (fp->block_offset & 0xFFFF))
+
+	/**
+	 * Set the file to read from the location specified by _pos_.
+	 *
+	 * @param fp     BGZF file handler
+	 * @param pos    virtual file offset returned by bgzf_tell()
+	 * @param whence must be SEEK_SET
+	 * @return       0 on success and -1 on error
+	 */
+	int64_t bgzf_seek(BGZF *fp, int64_t pos, int whence);
+
+	/**
+	 * Check if the BGZF end-of-file (EOF) marker is present
+	 *
+	 * @param fp    BGZF file handler opened for reading
+	 * @return      1 if EOF is present; 0 if not or on I/O error
+	 */
+	int bgzf_check_EOF(BGZF *fp);
+
+	/**
+	 * Check if a file is in the BGZF format
+	 *
+	 * @param fn    file name
+	 * @return      1 if _fn_ is BGZF; 0 if not or on I/O error
+	 */
+	 int bgzf_is_bgzf(const char *fn);
+
+	/*********************
+	 * Advanced routines *
+	 *********************/
+
+	/**
+	 * Set the cache size. Only effective when compiled with -DBGZF_CACHE.
+	 *
+	 * @param fp    BGZF file handler
+	 * @param size  size of cache in bytes; 0 to disable caching (default)
+	 */
+	void bgzf_set_cache_size(BGZF *fp, int size);
+
+	/**
+	 * Flush the file if the remaining buffer size is smaller than _size_ 
+	 */
+	int bgzf_flush_try(BGZF *fp, ssize_t size);
+
+	/**
+	 * Read one byte from a BGZF file. It is faster than bgzf_read()
+	 * @param fp     BGZF file handler
+	 * @return       byte read; -1 on end-of-file or error
+	 */
+	int bgzf_getc(BGZF *fp);
+
+	/**
+	 * Read one line from a BGZF file. It is faster than bgzf_getc()
+	 *
+	 * @param fp     BGZF file handler
+	 * @param delim  delimitor
+	 * @param str    string to write to; must be initialized
+	 * @return       length of the string; 0 on end-of-file; negative on error
+	 */
+	int bgzf_getline(BGZF *fp, int delim, kstring_t *str);
+
+	/**
+	 * Read the next BGZF block.
+	 */
+	int bgzf_read_block(BGZF *fp);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/ext/klib/cpp/kavl.hpp b/ext/klib/cpp/kavl.hpp
new file mode 100644
index 0000000..9fa497a
--- /dev/null
+++ b/ext/klib/cpp/kavl.hpp
@@ -0,0 +1,202 @@
+#ifndef KAVL_HPP
+#define KAVL_HPP
+
+#include <functional>
+
+namespace klib {
+
+template<class T, typename Less = std::less<T> >
+class Avl {
+	static const int MAX_DEPTH = 64;
+	struct Node {
+		T data;
+		signed char balance;
+		unsigned size;
+		Node *p[2];
+	};
+	Node *root;
+	inline int cmp_func(const T &x, const T &y) {
+		return Less()(y, x) - Less()(x, y);
+	}
+	inline unsigned child_size(Node *p, int dir) {
+		return p->p[dir]? p->p[dir]->size : 0;
+	};
+	// one rotation: (a,(b,c)q)p => ((a,b)p,c)q
+	inline Node *rotate1(Node *p, int dir) { // dir=0 to left; dir=1 to right
+		int opp = 1 - dir; // opposite direction
+		Node *q = p->p[opp];
+		unsigned size_p = p->size;
+		p->size -= q->size - child_size(q, dir);
+		q->size = size_p;
+		p->p[opp] = q->p[dir];
+		q->p[dir] = p;
+		return q;
+	};
+	// two consecutive rotations: (a,((b,c)r,d)q)p => ((a,b)p,(c,d)q)r
+	inline Node *rotate2(Node *p, int dir) {
+		int b1, opp = 1 - dir;
+		Node *q = p->p[opp], *r = q->p[dir];
+		unsigned size_x_dir = child_size(r, dir);
+		r->size = p->size;
+		p->size -= q->size - size_x_dir;
+		q->size -= size_x_dir + 1;
+		p->p[opp] = r->p[dir];
+		r->p[dir] = p;
+		q->p[dir] = r->p[opp];
+		r->p[opp] = q;
+		b1 = dir == 0? +1 : -1;
+		if (r->balance == b1) q->balance = 0, p->balance = -b1;
+		else if (r->balance == 0) q->balance = p->balance = 0;
+		else q->balance = b1, p->balance = 0;
+		r->balance = 0;
+		return r;
+	};
+	void destroy(Node *r) {
+		Node *p, *q;
+		for (p = r; p; p = q) {
+			if (p->p[0] == 0) {
+				q = p->p[1];
+				delete p;
+			} else {
+				q = p->p[0];
+				p->p[0] = q->p[1];
+				q->p[1] = p;
+			}
+		}
+	};
+public:
+	Avl() : root(NULL) {};
+	~Avl() { destroy(root); };
+	unsigned size() const { return root? root->size : 0; }
+	T *find(const T &data, unsigned *cnt_ = NULL) {
+		Node *p = root;
+		unsigned cnt = 0;
+		while (p != 0) {
+			int cmp = cmp_func(data, p->data);
+			if (cmp >= 0) cnt += child_size(p, 0) + 1;
+			if (cmp < 0) p = p->p[0];
+			else if (cmp > 0) p = p->p[1];
+			else break;
+		}
+		if (cnt_) *cnt_ = cnt;
+		return p? &p->data : NULL;
+	};
+	T *insert(const T &data, bool *is_new = NULL, unsigned *cnt_ = NULL) {
+		unsigned char stack[MAX_DEPTH];
+		Node *path[MAX_DEPTH];
+		Node *bp, *bq;
+		Node *x, *p, *q, *r = 0; // _r_ is potentially the new root
+		int i, which = 0, top, b1, path_len;
+		unsigned cnt = 0;
+		bp = root, bq = 0;
+		if (is_new) *is_new = false;
+		// find the insertion location
+		for (p = bp, q = bq, top = path_len = 0; p; q = p, p = p->p[which]) {
+			int cmp = cmp_func(data, p->data);
+			if (cmp >= 0) cnt += child_size(p, 0) + 1;
+			if (cmp == 0) {
+				if (cnt_) *cnt_ = cnt;
+				return &p->data;
+			}
+			if (p->balance != 0)
+				bq = q, bp = p, top = 0;
+			stack[top++] = which = (cmp > 0);
+			path[path_len++] = p;
+		}
+		if (cnt_) *cnt_ = cnt;
+		x = new Node;
+		x->data = data, x->balance = 0, x->size = 1, x->p[0] = x->p[1] = 0;
+		if (is_new) *is_new = true;
+		if (q == 0) root = x;
+		else q->p[which] = x;
+		if (bp == 0) return &x->data;
+		for (i = 0; i < path_len; ++i) ++path[i]->size;
+		for (p = bp, top = 0; p != x; p = p->p[stack[top]], ++top) /* update balance factors */
+			if (stack[top] == 0) --p->balance;
+			else ++p->balance;
+		if (bp->balance > -2 && bp->balance < 2) return &x->data; /* no re-balance needed */
+		// re-balance
+		which = (bp->balance < 0);
+		b1 = which == 0? +1 : -1;
+		q = bp->p[1 - which];
+		if (q->balance == b1) {
+			r = rotate1(bp, which);
+			q->balance = bp->balance = 0;
+		} else r = rotate2(bp, which);
+		if (bq == 0) root = r;
+		else bq->p[bp != bq->p[0]] = r;
+		return &x->data;
+	};
+	bool erase(const T &data) {
+		Node *p, *path[MAX_DEPTH], fake;
+		unsigned char dir[MAX_DEPTH];
+		int i, d = 0, cmp;
+		fake.p[0] = root, fake.p[1] = 0;
+		for (cmp = -1, p = &fake; cmp; cmp = cmp_func(data, p->data)) {
+			int which = (cmp > 0);
+			dir[d] = which;
+			path[d++] = p;
+			p = p->p[which];
+			if (p == 0) return false;
+		}
+		for (i = 1; i < d; ++i) --path[i]->size;
+		if (p->p[1] == 0) { // ((1,.)2,3)4 => (1,3)4; p=2
+			path[d-1]->p[dir[d-1]] = p->p[0];
+		} else {
+			Node *q = p->p[1];
+			if (q->p[0] == 0) { // ((1,2)3,4)5 => ((1)2,4)5; p=3
+				q->p[0] = p->p[0];
+				q->balance = p->balance;
+				path[d-1]->p[dir[d-1]] = q;
+				path[d] = q, dir[d++] = 1;
+				q->size = p->size - 1;
+			} else { // ((1,((.,2)3,4)5)6,7)8 => ((1,(2,4)5)3,7)8; p=6
+				Node *r;
+				int e = d++; // backup _d_
+				for (;;) {
+					dir[d] = 0;
+					path[d++] = q;
+					r = q->p[0];
+					if (r->p[0] == 0) break;
+					q = r;
+				}
+				r->p[0] = p->p[0];
+				q->p[0] = r->p[1];
+				r->p[1] = p->p[1];
+				r->balance = p->balance;
+				path[e-1]->p[dir[e-1]] = r;
+				path[e] = r, dir[e] = 1;
+				for (i = e + 1; i < d; ++i) --path[i]->size;
+				r->size = p->size - 1;
+			}
+		}
+		while (--d > 0) {
+			Node *q = path[d];
+			int which, other, b1 = 1, b2 = 2;
+			which = dir[d], other = 1 - which;
+			if (which) b1 = -b1, b2 = -b2;
+			q->balance += b1;
+			if (q->balance == b1) break;
+			else if (q->balance == b2) {
+				Node *r = q->p[other];
+				if (r->balance == -b1) {
+					path[d-1]->p[dir[d-1]] = rotate2(q, which);
+				} else {
+					path[d-1]->p[dir[d-1]] = rotate1(q, which);
+					if (r->balance == 0) {
+						r->balance = -b1;
+						q->balance = b1;
+						break;
+					} else r->balance = q->balance = 0;
+				}
+			}
+		}
+		root = fake.p[0];
+		delete p;
+		return true;
+	};
+};
+
+} // end of namespace klib
+
+#endif
diff --git a/ext/klib/cpp/khash.hpp b/ext/klib/cpp/khash.hpp
new file mode 100644
index 0000000..a57d6de
--- /dev/null
+++ b/ext/klib/cpp/khash.hpp
@@ -0,0 +1,163 @@
+#ifndef KHASH_HPP
+#define KHASH_HPP
+
+#include <memory>
+#include <functional>
+#include <cstdlib> // for malloc() etc
+#include <cstring> // for memset()
+
+#include <stdint.h> // for uint32_t
+
+namespace klib {
+
+#ifndef kroundup32 // FIXME: doesn't work for 64-bit integers
+#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
+#endif
+
+#define __ac_isempty(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&2)
+#define __ac_isdel(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&1)
+#define __ac_isempty(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&2)
+#define __ac_isdel(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&1)
+#define __ac_iseither(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&3)
+#define __ac_set_isdel_false(flag, i) (flag[i>>4]&=~(1ul<<((i&0xfU)<<1)))
+#define __ac_set_isempty_false(flag, i) (flag[i>>4]&=~(2ul<<((i&0xfU)<<1)))
+#define __ac_set_isboth_false(flag, i) (flag[i>>4]&=~(3ul<<((i&0xfU)<<1)))
+#define __ac_set_isdel_true(flag, i) (flag[i>>4]|=1ul<<((i&0xfU)<<1))
+
+#define __ac_fsize(m) ((m) < 16? 1 : (m)>>4)
+
+template<class T, class Hash, class Eq = std::equal_to<T>, typename khint_t = uint32_t>
+class KHash {
+	khint_t n_buckets, count, n_occupied, upper_bound;
+	uint32_t *flags;
+	T *keys;
+public:
+	KHash() : n_buckets(0), count(0), n_occupied(0), upper_bound(0), flags(NULL), keys(NULL) {};
+	~KHash() { std::free(flags); std::free(keys); };
+	khint_t capacity(void) const { return n_buckets; };
+	khint_t size(void) const { return count; };
+	khint_t begin(void) const { return 0; };
+	khint_t end(void) const { return n_buckets; };
+
+	void exist(khint_t x) const { return !__ac_iseither(flags, x); };
+	T &at(khint_t x) { return keys[x]; };
+
+	khint_t get(const T &key) const {
+		if (n_buckets) {
+			khint_t k, i, last, mask, step = 0;
+			mask = n_buckets - 1;
+			k = Hash()(key); i = k & mask;
+			last = i;
+			while (!__ac_isempty(flags, i) && (__ac_isdel(flags, i) || !Eq()(keys[i], key))) {
+				i = (i + (++step)) & mask;
+				if (i == last) return n_buckets;
+			}
+			return __ac_iseither(flags, i)? n_buckets : i;
+		} else return 0;
+	};
+
+	int resize(khint_t new_n_buckets) {
+		uint32_t *new_flags = 0;
+		khint_t j = 1;
+		{
+			kroundup32(new_n_buckets);
+			if (new_n_buckets < 4) new_n_buckets = 4;
+			if (count >= (new_n_buckets>>1) + (new_n_buckets>>2)) j = 0;	/* requested count is too small */
+			else { /* hash table count to be changed (shrink or expand); rehash */
+				new_flags = (uint32_t*)std::malloc(__ac_fsize(new_n_buckets) * sizeof(uint32_t));
+				if (!new_flags) return -1;
+				::memset(new_flags, 0xaa, __ac_fsize(new_n_buckets) * sizeof(uint32_t));
+				if (n_buckets < new_n_buckets) {	/* expand */
+					T *new_keys = (T*)std::realloc((void *)keys, new_n_buckets * sizeof(T));
+					if (!new_keys) { std::free(new_flags); return -1; }
+					keys = new_keys;
+				} /* otherwise shrink */
+			}
+		}
+		if (j) { /* rehashing is needed */
+			for (j = 0; j != n_buckets; ++j) {
+				if (__ac_iseither(flags, j) == 0) {
+					T key = keys[j];
+					khint_t new_mask;
+					new_mask = new_n_buckets - 1;
+					__ac_set_isdel_true(flags, j);
+					while (1) { /* kick-out process; sort of like in Cuckoo hashing */
+						khint_t k, i, step = 0;
+						k = Hash()(key);
+						i = k & new_mask;
+						while (!__ac_isempty(new_flags, i)) i = (i + (++step)) & new_mask;
+						__ac_set_isempty_false(new_flags, i);
+						if (i < n_buckets && __ac_iseither(flags, i) == 0) { /* kick out the existing element */
+							{ T tmp = keys[i]; keys[i] = key; key = tmp; }
+							__ac_set_isdel_true(flags, i); /* mark it as deleted in the old hash table */
+						} else { /* write the element and jump out of the loop */
+							keys[i] = key;
+							break;
+						}
+					}
+				}
+			}
+			if (n_buckets > new_n_buckets) /* shrink the hash table */
+				keys = (T*)std::realloc((void *)keys, new_n_buckets * sizeof(T));
+			std::free(flags); /* free the working space */
+			flags = new_flags;
+			n_buckets = new_n_buckets;
+			n_occupied = count;
+			upper_bound = (n_buckets>>1) + (n_buckets>>2);
+		}
+		return 0;
+	};
+
+	khint_t put(const T &key, int *ret) {
+		khint_t x;
+		if (n_occupied >= upper_bound) { /* update the hash table */
+			if (n_buckets > (count<<1)) {
+				if (resize(n_buckets - 1) < 0) { /* clear "deleted" elements */
+					*ret = -1; return n_buckets;
+				}
+			} else if (resize(n_buckets + 1) < 0) { /* expand the hash table */
+				*ret = -1; return n_buckets;
+			}
+		} /* TODO: to implement automatically shrinking; resize() already support shrinking */
+		{
+			khint_t k, i, site, last, mask = n_buckets - 1, step = 0;
+			x = site = n_buckets; k = Hash()(key); i = k & mask;
+			if (__ac_isempty(flags, i)) x = i; /* for speed up */
+			else {
+				last = i;
+				while (!__ac_isempty(flags, i) && (__ac_isdel(flags, i) || !Eq()(keys[i], key))) {
+					if (__ac_isdel(flags, i)) site = i;
+					i = (i + (++step)) & mask;
+					if (i == last) { x = site; break; }
+				}
+				if (x == n_buckets) {
+					if (__ac_isempty(flags, i) && site != n_buckets) x = site;
+					else x = i;
+				}
+			}
+		}
+		if (__ac_isempty(flags, x)) { /* not present at all */
+			keys[x] = key;
+			__ac_set_isboth_false(flags, x);
+			++count; ++n_occupied;
+			*ret = 1;
+		} else if (__ac_isdel(flags, x)) { /* deleted */
+			keys[x] = key;
+			__ac_set_isboth_false(flags, x);
+			++count;
+			*ret = 2;
+		} else *ret = 0; /* Don't touch keys[x] if present and not deleted */
+		return x;
+	};
+
+	void del(khint_t x) {
+		if (x != n_buckets && !__ac_iseither(flags, x)) {
+			__ac_set_isdel_true(flags, x);
+			--count;
+		}
+	};
+};
+
+} // end of namespace klib
+
+#endif
diff --git a/ext/klib/cpp/khashl.hpp b/ext/klib/cpp/khashl.hpp
new file mode 100644
index 0000000..8b870d6
--- /dev/null
+++ b/ext/klib/cpp/khashl.hpp
@@ -0,0 +1,258 @@
+#ifndef __AC_KHASHL_HPP
+#define __AC_KHASHL_HPP
+
+#include <functional> // for std::equal_to
+#include <cstdlib>    // for malloc() etc
+#include <cstring>    // for memset()
+#include <stdint.h>   // for uint32_t
+
+/* // ==> Code example <==
+#include <cstdio>
+#include "khashl.hpp"
+
+int main(void)
+{
+	klib::KHashMap<uint32_t, int, std::hash<uint32_t> > h; // NB: C++98 doesn't have std::hash
+	uint32_t k;
+	int absent;
+	h[43] = 1, h[53] = 2, h[63] = 3, h[73] = 4;       // one way to insert
+	k = h.put(53, &absent), h.value(k) = -2;          // another way to insert
+	if (!absent) printf("already in the table\n");    //   which allows to test presence
+	if (h.get(33) == h.end()) printf("not found!\n"); // test presence without insertion
+	h.del(h.get(43));               // deletion
+	for (k = 0; k != h.end(); ++k)  // traversal
+		if (h.occupied(k))          // some buckets are not occupied; skip them
+			printf("%u => %d\n", h.key(k), h.value(k));
+	return 0;
+}
+*/
+
+namespace klib {
+
+/***********
+ * HashSet *
+ ***********/
+
+template<class T, class Hash, class Eq = std::equal_to<T>, typename khint_t = uint32_t>
+class KHashSet {
+	khint_t bits, count;
+	uint32_t *used;
+	T *keys;
+	static inline uint32_t __kh_used(const uint32_t *flag, khint_t i) { return flag[i>>5] >> (i&0x1fU) & 1U; };
+	static inline void __kh_set_used(uint32_t *flag, khint_t i) { flag[i>>5] |= 1U<<(i&0x1fU); };
+	static inline void __kh_set_unused(uint32_t *flag, khint_t i) { flag[i>>5] &= ~(1U<<(i&0x1fU)); };
+	static inline khint_t __kh_fsize(khint_t m) { return m<32? 1 : m>>5; }
+	static inline khint_t __kh_h2b(uint32_t hash, khint_t bits) { return hash * 2654435769U >> (32 - bits); }
+	static inline khint_t __kh_h2b(uint64_t hash, khint_t bits) { return hash * 11400714819323198485ULL >> (64 - bits); }
+public:
+	KHashSet() : bits(0), count(0), used(0), keys(0) {};
+	~KHashSet() { std::free(used); std::free(keys); };
+	inline khint_t n_buckets() const { return used? khint_t(1) << bits : 0; }
+	inline khint_t end() const { return n_buckets(); }
+	inline khint_t size() const { return count; }
+	inline T &key(khint_t x) { return keys[x]; };
+	inline bool occupied(khint_t x) const { return (__kh_used(used, x) != 0); }
+	void clear(void) { 
+		if (!used) return;
+		memset(used, 0, __kh_fsize(n_buckets()) * sizeof(uint32_t));
+		count = 0;
+	}
+	khint_t get(const T &key) const {
+		khint_t i, last, mask, nb;
+		if (keys == 0) return 0;
+		nb = n_buckets();
+		mask = nb - khint_t(1);
+		i = last = __kh_h2b(Hash()(key), bits);
+		while (__kh_used(used, i) && !Eq()(keys[i], key)) {
+			i = (i + khint_t(1)) & mask;
+			if (i == last) return nb;
+		}
+		return !__kh_used(used, i)? nb : i;
+	}
+	int resize(khint_t new_nb) {
+		uint32_t *new_used = 0;
+		khint_t j = 0, x = new_nb, nb, new_bits, new_mask;
+		while ((x >>= khint_t(1)) != 0) ++j;
+		if (new_nb & (new_nb - 1)) ++j;
+		new_bits = j > 2? j : 2;
+		new_nb = khint_t(1) << new_bits;
+		if (count > (new_nb>>1) + (new_nb>>2)) return 0; /* requested size is too small */
+		new_used = (uint32_t*)std::malloc(__kh_fsize(new_nb) * sizeof(uint32_t));
+		memset(new_used, 0, __kh_fsize(new_nb) * sizeof(uint32_t));
+		if (!new_used) return -1; /* not enough memory */
+		nb = n_buckets();
+		if (nb < new_nb) { /* expand */
+			T *new_keys = (T*)std::realloc(keys, new_nb * sizeof(T));
+			if (!new_keys) { std::free(new_used); return -1; }
+			keys = new_keys;
+		} /* otherwise shrink */
+		new_mask = new_nb - 1;
+		for (j = 0; j != nb; ++j) {
+			if (!__kh_used(used, j)) continue;
+			T key = keys[j];
+			__kh_set_unused(used, j);
+			while (1) { /* kick-out process; sort of like in Cuckoo hashing */
+				khint_t i;
+				i = __kh_h2b(Hash()(key), new_bits);
+				while (__kh_used(new_used, i)) i = (i + khint_t(1)) & new_mask;
+				__kh_set_used(new_used, i);
+				if (i < nb && __kh_used(used, i)) { /* kick out the existing element */
+					{ T tmp = keys[i]; keys[i] = key; key = tmp; }
+					__kh_set_unused(used, i); /* mark it as deleted in the old hash table */
+				} else { /* write the element and jump out of the loop */
+					keys[i] = key;
+					break;
+				}
+			}
+		}
+		if (nb > new_nb) /* shrink the hash table */
+			keys = (T*)std::realloc(keys, new_nb * sizeof(T));
+		std::free(used); /* free the working space */
+		used = new_used, bits = new_bits;
+		return 0;
+	}
+	khint_t put(const T &key, int *absent_ = 0) {
+		khint_t nb, i, last, mask;
+		int absent = -1;
+		nb = n_buckets();
+		if (count >= (nb>>1) + (nb>>2)) { /* rehashing */
+			if (resize(nb + khint_t(1)) < 0) {
+				if (absent_) *absent_ = -1;
+				return nb;
+			}
+			nb = n_buckets();
+		} /* TODO: to implement automatically shrinking; resize() already support shrinking */
+		mask = nb - 1;
+		i = last = __kh_h2b(Hash()(key), bits);
+		while (__kh_used(used, i) && !Eq()(keys[i], key)) {
+			i = (i + 1U) & mask;
+			if (i == last) break;
+		}
+		if (!__kh_used(used, i)) { /* not present at all */
+			keys[i] = key;
+			__kh_set_used(used, i);
+			++count, absent = 1;
+		} else absent = 0; /* Don't touch keys[i] if present */
+		if (absent_) *absent_ = absent;
+		return i;
+	}
+	int del(khint_t i) {
+		khint_t j = i, k, mask, nb = n_buckets();
+		if (keys == 0 || i >= nb) return 0;
+		mask = nb - khint_t(1);
+		while (1) {
+			j = (j + khint_t(1)) & mask;
+			if (j == i || !__kh_used(used, j)) break; /* j==i only when the table is completely full */
+			k = __kh_h2b(Hash()(keys[j]), bits);
+			if ((j > i && (k <= i || k > j)) || (j < i && (k <= i && k > j)))
+				keys[i] = keys[j], i = j;
+		}
+		__kh_set_unused(used, i);
+		--count;
+		return 1;
+	}
+};
+
+/***********
+ * HashMap *
+ ***********/
+
+template<class KType, class VType>
+struct KHashMapBucket { KType key; VType val; };
+
+template<class T, class Hash, typename khint_t>
+struct KHashMapHash { khint_t operator() (const T &a) const { return Hash()(a.key); } };
+
+template<class T, class Eq>
+struct KHashMapEq { bool operator() (const T &a, const T &b) const { return Eq()(a.key, b.key); } };
+
+template<class KType, class VType, class Hash, class Eq=std::equal_to<KType>, typename khint_t=uint32_t>
+class KHashMap : public KHashSet<KHashMapBucket<KType, VType>,
+		KHashMapHash<KHashMapBucket<KType, VType>, Hash, khint_t>,
+		KHashMapEq<KHashMapBucket<KType, VType>, Eq>, khint_t>
+{
+	typedef KHashMapBucket<KType, VType> bucket_t;
+	typedef KHashSet<bucket_t, KHashMapHash<bucket_t, Hash, khint_t>, KHashMapEq<bucket_t, Eq>, khint_t> hashset_t;
+public:
+	khint_t get(const KType &key) const {
+		bucket_t t = { key, VType() };
+		return hashset_t::get(t);
+	}
+	khint_t put(const KType &key, int *absent) {
+		bucket_t t = { key, VType() };
+		return hashset_t::put(t, absent);
+	}
+	inline KType &key(khint_t i) { return hashset_t::key(i).key; }
+	inline VType &value(khint_t i) { return hashset_t::key(i).val; }
+	inline VType &operator[] (const KType &key) {
+		bucket_t t = { key, VType() };
+		return value(hashset_t::put(t));
+	}
+};
+
+/****************************
+ * HashSet with cached hash *
+ ****************************/
+
+template<class KType, typename khint_t>
+struct KHashSetCachedBucket { KType key; khint_t hash; };
+
+template<class T, typename khint_t>
+struct KHashCachedHash { khint_t operator() (const T &a) const { return a.hash; } };
+
+template<class T, class Eq>
+struct KHashCachedEq { bool operator() (const T &a, const T &b) const { return a.hash == b.hash && Eq()(a.key, b.key); } };
+
+template<class KType, class Hash, class Eq = std::equal_to<KType>, typename khint_t = uint32_t>
+class KHashSetCached : public KHashSet<KHashSetCachedBucket<KType, khint_t>,
+		KHashCachedHash<KHashSetCachedBucket<KType, khint_t>, khint_t>,
+		KHashCachedEq<KHashSetCachedBucket<KType, khint_t>, Eq>, khint_t>
+{
+	typedef KHashSetCachedBucket<KType, khint_t> bucket_t;
+	typedef KHashSet<bucket_t, KHashCachedHash<bucket_t, khint_t>, KHashCachedEq<bucket_t, Eq>, khint_t> hashset_t;
+public:
+	khint_t get(const KType &key) const {
+		bucket_t t = { key, Hash()(key) };
+		return hashset_t::get(t);
+	}
+	khint_t put(const KType &key, int *absent) {
+		bucket_t t = { key, Hash()(key) };
+		return hashset_t::put(t, absent);
+	}
+	inline KType &key(khint_t i) { return hashset_t::key(i).key; }
+};
+
+/****************************
+ * HashMap with cached hash *
+ ****************************/
+
+template<class KType, class VType, typename khint_t>
+struct KHashMapCachedBucket { KType key; VType val; khint_t hash; };
+
+template<class KType, class VType, class Hash, class Eq = std::equal_to<KType>, typename khint_t = uint32_t>
+class KHashMapCached : public KHashSet<KHashMapCachedBucket<KType, VType, khint_t>,
+		KHashCachedHash<KHashMapCachedBucket<KType, VType, khint_t>, khint_t>,
+		KHashCachedEq<KHashMapCachedBucket<KType, VType, khint_t>, Eq>, khint_t>
+{
+	typedef KHashMapCachedBucket<KType, VType, khint_t> bucket_t;
+	typedef KHashSet<bucket_t, KHashCachedHash<bucket_t, khint_t>, KHashCachedEq<bucket_t, Eq>, khint_t> hashset_t;
+public:
+	khint_t get(const KType &key) const {
+		bucket_t t = { key, VType(), Hash()(key) };
+		return hashset_t::get(t);
+	}
+	khint_t put(const KType &key, int *absent) {
+		bucket_t t = { key, VType(), Hash()(key) };
+		return hashset_t::put(t, absent);
+	}
+	inline KType &key(khint_t i) { return hashset_t::key(i).key; }
+	inline VType &value(khint_t i) { return hashset_t::key(i).val; }
+	inline VType &operator[] (const KType &key) {
+		bucket_t t = { key, VType(), Hash()(key) };
+		return value(hashset_t::put(t));
+	}
+};
+
+}
+
+#endif /* __AC_KHASHL_HPP */
diff --git a/ext/klib/kalloc.c b/ext/klib/kalloc.c
new file mode 100644
index 0000000..f5de41a
--- /dev/null
+++ b/ext/klib/kalloc.c
@@ -0,0 +1,224 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "kalloc.h"
+
+/* In kalloc, a *core* is a large chunk of contiguous memory. Each core is
+ * associated with a master header, which keeps the size of the current core
+ * and the pointer to next core. Kalloc allocates small *blocks* of memory from
+ * the cores and organizes free memory blocks in a circular single-linked list.
+ *
+ * In the following diagram, "@" stands for the header of a free block (of type
+ * header_t), "#" for the header of an allocated block (of type size_t), "-"
+ * for free memory, and "+" for allocated memory.
+ *
+ * master        This region is core 1.          master           This region is core 2.
+ *      |                                             |
+ *      *@-------#++++++#++++++++++++@--------        *@----------#++++++++++++#+++++++@------------
+ *       |                           |                 |                               |
+ *       p=p->ptr->ptr->ptr->ptr     p->ptr            p->ptr->ptr                     p->ptr->ptr->ptr
+ */
+typedef struct header_t {
+	size_t size;
+	struct header_t *ptr;
+} header_t;
+
+typedef struct {
+	void *par;
+	size_t min_core_size;
+	header_t base, *loop_head, *core_head; /* base is a zero-sized block always kept in the loop */
+} kmem_t;
+
+static void panic(const char *s)
+{
+	fprintf(stderr, "%s\n", s);
+	abort();
+}
+
+void *km_init2(void *km_par, size_t min_core_size)
+{
+	kmem_t *km;
+	km = (kmem_t*)kcalloc(km_par, 1, sizeof(kmem_t));
+	km->par = km_par;
+	if (km_par) km->min_core_size = min_core_size > 0? min_core_size : ((kmem_t*)km_par)->min_core_size - 2;
+	else km->min_core_size = min_core_size > 0? min_core_size : 0x80000;
+	return (void*)km;
+}
+
+void *km_init(void) { return km_init2(0, 0); }
+
+void km_destroy(void *_km)
+{
+	kmem_t *km = (kmem_t*)_km;
+	void *km_par;
+	header_t *p, *q;
+	if (km == NULL) return;
+	km_par = km->par;
+	for (p = km->core_head; p != NULL;) {
+		q = p->ptr;
+		kfree(km_par, p);
+		p = q;
+	}
+	kfree(km_par, km);
+}
+
+static header_t *morecore(kmem_t *km, size_t nu)
+{
+	header_t *q;
+	size_t bytes, *p;
+	nu = (nu + 1 + (km->min_core_size - 1)) / km->min_core_size * km->min_core_size; /* the first +1 for core header */
+	bytes = nu * sizeof(header_t);
+	q = (header_t*)kmalloc(km->par, bytes);
+	if (!q) panic("[morecore] insufficient memory");
+	q->ptr = km->core_head, q->size = nu, km->core_head = q;
+	p = (size_t*)(q + 1);
+	*p = nu - 1; /* the size of the free block; -1 because the first unit is used for the core header */
+	kfree(km, p + 1); /* initialize the new "core"; NB: the core header is not looped. */
+	return km->loop_head;
+}
+
+void kfree(void *_km, void *ap) /* kfree() also adds a new core to the circular list */
+{
+	header_t *p, *q;
+	kmem_t *km = (kmem_t*)_km;
+	
+	if (!ap) return;
+	if (km == NULL) {
+		free(ap);
+		return;
+	}
+	p = (header_t*)((size_t*)ap - 1);
+	p->size = *((size_t*)ap - 1);
+	/* Find the pointer that points to the block to be freed. The following loop can stop on two conditions:
+	 *
+	 * a) "p>q && p<q->ptr": @------#++++++++#+++++++@-------    @---------------#+++++++@-------
+	 *    (can also be in    |      |                |        -> |                       |
+	 *     two cores)        q      p           q->ptr           q                  q->ptr
+	 *
+	 *                       @--------    #+++++++++@--------    @--------    @------------------
+	 *                       |            |         |         -> |            |
+	 *                       q            p    q->ptr            q       q->ptr
+	 *
+	 * b) "q>=q->ptr && (p>q || p<q->ptr)":  @-------#+++++   @--------#+++++++     @-------#+++++   @----------------
+	 *                                       |                |        |         -> |                |
+	 *                                  q->ptr                q        p       q->ptr                q
+	 *
+	 *                                       #+++++++@-----   #++++++++@-------     @-------------   #++++++++@-------
+	 *                                       |       |                 |         -> |                         |
+	 *                                       p  q->ptr                 q       q->ptr                         q
+	 */
+	for (q = km->loop_head; !(p > q && p < q->ptr); q = q->ptr)
+		if (q >= q->ptr && (p > q || p < q->ptr)) break;
+	if (p + p->size == q->ptr) { /* two adjacent blocks, merge p and q->ptr (the 2nd and 4th cases) */
+		p->size += q->ptr->size;
+		p->ptr = q->ptr->ptr;
+	} else if (p + p->size > q->ptr && q->ptr >= p) {
+		panic("[kfree] The end of the allocated block enters a free block.");
+	} else p->ptr = q->ptr; /* backup q->ptr */
+
+	if (q + q->size == p) { /* two adjacent blocks, merge q and p (the other two cases) */
+		q->size += p->size;
+		q->ptr = p->ptr;
+		km->loop_head = q;
+	} else if (q + q->size > p && p >= q) {
+		panic("[kfree] The end of a free block enters the allocated block.");
+	} else km->loop_head = p, q->ptr = p; /* in two cores, cannot be merged; create a new block in the list */
+}
+
+void *kmalloc(void *_km, size_t n_bytes)
+{
+	kmem_t *km = (kmem_t*)_km;
+	size_t n_units;
+	header_t *p, *q;
+
+	if (n_bytes == 0) return 0;
+	if (km == NULL) return malloc(n_bytes);
+	n_units = (n_bytes + sizeof(size_t) + sizeof(header_t) - 1) / sizeof(header_t); /* header+n_bytes requires at least this number of units */
+
+	if (!(q = km->loop_head)) /* the first time when kmalloc() is called, intialize it */
+		q = km->loop_head = km->base.ptr = &km->base;
+	for (p = q->ptr;; q = p, p = p->ptr) { /* search for a suitable block */
+		if (p->size >= n_units) { /* p->size if the size of current block. This line means the current block is large enough. */
+			if (p->size == n_units) q->ptr = p->ptr; /* no need to split the block */
+			else { /* split the block. NB: memory is allocated at the end of the block! */
+				p->size -= n_units; /* reduce the size of the free block */
+				p += p->size; /* p points to the allocated block */
+				*(size_t*)p = n_units; /* set the size */
+			}
+			km->loop_head = q; /* set the end of chain */
+			return (size_t*)p + 1;
+		}
+		if (p == km->loop_head) { /* then ask for more "cores" */
+			if ((p = morecore(km, n_units)) == 0) return 0;
+		}
+	}
+}
+
+void *kcalloc(void *_km, size_t count, size_t size)
+{
+	kmem_t *km = (kmem_t*)_km;
+	void *p;
+	if (size == 0 || count == 0) return 0;
+	if (km == NULL) return calloc(count, size);
+	p = kmalloc(km, count * size);
+	memset(p, 0, count * size);
+	return p;
+}
+
+void *krealloc(void *_km, void *ap, size_t n_bytes) // TODO: this can be made more efficient in principle
+{
+	kmem_t *km = (kmem_t*)_km;
+	size_t cap, *p, *q;
+
+	if (n_bytes == 0) {
+		kfree(km, ap); return 0;
+	}
+	if (km == NULL) return realloc(ap, n_bytes);
+	if (ap == NULL) return kmalloc(km, n_bytes);
+	p = (size_t*)ap - 1;
+	cap = (*p) * sizeof(header_t) - sizeof(size_t);
+	if (cap >= n_bytes) return ap; /* TODO: this prevents shrinking */
+	q = (size_t*)kmalloc(km, n_bytes);
+	memcpy(q, ap, cap);
+	kfree(km, ap);
+	return q;
+}
+
+void *krelocate(void *km, void *ap, size_t n_bytes)
+{
+	void *p;
+	if (km == 0 || ap == 0) return ap;
+	p = kmalloc(km, n_bytes);
+	memcpy(p, ap, n_bytes);
+	kfree(km, ap);
+	return p;
+}
+
+void km_stat(const void *_km, km_stat_t *s)
+{
+	kmem_t *km = (kmem_t*)_km;
+	header_t *p;
+	memset(s, 0, sizeof(km_stat_t));
+	if (km == NULL || km->loop_head == NULL) return;
+	for (p = km->loop_head;; p = p->ptr) {
+		s->available += p->size * sizeof(header_t);
+		if (p->size != 0) ++s->n_blocks; /* &kmem_t::base is always one of the cores. It is zero-sized. */
+		if (p->ptr > p && p + p->size > p->ptr)
+			panic("[km_stat] The end of a free block enters another free block.");
+		if (p->ptr == km->loop_head) break;
+	}
+	for (p = km->core_head; p != NULL; p = p->ptr) {
+		size_t size = p->size * sizeof(header_t);
+		++s->n_cores;
+		s->capacity += size;
+		s->largest = s->largest > size? s->largest : size;
+	}
+}
+
+void km_stat_print(const void *km)
+{
+	km_stat_t st;
+	km_stat(km, &st);
+	fprintf(stderr, "[km_stat] cap=%ld, avail=%ld, largest=%ld, n_core=%ld, n_block=%ld\n",
+			st.capacity, st.available, st.largest, st.n_blocks, st.n_cores);
+}
diff --git a/ext/klib/kalloc.h b/ext/klib/kalloc.h
new file mode 100644
index 0000000..4378672
--- /dev/null
+++ b/ext/klib/kalloc.h
@@ -0,0 +1,87 @@
+#ifndef _KALLOC_H_
+#define _KALLOC_H_
+
+#include <stddef.h> /* for size_t */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+	size_t capacity, available, n_blocks, n_cores, largest;
+} km_stat_t;
+
+void *kmalloc(void *km, size_t size);
+void *krealloc(void *km, void *ptr, size_t size);
+void *krelocate(void *km, void *ap, size_t n_bytes);
+void *kcalloc(void *km, size_t count, size_t size);
+void kfree(void *km, void *ptr);
+
+void *km_init(void);
+void *km_init2(void *km_par, size_t min_core_size);
+void km_destroy(void *km);
+void km_stat(const void *_km, km_stat_t *s);
+void km_stat_print(const void *km);
+
+#ifdef __cplusplus
+}
+#endif
+
+#define Kmalloc(km, type, cnt)       ((type*)kmalloc((km), (cnt) * sizeof(type)))
+#define Kcalloc(km, type, cnt)       ((type*)kcalloc((km), (cnt), sizeof(type)))
+#define Krealloc(km, type, ptr, cnt) ((type*)krealloc((km), (ptr), (cnt) * sizeof(type)))
+
+#define Kexpand(km, type, a, m) do { \
+		(m) = (m) >= 4? (m) + ((m)>>1) : 16; \
+		(a) = Krealloc(km, type, (a), (m)); \
+	} while (0)
+
+#define KMALLOC(km, ptr, len) ((ptr) = (__typeof__(ptr))kmalloc((km), (len) * sizeof(*(ptr))))
+#define KCALLOC(km, ptr, len) ((ptr) = (__typeof__(ptr))kcalloc((km), (len), sizeof(*(ptr))))
+#define KREALLOC(km, ptr, len) ((ptr) = (__typeof__(ptr))krealloc((km), (ptr), (len) * sizeof(*(ptr))))
+
+#define KEXPAND(km, a, m) do { \
+		(m) = (m) >= 4? (m) + ((m)>>1) : 16; \
+		KREALLOC((km), (a), (m)); \
+	} while (0)
+
+#ifndef klib_unused
+#if (defined __clang__ && __clang_major__ >= 3) || (defined __GNUC__ && __GNUC__ >= 3)
+#define klib_unused __attribute__ ((__unused__))
+#else
+#define klib_unused
+#endif
+#endif /* klib_unused */
+
+#define KALLOC_POOL_INIT2(SCOPE, name, kmptype_t) \
+	typedef struct { \
+		size_t cnt, n, max; \
+		kmptype_t **buf; \
+		void *km; \
+	} kmp_##name##_t; \
+	SCOPE kmp_##name##_t *kmp_init_##name(void *km) { \
+		kmp_##name##_t *mp; \
+		mp = Kcalloc(km, kmp_##name##_t, 1); \
+		mp->km = km; \
+		return mp; \
+	} \
+	SCOPE void kmp_destroy_##name(kmp_##name##_t *mp) { \
+		size_t k; \
+		for (k = 0; k < mp->n; ++k) kfree(mp->km, mp->buf[k]); \
+		kfree(mp->km, mp->buf); kfree(mp->km, mp); \
+	} \
+	SCOPE kmptype_t *kmp_alloc_##name(kmp_##name##_t *mp) { \
+		++mp->cnt; \
+		if (mp->n == 0) return (kmptype_t*)kcalloc(mp->km, 1, sizeof(kmptype_t)); \
+		return mp->buf[--mp->n]; \
+	} \
+	SCOPE void kmp_free_##name(kmp_##name##_t *mp, kmptype_t *p) { \
+		--mp->cnt; \
+		if (mp->n == mp->max) Kexpand(mp->km, kmptype_t*, mp->buf, mp->max); \
+		mp->buf[mp->n++] = p; \
+	}
+
+#define KALLOC_POOL_INIT(name, kmptype_t) \
+	KALLOC_POOL_INIT2(static inline klib_unused, name, kmptype_t)
+
+#endif
diff --git a/ext/klib/kavl-lite.h b/ext/klib/kavl-lite.h
new file mode 100644
index 0000000..240ac8a
--- /dev/null
+++ b/ext/klib/kavl-lite.h
@@ -0,0 +1,306 @@
+/* The MIT License
+
+   Copyright (c) 2021 by Attractive Chaos <attractor@live.co.uk>
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be
+   included in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+*/
+
+/* An example:
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include "kavl-lite.h"
+
+struct my_node {
+  char key;
+  KAVLL_HEAD(struct my_node) head;
+};
+#define my_cmp(p, q) (((q)->key < (p)->key) - ((p)->key < (q)->key))
+KAVLL_INIT(my, struct my_node, head, my_cmp)
+
+int main(void) {
+  const char *str = "MNOLKQOPHIA"; // from wiki, except a duplicate
+  struct my_node *root = 0;
+  int i, l = strlen(str);
+  for (i = 0; i < l; ++i) {        // insert in the input order
+    struct my_node *q, *p = malloc(sizeof(*p));
+    p->key = str[i];
+    q = my_insert(&root, p);
+    if (p != q) free(p);           // if already present, free
+  }
+  my_itr_t itr;
+  my_itr_first(root, &itr);  // place at first
+  do {                             // traverse
+    const struct my_node *p = kavll_at(&itr);
+    putchar(p->key);
+    free((void*)p);                // free node
+  } while (my_itr_next(&itr));
+  putchar('\n');
+  return 0;
+}
+*/
+
+#ifndef KAVL_LITE_H
+#define KAVL_LITE_H
+
+#ifdef __STRICT_ANSI__
+#define inline __inline__
+#endif
+
+#define KAVLL_MAX_DEPTH 64
+
+#define KAVLL_HEAD(__type) \
+	struct { \
+		__type *p[2]; \
+		signed char balance; /* balance factor */ \
+	}
+
+#define __KAVLL_FIND(pre, __scope, __type, __head,  __cmp) \
+	__scope __type *pre##_find(const __type *root, const __type *x) { \
+		const __type *p = root; \
+		while (p != 0) { \
+			int cmp; \
+			cmp = __cmp(x, p); \
+			if (cmp < 0) p = p->__head.p[0]; \
+			else if (cmp > 0) p = p->__head.p[1]; \
+			else break; \
+		} \
+		return (__type*)p; \
+	}
+
+#define __KAVLL_ROTATE(pre, __type, __head) \
+	/* one rotation: (a,(b,c)q)p => ((a,b)p,c)q */ \
+	static inline __type *pre##_rotate1(__type *p, int dir) { /* dir=0 to left; dir=1 to right */ \
+		int opp = 1 - dir; /* opposite direction */ \
+		__type *q = p->__head.p[opp]; \
+		p->__head.p[opp] = q->__head.p[dir]; \
+		q->__head.p[dir] = p; \
+		return q; \
+	} \
+	/* two consecutive rotations: (a,((b,c)r,d)q)p => ((a,b)p,(c,d)q)r */ \
+	static inline __type *pre##_rotate2(__type *p, int dir) { \
+		int b1, opp = 1 - dir; \
+		__type *q = p->__head.p[opp], *r = q->__head.p[dir]; \
+		p->__head.p[opp] = r->__head.p[dir]; \
+		r->__head.p[dir] = p; \
+		q->__head.p[dir] = r->__head.p[opp]; \
+		r->__head.p[opp] = q; \
+		b1 = dir == 0? +1 : -1; \
+		if (r->__head.balance == b1) q->__head.balance = 0, p->__head.balance = -b1; \
+		else if (r->__head.balance == 0) q->__head.balance = p->__head.balance = 0; \
+		else q->__head.balance = b1, p->__head.balance = 0; \
+		r->__head.balance = 0; \
+		return r; \
+	}
+
+#define __KAVLL_INSERT(pre, __scope, __type, __head, __cmp) \
+	__scope __type *pre##_insert(__type **root_, __type *x) { \
+		unsigned char stack[KAVLL_MAX_DEPTH]; \
+		__type *path[KAVLL_MAX_DEPTH]; \
+		__type *bp, *bq; \
+		__type *p, *q, *r = 0; /* _r_ is potentially the new root */ \
+		int which = 0, top, b1, path_len; \
+		bp = *root_, bq = 0; \
+		/* find the insertion location */ \
+		for (p = bp, q = bq, top = path_len = 0; p; q = p, p = p->__head.p[which]) { \
+			int cmp; \
+			cmp = __cmp(x, p); \
+			if (cmp == 0) return p; \
+			if (p->__head.balance != 0) \
+				bq = q, bp = p, top = 0; \
+			stack[top++] = which = (cmp > 0); \
+			path[path_len++] = p; \
+		} \
+		x->__head.balance = 0, x->__head.p[0] = x->__head.p[1] = 0; \
+		if (q == 0) *root_ = x; \
+		else q->__head.p[which] = x; \
+		if (bp == 0) return x; \
+		for (p = bp, top = 0; p != x; p = p->__head.p[stack[top]], ++top) /* update balance factors */ \
+			if (stack[top] == 0) --p->__head.balance; \
+			else ++p->__head.balance; \
+		if (bp->__head.balance > -2 && bp->__head.balance < 2) return x; /* no re-balance needed */ \
+		/* re-balance */ \
+		which = (bp->__head.balance < 0); \
+		b1 = which == 0? +1 : -1; \
+		q = bp->__head.p[1 - which]; \
+		if (q->__head.balance == b1) { \
+			r = pre##_rotate1(bp, which); \
+			q->__head.balance = bp->__head.balance = 0; \
+		} else r = pre##_rotate2(bp, which); \
+		if (bq == 0) *root_ = r; \
+		else bq->__head.p[bp != bq->__head.p[0]] = r; \
+		return x; \
+	}
+
+#define __KAVLL_ERASE(pre, __scope, __type, __head, __cmp) \
+	__scope __type *pre##_erase(__type **root_, const __type *x) { \
+		__type *p, *path[KAVLL_MAX_DEPTH], fake; \
+		unsigned char dir[KAVLL_MAX_DEPTH]; \
+		int d = 0, cmp; \
+		fake.__head.p[0] = *root_, fake.__head.p[1] = 0; \
+		if (x) { \
+			for (cmp = -1, p = &fake; cmp; cmp = __cmp(x, p)) { \
+				int which = (cmp > 0); \
+				dir[d] = which; \
+				path[d++] = p; \
+				p = p->__head.p[which]; \
+				if (p == 0) return 0; \
+			} \
+		} else { \
+			for (p = &fake; p; p = p->__head.p[0]) \
+				dir[d] = 0, path[d++] = p; \
+			p = path[--d]; \
+		} \
+		if (p->__head.p[1] == 0) { /* ((1,.)2,3)4 => (1,3)4; p=2 */ \
+			path[d-1]->__head.p[dir[d-1]] = p->__head.p[0]; \
+		} else { \
+			__type *q = p->__head.p[1]; \
+			if (q->__head.p[0] == 0) { /* ((1,2)3,4)5 => ((1)2,4)5; p=3 */ \
+				q->__head.p[0] = p->__head.p[0]; \
+				q->__head.balance = p->__head.balance; \
+				path[d-1]->__head.p[dir[d-1]] = q; \
+				path[d] = q, dir[d++] = 1; \
+			} else { /* ((1,((.,2)3,4)5)6,7)8 => ((1,(2,4)5)3,7)8; p=6 */ \
+				__type *r; \
+				int e = d++; /* backup _d_ */\
+				for (;;) { \
+					dir[d] = 0; \
+					path[d++] = q; \
+					r = q->__head.p[0]; \
+					if (r->__head.p[0] == 0) break; \
+					q = r; \
+				} \
+				r->__head.p[0] = p->__head.p[0]; \
+				q->__head.p[0] = r->__head.p[1]; \
+				r->__head.p[1] = p->__head.p[1]; \
+				r->__head.balance = p->__head.balance; \
+				path[e-1]->__head.p[dir[e-1]] = r; \
+				path[e] = r, dir[e] = 1; \
+			} \
+		} \
+		while (--d > 0) { \
+			__type *q = path[d]; \
+			int which, other, b1 = 1, b2 = 2; \
+			which = dir[d], other = 1 - which; \
+			if (which) b1 = -b1, b2 = -b2; \
+			q->__head.balance += b1; \
+			if (q->__head.balance == b1) break; \
+			else if (q->__head.balance == b2) { \
+				__type *r = q->__head.p[other]; \
+				if (r->__head.balance == -b1) { \
+					path[d-1]->__head.p[dir[d-1]] = pre##_rotate2(q, which); \
+				} else { \
+					path[d-1]->__head.p[dir[d-1]] = pre##_rotate1(q, which); \
+					if (r->__head.balance == 0) { \
+						r->__head.balance = -b1; \
+						q->__head.balance = b1; \
+						break; \
+					} else r->__head.balance = q->__head.balance = 0; \
+				} \
+			} \
+		} \
+		*root_ = fake.__head.p[0]; \
+		return p; \
+	}
+
+#define kavll_free(__type, __head, __root, __free) do { \
+		__type *_p, *_q; \
+		for (_p = __root; _p; _p = _q) { \
+			if (_p->__head.p[0] == 0) { \
+				_q = _p->__head.p[1]; \
+				__free(_p); \
+			} else { \
+				_q = _p->__head.p[0]; \
+				_p->__head.p[0] = _q->__head.p[1]; \
+				_q->__head.p[1] = _p; \
+			} \
+		} \
+	} while (0)
+
+#define kavll_size(__type, __head, __root, __cnt) do { \
+		__type *_p, *_q; \
+		*(__cnt) = 0; \
+		for (_p = __root; _p; _p = _q) { \
+			if (_p->__head.p[0] == 0) { \
+				_q = _p->__head.p[1]; \
+				++*(__cnt); \
+			} else { \
+				_q = _p->__head.p[0]; \
+				_p->__head.p[0] = _q->__head.p[1]; \
+				_q->__head.p[1] = _p; \
+			} \
+		} \
+	} while (0)
+
+#define __KAVLL_ITR(pre, __scope, __type, __head, __cmp) \
+	typedef struct pre##_itr_t { \
+		const __type *stack[KAVLL_MAX_DEPTH], **top, *right; /* _right_ points to the right child of *top */ \
+	} pre##_itr_t; \
+	__scope void pre##_itr_first(const __type *root, struct pre##_itr_t *itr) { \
+		const __type *p; \
+		for (itr->top = itr->stack - 1, p = root; p; p = p->__head.p[0]) \
+			*++itr->top = p; \
+		itr->right = (*itr->top)->__head.p[1]; \
+	} \
+	__scope int pre##_itr_find(const __type *root, const __type *x, struct pre##_itr_t *itr) { \
+		const __type *p = root; \
+		itr->top = itr->stack - 1; \
+		while (p != 0) { \
+			int cmp; \
+			cmp = __cmp(x, p); \
+			if (cmp < 0) *++itr->top = p, p = p->__head.p[0]; \
+			else if (cmp > 0) p = p->__head.p[1]; \
+			else break; \
+		} \
+		if (p) { \
+			*++itr->top = p; \
+			itr->right = p->__head.p[1]; \
+			return 1; \
+		} else if (itr->top >= itr->stack) { \
+			itr->right = (*itr->top)->__head.p[1]; \
+			return 0; \
+		} else return 0; \
+	} \
+	__scope int pre##_itr_next(struct pre##_itr_t *itr) { \
+		for (;;) { \
+			const __type *p; \
+			for (p = itr->right, --itr->top; p; p = p->__head.p[0]) \
+				*++itr->top = p; \
+			if (itr->top < itr->stack) return 0; \
+			itr->right = (*itr->top)->__head.p[1]; \
+			return 1; \
+		} \
+	}
+
+#define kavll_at(itr) ((itr)->top < (itr)->stack? 0 : *(itr)->top)
+
+#define KAVLL_INIT2(pre, __scope, __type, __head, __cmp) \
+	__KAVLL_FIND(pre, __scope, __type, __head,  __cmp) \
+	__KAVLL_ROTATE(pre, __type, __head) \
+	__KAVLL_INSERT(pre, __scope, __type, __head, __cmp) \
+	__KAVLL_ERASE(pre, __scope, __type, __head, __cmp) \
+	__KAVLL_ITR(pre, __scope, __type, __head, __cmp)
+
+#define KAVLL_INIT(pre, __type, __head, __cmp) \
+	KAVLL_INIT2(pre,, __type, __head, __cmp)
+
+#endif
diff --git a/ext/klib/kavl.h b/ext/klib/kavl.h
new file mode 100644
index 0000000..f3520fd
--- /dev/null
+++ b/ext/klib/kavl.h
@@ -0,0 +1,400 @@
+/* The MIT License
+
+   Copyright (c) 2018 by Attractive Chaos <attractor@live.co.uk>
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be
+   included in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+*/
+
+/* An example:
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include "kavl.h"
+
+struct my_node {
+  char key;
+  KAVL_HEAD(struct my_node) head;
+};
+#define my_cmp(p, q) (((q)->key < (p)->key) - ((p)->key < (q)->key))
+KAVL_INIT(my, struct my_node, head, my_cmp)
+
+int main(void) {
+  const char *str = "MNOLKQOPHIA"; // from wiki, except a duplicate
+  struct my_node *root = 0;
+  int i, l = strlen(str);
+  for (i = 0; i < l; ++i) {        // insert in the input order
+    struct my_node *q, *p = malloc(sizeof(*p));
+    p->key = str[i];
+    q = kavl_insert(my, &root, p, 0);
+    if (p != q) free(p);           // if already present, free
+  }
+  kavl_itr_t(my) itr;
+  kavl_itr_first(my, root, &itr);  // place at first
+  do {                             // traverse
+    const struct my_node *p = kavl_at(&itr);
+    putchar(p->key);
+    free((void*)p);                // free node
+  } while (kavl_itr_next(my, &itr));
+  putchar('\n');
+  return 0;
+}
+*/
+
+#ifndef KAVL_H
+#define KAVL_H
+
+#ifdef __STRICT_ANSI__
+#define inline __inline__
+#endif
+
+#define KAVL_MAX_DEPTH 64
+
+#define kavl_size(head, p) ((p)? (p)->head.size : 0)
+#define kavl_size_child(head, q, i) ((q)->head.p[(i)]? (q)->head.p[(i)]->head.size : 0)
+
+#define KAVL_HEAD(__type) \
+	struct { \
+		__type *p[2]; \
+		signed char balance; /* balance factor */ \
+		unsigned size; /* #elements in subtree */ \
+	}
+
+#define __KAVL_FIND(suf, __scope, __type, __head,  __cmp) \
+	__scope __type *kavl_find_##suf(const __type *root, const __type *x, unsigned *cnt_) { \
+		const __type *p = root; \
+		unsigned cnt = 0; \
+		while (p != 0) { \
+			int cmp; \
+			cmp = __cmp(x, p); \
+			if (cmp >= 0) cnt += kavl_size_child(__head, p, 0) + 1; \
+			if (cmp < 0) p = p->__head.p[0]; \
+			else if (cmp > 0) p = p->__head.p[1]; \
+			else break; \
+		} \
+		if (cnt_) *cnt_ = cnt; \
+		return (__type*)p; \
+	}
+
+#define __KAVL_ROTATE(suf, __type, __head) \
+	/* one rotation: (a,(b,c)q)p => ((a,b)p,c)q */ \
+	static inline __type *kavl_rotate1_##suf(__type *p, int dir) { /* dir=0 to left; dir=1 to right */ \
+		int opp = 1 - dir; /* opposite direction */ \
+		__type *q = p->__head.p[opp]; \
+		unsigned size_p = p->__head.size; \
+		p->__head.size -= q->__head.size - kavl_size_child(__head, q, dir); \
+		q->__head.size = size_p; \
+		p->__head.p[opp] = q->__head.p[dir]; \
+		q->__head.p[dir] = p; \
+		return q; \
+	} \
+	/* two consecutive rotations: (a,((b,c)r,d)q)p => ((a,b)p,(c,d)q)r */ \
+	static inline __type *kavl_rotate2_##suf(__type *p, int dir) { \
+		int b1, opp = 1 - dir; \
+		__type *q = p->__head.p[opp], *r = q->__head.p[dir]; \
+		unsigned size_x_dir = kavl_size_child(__head, r, dir); \
+		r->__head.size = p->__head.size; \
+		p->__head.size -= q->__head.size - size_x_dir; \
+		q->__head.size -= size_x_dir + 1; \
+		p->__head.p[opp] = r->__head.p[dir]; \
+		r->__head.p[dir] = p; \
+		q->__head.p[dir] = r->__head.p[opp]; \
+		r->__head.p[opp] = q; \
+		b1 = dir == 0? +1 : -1; \
+		if (r->__head.balance == b1) q->__head.balance = 0, p->__head.balance = -b1; \
+		else if (r->__head.balance == 0) q->__head.balance = p->__head.balance = 0; \
+		else q->__head.balance = b1, p->__head.balance = 0; \
+		r->__head.balance = 0; \
+		return r; \
+	}
+
+#define __KAVL_INSERT(suf, __scope, __type, __head, __cmp) \
+	__scope __type *kavl_insert_##suf(__type **root_, __type *x, unsigned *cnt_) { \
+		unsigned char stack[KAVL_MAX_DEPTH]; \
+		__type *path[KAVL_MAX_DEPTH]; \
+		__type *bp, *bq; \
+		__type *p, *q, *r = 0; /* _r_ is potentially the new root */ \
+		int i, which = 0, top, b1, path_len; \
+		unsigned cnt = 0; \
+		bp = *root_, bq = 0; \
+		/* find the insertion location */ \
+		for (p = bp, q = bq, top = path_len = 0; p; q = p, p = p->__head.p[which]) { \
+			int cmp; \
+			cmp = __cmp(x, p); \
+			if (cmp >= 0) cnt += kavl_size_child(__head, p, 0) + 1; \
+			if (cmp == 0) { \
+				if (cnt_) *cnt_ = cnt; \
+				return p; \
+			} \
+			if (p->__head.balance != 0) \
+				bq = q, bp = p, top = 0; \
+			stack[top++] = which = (cmp > 0); \
+			path[path_len++] = p; \
+		} \
+		if (cnt_) *cnt_ = cnt; \
+		x->__head.balance = 0, x->__head.size = 1, x->__head.p[0] = x->__head.p[1] = 0; \
+		if (q == 0) *root_ = x; \
+		else q->__head.p[which] = x; \
+		if (bp == 0) return x; \
+		for (i = 0; i < path_len; ++i) ++path[i]->__head.size; \
+		for (p = bp, top = 0; p != x; p = p->__head.p[stack[top]], ++top) /* update balance factors */ \
+			if (stack[top] == 0) --p->__head.balance; \
+			else ++p->__head.balance; \
+		if (bp->__head.balance > -2 && bp->__head.balance < 2) return x; /* no re-balance needed */ \
+		/* re-balance */ \
+		which = (bp->__head.balance < 0); \
+		b1 = which == 0? +1 : -1; \
+		q = bp->__head.p[1 - which]; \
+		if (q->__head.balance == b1) { \
+			r = kavl_rotate1_##suf(bp, which); \
+			q->__head.balance = bp->__head.balance = 0; \
+		} else r = kavl_rotate2_##suf(bp, which); \
+		if (bq == 0) *root_ = r; \
+		else bq->__head.p[bp != bq->__head.p[0]] = r; \
+		return x; \
+	}
+
+#define __KAVL_ERASE(suf, __scope, __type, __head, __cmp) \
+	__scope __type *kavl_erase_##suf(__type **root_, const __type *x, unsigned *cnt_) { \
+		__type *p, *path[KAVL_MAX_DEPTH], fake; \
+		unsigned char dir[KAVL_MAX_DEPTH]; \
+		int i, d = 0, cmp; \
+		unsigned cnt = 0; \
+		fake.__head.p[0] = *root_, fake.__head.p[1] = 0; \
+		if (cnt_) *cnt_ = 0; \
+		if (x) { \
+			for (cmp = -1, p = &fake; cmp; cmp = __cmp(x, p)) { \
+				int which = (cmp > 0); \
+				if (cmp > 0) cnt += kavl_size_child(__head, p, 0) + 1; \
+				dir[d] = which; \
+				path[d++] = p; \
+				p = p->__head.p[which]; \
+				if (p == 0) { \
+					if (cnt_) *cnt_ = 0; \
+					return 0; \
+				} \
+			} \
+			cnt += kavl_size_child(__head, p, 0) + 1; /* because p==x is not counted */ \
+		} else { \
+			for (p = &fake, cnt = 1; p; p = p->__head.p[0]) \
+				dir[d] = 0, path[d++] = p; \
+			p = path[--d]; \
+		} \
+		if (cnt_) *cnt_ = cnt; \
+		for (i = 1; i < d; ++i) --path[i]->__head.size; \
+		if (p->__head.p[1] == 0) { /* ((1,.)2,3)4 => (1,3)4; p=2 */ \
+			path[d-1]->__head.p[dir[d-1]] = p->__head.p[0]; \
+		} else { \
+			__type *q = p->__head.p[1]; \
+			if (q->__head.p[0] == 0) { /* ((1,2)3,4)5 => ((1)2,4)5; p=3 */ \
+				q->__head.p[0] = p->__head.p[0]; \
+				q->__head.balance = p->__head.balance; \
+				path[d-1]->__head.p[dir[d-1]] = q; \
+				path[d] = q, dir[d++] = 1; \
+				q->__head.size = p->__head.size - 1; \
+			} else { /* ((1,((.,2)3,4)5)6,7)8 => ((1,(2,4)5)3,7)8; p=6 */ \
+				__type *r; \
+				int e = d++; /* backup _d_ */\
+				for (;;) { \
+					dir[d] = 0; \
+					path[d++] = q; \
+					r = q->__head.p[0]; \
+					if (r->__head.p[0] == 0) break; \
+					q = r; \
+				} \
+				r->__head.p[0] = p->__head.p[0]; \
+				q->__head.p[0] = r->__head.p[1]; \
+				r->__head.p[1] = p->__head.p[1]; \
+				r->__head.balance = p->__head.balance; \
+				path[e-1]->__head.p[dir[e-1]] = r; \
+				path[e] = r, dir[e] = 1; \
+				for (i = e + 1; i < d; ++i) --path[i]->__head.size; \
+				r->__head.size = p->__head.size - 1; \
+			} \
+		} \
+		while (--d > 0) { \
+			__type *q = path[d]; \
+			int which, other, b1 = 1, b2 = 2; \
+			which = dir[d], other = 1 - which; \
+			if (which) b1 = -b1, b2 = -b2; \
+			q->__head.balance += b1; \
+			if (q->__head.balance == b1) break; \
+			else if (q->__head.balance == b2) { \
+				__type *r = q->__head.p[other]; \
+				if (r->__head.balance == -b1) { \
+					path[d-1]->__head.p[dir[d-1]] = kavl_rotate2_##suf(q, which); \
+				} else { \
+					path[d-1]->__head.p[dir[d-1]] = kavl_rotate1_##suf(q, which); \
+					if (r->__head.balance == 0) { \
+						r->__head.balance = -b1; \
+						q->__head.balance = b1; \
+						break; \
+					} else r->__head.balance = q->__head.balance = 0; \
+				} \
+			} \
+		} \
+		*root_ = fake.__head.p[0]; \
+		return p; \
+	}
+
+#define kavl_free(__type, __head, __root, __free) do { \
+		__type *_p, *_q; \
+		for (_p = __root; _p; _p = _q) { \
+			if (_p->__head.p[0] == 0) { \
+				_q = _p->__head.p[1]; \
+				__free(_p); \
+			} else { \
+				_q = _p->__head.p[0]; \
+				_p->__head.p[0] = _q->__head.p[1]; \
+				_q->__head.p[1] = _p; \
+			} \
+		} \
+	} while (0)
+
+#define __KAVL_ITR(suf, __scope, __type, __head, __cmp) \
+	struct kavl_itr_##suf { \
+		const __type *stack[KAVL_MAX_DEPTH], **top, *right; /* _right_ points to the right child of *top */ \
+	}; \
+	__scope void kavl_itr_first_##suf(const __type *root, struct kavl_itr_##suf *itr) { \
+		const __type *p; \
+		for (itr->top = itr->stack - 1, p = root; p; p = p->__head.p[0]) \
+			*++itr->top = p; \
+		itr->right = (*itr->top)->__head.p[1]; \
+	} \
+	__scope int kavl_itr_find_##suf(const __type *root, const __type *x, struct kavl_itr_##suf *itr) { \
+		const __type *p = root; \
+		itr->top = itr->stack - 1; \
+		while (p != 0) { \
+			int cmp; \
+			cmp = __cmp(x, p); \
+			if (cmp < 0) *++itr->top = p, p = p->__head.p[0]; \
+			else if (cmp > 0) p = p->__head.p[1]; \
+			else break; \
+		} \
+		if (p) { \
+			*++itr->top = p; \
+			itr->right = p->__head.p[1]; \
+			return 1; \
+		} else if (itr->top >= itr->stack) { \
+			itr->right = (*itr->top)->__head.p[1]; \
+			return 0; \
+		} else return 0; \
+	} \
+	__scope int kavl_itr_next_##suf(struct kavl_itr_##suf *itr) { \
+		for (;;) { \
+			const __type *p; \
+			for (p = itr->right, --itr->top; p; p = p->__head.p[0]) \
+				*++itr->top = p; \
+			if (itr->top < itr->stack) return 0; \
+			itr->right = (*itr->top)->__head.p[1]; \
+			return 1; \
+		} \
+	}
+
+/**
+ * Insert a node to the tree
+ *
+ * @param suf     name suffix used in KAVL_INIT()
+ * @param proot   pointer to the root of the tree (in/out: root may change)
+ * @param x       node to insert (in)
+ * @param cnt     number of nodes smaller than or equal to _x_; can be NULL (out)
+ *
+ * @return _x_ if not present in the tree, or the node equal to x.
+ */
+#define kavl_insert(suf, proot, x, cnt) kavl_insert_##suf(proot, x, cnt)
+
+/**
+ * Find a node in the tree
+ *
+ * @param suf     name suffix used in KAVL_INIT()
+ * @param root    root of the tree
+ * @param x       node value to find (in)
+ * @param cnt     number of nodes smaller than or equal to _x_; can be NULL (out)
+ *
+ * @return node equal to _x_ if present, or NULL if absent
+ */
+#define kavl_find(suf, root, x, cnt) kavl_find_##suf(root, x, cnt)
+
+/**
+ * Delete a node from the tree
+ *
+ * @param suf     name suffix used in KAVL_INIT()
+ * @param proot   pointer to the root of the tree (in/out: root may change)
+ * @param x       node value to delete; if NULL, delete the first node (in)
+ *
+ * @return node removed from the tree if present, or NULL if absent
+ */
+#define kavl_erase(suf, proot, x, cnt) kavl_erase_##suf(proot, x, cnt)
+#define kavl_erase_first(suf, proot) kavl_erase_##suf(proot, 0, 0)
+
+#define kavl_itr_t(suf) struct kavl_itr_##suf
+
+/**
+ * Place the iterator at the smallest object
+ *
+ * @param suf     name suffix used in KAVL_INIT()
+ * @param root    root of the tree
+ * @param itr     iterator
+ */
+#define kavl_itr_first(suf, root, itr) kavl_itr_first_##suf(root, itr)
+
+/**
+ * Place the iterator at the object equal to or greater than the query
+ *
+ * @param suf     name suffix used in KAVL_INIT()
+ * @param root    root of the tree
+ * @param x       query (in)
+ * @param itr     iterator (out)
+ *
+ * @return 1 if find; 0 otherwise. kavl_at(itr) is NULL if and only if query is
+ *         larger than all objects in the tree
+ */
+#define kavl_itr_find(suf, root, x, itr) kavl_itr_find_##suf(root, x, itr)
+
+/**
+ * Move to the next object in order
+ *
+ * @param itr     iterator (modified)
+ *
+ * @return 1 if there is a next object; 0 otherwise
+ */
+#define kavl_itr_next(suf, itr) kavl_itr_next_##suf(itr)
+
+/**
+ * Return the pointer at the iterator
+ *
+ * @param itr     iterator
+ *
+ * @return pointer if present; NULL otherwise
+ */
+#define kavl_at(itr) ((itr)->top < (itr)->stack? 0 : *(itr)->top)
+
+#define KAVL_INIT2(suf, __scope, __type, __head, __cmp) \
+	__KAVL_FIND(suf, __scope, __type, __head,  __cmp) \
+	__KAVL_ROTATE(suf, __type, __head) \
+	__KAVL_INSERT(suf, __scope, __type, __head, __cmp) \
+	__KAVL_ERASE(suf, __scope, __type, __head, __cmp) \
+	__KAVL_ITR(suf, __scope, __type, __head, __cmp)
+
+#define KAVL_INIT(suf, __type, __head, __cmp) \
+	KAVL_INIT2(suf,, __type, __head, __cmp)
+
+#endif
diff --git a/ext/klib/kbit.h b/ext/klib/kbit.h
new file mode 100644
index 0000000..3793cf8
--- /dev/null
+++ b/ext/klib/kbit.h
@@ -0,0 +1,30 @@
+#ifndef KBIT_H
+#define KBIT_H
+
+#include <stdint.h>
+
+static inline uint64_t kbi_popcount64(uint64_t y) // standard popcount; from wikipedia
+{
+	y -= ((y >> 1) & 0x5555555555555555ull);
+	y = (y & 0x3333333333333333ull) + (y >> 2 & 0x3333333333333333ull);
+	return ((y + (y >> 4)) & 0xf0f0f0f0f0f0f0full) * 0x101010101010101ull >> 56;
+}
+
+static inline uint64_t kbi_DNAcount64(uint64_t y, int c) // count #A/C/G/T from a 2-bit encoded integer; from BWA
+{
+	// reduce nucleotide counting to bits counting
+	y = ((c&2)? y : ~y) >> 1 & ((c&1)? y : ~y) & 0x5555555555555555ull;
+	// count the number of 1s in y
+	y = (y & 0x3333333333333333ull) + (y >> 2 & 0x3333333333333333ull);
+	return ((y + (y >> 4)) & 0xf0f0f0f0f0f0f0full) * 0x101010101010101ull >> 56;
+}
+
+#ifndef kroundup32 // round a 32-bit integer to the next closet integer; from "bit twiddling hacks"
+#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
+#endif
+
+#ifndef kbi_swap
+#define kbi_swap(a, b) (((a) ^= (b)), ((b) ^= (a)), ((a) ^= (b))) // from "bit twiddling hacks"
+#endif
+
+#endif
diff --git a/ext/klib/kbtree.h b/ext/klib/kbtree.h
new file mode 100644
index 0000000..8b4f917
--- /dev/null
+++ b/ext/klib/kbtree.h
@@ -0,0 +1,437 @@
+/*-
+ * Copyright 1997-1999, 2001, John-Mark Gurney.
+ *           2008-2009, Attractive Chaos <attractor@live.co.uk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef __AC_KBTREE_H
+#define __AC_KBTREE_H
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+
+#define KB_MAX_DEPTH 64
+
+typedef struct {
+	int32_t is_internal:1, n:31;
+} kbnode_t;
+
+typedef struct {
+	kbnode_t *x;
+	int i;
+} kbpos_t;
+
+typedef struct {
+	kbpos_t stack[KB_MAX_DEPTH], *p;
+} kbitr_t;
+
+#define	__KB_KEY(type, x)	((type*)((char*)x + 4))
+#define __KB_PTR(btr, x)	((kbnode_t**)((char*)x + btr->off_ptr))
+
+#define __KB_TREE_T(name)						\
+	typedef struct {							\
+		kbnode_t *root;							\
+		int	off_key, off_ptr, ilen, elen;		\
+		int	n, t;								\
+		int	n_keys, n_nodes;					\
+	} kbtree_##name##_t;
+
+#define __KB_INIT(name, key_t)											\
+	kbtree_##name##_t *kb_init_##name(int size)							\
+	{																	\
+		kbtree_##name##_t *b;											\
+		b = (kbtree_##name##_t*)calloc(1, sizeof(kbtree_##name##_t));	\
+		b->t = ((size - 4 - sizeof(void*)) / (sizeof(void*) + sizeof(key_t)) + 1) >> 1; \
+		if (b->t < 2) {													\
+			free(b); return 0;											\
+		}																\
+		b->n = 2 * b->t - 1;											\
+		b->off_ptr = 4 + b->n * sizeof(key_t);							\
+		b->ilen = (4 + sizeof(void*) + b->n * (sizeof(void*) + sizeof(key_t)) + 3) >> 2 << 2; \
+		b->elen = (b->off_ptr + 3) >> 2 << 2;							\
+		b->root = (kbnode_t*)calloc(1, b->ilen);						\
+		++b->n_nodes;													\
+		return b;														\
+	}
+
+#define __kb_destroy(b) do {											\
+		int i, max = 8;													\
+		kbnode_t *x, **top, **stack = 0;								\
+		if (b) {														\
+			top = stack = (kbnode_t**)calloc(max, sizeof(kbnode_t*));	\
+			*top++ = (b)->root;											\
+			while (top != stack) {										\
+				x = *--top;												\
+				if (x->is_internal == 0) { free(x); continue; }			\
+				for (i = 0; i <= x->n; ++i)								\
+					if (__KB_PTR(b, x)[i]) {							\
+						if (top - stack == max) {						\
+							max <<= 1;									\
+							stack = (kbnode_t**)realloc(stack, max * sizeof(kbnode_t*)); \
+							top = stack + (max>>1);						\
+						}												\
+						*top++ = __KB_PTR(b, x)[i];						\
+					}													\
+				free(x);												\
+			}															\
+		}																\
+		free(b); free(stack);											\
+	} while (0)
+
+#define __KB_GET_AUX1(name, key_t, __cmp)								\
+	static inline int __kb_getp_aux_##name(const kbnode_t * __restrict x, const key_t * __restrict k, int *r) \
+	{																	\
+		int tr, *rr, begin = 0, end = x->n;								\
+		if (x->n == 0) return -1;										\
+		rr = r? r : &tr;												\
+		while (begin < end) {											\
+			int mid = (begin + end) >> 1;								\
+			if (__cmp(__KB_KEY(key_t, x)[mid], *k) < 0) begin = mid + 1; \
+			else end = mid;												\
+		}																\
+		if (begin == x->n) { *rr = 1; return x->n - 1; }				\
+		if ((*rr = __cmp(*k, __KB_KEY(key_t, x)[begin])) < 0) --begin;	\
+		return begin;													\
+	}
+
+#define __KB_GET(name, key_t)											\
+	static key_t *kb_getp_##name(kbtree_##name##_t *b, const key_t * __restrict k) \
+	{																	\
+		int i, r = 0;													\
+		kbnode_t *x = b->root;											\
+		while (x) {														\
+			i = __kb_getp_aux_##name(x, k, &r);							\
+			if (i >= 0 && r == 0) return &__KB_KEY(key_t, x)[i];		\
+			if (x->is_internal == 0) return 0;							\
+			x = __KB_PTR(b, x)[i + 1];									\
+		}																\
+		return 0;														\
+	}																	\
+	static inline key_t *kb_get_##name(kbtree_##name##_t *b, const key_t k) \
+	{																	\
+		return kb_getp_##name(b, &k);									\
+	}
+
+#define __KB_INTERVAL(name, key_t)										\
+	static void kb_intervalp_##name(kbtree_##name##_t *b, const key_t * __restrict k, key_t **lower, key_t **upper)	\
+	{																	\
+		int i, r = 0;													\
+		kbnode_t *x = b->root;											\
+		*lower = *upper = 0;											\
+		while (x) {														\
+			i = __kb_getp_aux_##name(x, k, &r);							\
+			if (i >= 0 && r == 0) {										\
+				*lower = *upper = &__KB_KEY(key_t, x)[i];				\
+				return;													\
+			}															\
+			if (i >= 0) *lower = &__KB_KEY(key_t, x)[i];				\
+			if (i < x->n - 1) *upper = &__KB_KEY(key_t, x)[i + 1];		\
+			if (x->is_internal == 0) return;							\
+			x = __KB_PTR(b, x)[i + 1];									\
+		}																\
+	}																	\
+	static inline void kb_interval_##name(kbtree_##name##_t *b, const key_t k, key_t **lower, key_t **upper) \
+	{																	\
+		kb_intervalp_##name(b, &k, lower, upper);						\
+	}
+
+#define __KB_PUT(name, key_t, __cmp)									\
+	/* x must be an internal node */									\
+	static void __kb_split_##name(kbtree_##name##_t *b, kbnode_t *x, int i, kbnode_t *y) \
+	{																	\
+		kbnode_t *z;													\
+		z = (kbnode_t*)calloc(1, y->is_internal? b->ilen : b->elen);	\
+		++b->n_nodes;													\
+		z->is_internal = y->is_internal;								\
+		z->n = b->t - 1;												\
+		memcpy(__KB_KEY(key_t, z), __KB_KEY(key_t, y) + b->t, sizeof(key_t) * (b->t - 1)); \
+		if (y->is_internal) memcpy(__KB_PTR(b, z), __KB_PTR(b, y) + b->t, sizeof(void*) * b->t); \
+		y->n = b->t - 1;												\
+		memmove(__KB_PTR(b, x) + i + 2, __KB_PTR(b, x) + i + 1, sizeof(void*) * (x->n - i)); \
+		__KB_PTR(b, x)[i + 1] = z;										\
+		memmove(__KB_KEY(key_t, x) + i + 1, __KB_KEY(key_t, x) + i, sizeof(key_t) * (x->n - i)); \
+		__KB_KEY(key_t, x)[i] = __KB_KEY(key_t, y)[b->t - 1];			\
+		++x->n;															\
+	}																	\
+	static key_t *__kb_putp_aux_##name(kbtree_##name##_t *b, kbnode_t *x, const key_t * __restrict k) \
+	{																	\
+		int i = x->n - 1;												\
+		key_t *ret;														\
+		if (x->is_internal == 0) {										\
+			i = __kb_getp_aux_##name(x, k, 0);							\
+			if (i != x->n - 1)											\
+				memmove(__KB_KEY(key_t, x) + i + 2, __KB_KEY(key_t, x) + i + 1, (x->n - i - 1) * sizeof(key_t)); \
+			ret = &__KB_KEY(key_t, x)[i + 1];							\
+			*ret = *k;													\
+			++x->n;														\
+		} else {														\
+			i = __kb_getp_aux_##name(x, k, 0) + 1;						\
+			if (__KB_PTR(b, x)[i]->n == 2 * b->t - 1) {					\
+				__kb_split_##name(b, x, i, __KB_PTR(b, x)[i]);			\
+				if (__cmp(*k, __KB_KEY(key_t, x)[i]) > 0) ++i;			\
+			}															\
+			ret = __kb_putp_aux_##name(b, __KB_PTR(b, x)[i], k);		\
+		}																\
+		return ret; 													\
+	}																	\
+	static key_t *kb_putp_##name(kbtree_##name##_t *b, const key_t * __restrict k) \
+	{																	\
+		kbnode_t *r, *s;												\
+		++b->n_keys;													\
+		r = b->root;													\
+		if (r->n == 2 * b->t - 1) {										\
+			++b->n_nodes;												\
+			s = (kbnode_t*)calloc(1, b->ilen);							\
+			b->root = s; s->is_internal = 1; s->n = 0;					\
+			__KB_PTR(b, s)[0] = r;										\
+			__kb_split_##name(b, s, 0, r);								\
+			r = s;														\
+		}																\
+		return __kb_putp_aux_##name(b, r, k);							\
+	}																	\
+	static inline void kb_put_##name(kbtree_##name##_t *b, const key_t k) \
+	{																	\
+		kb_putp_##name(b, &k);											\
+	}
+
+
+#define __KB_DEL(name, key_t)											\
+	static key_t __kb_delp_aux_##name(kbtree_##name##_t *b, kbnode_t *x, const key_t * __restrict k, int s) \
+	{																	\
+		int yn, zn, i, r = 0;											\
+		kbnode_t *xp, *y, *z;											\
+		key_t kp;														\
+		if (x == 0) return *k;											\
+		if (s) { /* s can only be 0, 1 or 2 */							\
+			r = x->is_internal == 0? 0 : s == 1? 1 : -1;				\
+			i = s == 1? x->n - 1 : -1;									\
+		} else i = __kb_getp_aux_##name(x, k, &r);						\
+		if (x->is_internal == 0) {										\
+			if (s == 2) ++i;											\
+			kp = __KB_KEY(key_t, x)[i];									\
+			memmove(__KB_KEY(key_t, x) + i, __KB_KEY(key_t, x) + i + 1, (x->n - i - 1) * sizeof(key_t)); \
+			--x->n;														\
+			return kp;													\
+		}																\
+		if (r == 0) {													\
+			if ((yn = __KB_PTR(b, x)[i]->n) >= b->t) {					\
+				xp = __KB_PTR(b, x)[i];									\
+				kp = __KB_KEY(key_t, x)[i];								\
+				__KB_KEY(key_t, x)[i] = __kb_delp_aux_##name(b, xp, 0, 1); \
+				return kp;												\
+			} else if ((zn = __KB_PTR(b, x)[i + 1]->n) >= b->t) {		\
+				xp = __KB_PTR(b, x)[i + 1];								\
+				kp = __KB_KEY(key_t, x)[i];								\
+				__KB_KEY(key_t, x)[i] = __kb_delp_aux_##name(b, xp, 0, 2); \
+				return kp;												\
+			} else if (yn == b->t - 1 && zn == b->t - 1) {				\
+				y = __KB_PTR(b, x)[i]; z = __KB_PTR(b, x)[i + 1];		\
+				__KB_KEY(key_t, y)[y->n++] = *k;						\
+				memmove(__KB_KEY(key_t, y) + y->n, __KB_KEY(key_t, z), z->n * sizeof(key_t)); \
+				if (y->is_internal) memmove(__KB_PTR(b, y) + y->n, __KB_PTR(b, z), (z->n + 1) * sizeof(void*)); \
+				y->n += z->n;											\
+				memmove(__KB_KEY(key_t, x) + i, __KB_KEY(key_t, x) + i + 1, (x->n - i - 1) * sizeof(key_t)); \
+				memmove(__KB_PTR(b, x) + i + 1, __KB_PTR(b, x) + i + 2, (x->n - i - 1) * sizeof(void*)); \
+				--x->n;													\
+				free(z);												\
+				return __kb_delp_aux_##name(b, y, k, s);				\
+			}															\
+		}																\
+		++i;															\
+		if ((xp = __KB_PTR(b, x)[i])->n == b->t - 1) {					\
+			if (i > 0 && (y = __KB_PTR(b, x)[i - 1])->n >= b->t) {		\
+				memmove(__KB_KEY(key_t, xp) + 1, __KB_KEY(key_t, xp), xp->n * sizeof(key_t)); \
+				if (xp->is_internal) memmove(__KB_PTR(b, xp) + 1, __KB_PTR(b, xp), (xp->n + 1) * sizeof(void*)); \
+				__KB_KEY(key_t, xp)[0] = __KB_KEY(key_t, x)[i - 1];		\
+				__KB_KEY(key_t, x)[i - 1] = __KB_KEY(key_t, y)[y->n - 1]; \
+				if (xp->is_internal) __KB_PTR(b, xp)[0] = __KB_PTR(b, y)[y->n]; \
+				--y->n; ++xp->n;										\
+			} else if (i < x->n && (y = __KB_PTR(b, x)[i + 1])->n >= b->t) { \
+				__KB_KEY(key_t, xp)[xp->n++] = __KB_KEY(key_t, x)[i];	\
+				__KB_KEY(key_t, x)[i] = __KB_KEY(key_t, y)[0];			\
+				if (xp->is_internal) __KB_PTR(b, xp)[xp->n] = __KB_PTR(b, y)[0]; \
+				--y->n;													\
+				memmove(__KB_KEY(key_t, y), __KB_KEY(key_t, y) + 1, y->n * sizeof(key_t)); \
+				if (y->is_internal) memmove(__KB_PTR(b, y), __KB_PTR(b, y) + 1, (y->n + 1) * sizeof(void*)); \
+			} else if (i > 0 && (y = __KB_PTR(b, x)[i - 1])->n == b->t - 1) { \
+				__KB_KEY(key_t, y)[y->n++] = __KB_KEY(key_t, x)[i - 1];	\
+				memmove(__KB_KEY(key_t, y) + y->n, __KB_KEY(key_t, xp), xp->n * sizeof(key_t));	\
+				if (y->is_internal) memmove(__KB_PTR(b, y) + y->n, __KB_PTR(b, xp), (xp->n + 1) * sizeof(void*)); \
+				y->n += xp->n;											\
+				memmove(__KB_KEY(key_t, x) + i - 1, __KB_KEY(key_t, x) + i, (x->n - i) * sizeof(key_t)); \
+				memmove(__KB_PTR(b, x) + i, __KB_PTR(b, x) + i + 1, (x->n - i) * sizeof(void*)); \
+				--x->n;													\
+				free(xp);												\
+				xp = y;													\
+			} else if (i < x->n && (y = __KB_PTR(b, x)[i + 1])->n == b->t - 1) { \
+				__KB_KEY(key_t, xp)[xp->n++] = __KB_KEY(key_t, x)[i];	\
+				memmove(__KB_KEY(key_t, xp) + xp->n, __KB_KEY(key_t, y), y->n * sizeof(key_t));	\
+				if (xp->is_internal) memmove(__KB_PTR(b, xp) + xp->n, __KB_PTR(b, y), (y->n + 1) * sizeof(void*)); \
+				xp->n += y->n;											\
+				memmove(__KB_KEY(key_t, x) + i, __KB_KEY(key_t, x) + i + 1, (x->n - i - 1) * sizeof(key_t)); \
+				memmove(__KB_PTR(b, x) + i + 1, __KB_PTR(b, x) + i + 2, (x->n - i - 1) * sizeof(void*)); \
+				--x->n;													\
+				free(y);												\
+			}															\
+		}																\
+		return __kb_delp_aux_##name(b, xp, k, s);						\
+	}																	\
+	static key_t kb_delp_##name(kbtree_##name##_t *b, const key_t * __restrict k) \
+	{																	\
+		kbnode_t *x;													\
+		key_t ret;														\
+		ret = __kb_delp_aux_##name(b, b->root, k, 0);					\
+		--b->n_keys;													\
+		if (b->root->n == 0 && b->root->is_internal) {					\
+			--b->n_nodes;												\
+			x = b->root;												\
+			b->root = __KB_PTR(b, x)[0];								\
+			free(x);													\
+		}																\
+		return ret;														\
+	}																	\
+	static inline key_t kb_del_##name(kbtree_##name##_t *b, const key_t k) \
+	{																	\
+		return kb_delp_##name(b, &k);									\
+	}
+
+#define __KB_ITR(name, key_t) \
+	static inline void kb_itr_first_##name(kbtree_##name##_t *b, kbitr_t *itr) \
+	{ \
+		itr->p = 0; \
+		if (b->n_keys == 0) return; \
+		itr->p = itr->stack; \
+		itr->p->x = b->root; itr->p->i = 0; \
+		while (itr->p->x->is_internal && __KB_PTR(b, itr->p->x)[0] != 0) { \
+			kbnode_t *x = itr->p->x; \
+			++itr->p; \
+			itr->p->x = __KB_PTR(b, x)[0]; itr->p->i = 0; \
+		} \
+	} \
+	static int kb_itr_get_##name(kbtree_##name##_t *b, const key_t * __restrict k, kbitr_t *itr) \
+	{ \
+		int i, r = 0; \
+		itr->p = itr->stack; \
+		itr->p->x = b->root; itr->p->i = 0; \
+		while (itr->p->x) { \
+			i = __kb_getp_aux_##name(itr->p->x, k, &r); \
+			if (i >= 0 && r == 0) return 0; \
+			if (itr->p->x->is_internal == 0) return -1; \
+			itr->p[1].x = __KB_PTR(b, itr->p->x)[i + 1]; \
+			itr->p[1].i = i; \
+			++itr->p; \
+		} \
+		return -1; \
+	} \
+	static inline int kb_itr_next_##name(kbtree_##name##_t *b, kbitr_t *itr) \
+	{ \
+		if (itr->p < itr->stack) return 0; \
+		for (;;) { \
+			++itr->p->i; \
+			while (itr->p->x && itr->p->i <= itr->p->x->n) { \
+				itr->p[1].i = 0; \
+				itr->p[1].x = itr->p->x->is_internal? __KB_PTR(b, itr->p->x)[itr->p->i] : 0; \
+				++itr->p; \
+			} \
+			--itr->p; \
+			if (itr->p < itr->stack) return 0; \
+			if (itr->p->x && itr->p->i < itr->p->x->n) return 1; \
+		} \
+	}
+
+#define KBTREE_INIT(name, key_t, __cmp)			\
+	__KB_TREE_T(name)							\
+	__KB_INIT(name, key_t)						\
+	__KB_GET_AUX1(name, key_t, __cmp)			\
+	__KB_GET(name, key_t)						\
+	__KB_INTERVAL(name, key_t)					\
+	__KB_PUT(name, key_t, __cmp)				\
+	__KB_DEL(name, key_t) \
+	__KB_ITR(name, key_t)
+
+#define KB_DEFAULT_SIZE 512
+
+#define kbtree_t(name) kbtree_##name##_t
+#define kb_init(name, s) kb_init_##name(s)
+#define kb_destroy(name, b) __kb_destroy(b)
+#define kb_get(name, b, k) kb_get_##name(b, k)
+#define kb_put(name, b, k) kb_put_##name(b, k)
+#define kb_del(name, b, k) kb_del_##name(b, k)
+#define kb_interval(name, b, k, l, u) kb_interval_##name(b, k, l, u)
+#define kb_getp(name, b, k) kb_getp_##name(b, k)
+#define kb_putp(name, b, k) kb_putp_##name(b, k)
+#define kb_delp(name, b, k) kb_delp_##name(b, k)
+#define kb_intervalp(name, b, k, l, u) kb_intervalp_##name(b, k, l, u)
+
+#define kb_itr_first(name, b, i) kb_itr_first_##name(b, i)
+#define kb_itr_get(name, b, k, i) kb_itr_get_##name(b, k, i)
+#define kb_itr_next(name, b, i) kb_itr_next_##name(b, i)
+#define kb_itr_key(type, itr) __KB_KEY(type, (itr)->p->x)[(itr)->p->i]
+#define kb_itr_valid(itr) ((itr)->p >= (itr)->stack)
+
+#define kb_size(b) ((b)->n_keys)
+
+#define kb_generic_cmp(a, b) (((b) < (a)) - ((a) < (b)))
+#define kb_str_cmp(a, b) strcmp(a, b)
+
+/* The following is *DEPRECATED*!!! Use the iterator interface instead! */
+
+typedef struct {
+	kbnode_t *x;
+	int i;
+} __kbstack_t;
+
+#define __kb_traverse(key_t, b, __func) do {							\
+		int __kmax = 8;													\
+		__kbstack_t *__kstack, *__kp;									\
+		__kp = __kstack = (__kbstack_t*)calloc(__kmax, sizeof(__kbstack_t)); \
+		__kp->x = (b)->root; __kp->i = 0;								\
+		for (;;) {														\
+			while (__kp->x && __kp->i <= __kp->x->n) {					\
+				if (__kp - __kstack == __kmax - 1) {					\
+					__kmax <<= 1;										\
+					__kstack = (__kbstack_t*)realloc(__kstack, __kmax * sizeof(__kbstack_t)); \
+					__kp = __kstack + (__kmax>>1) - 1;					\
+				}														\
+				(__kp+1)->i = 0; (__kp+1)->x = __kp->x->is_internal? __KB_PTR(b, __kp->x)[__kp->i] : 0; \
+				++__kp;													\
+			}															\
+			--__kp;														\
+			if (__kp >= __kstack) {										\
+				if (__kp->x && __kp->i < __kp->x->n) __func(&__KB_KEY(key_t, __kp->x)[__kp->i]); \
+				++__kp->i;												\
+			} else break;												\
+		}																\
+		free(__kstack);													\
+	} while (0)
+
+#define __kb_get_first(key_t, b, ret) do {	\
+		kbnode_t *__x = (b)->root;			\
+		while (__KB_PTR(b, __x)[0] != 0)	\
+			__x = __KB_PTR(b, __x)[0];		\
+		(ret) = __KB_KEY(key_t, __x)[0];	\
+	} while (0)
+
+#endif
diff --git a/ext/klib/kdq.h b/ext/klib/kdq.h
new file mode 100644
index 0000000..edd55b5
--- /dev/null
+++ b/ext/klib/kdq.h
@@ -0,0 +1,128 @@
+#ifndef __AC_KDQ_H
+#define __AC_KDQ_H
+
+#include <stdlib.h>
+#include <string.h>
+
+#define __KDQ_TYPE(type) \
+	typedef struct { \
+		size_t front:58, bits:6, count, mask; \
+		type *a; \
+	} kdq_##type##_t;
+
+#define kdq_t(type) kdq_##type##_t
+#define kdq_size(q) ((q)->count)
+#define kdq_first(q) ((q)->a[(q)->front])
+#define kdq_last(q) ((q)->a[((q)->front + (q)->count - 1) & (q)->mask])
+#define kdq_at(q, i) ((q)->a[((q)->front + (i)) & (q)->mask])
+
+#define __KDQ_IMPL(type, SCOPE) \
+	SCOPE kdq_##type##_t *kdq_init_##type() \
+	{ \
+		kdq_##type##_t *q; \
+		q = (kdq_##type##_t*)calloc(1, sizeof(kdq_##type##_t)); \
+		q->bits = 2, q->mask = (1ULL<<q->bits) - 1; \
+		q->a = (type*)malloc((1<<q->bits) * sizeof(type)); \
+		return q; \
+	} \
+	SCOPE void kdq_destroy_##type(kdq_##type##_t *q) \
+	{ \
+		if (q == 0) return; \
+		free(q->a); free(q); \
+	} \
+	SCOPE int kdq_resize_##type(kdq_##type##_t *q, int new_bits) \
+	{ \
+		size_t new_size = 1ULL<<new_bits, old_size = 1ULL<<q->bits; \
+		if (new_size < q->count) { /* not big enough */ \
+			int i; \
+			for (i = 0; i < 64; ++i) \
+				if (1ULL<<i > q->count) break; \
+			new_bits = i, new_size = 1ULL<<new_bits; \
+		} \
+		if (new_bits == q->bits) return q->bits; /* unchanged */ \
+		if (new_bits > q->bits) q->a = (type*)realloc(q->a, (1ULL<<new_bits) * sizeof(type)); \
+		if (q->front + q->count <= old_size) { /* unwrapped */ \
+			if (q->front + q->count > new_size) /* only happens for shrinking */ \
+				memmove(q->a, q->a + new_size, (q->front + q->count - new_size) * sizeof(type)); \
+		} else { /* wrapped */ \
+			memmove(q->a + (new_size - (old_size - q->front)), q->a + q->front, (old_size - q->front) * sizeof(type)); \
+			q->front = new_size - (old_size - q->front); \
+		} \
+		q->bits = new_bits, q->mask = (1ULL<<q->bits) - 1; \
+		if (new_bits < q->bits) q->a = (type*)realloc(q->a, (1ULL<<new_bits) * sizeof(type)); \
+		return q->bits; \
+	} \
+	SCOPE type *kdq_pushp_##type(kdq_##type##_t *q) \
+	{ \
+		if (q->count == 1ULL<<q->bits) kdq_resize_##type(q, q->bits + 1); \
+		return &q->a[((q->count++) + q->front) & (q)->mask]; \
+	} \
+	SCOPE void kdq_push_##type(kdq_##type##_t *q, type v) \
+	{ \
+		if (q->count == 1ULL<<q->bits) kdq_resize_##type(q, q->bits + 1); \
+		q->a[((q->count++) + q->front) & (q)->mask] = v; \
+	} \
+	SCOPE type *kdq_unshiftp_##type(kdq_##type##_t *q) \
+	{ \
+		if (q->count == 1ULL<<q->bits) kdq_resize_##type(q, q->bits + 1); \
+		++q->count; \
+		q->front = q->front? q->front - 1 : (1ULL<<q->bits) - 1; \
+		return &q->a[q->front]; \
+	} \
+	SCOPE void kdq_unshift_##type(kdq_##type##_t *q, type v) \
+	{ \
+		type *p; \
+		p = kdq_unshiftp_##type(q); \
+		*p = v; \
+	} \
+	SCOPE type *kdq_pop_##type(kdq_##type##_t *q) \
+	{ \
+		return q->count? &q->a[((--q->count) + q->front) & q->mask] : 0; \
+	} \
+	SCOPE type *kdq_shift_##type(kdq_##type##_t *q) \
+	{ \
+		type *d = 0; \
+		if (q->count == 0) return 0; \
+		d = &q->a[q->front++]; \
+		q->front &= q->mask; \
+		--q->count; \
+		return d; \
+	}
+
+#define KDQ_INIT2(type, SCOPE) \
+	__KDQ_TYPE(type) \
+	__KDQ_IMPL(type, SCOPE)
+
+#ifndef klib_unused
+#if (defined __clang__ && __clang_major__ >= 3) || (defined __GNUC__ && __GNUC__ >= 3)
+#define klib_unused __attribute__ ((__unused__))
+#else
+#define klib_unused
+#endif
+#endif /* klib_unused */
+
+#define KDQ_INIT(type) KDQ_INIT2(type, static inline klib_unused)
+
+#define KDQ_DECLARE(type) \
+	__KDQ_TYPE(type) \
+	kdq_##type##_t *kdq_init_##type(); \
+	void kdq_destroy_##type(kdq_##type##_t *q); \
+	int kdq_resize_##type(kdq_##type##_t *q, int new_bits); \
+	type *kdq_pushp_##type(kdq_##type##_t *q); \
+	void kdq_push_##type(kdq_##type##_t *q, type v); \
+	type *kdq_unshiftp_##type(kdq_##type##_t *q); \
+	void kdq_unshift_##type(kdq_##type##_t *q, type v); \
+	type *kdq_pop_##type(kdq_##type##_t *q); \
+	type *kdq_shift_##type(kdq_##type##_t *q);
+
+#define kdq_init(type) kdq_init_##type()
+#define kdq_destroy(type, q) kdq_destroy_##type(q)
+#define kdq_resize(type, q, new_bits) kdq_resize_##type(q, new_bits)
+#define kdq_pushp(type, q) kdq_pushp_##type(q)
+#define kdq_push(type, q, v) kdq_push_##type(q, v)
+#define kdq_pop(type, q) kdq_pop_##type(q)
+#define kdq_unshiftp(type, q) kdq_unshiftp_##type(q)
+#define kdq_unshift(type, q, v) kdq_unshift_##type(q, v)
+#define kdq_shift(type, q) kdq_shift_##type(q)
+
+#endif
diff --git a/ext/klib/keigen.c b/ext/klib/keigen.c
new file mode 100644
index 0000000..5dd2ddc
--- /dev/null
+++ b/ext/klib/keigen.c
@@ -0,0 +1,186 @@
+#include <math.h>
+#include <stdlib.h>
+#include "keigen.h"
+
+void ke_core_strq(int n, double *q, double *b, double *c)
+{
+	int	i, j, k, u, v;
+	double h, f, g, h2;
+	for (i = n - 1; i >= 1; i--) {
+		h = 0.0;
+		if (i > 1)
+			for (k = 0; k < i; k++) {
+				u = i * n + k;
+				h = h + q[u] * q[u];
+			}
+		if (h + 1.0 == 1.0) {
+			c[i] = 0.0;
+			if (i == 1)
+				c[i] = q[i * n + i - 1];
+			b[i] = 0.0;
+		} else {
+			c[i] = sqrt(h);
+			u = i * n + i - 1;
+			if (q[u] > 0.0)
+				c[i] = -c[i];
+			h = h - q[u] * c[i];
+			q[u] = q[u] - c[i];
+			f = 0.0;
+			for (j = 0; j < i; j++) {
+				q[j * n + i] = q[i * n + j] / h;
+				g = 0.0;
+				for (k = 0; k <= j; k++)
+					g = g + q[j * n + k] * q[i * n + k];
+				if (j + 1 < i)
+					for (k = j + 1; k <= i - 1; k++)
+						g = g + q[k * n + j] * q[i * n + k];
+				c[j] = g / h;
+				f = f + g * q[j * n + i];
+			}
+			h2 = f / (h + h);
+			for (j = 0; j < i; j++) {
+				f = q[i * n + j];
+				g = c[j] - h2 * f;
+				c[j] = g;
+				for (k = 0; k <= j; k++) {
+					u = j * n + k;
+					q[u] = q[u] - f * c[k] - g * q[i * n + k];
+				}
+			}
+			b[i] = h;
+		}
+	}
+	for (i = 0; i < n - 1; i++)
+		c[i] = c[i + 1];
+	c[n - 1] = 0.0;
+	b[0] = 0.0;
+	for (i = 0; i < n; i++) {
+		if (b[i] != 0.0 && i - 1 >= 0)
+			for (j = 0; j < i; j++) {
+				g = 0.0;
+				for (k = 0; k < i; k++)
+					g = g + q[i * n + k] * q[k * n + j];
+				for (k = 0; k < i; k++) {
+					u = k * n + j;
+					q[u] = q[u] - g * q[k * n + i];
+				}
+			}
+		u = i * n + i;
+		b[i] = q[u];
+		q[u] = 1.0;
+		if (i - 1 >= 0)
+			for (j = 0; j < i; j++) {
+				q[i * n + j] = 0.0;
+				q[j * n + i] = 0.0;
+			}
+	}
+}
+
+int ke_core_sstq(int n, double *b, double *c, double *q, int cal_ev, double eps, int l)
+{
+	int i, j, k, m, it, u, v;
+	double d, f, h, g, p, r, e, s;
+	c[n - 1] = 0.0;
+	d = 0.0;
+	f = 0.0;
+	for (j = 0; j < n; j++) {
+		it = 0;
+		h = eps * (fabs(b[j]) + fabs(c[j]));
+		if (h > d)
+			d = h;
+		m = j;
+		while (m < n && fabs(c[m]) > d)
+			m = m + 1;
+		if (m != j) {
+			do {
+				if (it == l) return KE_EXCESS_ITER;
+				it = it + 1;
+				g = b[j];
+				p = (b[j + 1] - g) / (2.0 * c[j]);
+				r = sqrt(p * p + 1.0);
+				if (p >= 0.0)
+					b[j] = c[j] / (p + r);
+				else
+					b[j] = c[j] / (p - r);
+				h = g - b[j];
+				for (i = j + 1; i < n; i++)
+					b[i] = b[i] - h;
+				f = f + h;
+				p = b[m];
+				e = 1.0;
+				s = 0.0;
+				for (i = m - 1; i >= j; i--) {
+					g = e * c[i];
+					h = e * p;
+					if (fabs(p) >= fabs(c[i])) {
+						e = c[i] / p;
+						r = sqrt(e * e + 1.0);
+						c[i + 1] = s * p * r;
+						s = e / r;
+						e = 1.0 / r;
+					} else {
+						e = p / c[i];
+						r = sqrt(e * e + 1.0);
+						c[i + 1] = s * c[i] * r;
+						s = 1.0 / r;
+						e = e / r;
+					}
+					p = e * b[i] - s * g;
+					b[i + 1] = h + s * (e * g + s * b[i]);
+					if (cal_ev) {
+						for (k = 0; k < n; k++) {
+							u = k * n + i + 1;
+							v = u - 1;
+							h = q[u];
+							q[u] = s * q[v] + e * h;
+							q[v] = e * q[v] - s * h;
+						}
+					}
+				}
+				c[j] = s * p;
+				b[j] = e * p;
+			}
+			while (fabs(c[j]) > d);
+		}
+		b[j] = b[j] + f;
+	}
+	for (i = 0; i < n; i++) {
+		k = i;
+		p = b[i];
+		if (i + 1 < n) {
+			j = i + 1;
+			while (j < n && b[j] <= p) {
+				k = j;
+				p = b[j];
+				j = j + 1;
+			}
+		}
+		if (k != i) {
+			b[k] = b[i];
+			b[i] = p;
+			for (j = 0; j < n; j++) {
+				u = j * n + i;
+				v = j * n + k;
+				p = q[u];
+				q[u] = q[v];
+				q[v] = p;
+			}
+		}
+	}
+	return 0;
+}
+
+#define MALLOC(type, size) ((type*)malloc(size * sizeof(type)))
+
+int ke_eigen_sd(int n, double *a, double *v, int cal_ev, double eps, int max_iter)
+{
+	double *c;
+	int r;
+	if (1.0 + eps <= 1.0) eps = 1e-7;
+	if (max_iter <= 0) max_iter = 50;
+	c = MALLOC(double, n);
+	ke_core_strq(n, a, v, c);
+	r = ke_core_sstq(n, v, c, a, cal_ev, eps, max_iter);
+	free(c);
+	return r;
+}
diff --git a/ext/klib/keigen.h b/ext/klib/keigen.h
new file mode 100644
index 0000000..14aa53d
--- /dev/null
+++ b/ext/klib/keigen.h
@@ -0,0 +1,53 @@
+#ifndef KEIGEN_H
+#define KEIGEN_H
+
+#define KE_EXCESS_ITER (-1)
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Compute eigenvalues/vectors for a dense symmetric matrix
+ *
+ * @param n       dimension
+ * @param a       input matrix and eigenvalues on return ([n*n]; in & out)
+ * @param v       eigenvalues ([n]; out)
+ * @param cal_ev  compute eigenvectos or not (faster without vectors)
+ * @param eps     precision (<=0 for default)
+ * @param max_itr max iteration (<=0 for detaul)
+ *
+ * @return 0 on success; KE_EXCESS_ITER if too many iterations
+ */
+int ke_eigen_sd(int n, double *a, double *v, int cal_ev, double eps, int max_iter);
+
+/**
+ * Transform a real symmetric matrix to a tridiagonal matrix
+ *
+ * @param n       dimension
+ * @param q       input matrix and transformation matrix ([n*n]; in & out)
+ * @param b       diagonal ([n]; out)
+ * @param c       subdiagonal ([n]; out)
+ */
+void ke_core_strq(int n, double *q, double *b, double *c);
+
+/**
+ * Compute eigenvalues and eigenvectors for a tridiagonal matrix
+ *
+ * @param n       dimension
+ * @param b       diagonal and eigenvalues on return ([n]; in & out)
+ * @param c       subdiagonal ([n]; in)
+ * @param q       transformation matrix and eigenvectors on return ([n*n]; in & out)
+ * @param cal_ev  compute eigenvectors or not (faster without vectors)
+ * @param eps     precision
+ * @param l       max iterations
+ *
+ * @return 0 on success; KE_EXCESS_ITER if too many iterations
+ */
+int ke_core_sstq(int n, double *b, double *c, double *q, int cal_ev, double eps, int l);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/ext/klib/ketopt.h b/ext/klib/ketopt.h
new file mode 100644
index 0000000..73dd2dc
--- /dev/null
+++ b/ext/klib/ketopt.h
@@ -0,0 +1,120 @@
+#ifndef KETOPT_H
+#define KETOPT_H
+
+#include <string.h> /* for strchr() and strncmp() */
+
+#define ko_no_argument       0
+#define ko_required_argument 1
+#define ko_optional_argument 2
+
+typedef struct {
+	int ind;   /* equivalent to optind */
+	int opt;   /* equivalent to optopt */
+	char *arg; /* equivalent to optarg */
+	int longidx; /* index of a long option; or -1 if short */
+	/* private variables not intended for external uses */
+	int i, pos, n_args;
+} ketopt_t;
+
+typedef struct {
+	char *name;
+	int has_arg;
+	int val;
+} ko_longopt_t;
+
+static ketopt_t KETOPT_INIT = { 1, 0, 0, -1, 1, 0, 0 };
+
+static void ketopt_permute(char *argv[], int j, int n) /* move argv[j] over n elements to the left */
+{
+	int k;
+	char *p = argv[j];
+	for (k = 0; k < n; ++k)
+		argv[j - k] = argv[j - k - 1];
+	argv[j - k] = p;
+}
+
+/**
+ * Parse command-line options and arguments
+ *
+ * This fuction has a similar interface to GNU's getopt_long(). Each call
+ * parses one option and returns the option name.  s->arg points to the option
+ * argument if present. The function returns -1 when all command-line arguments
+ * are parsed. In this case, s->ind is the index of the first non-option
+ * argument.
+ *
+ * @param s         status; shall be initialized to KETOPT_INIT on the first call
+ * @param argc      length of argv[]
+ * @param argv      list of command-line arguments; argv[0] is ignored
+ * @param permute   non-zero to move options ahead of non-option arguments
+ * @param ostr      option string
+ * @param longopts  long options
+ *
+ * @return ASCII for a short option; ko_longopt_t::val for a long option; -1 if
+ *         argv[] is fully processed; '?' for an unknown option or an ambiguous
+ *         long option; ':' if an option argument is missing
+ */
+static int ketopt(ketopt_t *s, int argc, char *argv[], int permute, const char *ostr, const ko_longopt_t *longopts)
+{
+	int opt = -1, i0, j;
+	if (permute) {
+		while (s->i < argc && (argv[s->i][0] != '-' || argv[s->i][1] == '\0'))
+			++s->i, ++s->n_args;
+	}
+	s->arg = 0, s->longidx = -1, i0 = s->i;
+	if (s->i >= argc || argv[s->i][0] != '-' || argv[s->i][1] == '\0') {
+		s->ind = s->i - s->n_args;
+		return -1;
+	}
+	if (argv[s->i][0] == '-' && argv[s->i][1] == '-') { /* "--" or a long option */
+		if (argv[s->i][2] == '\0') { /* a bare "--" */
+			ketopt_permute(argv, s->i, s->n_args);
+			++s->i, s->ind = s->i - s->n_args;
+			return -1;
+		}
+		s->opt = 0, opt = '?', s->pos = -1;
+		if (longopts) { /* parse long options */
+			int k, n_exact = 0, n_partial = 0;
+			const ko_longopt_t *o = 0, *o_exact = 0, *o_partial = 0;
+			for (j = 2; argv[s->i][j] != '\0' && argv[s->i][j] != '='; ++j) {} /* find the end of the option name */
+			for (k = 0; longopts[k].name != 0; ++k)
+				if (strncmp(&argv[s->i][2], longopts[k].name, j - 2) == 0) {
+					if (longopts[k].name[j - 2] == 0) ++n_exact, o_exact = &longopts[k];
+					else ++n_partial, o_partial = &longopts[k];
+				}
+			if (n_exact > 1 || (n_exact == 0 && n_partial > 1)) return '?';
+			o = n_exact == 1? o_exact : n_partial == 1? o_partial : 0;
+			if (o) {
+				s->opt = opt = o->val, s->longidx = o - longopts;
+				if (argv[s->i][j] == '=') s->arg = &argv[s->i][j + 1];
+				if (o->has_arg == 1 && argv[s->i][j] == '\0') {
+					if (s->i < argc - 1) s->arg = argv[++s->i];
+					else opt = ':'; /* missing option argument */
+				}
+			}
+		}
+	} else { /* a short option */
+		const char *p;
+		if (s->pos == 0) s->pos = 1;
+		opt = s->opt = argv[s->i][s->pos++];
+		p = strchr((char*)ostr, opt);
+		if (p == 0) {
+			opt = '?'; /* unknown option */
+		} else if (p[1] == ':') {
+			if (argv[s->i][s->pos] == 0) {
+				if (s->i < argc - 1) s->arg = argv[++s->i];
+				else opt = ':'; /* missing option argument */
+			} else s->arg = &argv[s->i][s->pos];
+			s->pos = -1;
+		}
+	}
+	if (s->pos < 0 || argv[s->i][s->pos] == 0) {
+		++s->i, s->pos = 0;
+		if (s->n_args > 0) /* permute */
+			for (j = i0; j < s->i; ++j)
+				ketopt_permute(argv, j, s->n_args);
+	}
+	s->ind = s->i - s->n_args;
+	return opt;
+}
+
+#endif
diff --git a/ext/klib/kexpr.c b/ext/klib/kexpr.c
new file mode 100644
index 0000000..79f110f
--- /dev/null
+++ b/ext/klib/kexpr.c
@@ -0,0 +1,586 @@
+#include <string.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <assert.h>
+#include <stdio.h>
+#include <ctype.h>
+#include <math.h>
+#include "kexpr.h"
+
+/***************
+ * Definitions *
+ ***************/
+
+#define KEO_NULL  0
+#define KEO_POS   1
+#define KEO_NEG   2
+#define KEO_BNOT  3
+#define KEO_LNOT  4
+#define KEO_POW   5
+#define KEO_MUL   6
+#define KEO_DIV   7
+#define KEO_IDIV  8
+#define KEO_MOD   9
+#define KEO_ADD  10
+#define KEO_SUB  11
+#define KEO_LSH  12
+#define KEO_RSH  13
+#define KEO_LT   14
+#define KEO_LE   15
+#define KEO_GT   16
+#define KEO_GE   17
+#define KEO_EQ   18
+#define KEO_NE   19
+#define KEO_BAND 20
+#define KEO_BXOR 21
+#define KEO_BOR  22
+#define KEO_LAND 23
+#define KEO_LOR  24
+
+#define KET_NULL  0
+#define KET_VAL   1
+#define KET_OP    2
+#define KET_FUNC  3
+
+#define KEF_NULL  0
+#define KEF_REAL  1
+
+struct ke1_s;
+
+typedef struct ke1_s {
+	uint32_t ttype:16, vtype:10, assigned:1, user_func:5; // ttype: token type; vtype: value type
+	int32_t op:8, n_args:24; // op: operator, n_args: number of arguments
+	char *name; // variable name or function name
+	union {
+		void (*builtin)(struct ke1_s *a, struct ke1_s *b); // execution function
+		double (*real_func1)(double);
+		double (*real_func2)(double, double);
+	} f;
+	double r;
+	int64_t i;
+	char *s;
+} ke1_t;
+
+static int ke_op[25] = {
+	0,
+	1<<1|1, 1<<1|1, 1<<1|1, 1<<1|1, // unary operators
+	2<<1|1, // pow()
+	3<<1, 3<<1, 3<<1, 3<<1, // * / // %
+	4<<1, 4<<1, // + and -
+	5<<1, 5<<1, // << and >>
+	6<<1, 6<<1, 6<<1, 6<<1, // < > <= >=
+	7<<1, 7<<1, // == !=
+	8<<1, // &
+	9<<1, // ^
+	10<<1,// |
+	11<<1,// &&
+	12<<1 // ||
+};
+
+static const char *ke_opstr[] = {
+	"",
+	"+(1)", "-(1)", "~", "!",
+	"**",
+	"*", "/", "//", "%",
+	"+", "-",
+	"<<", ">>",
+	"<", "<=", ">", ">=",
+	"==", "!=",
+	"&",
+	"^",
+	"|",
+	"&&",
+	"||"
+};
+
+struct kexpr_s {
+	int n;
+	ke1_t *e;
+};
+
+/**********************
+ * Operator functions *
+ **********************/
+
+#define KE_GEN_CMP(_type, _op) \
+	static void ke_op_##_type(ke1_t *p, ke1_t *q) { \
+		if (p->vtype == KEV_STR && q->vtype == KEV_STR) p->i = (strcmp(p->s, q->s) _op 0); \
+		else p->i = p->vtype == KEV_REAL || q->vtype == KEV_REAL? (p->r _op q->r) : (p->i _op q->i); \
+		p->r = (double)p->i; \
+		p->vtype = KEV_INT; \
+	}
+
+KE_GEN_CMP(KEO_LT, <)
+KE_GEN_CMP(KEO_LE, <=)
+KE_GEN_CMP(KEO_GT, >)
+KE_GEN_CMP(KEO_GE, >=)
+KE_GEN_CMP(KEO_EQ, ==)
+KE_GEN_CMP(KEO_NE, !=)
+
+#define KE_GEN_BIN_INT(_type, _op) \
+	static void ke_op_##_type(ke1_t *p, ke1_t *q) { \
+		p->i _op q->i; p->r = (double)p->i; \
+		p->vtype = KEV_INT; \
+	}
+
+KE_GEN_BIN_INT(KEO_BAND, &=)
+KE_GEN_BIN_INT(KEO_BOR, |=)
+KE_GEN_BIN_INT(KEO_BXOR, ^=)
+KE_GEN_BIN_INT(KEO_LSH, <<=)
+KE_GEN_BIN_INT(KEO_RSH, >>=)
+KE_GEN_BIN_INT(KEO_MOD, %=)
+KE_GEN_BIN_INT(KEO_IDIV, /=)
+
+#define KE_GEN_BIN_BOTH(_type, _op) \
+	static void ke_op_##_type(ke1_t *p, ke1_t *q) { \
+		p->i _op q->i; p->r _op q->r; \
+		p->vtype = p->vtype == KEV_REAL || q->vtype == KEV_REAL? KEV_REAL : KEV_INT; \
+	}
+
+KE_GEN_BIN_BOTH(KEO_ADD, +=)
+KE_GEN_BIN_BOTH(KEO_SUB, -=)
+KE_GEN_BIN_BOTH(KEO_MUL, *=)
+
+static void ke_op_KEO_DIV(ke1_t *p, ke1_t *q)  { p->r /= q->r, p->i = (int64_t)(p->r + .5); p->vtype = KEV_REAL; }
+static void ke_op_KEO_LAND(ke1_t *p, ke1_t *q) { p->i = (p->i && q->i); p->r = p->i; p->vtype = KEV_INT; }
+static void ke_op_KEO_LOR(ke1_t *p, ke1_t *q)  { p->i = (p->i || q->i); p->r = p->i; p->vtype = KEV_INT; }
+static void ke_op_KEO_POW(ke1_t *p, ke1_t *q)  { p->r = pow(p->r, q->r), p->i = (int64_t)(p->r + .5); p->vtype = p->vtype == KEV_REAL || q->vtype == KEV_REAL? KEV_REAL : KEV_INT; }
+static void ke_op_KEO_BNOT(ke1_t *p, ke1_t *q) { p->i = ~p->i; p->r = (double)p->i; p->vtype = KEV_INT; }
+static void ke_op_KEO_LNOT(ke1_t *p, ke1_t *q) { p->i = !p->i; p->r = (double)p->i; p->vtype = KEV_INT; }
+static void ke_op_KEO_POS(ke1_t *p, ke1_t *q)  { } // do nothing
+static void ke_op_KEO_NEG(ke1_t *p, ke1_t *q)  { p->i = -p->i, p->r = -p->r; }
+
+static void ke_func1_abs(ke1_t *p, ke1_t *q) { if (p->vtype == KEV_INT) p->i = abs(p->i), p->r = (double)p->i; else p->r = fabs(p->r), p->i = (int64_t)(p->r + .5); }
+
+/**********
+ * Parser *
+ **********/
+
+static inline char *mystrndup(const char *src, int n)
+{
+	char *dst;
+	dst = (char*)calloc(n + 1, 1);
+	strncpy(dst, src, n);
+	return dst;
+}
+
+// parse a token except "(", ")" and ","
+static ke1_t ke_read_token(char *p, char **r, int *err, int last_is_val) // it doesn't parse parentheses
+{
+	char *q = p;
+	ke1_t e;
+	memset(&e, 0, sizeof(ke1_t));
+	if (isalpha(*p) || *p == '_') { // a variable or a function
+		for (; *p && (*p == '_' || isalnum(*p)); ++p);
+		if (*p == '(') e.ttype = KET_FUNC, e.n_args = 1;
+		else e.ttype = KET_VAL, e.vtype = KEV_REAL;
+		e.name = mystrndup(q, p - q);
+		e.i = 0, e.r = 0.;
+		*r = p;
+	} else if (isdigit(*p) || *p == '.') { // a number
+		long x;
+		double y;
+		char *pp;
+		e.ttype = KET_VAL;
+		y = strtod(q, &p);
+		x = strtol(q, &pp, 0); // FIXME: check int/double parsing errors
+		if (q == p && q == pp) { // parse error
+			*err |= KEE_NUM;
+		} else if (p > pp) { // has "." or "[eE]"; then it is a real number
+			e.vtype = KEV_REAL;
+			e.i = (int64_t)(y + .5), e.r = y;
+			*r = p;
+		} else {
+			e.vtype = KEV_INT;
+			e.i = x, e.r = y;
+			*r = pp;
+		}
+	} else if (*p == '"' || *p == '\'') { // a string value
+		int c = *p;
+		for (++p; *p && *p != c; ++p)
+			if (*p == '\\') ++p; // escaping
+		if (*p == c) {
+			e.ttype = KET_VAL, e.vtype = KEV_STR;
+			e.s = mystrndup(q + 1, p - q - 1);
+			*r = p + 1;
+		} else *err |= KEE_UNQU, *r = p;
+	} else { // an operator
+		e.ttype = KET_OP;
+		if (*p == '*' && p[1] == '*') e.op = KEO_POW, e.f.builtin = ke_op_KEO_POW, e.n_args = 2, *r = q + 2;
+		else if (*p == '*') e.op = KEO_MUL, e.f.builtin = ke_op_KEO_MUL, e.n_args = 2, *r = q + 1; // FIXME: NOT working for unary operators
+		else if (*p == '/' && p[1] == '/') e.op = KEO_IDIV, e.f.builtin = ke_op_KEO_IDIV, e.n_args = 2, *r = q + 2;
+		else if (*p == '/') e.op = KEO_DIV, e.f.builtin = ke_op_KEO_DIV, e.n_args = 2, *r = q + 1;
+		else if (*p == '%') e.op = KEO_MOD, e.f.builtin = ke_op_KEO_MOD, e.n_args = 2, *r = q + 1;
+		else if (*p == '+') {
+			if (last_is_val) e.op = KEO_ADD, e.f.builtin = ke_op_KEO_ADD, e.n_args = 2;
+			else e.op = KEO_POS, e.f.builtin = ke_op_KEO_POS, e.n_args = 1;
+			*r = q + 1;
+		} else if (*p == '-') {
+			if (last_is_val) e.op = KEO_SUB, e.f.builtin = ke_op_KEO_SUB, e.n_args = 2;
+			else e.op = KEO_NEG, e.f.builtin = ke_op_KEO_NEG, e.n_args = 1;
+			*r = q + 1;
+		} else if (*p == '=' && p[1] == '=') e.op = KEO_EQ, e.f.builtin = ke_op_KEO_EQ, e.n_args = 2, *r = q + 2;
+		else if (*p == '!' && p[1] == '=') e.op = KEO_NE, e.f.builtin = ke_op_KEO_NE, e.n_args = 2, *r = q + 2;
+		else if (*p == '<' && p[1] == '>') e.op = KEO_NE, e.f.builtin = ke_op_KEO_NE, e.n_args = 2, *r = q + 2;
+		else if (*p == '>' && p[1] == '=') e.op = KEO_GE, e.f.builtin = ke_op_KEO_GE, e.n_args = 2, *r = q + 2;
+		else if (*p == '<' && p[1] == '=') e.op = KEO_LE, e.f.builtin = ke_op_KEO_LE, e.n_args = 2, *r = q + 2;
+		else if (*p == '>' && p[1] == '>') e.op = KEO_RSH, e.f.builtin = ke_op_KEO_RSH, e.n_args = 2, *r = q + 2;
+		else if (*p == '<' && p[1] == '<') e.op = KEO_LSH, e.f.builtin = ke_op_KEO_LSH, e.n_args = 2, *r = q + 2;
+		else if (*p == '>') e.op = KEO_GT, e.f.builtin = ke_op_KEO_GT, e.n_args = 2, *r = q + 1;
+		else if (*p == '<') e.op = KEO_LT, e.f.builtin = ke_op_KEO_LT, e.n_args = 2, *r = q + 1;
+		else if (*p == '|' && p[1] == '|') e.op = KEO_LOR, e.f.builtin = ke_op_KEO_LOR, e.n_args = 2, *r = q + 2;
+		else if (*p == '&' && p[1] == '&') e.op = KEO_LAND, e.f.builtin = ke_op_KEO_LAND, e.n_args = 2, *r = q + 2;
+		else if (*p == '|') e.op = KEO_BOR, e.f.builtin = ke_op_KEO_BOR, e.n_args = 2, *r = q + 1;
+		else if (*p == '&') e.op = KEO_BAND, e.f.builtin = ke_op_KEO_BAND, e.n_args = 2, *r = q + 1;
+		else if (*p == '^') e.op = KEO_BXOR, e.f.builtin = ke_op_KEO_BXOR, e.n_args = 2, *r = q + 1;
+		else if (*p == '~') e.op = KEO_BNOT, e.f.builtin = ke_op_KEO_BNOT, e.n_args = 1, *r = q + 1;
+		else if (*p == '!') e.op = KEO_LNOT, e.f.builtin = ke_op_KEO_LNOT, e.n_args = 1, *r = q + 1;
+		else e.ttype = KET_NULL, *err |= KEE_UNOP;
+	}
+	return e;
+}
+
+static inline ke1_t *push_back(ke1_t **a, int *n, int *m)
+{
+	if (*n == *m) {
+		int old_m = *m;
+		*m = *m? *m<<1 : 8;
+		*a = (ke1_t*)realloc(*a, *m * sizeof(ke1_t));
+		memset(*a + old_m, 0, (*m - old_m) * sizeof(ke1_t));
+	}
+	return &(*a)[(*n)++];
+}
+
+static ke1_t *ke_parse_core(const char *_s, int *_n, int *err)
+{
+	char *s, *p, *q;
+	int n_out, m_out, n_op, m_op, last_is_val = 0;
+	ke1_t *out, *op, *t, *u;
+
+	*err = 0; *_n = 0;
+	s = strdup(_s); // make a copy
+	for (p = q = s; *p; ++p) // squeeze out spaces
+		if (!isspace(*p)) *q++ = *p;
+	*q++ = 0;
+
+	out = op = 0;
+	n_out = m_out = n_op = m_op = 0;
+	p = s;
+	while (*p) {
+		if (*p == '(') {
+			t = push_back(&op, &n_op, &m_op); // push to the operator stack
+			t->op = -1, t->ttype = KET_NULL; // ->op < 0 for a left parenthsis
+			++p;
+		} else if (*p == ')') {
+			while (n_op > 0 && op[n_op-1].op >= 0) { // move operators to the output until we see a left parenthesis
+				u = push_back(&out, &n_out, &m_out);
+				*u = op[--n_op];
+			}
+			if (n_op == 0) { // error: extra right parenthesis
+				*err |= KEE_UNRP;
+				break;
+			} else --n_op; // pop out '('
+			if (n_op > 0 && op[n_op-1].ttype == KET_FUNC) { // the top of the operator stack is a function
+				u = push_back(&out, &n_out, &m_out); // move it to the output
+				*u = op[--n_op];
+				if (u->n_args == 1 && strcmp(u->name, "abs") == 0) u->f.builtin = ke_func1_abs;
+			}
+			++p;
+		} else if (*p == ',') { // function arguments separator
+			while (n_op > 0 && op[n_op-1].op >= 0) {
+				u = push_back(&out, &n_out, &m_out);
+				*u = op[--n_op];
+			}
+			if (n_op < 2 || op[n_op-2].ttype != KET_FUNC) { // we should at least see a function and a left parenthesis
+				*err |= KEE_FUNC;
+				break;
+			}
+			++op[n_op-2].n_args;
+			++p;
+		} else { // output-able token
+			ke1_t v;
+			v = ke_read_token(p, &p, err, last_is_val);
+			if (*err) break;
+			if (v.ttype == KET_VAL) {
+				u = push_back(&out, &n_out, &m_out);
+				*u = v;
+				last_is_val = 1;
+			} else if (v.ttype == KET_FUNC) {
+				t = push_back(&op, &n_op, &m_op);
+				*t = v;
+				last_is_val = 0;
+			} else if (v.ttype == KET_OP) {
+				int oi = ke_op[v.op];
+				while (n_op > 0 && op[n_op-1].ttype == KET_OP) {
+					int pre = ke_op[op[n_op-1].op]>>1;
+					if (((oi&1) && oi>>1 <= pre) || (!(oi&1) && oi>>1 < pre)) break;
+					u = push_back(&out, &n_out, &m_out);
+					*u = op[--n_op];
+				}
+				t = push_back(&op, &n_op, &m_op);
+				*t = v;
+				last_is_val = 0;
+			}
+		}
+	}
+
+	if (*err == 0) {
+		while (n_op > 0 && op[n_op-1].op >= 0) {
+			u = push_back(&out, &n_out, &m_out);
+			*u = op[--n_op];
+		}
+		if (n_op > 0) *err |= KEE_UNLP;
+	}
+	
+	if (*err == 0) { // then check if the number of args is correct
+		int i, n;
+		for (i = n = 0; i < n_out; ++i) {
+			ke1_t *e = &out[i];
+			if (e->ttype == KET_VAL) ++n;
+			else n -= e->n_args - 1;
+		}
+		if (n != 1) *err |= KEE_ARG;
+	}
+
+	free(op); free(s);
+	if (*err) {
+		free(out);
+		return 0;
+	}
+	*_n = n_out;
+	return out;
+}
+
+kexpr_t *ke_parse(const char *_s, int *err)
+{
+	int n;
+	ke1_t *e;
+	kexpr_t *ke;
+	e = ke_parse_core(_s, &n, err);
+	if (*err) return 0;
+	ke = (kexpr_t*)calloc(1, sizeof(kexpr_t));
+	ke->n = n, ke->e = e;
+	return ke;
+}
+
+int ke_eval(const kexpr_t *ke, int64_t *_i, double *_r, const char **_p, int *ret_type)
+{
+	ke1_t *stack, *p, *q;
+	int i, top = 0, err = 0;
+	*_i = 0, *_r = 0., *ret_type = 0;
+	for (i = 0; i < ke->n; ++i) {
+		ke1_t *e = &ke->e[i];
+		if ((e->ttype == KET_OP || e->ttype == KET_FUNC) && e->f.builtin == 0) err |= KEE_UNFUNC;
+		else if (e->ttype == KET_VAL && e->name && e->assigned == 0) err |= KEE_UNVAR;
+	}
+	stack = (ke1_t*)malloc(ke->n * sizeof(ke1_t));
+	for (i = 0; i < ke->n; ++i) {
+		ke1_t *e = &ke->e[i];
+		if (e->ttype == KET_OP || e->ttype == KET_FUNC) {
+			if (e->n_args == 2 && e->f.builtin) {
+				q = &stack[--top], p = &stack[top-1];
+				if (e->user_func) {
+					if (e->user_func == KEF_REAL)
+						p->r = e->f.real_func2(p->r, q->r), p->i = (int64_t)(p->r + .5), p->vtype = KEV_REAL;
+				} else e->f.builtin(p, q);
+			} else if (e->n_args == 1 && e->f.builtin) {
+				p = &stack[top-1];
+				if (e->user_func) {
+					if (e->user_func == KEF_REAL)
+						p->r = e->f.real_func1(p->r), p->i = (int64_t)(p->r + .5), p->vtype = KEV_REAL;
+				} else e->f.builtin(&stack[top-1], 0);
+			} else top -= e->n_args - 1;
+		} else stack[top++] = *e;
+	}
+	*ret_type = stack->vtype;
+	*_i = stack->i, *_r = stack->r, *_p = stack->s;
+	free(stack);
+	return err;
+}	
+
+int64_t ke_eval_int(const kexpr_t *ke, int *err)
+{
+	int int_ret;
+	int64_t i;
+	double r;
+	const char *s;
+	*err = ke_eval(ke, &i, &r, &s, &int_ret);
+	return i;
+}
+
+double ke_eval_real(const kexpr_t *ke, int *err)
+{
+	int int_ret;
+	int64_t i;
+	double r;
+	const char *s;
+	*err = ke_eval(ke, &i, &r, &s, &int_ret);
+	return r;
+}
+
+void ke_destroy(kexpr_t *ke)
+{
+	int i;
+	if (ke == 0) return;
+	for (i = 0; i < ke->n; ++i) {
+		free(ke->e[i].name);
+		free(ke->e[i].s);
+	}
+	free(ke->e); free(ke);
+}
+
+int ke_set_int(kexpr_t *ke, const char *var, int64_t y)
+{
+	int i, n = 0;
+	double yy = (double)y;
+	for (i = 0; i < ke->n; ++i) {
+		ke1_t *e = &ke->e[i];
+		if (e->ttype == KET_VAL && e->name && strcmp(e->name, var) == 0)
+			e->i = y, e->r = yy, e->vtype = KEV_INT, e->assigned = 1, ++n;
+	}
+	return n;
+}
+
+int ke_set_real(kexpr_t *ke, const char *var, double x)
+{
+	int i, n = 0;
+	int64_t xx = (int64_t)(x + .5);
+	for (i = 0; i < ke->n; ++i) {
+		ke1_t *e = &ke->e[i];
+		if (e->ttype == KET_VAL && e->name && strcmp(e->name, var) == 0)
+			e->r = x, e->i = xx, e->vtype = KEV_REAL, e->assigned = 1, ++n;
+	}
+	return n;
+}
+
+int ke_set_str(kexpr_t *ke, const char *var, const char *x)
+{
+	int i, n = 0;
+	for (i = 0; i < ke->n; ++i) {
+		ke1_t *e = &ke->e[i];
+		if (e->ttype == KET_VAL && e->name && strcmp(e->name, var) == 0) {
+			if (e->vtype == KEV_STR) free(e->s);
+			e->s = strdup(x);
+			e->i = 0, e->r = 0., e->assigned = 1;
+			e->vtype = KEV_STR;
+			++n;
+		}
+	}
+	return n;
+}
+
+int ke_set_real_func1(kexpr_t *ke, const char *name, double (*func)(double))
+{
+	int i, n = 0;
+	for (i = 0; i < ke->n; ++i) {
+		ke1_t *e = &ke->e[i];
+		if (e->ttype == KET_FUNC && e->n_args == 1 && strcmp(e->name, name) == 0)
+			e->f.real_func1 = func, e->user_func = KEF_REAL, ++n;
+	}
+	return n;
+}
+
+int ke_set_real_func2(kexpr_t *ke, const char *name, double (*func)(double, double))
+{
+	int i, n = 0;
+	for (i = 0; i < ke->n; ++i) {
+		ke1_t *e = &ke->e[i];
+		if (e->ttype == KET_FUNC && e->n_args == 2 && strcmp(e->name, name) == 0)
+			e->f.real_func2 = func, e->user_func = KEF_REAL, ++n;
+	}
+	return n;
+}
+
+int ke_set_default_func(kexpr_t *ke)
+{
+	int n = 0;
+	n += ke_set_real_func1(ke, "exp", exp);
+	n += ke_set_real_func1(ke, "log", log);
+	n += ke_set_real_func1(ke, "log10", log10);
+	n += ke_set_real_func1(ke, "sqrt", sqrt);
+	n += ke_set_real_func1(ke, "sin", sin);
+	n += ke_set_real_func1(ke, "cos", cos);
+	n += ke_set_real_func1(ke, "tan", tan);
+	n += ke_set_real_func2(ke, "pow", pow);
+	return n;
+}
+
+void ke_unset(kexpr_t *ke)
+{
+	int i;
+	for (i = 0; i < ke->n; ++i) {
+		ke1_t *e = &ke->e[i];
+		if (e->ttype == KET_VAL && e->name) e->assigned = 0;
+	}
+}
+
+void ke_print(const kexpr_t *ke)
+{
+	int i;
+	if (ke == 0) return;
+	for (i = 0; i < ke->n; ++i) {
+		const ke1_t *u = &ke->e[i];
+		if (i) putchar(' ');
+		if (u->ttype == KET_VAL) {
+			if (u->name) printf("%s", u->name);
+			else if (u->vtype == KEV_REAL) printf("%g", u->r);
+			else if (u->vtype == KEV_INT) printf("%lld", (long long)u->i);
+			else if (u->vtype == KEV_STR) printf("\"%s\"", u->s);
+		} else if (u->ttype == KET_OP) {
+			printf("%s", ke_opstr[u->op]);
+		} else if (u->ttype == KET_FUNC) {
+			printf("%s(%d)", u->name, u->n_args);
+		}
+	}
+	putchar('\n');
+}
+
+
+#ifdef KE_MAIN
+#include <unistd.h>
+
+int main(int argc, char *argv[])
+{
+	int c, err, to_print = 0, is_int = 0;
+	kexpr_t *ke;
+
+	while ((c = getopt(argc, argv, "pi")) >= 0) {
+		if (c == 'p') to_print = 1;
+		else if (c == 'i') is_int = 1;
+	}
+	if (optind == argc) {
+		fprintf(stderr, "Usage: %s [-pi] <expr>\n", argv[0]);
+		return 1;
+	}
+	ke = ke_parse(argv[optind], &err);
+	ke_set_default_func(ke);
+	if (err) {
+		fprintf(stderr, "Parse error: 0x%x\n", err);
+		return 1;
+	}
+	if (!to_print) {
+		int64_t vi;
+		double vr;
+		const char *vs;
+		int i, ret_type;
+		if (argc - optind > 1) {
+			for (i = optind + 1; i < argc; ++i) {
+				char *p, *s = argv[i];
+				for (p = s; *p && *p != '='; ++p);
+				if (*p == 0) continue; // not an assignment
+				*p = 0;
+				ke_set_real(ke, s, strtod(p+1, &p));
+			}
+		}
+		err |= ke_eval(ke, &vi, &vr, &vs, &ret_type);
+		if (err & KEE_UNFUNC)
+			fprintf(stderr, "Evaluation warning: an undefined function returns the first function argument.\n");
+		if (err & KEE_UNVAR) fprintf(stderr, "Evaluation warning: unassigned variables are set to 0.\n");
+		if (ret_type == KEV_INT) printf("%lld\n", (long long)vi);
+		else if (ret_type == KEV_REAL) printf("%g\n", vr);
+		else printf("%s\n", vs);
+	} else ke_print(ke);
+	ke_destroy(ke);
+	return 0;
+}
+#endif
diff --git a/ext/klib/kexpr.h b/ext/klib/kexpr.h
new file mode 100644
index 0000000..8023d4c
--- /dev/null
+++ b/ext/klib/kexpr.h
@@ -0,0 +1,68 @@
+#ifndef KEXPR_H
+#define KEXPR_H
+
+#include <stdint.h>
+
+struct kexpr_s;
+typedef struct kexpr_s kexpr_t;
+
+// Parse errors
+#define KEE_UNQU    0x01 // unmatched quotation marks
+#define KEE_UNLP    0x02 // unmatched left parentheses
+#define KEE_UNRP    0x04 // unmatched right parentheses
+#define KEE_UNOP    0x08 // unknown operators
+#define KEE_FUNC    0x10 // wrong function syntax
+#define KEE_ARG     0x20
+#define KEE_NUM     0x40 // fail to parse a number
+
+// Evaluation errors
+#define KEE_UNFUNC  0x40 // undefined function
+#define KEE_UNVAR   0x80 // unassigned variable
+
+// Return type
+#define KEV_REAL  1
+#define KEV_INT   2
+#define KEV_STR   3
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+	// parse an expression and return errors in $err
+	kexpr_t *ke_parse(const char *_s, int *err);
+
+	// free memory allocated during parsing
+	void ke_destroy(kexpr_t *ke);
+
+	// set a variable to integer value and return the occurrence of the variable
+	int ke_set_int(kexpr_t *ke, const char *var, int64_t x);
+
+	// set a variable to real value and return the occurrence of the variable
+	int ke_set_real(kexpr_t *ke, const char *var, double x);
+
+	// set a variable to string value and return the occurrence of the variable
+	int ke_set_str(kexpr_t *ke, const char *var, const char *x);
+
+	// set a user-defined function
+	int ke_set_real_func1(kexpr_t *ke, const char *name, double (*func)(double));
+	int ke_set_real_func2(kexpr_t *ke, const char *name, double (*func)(double, double));
+
+	// set default math functions
+	int ke_set_default_func(kexpr_t *ke);
+
+	// mark all variable as unset
+	void ke_unset(kexpr_t *e);
+
+	// evaluate expression; return error code; final value is returned via pointers
+	int ke_eval(const kexpr_t *ke, int64_t *_i, double *_r, const char **_s, int *ret_type);
+	int64_t ke_eval_int(const kexpr_t *ke, int *err);
+	double ke_eval_real(const kexpr_t *ke, int *err);
+
+	// print the expression in Reverse Polish notation (RPN)
+	void ke_print(const kexpr_t *ke);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/ext/klib/kgraph.h b/ext/klib/kgraph.h
new file mode 100644
index 0000000..af008ef
--- /dev/null
+++ b/ext/klib/kgraph.h
@@ -0,0 +1,79 @@
+#ifndef AC_KGRAPH_H
+#define AC_KGRAPH_H
+
+#include <stdint.h>
+#include <stdlib.h>
+#include "khash.h"
+#include "kbtree.h"
+
+typedef unsigned kgint_t;
+
+#define kgraph_t(name) kh_##name##_t
+
+#define __KG_BASIC(name, SCOPE, vertex_t, arc_t, ehn) \
+	SCOPE kgraph_t(name) *kg_init_##name(void) { return kh_init(name); } \
+	SCOPE void kg_destroy_##name(kgraph_t(name) *g) { \
+		khint_t k; \
+		if (g == 0) return; \
+		for (k = kh_begin(g); k != kh_end(g); ++k) \
+			if (kh_exist(g, k)) kh_destroy(ehn, kh_val(g, k)._arc); \
+		kh_destroy(name, g); \
+	} \
+	SCOPE vertex_t *kg_get_v_##name(kgraph_t(name) *g, kgint_t v) { \
+		khint_t k = kh_get(name, g, v); \
+		return k == kh_end(g)? 0 : &kh_val(g, k); \
+	} \
+	SCOPE vertex_t *kg_put_v_##name(kgraph_t(name) *g, kgint_t v, int *absent) { \
+		khint_t k; \
+		k = kh_put(name, g, v, absent); \
+		if (*absent) kh_val(g, k)._arc = kh_init(ehn); \
+		return &kh_val(g, k); \
+	} \
+	SCOPE void kg_put_a_##name(kgraph_t(name) *g, kgint_t vbeg, kgint_t vend, int dir, arc_t **pb, arc_t **pe) { \
+		vertex_t *p; \
+		khint_t k; \
+		int absent; \
+		p = kg_put_v_##name(g, vbeg, &absent); \
+		k = kh_put(ehn, p->_arc, vend<<2|dir, &absent); \
+		*pb = &kh_val(p->_arc, k); \
+		p = kg_put_v_##name(g, vend, &absent); \
+		k = kh_put(ehn, p->_arc, vbeg<<2|(~dir&3), &absent); \
+		*pe = &kh_val(p->_arc, k); \
+	} \
+	SCOPE vertex_t *kg_del_v_##name(kgraph_t(name) *g, kgint_t v) { \
+		khint_t k, k0, k2, k3; \
+		khash_t(ehn) *h; \
+		k0 = k = kh_get(name, g, v); \
+		if (k == kh_end(g)) return 0; /* not present in the graph */ \
+		h = kh_val(g, k)._arc; \
+		for (k = kh_begin(h); k != kh_end(h); ++k) /* remove v from its neighbors */ \
+			if (kh_exist(h, k)) { \
+				k2 = kh_get(name, g, kh_key(h, k)>>2); \
+				/* assert(k2 != kh_end(g)); */ \
+				k3 = kh_get(ehn, kh_val(g, k2)._arc, v<<2|(~kh_key(h, k)&3)); \
+				/* assert(k3 != kh_end(kh_val(g, k2)._arc)); */ \
+				kh_del(ehn, kh_val(g, k2)._arc, k3); \
+			} \
+		kh_destroy(ehn, h); \
+		kh_del(name, g, k0); \
+		return &kh_val(g, k0); \
+	}
+
+#define KGRAPH_PRINT(name, SCOPE) \
+	SCOPE void kg_print_##name(kgraph_t(name) *g) { \
+		khint_t k, k2; \
+		for (k = kh_begin(g); k != kh_end(g); ++k) \
+			if (kh_exist(g, k)) { \
+				printf("v %u\n", kh_key(g, k)); \
+				for (k2 = kh_begin(kh_val(g, k)._arc); k2 != kh_end(kh_val(g, k)._arc); ++k2) \
+					if (kh_exist(kh_val(g, k)._arc, k2) && kh_key(g, k) < kh_key(kh_val(g, k)._arc, k2)>>2) \
+						printf("a %u%c%c%u\n", kh_key(g, k), "><"[kh_key(kh_val(g, k)._arc, k2)>>1&1], \
+								"><"[kh_key(kh_val(g, k)._arc, k2)&1], kh_key(kh_val(g, k)._arc, k2)>>2); \
+			} \
+	}
+
+#define KGRAPH_INIT(name, SCOPE, vertex_t, arc_t, ehn) \
+	KHASH_INIT2(name, SCOPE, kgint_t, vertex_t, 1, kh_int_hash_func, kh_int_hash_equal) \
+	__KG_BASIC(name, SCOPE, vertex_t, arc_t, ehn)
+
+#endif
diff --git a/ext/klib/khash.h b/ext/klib/khash.h
new file mode 100644
index 0000000..f75f347
--- /dev/null
+++ b/ext/klib/khash.h
@@ -0,0 +1,627 @@
+/* The MIT License
+
+   Copyright (c) 2008, 2009, 2011 by Attractive Chaos <attractor@live.co.uk>
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be
+   included in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+*/
+
+/*
+  An example:
+
+#include "khash.h"
+KHASH_MAP_INIT_INT(32, char)
+int main() {
+	int ret, is_missing;
+	khiter_t k;
+	khash_t(32) *h = kh_init(32);
+	k = kh_put(32, h, 5, &ret);
+	kh_value(h, k) = 10;
+	k = kh_get(32, h, 10);
+	is_missing = (k == kh_end(h));
+	k = kh_get(32, h, 5);
+	kh_del(32, h, k);
+	for (k = kh_begin(h); k != kh_end(h); ++k)
+		if (kh_exist(h, k)) kh_value(h, k) = 1;
+	kh_destroy(32, h);
+	return 0;
+}
+*/
+
+/*
+  2013-05-02 (0.2.8):
+
+	* Use quadratic probing. When the capacity is power of 2, stepping function
+	  i*(i+1)/2 guarantees to traverse each bucket. It is better than double
+	  hashing on cache performance and is more robust than linear probing.
+
+	  In theory, double hashing should be more robust than quadratic probing.
+	  However, my implementation is probably not for large hash tables, because
+	  the second hash function is closely tied to the first hash function,
+	  which reduce the effectiveness of double hashing.
+
+	Reference: http://research.cs.vt.edu/AVresearch/hashing/quadratic.php
+
+  2011-12-29 (0.2.7):
+
+    * Minor code clean up; no actual effect.
+
+  2011-09-16 (0.2.6):
+
+	* The capacity is a power of 2. This seems to dramatically improve the
+	  speed for simple keys. Thank Zilong Tan for the suggestion. Reference:
+
+	   - http://code.google.com/p/ulib/
+	   - http://nothings.org/computer/judy/
+
+	* Allow to optionally use linear probing which usually has better
+	  performance for random input. Double hashing is still the default as it
+	  is more robust to certain non-random input.
+
+	* Added Wang's integer hash function (not used by default). This hash
+	  function is more robust to certain non-random input.
+
+  2011-02-14 (0.2.5):
+
+    * Allow to declare global functions.
+
+  2009-09-26 (0.2.4):
+
+    * Improve portability
+
+  2008-09-19 (0.2.3):
+
+	* Corrected the example
+	* Improved interfaces
+
+  2008-09-11 (0.2.2):
+
+	* Improved speed a little in kh_put()
+
+  2008-09-10 (0.2.1):
+
+	* Added kh_clear()
+	* Fixed a compiling error
+
+  2008-09-02 (0.2.0):
+
+	* Changed to token concatenation which increases flexibility.
+
+  2008-08-31 (0.1.2):
+
+	* Fixed a bug in kh_get(), which has not been tested previously.
+
+  2008-08-31 (0.1.1):
+
+	* Added destructor
+*/
+
+
+#ifndef __AC_KHASH_H
+#define __AC_KHASH_H
+
+/*!
+  @header
+
+  Generic hash table library.
+ */
+
+#define AC_VERSION_KHASH_H "0.2.8"
+
+#include <stdlib.h>
+#include <string.h>
+#include <limits.h>
+
+/* compiler specific configuration */
+
+#if UINT_MAX == 0xffffffffu
+typedef unsigned int khint32_t;
+#elif ULONG_MAX == 0xffffffffu
+typedef unsigned long khint32_t;
+#endif
+
+#if ULONG_MAX == ULLONG_MAX
+typedef unsigned long khint64_t;
+#else
+typedef unsigned long long khint64_t;
+#endif
+
+#ifndef kh_inline
+#ifdef _MSC_VER
+#define kh_inline __inline
+#else
+#define kh_inline inline
+#endif
+#endif /* kh_inline */
+
+#ifndef klib_unused
+#if (defined __clang__ && __clang_major__ >= 3) || (defined __GNUC__ && __GNUC__ >= 3)
+#define klib_unused __attribute__ ((__unused__))
+#else
+#define klib_unused
+#endif
+#endif /* klib_unused */
+
+typedef khint32_t khint_t;
+typedef khint_t khiter_t;
+
+#define __ac_isempty(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&2)
+#define __ac_isdel(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&1)
+#define __ac_iseither(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&3)
+#define __ac_set_isdel_false(flag, i) (flag[i>>4]&=~(1ul<<((i&0xfU)<<1)))
+#define __ac_set_isempty_false(flag, i) (flag[i>>4]&=~(2ul<<((i&0xfU)<<1)))
+#define __ac_set_isboth_false(flag, i) (flag[i>>4]&=~(3ul<<((i&0xfU)<<1)))
+#define __ac_set_isdel_true(flag, i) (flag[i>>4]|=1ul<<((i&0xfU)<<1))
+
+#define __ac_fsize(m) ((m) < 16? 1 : (m)>>4)
+
+#ifndef kroundup32
+#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
+#endif
+
+#ifndef kcalloc
+#define kcalloc(N,Z) calloc(N,Z)
+#endif
+#ifndef kmalloc
+#define kmalloc(Z) malloc(Z)
+#endif
+#ifndef krealloc
+#define krealloc(P,Z) realloc(P,Z)
+#endif
+#ifndef kfree
+#define kfree(P) free(P)
+#endif
+
+static const double __ac_HASH_UPPER = 0.77;
+
+#define __KHASH_TYPE(name, khkey_t, khval_t) \
+	typedef struct kh_##name##_s { \
+		khint_t n_buckets, size, n_occupied, upper_bound; \
+		khint32_t *flags; \
+		khkey_t *keys; \
+		khval_t *vals; \
+	} kh_##name##_t;
+
+#define __KHASH_PROTOTYPES(name, khkey_t, khval_t)	 					\
+	extern kh_##name##_t *kh_init_##name(void);							\
+	extern void kh_destroy_##name(kh_##name##_t *h);					\
+	extern void kh_clear_##name(kh_##name##_t *h);						\
+	extern khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key); 	\
+	extern int kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets); \
+	extern khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret); \
+	extern void kh_del_##name(kh_##name##_t *h, khint_t x);
+
+#define __KHASH_IMPL(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \
+	SCOPE kh_##name##_t *kh_init_##name(void) {							\
+		return (kh_##name##_t*)kcalloc(1, sizeof(kh_##name##_t));		\
+	}																	\
+	SCOPE void kh_destroy_##name(kh_##name##_t *h)						\
+	{																	\
+		if (h) {														\
+			kfree((void *)h->keys); kfree(h->flags);					\
+			kfree((void *)h->vals);										\
+			kfree(h);													\
+		}																\
+	}																	\
+	SCOPE void kh_clear_##name(kh_##name##_t *h)						\
+	{																	\
+		if (h && h->flags) {											\
+			memset(h->flags, 0xaa, __ac_fsize(h->n_buckets) * sizeof(khint32_t)); \
+			h->size = h->n_occupied = 0;								\
+		}																\
+	}																	\
+	SCOPE khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key) 	\
+	{																	\
+		if (h->n_buckets) {												\
+			khint_t k, i, last, mask, step = 0; \
+			mask = h->n_buckets - 1;									\
+			k = __hash_func(key); i = k & mask;							\
+			last = i; \
+			while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \
+				i = (i + (++step)) & mask; \
+				if (i == last) return h->n_buckets;						\
+			}															\
+			return __ac_iseither(h->flags, i)? h->n_buckets : i;		\
+		} else return 0;												\
+	}																	\
+	SCOPE int kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets) \
+	{ /* This function uses 0.25*n_buckets bytes of working space instead of [sizeof(key_t+val_t)+.25]*n_buckets. */ \
+		khint32_t *new_flags = 0;										\
+		khint_t j = 1;													\
+		{																\
+			kroundup32(new_n_buckets); 									\
+			if (new_n_buckets < 4) new_n_buckets = 4;					\
+			if (h->size >= (khint_t)(new_n_buckets * __ac_HASH_UPPER + 0.5)) j = 0;	/* requested size is too small */ \
+			else { /* hash table size to be changed (shrink or expand); rehash */ \
+				new_flags = (khint32_t*)kmalloc(__ac_fsize(new_n_buckets) * sizeof(khint32_t));	\
+				if (!new_flags) return -1;								\
+				memset(new_flags, 0xaa, __ac_fsize(new_n_buckets) * sizeof(khint32_t)); \
+				if (h->n_buckets < new_n_buckets) {	/* expand */		\
+					khkey_t *new_keys = (khkey_t*)krealloc((void *)h->keys, new_n_buckets * sizeof(khkey_t)); \
+					if (!new_keys) { kfree(new_flags); return -1; }		\
+					h->keys = new_keys;									\
+					if (kh_is_map) {									\
+						khval_t *new_vals = (khval_t*)krealloc((void *)h->vals, new_n_buckets * sizeof(khval_t)); \
+						if (!new_vals) { kfree(new_flags); return -1; }	\
+						h->vals = new_vals;								\
+					}													\
+				} /* otherwise shrink */								\
+			}															\
+		}																\
+		if (j) { /* rehashing is needed */								\
+			for (j = 0; j != h->n_buckets; ++j) {						\
+				if (__ac_iseither(h->flags, j) == 0) {					\
+					khkey_t key = h->keys[j];							\
+					khval_t val;										\
+					khint_t new_mask;									\
+					new_mask = new_n_buckets - 1; 						\
+					if (kh_is_map) val = h->vals[j];					\
+					__ac_set_isdel_true(h->flags, j);					\
+					while (1) { /* kick-out process; sort of like in Cuckoo hashing */ \
+						khint_t k, i, step = 0; \
+						k = __hash_func(key);							\
+						i = k & new_mask;								\
+						while (!__ac_isempty(new_flags, i)) i = (i + (++step)) & new_mask; \
+						__ac_set_isempty_false(new_flags, i);			\
+						if (i < h->n_buckets && __ac_iseither(h->flags, i) == 0) { /* kick out the existing element */ \
+							{ khkey_t tmp = h->keys[i]; h->keys[i] = key; key = tmp; } \
+							if (kh_is_map) { khval_t tmp = h->vals[i]; h->vals[i] = val; val = tmp; } \
+							__ac_set_isdel_true(h->flags, i); /* mark it as deleted in the old hash table */ \
+						} else { /* write the element and jump out of the loop */ \
+							h->keys[i] = key;							\
+							if (kh_is_map) h->vals[i] = val;			\
+							break;										\
+						}												\
+					}													\
+				}														\
+			}															\
+			if (h->n_buckets > new_n_buckets) { /* shrink the hash table */ \
+				h->keys = (khkey_t*)krealloc((void *)h->keys, new_n_buckets * sizeof(khkey_t)); \
+				if (kh_is_map) h->vals = (khval_t*)krealloc((void *)h->vals, new_n_buckets * sizeof(khval_t)); \
+			}															\
+			kfree(h->flags); /* free the working space */				\
+			h->flags = new_flags;										\
+			h->n_buckets = new_n_buckets;								\
+			h->n_occupied = h->size;									\
+			h->upper_bound = (khint_t)(h->n_buckets * __ac_HASH_UPPER + 0.5); \
+		}																\
+		return 0;														\
+	}																	\
+	SCOPE khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret) \
+	{																	\
+		khint_t x;														\
+		if (h->n_occupied >= h->upper_bound) { /* update the hash table */ \
+			if (h->n_buckets > (h->size<<1)) {							\
+				if (kh_resize_##name(h, h->n_buckets - 1) < 0) { /* clear "deleted" elements */ \
+					*ret = -1; return h->n_buckets;						\
+				}														\
+			} else if (kh_resize_##name(h, h->n_buckets + 1) < 0) { /* expand the hash table */ \
+				*ret = -1; return h->n_buckets;							\
+			}															\
+		} /* TODO: to implement automatically shrinking; resize() already support shrinking */ \
+		{																\
+			khint_t k, i, site, last, mask = h->n_buckets - 1, step = 0; \
+			x = site = h->n_buckets; k = __hash_func(key); i = k & mask; \
+			if (__ac_isempty(h->flags, i)) x = i; /* for speed up */	\
+			else {														\
+				last = i; \
+				while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \
+					if (__ac_isdel(h->flags, i)) site = i;				\
+					i = (i + (++step)) & mask; \
+					if (i == last) { x = site; break; }					\
+				}														\
+				if (x == h->n_buckets) {								\
+					if (__ac_isempty(h->flags, i) && site != h->n_buckets) x = site; \
+					else x = i;											\
+				}														\
+			}															\
+		}																\
+		if (__ac_isempty(h->flags, x)) { /* not present at all */		\
+			h->keys[x] = key;											\
+			__ac_set_isboth_false(h->flags, x);							\
+			++h->size; ++h->n_occupied;									\
+			*ret = 1;													\
+		} else if (__ac_isdel(h->flags, x)) { /* deleted */				\
+			h->keys[x] = key;											\
+			__ac_set_isboth_false(h->flags, x);							\
+			++h->size;													\
+			*ret = 2;													\
+		} else *ret = 0; /* Don't touch h->keys[x] if present and not deleted */ \
+		return x;														\
+	}																	\
+	SCOPE void kh_del_##name(kh_##name##_t *h, khint_t x)				\
+	{																	\
+		if (x != h->n_buckets && !__ac_iseither(h->flags, x)) {			\
+			__ac_set_isdel_true(h->flags, x);							\
+			--h->size;													\
+		}																\
+	}
+
+#define KHASH_DECLARE(name, khkey_t, khval_t)		 					\
+	__KHASH_TYPE(name, khkey_t, khval_t) 								\
+	__KHASH_PROTOTYPES(name, khkey_t, khval_t)
+
+#define KHASH_INIT2(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \
+	__KHASH_TYPE(name, khkey_t, khval_t) 								\
+	__KHASH_IMPL(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal)
+
+#define KHASH_INIT(name, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \
+	KHASH_INIT2(name, static kh_inline klib_unused, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal)
+
+/* --- BEGIN OF HASH FUNCTIONS --- */
+
+/*! @function
+  @abstract     Integer hash function
+  @param  key   The integer [khint32_t]
+  @return       The hash value [khint_t]
+ */
+#define kh_int_hash_func(key) (khint32_t)(key)
+/*! @function
+  @abstract     Integer comparison function
+ */
+#define kh_int_hash_equal(a, b) ((a) == (b))
+/*! @function
+  @abstract     64-bit integer hash function
+  @param  key   The integer [khint64_t]
+  @return       The hash value [khint_t]
+ */
+#define kh_int64_hash_func(key) (khint32_t)((key)>>33^(key)^(key)<<11)
+/*! @function
+  @abstract     64-bit integer comparison function
+ */
+#define kh_int64_hash_equal(a, b) ((a) == (b))
+/*! @function
+  @abstract     const char* hash function
+  @param  s     Pointer to a null terminated string
+  @return       The hash value
+ */
+static kh_inline khint_t __ac_X31_hash_string(const char *s)
+{
+	khint_t h = (khint_t)*s;
+	if (h) for (++s ; *s; ++s) h = (h << 5) - h + (khint_t)*s;
+	return h;
+}
+/*! @function
+  @abstract     Another interface to const char* hash function
+  @param  key   Pointer to a null terminated string [const char*]
+  @return       The hash value [khint_t]
+ */
+#define kh_str_hash_func(key) __ac_X31_hash_string(key)
+/*! @function
+  @abstract     Const char* comparison function
+ */
+#define kh_str_hash_equal(a, b) (strcmp(a, b) == 0)
+
+static kh_inline khint_t __ac_Wang_hash(khint_t key)
+{
+    key += ~(key << 15);
+    key ^=  (key >> 10);
+    key +=  (key << 3);
+    key ^=  (key >> 6);
+    key += ~(key << 11);
+    key ^=  (key >> 16);
+    return key;
+}
+#define kh_int_hash_func2(key) __ac_Wang_hash((khint_t)key)
+
+/* --- END OF HASH FUNCTIONS --- */
+
+/* Other convenient macros... */
+
+/*!
+  @abstract Type of the hash table.
+  @param  name  Name of the hash table [symbol]
+ */
+#define khash_t(name) kh_##name##_t
+
+/*! @function
+  @abstract     Initiate a hash table.
+  @param  name  Name of the hash table [symbol]
+  @return       Pointer to the hash table [khash_t(name)*]
+ */
+#define kh_init(name) kh_init_##name()
+
+/*! @function
+  @abstract     Destroy a hash table.
+  @param  name  Name of the hash table [symbol]
+  @param  h     Pointer to the hash table [khash_t(name)*]
+ */
+#define kh_destroy(name, h) kh_destroy_##name(h)
+
+/*! @function
+  @abstract     Reset a hash table without deallocating memory.
+  @param  name  Name of the hash table [symbol]
+  @param  h     Pointer to the hash table [khash_t(name)*]
+ */
+#define kh_clear(name, h) kh_clear_##name(h)
+
+/*! @function
+  @abstract     Resize a hash table.
+  @param  name  Name of the hash table [symbol]
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @param  s     New size [khint_t]
+ */
+#define kh_resize(name, h, s) kh_resize_##name(h, s)
+
+/*! @function
+  @abstract     Insert a key to the hash table.
+  @param  name  Name of the hash table [symbol]
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @param  k     Key [type of keys]
+  @param  r     Extra return code: -1 if the operation failed;
+                0 if the key is present in the hash table;
+                1 if the bucket is empty (never used); 2 if the element in
+				the bucket has been deleted [int*]
+  @return       Iterator to the inserted element [khint_t]
+ */
+#define kh_put(name, h, k, r) kh_put_##name(h, k, r)
+
+/*! @function
+  @abstract     Retrieve a key from the hash table.
+  @param  name  Name of the hash table [symbol]
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @param  k     Key [type of keys]
+  @return       Iterator to the found element, or kh_end(h) if the element is absent [khint_t]
+ */
+#define kh_get(name, h, k) kh_get_##name(h, k)
+
+/*! @function
+  @abstract     Remove a key from the hash table.
+  @param  name  Name of the hash table [symbol]
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @param  k     Iterator to the element to be deleted [khint_t]
+ */
+#define kh_del(name, h, k) kh_del_##name(h, k)
+
+/*! @function
+  @abstract     Test whether a bucket contains data.
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @param  x     Iterator to the bucket [khint_t]
+  @return       1 if containing data; 0 otherwise [int]
+ */
+#define kh_exist(h, x) (!__ac_iseither((h)->flags, (x)))
+
+/*! @function
+  @abstract     Get key given an iterator
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @param  x     Iterator to the bucket [khint_t]
+  @return       Key [type of keys]
+ */
+#define kh_key(h, x) ((h)->keys[x])
+
+/*! @function
+  @abstract     Get value given an iterator
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @param  x     Iterator to the bucket [khint_t]
+  @return       Value [type of values]
+  @discussion   For hash sets, calling this results in segfault.
+ */
+#define kh_val(h, x) ((h)->vals[x])
+
+/*! @function
+  @abstract     Alias of kh_val()
+ */
+#define kh_value(h, x) ((h)->vals[x])
+
+/*! @function
+  @abstract     Get the start iterator
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @return       The start iterator [khint_t]
+ */
+#define kh_begin(h) (khint_t)(0)
+
+/*! @function
+  @abstract     Get the end iterator
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @return       The end iterator [khint_t]
+ */
+#define kh_end(h) ((h)->n_buckets)
+
+/*! @function
+  @abstract     Get the number of elements in the hash table
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @return       Number of elements in the hash table [khint_t]
+ */
+#define kh_size(h) ((h)->size)
+
+/*! @function
+  @abstract     Get the number of buckets in the hash table
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @return       Number of buckets in the hash table [khint_t]
+ */
+#define kh_n_buckets(h) ((h)->n_buckets)
+
+/*! @function
+  @abstract     Iterate over the entries in the hash table
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @param  kvar  Variable to which key will be assigned
+  @param  vvar  Variable to which value will be assigned
+  @param  code  Block of code to execute
+ */
+#define kh_foreach(h, kvar, vvar, code) { khint_t __i;		\
+	for (__i = kh_begin(h); __i != kh_end(h); ++__i) {		\
+		if (!kh_exist(h,__i)) continue;						\
+		(kvar) = kh_key(h,__i);								\
+		(vvar) = kh_val(h,__i);								\
+		code;												\
+	} }
+
+/*! @function
+  @abstract     Iterate over the values in the hash table
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @param  vvar  Variable to which value will be assigned
+  @param  code  Block of code to execute
+ */
+#define kh_foreach_value(h, vvar, code) { khint_t __i;		\
+	for (__i = kh_begin(h); __i != kh_end(h); ++__i) {		\
+		if (!kh_exist(h,__i)) continue;						\
+		(vvar) = kh_val(h,__i);								\
+		code;												\
+	} }
+
+/* More convenient interfaces */
+
+/*! @function
+  @abstract     Instantiate a hash set containing integer keys
+  @param  name  Name of the hash table [symbol]
+ */
+#define KHASH_SET_INIT_INT(name)										\
+	KHASH_INIT(name, khint32_t, char, 0, kh_int_hash_func, kh_int_hash_equal)
+
+/*! @function
+  @abstract     Instantiate a hash map containing integer keys
+  @param  name  Name of the hash table [symbol]
+  @param  khval_t  Type of values [type]
+ */
+#define KHASH_MAP_INIT_INT(name, khval_t)								\
+	KHASH_INIT(name, khint32_t, khval_t, 1, kh_int_hash_func, kh_int_hash_equal)
+
+/*! @function
+  @abstract     Instantiate a hash set containing 64-bit integer keys
+  @param  name  Name of the hash table [symbol]
+ */
+#define KHASH_SET_INIT_INT64(name)										\
+	KHASH_INIT(name, khint64_t, char, 0, kh_int64_hash_func, kh_int64_hash_equal)
+
+/*! @function
+  @abstract     Instantiate a hash map containing 64-bit integer keys
+  @param  name  Name of the hash table [symbol]
+  @param  khval_t  Type of values [type]
+ */
+#define KHASH_MAP_INIT_INT64(name, khval_t)								\
+	KHASH_INIT(name, khint64_t, khval_t, 1, kh_int64_hash_func, kh_int64_hash_equal)
+
+typedef const char *kh_cstr_t;
+/*! @function
+  @abstract     Instantiate a hash map containing const char* keys
+  @param  name  Name of the hash table [symbol]
+ */
+#define KHASH_SET_INIT_STR(name)										\
+	KHASH_INIT(name, kh_cstr_t, char, 0, kh_str_hash_func, kh_str_hash_equal)
+
+/*! @function
+  @abstract     Instantiate a hash map containing const char* keys
+  @param  name  Name of the hash table [symbol]
+  @param  khval_t  Type of values [type]
+ */
+#define KHASH_MAP_INIT_STR(name, khval_t)								\
+	KHASH_INIT(name, kh_cstr_t, khval_t, 1, kh_str_hash_func, kh_str_hash_equal)
+
+#endif /* __AC_KHASH_H */
diff --git a/ext/klib/khashl.h b/ext/klib/khashl.h
new file mode 100644
index 0000000..7f2a81a
--- /dev/null
+++ b/ext/klib/khashl.h
@@ -0,0 +1,446 @@
+/* The MIT License
+
+   Copyright (c) 2019-2024 by Attractive Chaos <attractor@live.co.uk>
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be
+   included in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+*/
+
+#ifndef __AC_KHASHL_H
+#define __AC_KHASHL_H
+
+#define AC_VERSION_KHASHL_H "r20"
+
+#include <stdlib.h>
+#include <string.h>
+#include <limits.h>
+
+/************************************
+ * Compiler specific configurations *
+ ************************************/
+
+#if UINT_MAX == 0xffffffffu
+typedef unsigned int khint32_t;
+#elif ULONG_MAX == 0xffffffffu
+typedef unsigned long khint32_t;
+#endif
+
+#if ULONG_MAX == ULLONG_MAX
+typedef unsigned long khint64_t;
+#else
+typedef unsigned long long khint64_t;
+#endif
+
+#ifndef kh_inline
+#ifdef _MSC_VER
+#define kh_inline __inline
+#else
+#define kh_inline inline
+#endif
+#endif /* kh_inline */
+
+#ifndef klib_unused
+#if (defined __clang__ && __clang_major__ >= 3) || (defined __GNUC__ && __GNUC__ >= 3)
+#define klib_unused __attribute__ ((__unused__))
+#else
+#define klib_unused
+#endif
+#endif /* klib_unused */
+
+#define KH_LOCAL static kh_inline klib_unused
+
+typedef khint32_t khint_t;
+typedef const char *kh_cstr_t;
+
+/***********************
+ * Configurable macros *
+ ***********************/
+
+#ifndef kh_max_count
+#define kh_max_count(cap) (((cap)>>1) + ((cap)>>2)) /* default load factor: 75% */
+#endif
+
+#ifndef kh_packed
+#define kh_packed __attribute__ ((__packed__))
+#endif
+
+#ifndef kcalloc
+#define kcalloc(N,Z) calloc(N,Z)
+#endif
+#ifndef kmalloc
+#define kmalloc(Z) malloc(Z)
+#endif
+#ifndef krealloc
+#define krealloc(P,Z) realloc(P,Z)
+#endif
+#ifndef kfree
+#define kfree(P) free(P)
+#endif
+
+/****************************
+ * Simple private functions *
+ ****************************/
+
+#define __kh_used(flag, i)       (flag[i>>5] >> (i&0x1fU) & 1U)
+#define __kh_set_used(flag, i)   (flag[i>>5] |= 1U<<(i&0x1fU))
+#define __kh_set_unused(flag, i) (flag[i>>5] &= ~(1U<<(i&0x1fU)))
+
+#define __kh_fsize(m) ((m) < 32? 1 : (m)>>5)
+
+static kh_inline khint_t __kh_h2b(khint_t hash, khint_t bits) { return hash * 2654435769U >> (32 - bits); }
+
+/*******************
+ * Hash table base *
+ *******************/
+
+#define __KHASHL_TYPE(HType, khkey_t) \
+	typedef struct HType { \
+		khint_t bits, count; \
+		khint32_t *used; \
+		khkey_t *keys; \
+	} HType;
+
+#define __KHASHL_PROTOTYPES(HType, prefix, khkey_t) \
+	extern HType *prefix##_init(void); \
+	extern void prefix##_destroy(HType *h); \
+	extern void prefix##_clear(HType *h); \
+	extern khint_t prefix##_getp(const HType *h, const khkey_t *key); \
+	extern int prefix##_resize(HType *h, khint_t new_n_buckets); \
+	extern khint_t prefix##_putp(HType *h, const khkey_t *key, int *absent); \
+	extern void prefix##_del(HType *h, khint_t k);
+
+#define __KHASHL_IMPL_BASIC(SCOPE, HType, prefix) \
+	SCOPE HType *prefix##_init(void) { \
+		return (HType*)kcalloc(1, sizeof(HType)); \
+	} \
+	SCOPE void prefix##_destroy(HType *h) { \
+		if (!h) return; \
+		kfree((void *)h->keys); kfree(h->used); \
+		kfree(h); \
+	} \
+	SCOPE void prefix##_clear(HType *h) { \
+		if (h && h->used) { \
+			khint_t n_buckets = (khint_t)1U << h->bits; \
+			memset(h->used, 0, __kh_fsize(n_buckets) * sizeof(khint32_t)); \
+			h->count = 0; \
+		} \
+	}
+
+#define __KHASHL_IMPL_GET(SCOPE, HType, prefix, khkey_t, __hash_fn, __hash_eq) \
+	SCOPE khint_t prefix##_getp_core(const HType *h, const khkey_t *key, khint_t hash) { \
+		khint_t i, last, n_buckets, mask; \
+		if (h->keys == 0) return 0; \
+		n_buckets = (khint_t)1U << h->bits; \
+		mask = n_buckets - 1U; \
+		i = last = __kh_h2b(hash, h->bits); \
+		while (__kh_used(h->used, i) && !__hash_eq(h->keys[i], *key)) { \
+			i = (i + 1U) & mask; \
+			if (i == last) return n_buckets; \
+		} \
+		return !__kh_used(h->used, i)? n_buckets : i; \
+	} \
+	SCOPE khint_t prefix##_getp(const HType *h, const khkey_t *key) { return prefix##_getp_core(h, key, __hash_fn(*key)); } \
+	SCOPE khint_t prefix##_get(const HType *h, khkey_t key) { return prefix##_getp_core(h, &key, __hash_fn(key)); }
+
+#define __KHASHL_IMPL_RESIZE(SCOPE, HType, prefix, khkey_t, __hash_fn, __hash_eq) \
+	SCOPE int prefix##_resize(HType *h, khint_t new_n_buckets) { \
+		khint32_t *new_used = 0; \
+		khint_t j = 0, x = new_n_buckets, n_buckets, new_bits, new_mask; \
+		while ((x >>= 1) != 0) ++j; \
+		if (new_n_buckets & (new_n_buckets - 1)) ++j; \
+		new_bits = j > 2? j : 2; \
+		new_n_buckets = (khint_t)1U << new_bits; \
+		if (h->count > kh_max_count(new_n_buckets)) return 0; /* requested size is too small */ \
+		new_used = (khint32_t*)kmalloc(__kh_fsize(new_n_buckets) * sizeof(khint32_t)); \
+		memset(new_used, 0, __kh_fsize(new_n_buckets) * sizeof(khint32_t)); \
+		if (!new_used) return -1; /* not enough memory */ \
+		n_buckets = h->keys? (khint_t)1U<<h->bits : 0U; \
+		if (n_buckets < new_n_buckets) { /* expand */ \
+			khkey_t *new_keys = (khkey_t*)krealloc((void*)h->keys, new_n_buckets * sizeof(khkey_t)); \
+			if (!new_keys) { kfree(new_used); return -1; } \
+			h->keys = new_keys; \
+		} /* otherwise shrink */ \
+		new_mask = new_n_buckets - 1; \
+		for (j = 0; j != n_buckets; ++j) { \
+			khkey_t key; \
+			if (!__kh_used(h->used, j)) continue; \
+			key = h->keys[j]; \
+			__kh_set_unused(h->used, j); \
+			while (1) { /* kick-out process; sort of like in Cuckoo hashing */ \
+				khint_t i; \
+				i = __kh_h2b(__hash_fn(key), new_bits); \
+				while (__kh_used(new_used, i)) i = (i + 1) & new_mask; \
+				__kh_set_used(new_used, i); \
+				if (i < n_buckets && __kh_used(h->used, i)) { /* kick out the existing element */ \
+					{ khkey_t tmp = h->keys[i]; h->keys[i] = key; key = tmp; } \
+					__kh_set_unused(h->used, i); /* mark it as deleted in the old hash table */ \
+				} else { /* write the element and jump out of the loop */ \
+					h->keys[i] = key; \
+					break; \
+				} \
+			} \
+		} \
+		if (n_buckets > new_n_buckets) /* shrink the hash table */ \
+			h->keys = (khkey_t*)krealloc((void *)h->keys, new_n_buckets * sizeof(khkey_t)); \
+		kfree(h->used); /* free the working space */ \
+		h->used = new_used, h->bits = new_bits; \
+		return 0; \
+	}
+
+#define __KHASHL_IMPL_PUT(SCOPE, HType, prefix, khkey_t, __hash_fn, __hash_eq) \
+	SCOPE khint_t prefix##_putp_core(HType *h, const khkey_t *key, khint_t hash, int *absent) { \
+		khint_t n_buckets, i, last, mask; \
+		n_buckets = h->keys? (khint_t)1U<<h->bits : 0U; \
+		*absent = -1; \
+		if (h->count >= kh_max_count(n_buckets)) { /* rehashing */ \
+			if (prefix##_resize(h, n_buckets + 1U) < 0) \
+				return n_buckets; \
+			n_buckets = (khint_t)1U<<h->bits; \
+		} /* TODO: to implement automatically shrinking; resize() already support shrinking */ \
+		mask = n_buckets - 1; \
+		i = last = __kh_h2b(hash, h->bits); \
+		while (__kh_used(h->used, i) && !__hash_eq(h->keys[i], *key)) { \
+			i = (i + 1U) & mask; \
+			if (i == last) break; \
+		} \
+		if (!__kh_used(h->used, i)) { /* not present at all */ \
+			h->keys[i] = *key; \
+			__kh_set_used(h->used, i); \
+			++h->count; \
+			*absent = 1; \
+		} else *absent = 0; /* Don't touch h->keys[i] if present */ \
+		return i; \
+	} \
+	SCOPE khint_t prefix##_putp(HType *h, const khkey_t *key, int *absent) { return prefix##_putp_core(h, key, __hash_fn(*key), absent); } \
+	SCOPE khint_t prefix##_put(HType *h, khkey_t key, int *absent) { return prefix##_putp_core(h, &key, __hash_fn(key), absent); }
+
+#define __KHASHL_IMPL_DEL(SCOPE, HType, prefix, khkey_t, __hash_fn) \
+	SCOPE int prefix##_del(HType *h, khint_t i) { \
+		khint_t j = i, k, mask, n_buckets; \
+		if (h->keys == 0) return 0; \
+		n_buckets = (khint_t)1U<<h->bits; \
+		mask = n_buckets - 1U; \
+		while (1) { \
+			j = (j + 1U) & mask; \
+			if (j == i || !__kh_used(h->used, j)) break; /* j==i only when the table is completely full */ \
+			k = __kh_h2b(__hash_fn(h->keys[j]), h->bits); \
+			if ((j > i && (k <= i || k > j)) || (j < i && (k <= i && k > j))) \
+				h->keys[i] = h->keys[j], i = j; \
+		} \
+		__kh_set_unused(h->used, i); \
+		--h->count; \
+		return 1; \
+	}
+
+#define KHASHL_DECLARE(HType, prefix, khkey_t) \
+	__KHASHL_TYPE(HType, khkey_t) \
+	__KHASHL_PROTOTYPES(HType, prefix, khkey_t)
+
+#define KHASHL_INIT(SCOPE, HType, prefix, khkey_t, __hash_fn, __hash_eq) \
+	__KHASHL_TYPE(HType, khkey_t) \
+	__KHASHL_IMPL_BASIC(SCOPE, HType, prefix) \
+	__KHASHL_IMPL_GET(SCOPE, HType, prefix, khkey_t, __hash_fn, __hash_eq) \
+	__KHASHL_IMPL_RESIZE(SCOPE, HType, prefix, khkey_t, __hash_fn, __hash_eq) \
+	__KHASHL_IMPL_PUT(SCOPE, HType, prefix, khkey_t, __hash_fn, __hash_eq) \
+	__KHASHL_IMPL_DEL(SCOPE, HType, prefix, khkey_t, __hash_fn)
+
+/***************************
+ * Ensemble of hash tables *
+ ***************************/
+
+typedef struct {
+	khint_t sub, pos;
+} kh_ensitr_t;
+
+#define KHASHE_INIT(SCOPE, HType, prefix, khkey_t, __hash_fn, __hash_eq) \
+	KHASHL_INIT(KH_LOCAL, HType##_sub, prefix##_sub, khkey_t, __hash_fn, __hash_eq) \
+	typedef struct HType { \
+		khint64_t count:54, bits:8; \
+		HType##_sub *sub; \
+	} HType; \
+	SCOPE HType *prefix##_init(int bits) { \
+		HType *g; \
+		g = (HType*)kcalloc(1, sizeof(*g)); \
+		g->bits = bits; \
+		g->sub = (HType##_sub*)kcalloc(1U<<bits, sizeof(*g->sub)); \
+		return g; \
+	} \
+	SCOPE void prefix##_destroy(HType *g) { \
+		int t; \
+		if (!g) return; \
+		for (t = 0; t < 1<<g->bits; ++t) { kfree((void*)g->sub[t].keys); kfree(g->sub[t].used); } \
+		kfree(g->sub); kfree(g); \
+	} \
+	SCOPE kh_ensitr_t prefix##_getp(const HType *g, const khkey_t *key) { \
+		khint_t hash, low, ret; \
+		kh_ensitr_t r; \
+		HType##_sub *h; \
+		hash = __hash_fn(*key); \
+		low = hash & ((1U<<g->bits) - 1); \
+		h = &g->sub[low]; \
+		ret = prefix##_sub_getp_core(h, key, hash); \
+		if (ret == kh_end(h)) r.sub = low, r.pos = (khint_t)-1; \
+		else r.sub = low, r.pos = ret; \
+		return r; \
+	} \
+	SCOPE kh_ensitr_t prefix##_get(const HType *g, const khkey_t key) { return prefix##_getp(g, &key); } \
+	SCOPE kh_ensitr_t prefix##_putp(HType *g, const khkey_t *key, int *absent) { \
+		khint_t hash, low, ret; \
+		kh_ensitr_t r; \
+		HType##_sub *h; \
+		hash = __hash_fn(*key); \
+		low = hash & ((1U<<g->bits) - 1); \
+		h = &g->sub[low]; \
+		ret = prefix##_sub_putp_core(h, key, hash, absent); \
+		if (*absent) ++g->count; \
+		r.sub = low, r.pos = ret; \
+		return r; \
+	} \
+	SCOPE kh_ensitr_t prefix##_put(HType *g, const khkey_t key, int *absent) { return prefix##_putp(g, &key, absent); } \
+	SCOPE int prefix##_del(HType *g, kh_ensitr_t itr) { \
+		HType##_sub *h = &g->sub[itr.sub]; \
+		int ret; \
+		ret = prefix##_sub_del(h, itr.pos); \
+		if (ret) --g->count; \
+		return ret; \
+	}
+
+/*****************************
+ * More convenient interface *
+ *****************************/
+
+#define __kh_cached_hash(x) ((x).hash)
+
+#define KHASHL_SET_INIT(SCOPE, HType, prefix, khkey_t, __hash_fn, __hash_eq) \
+	KHASHL_INIT(SCOPE, HType, prefix, khkey_t, __hash_fn, __hash_eq)
+
+#define KHASHL_MAP_INIT(SCOPE, HType, prefix, khkey_t, kh_val_t, __hash_fn, __hash_eq) \
+	typedef struct { khkey_t key; kh_val_t val; } kh_packed HType##_m_bucket_t; \
+	static kh_inline khint_t prefix##_m_hash(HType##_m_bucket_t x) { return __hash_fn(x.key); } \
+	static kh_inline int prefix##_m_eq(HType##_m_bucket_t x, HType##_m_bucket_t y) { return __hash_eq(x.key, y.key); } \
+	KHASHL_INIT(KH_LOCAL, HType, prefix##_m, HType##_m_bucket_t, prefix##_m_hash, prefix##_m_eq) \
+	SCOPE HType *prefix##_init(void) { return prefix##_m_init(); } \
+	SCOPE void prefix##_destroy(HType *h) { prefix##_m_destroy(h); } \
+	SCOPE khint_t prefix##_get(const HType *h, khkey_t key) { HType##_m_bucket_t t; t.key = key; return prefix##_m_getp(h, &t); } \
+	SCOPE int prefix##_del(HType *h, khint_t k) { return prefix##_m_del(h, k); } \
+	SCOPE khint_t prefix##_put(HType *h, khkey_t key, int *absent) { HType##_m_bucket_t t; t.key = key; return prefix##_m_putp(h, &t, absent); }
+
+#define KHASHL_CSET_INIT(SCOPE, HType, prefix, khkey_t, __hash_fn, __hash_eq) \
+	typedef struct { khkey_t key; khint_t hash; } kh_packed HType##_cs_bucket_t; \
+	static kh_inline int prefix##_cs_eq(HType##_cs_bucket_t x, HType##_cs_bucket_t y) { return x.hash == y.hash && __hash_eq(x.key, y.key); } \
+	KHASHL_INIT(KH_LOCAL, HType, prefix##_cs, HType##_cs_bucket_t, __kh_cached_hash, prefix##_cs_eq) \
+	SCOPE HType *prefix##_init(void) { return prefix##_cs_init(); } \
+	SCOPE void prefix##_destroy(HType *h) { prefix##_cs_destroy(h); } \
+	SCOPE khint_t prefix##_get(const HType *h, khkey_t key) { HType##_cs_bucket_t t; t.key = key; t.hash = __hash_fn(key); return prefix##_cs_getp(h, &t); } \
+	SCOPE int prefix##_del(HType *h, khint_t k) { return prefix##_cs_del(h, k); } \
+	SCOPE khint_t prefix##_put(HType *h, khkey_t key, int *absent) { HType##_cs_bucket_t t; t.key = key, t.hash = __hash_fn(key); return prefix##_cs_putp(h, &t, absent); }
+
+#define KHASHL_CMAP_INIT(SCOPE, HType, prefix, khkey_t, kh_val_t, __hash_fn, __hash_eq) \
+	typedef struct { khkey_t key; kh_val_t val; khint_t hash; } kh_packed HType##_cm_bucket_t; \
+	static kh_inline int prefix##_cm_eq(HType##_cm_bucket_t x, HType##_cm_bucket_t y) { return x.hash == y.hash && __hash_eq(x.key, y.key); } \
+	KHASHL_INIT(KH_LOCAL, HType, prefix##_cm, HType##_cm_bucket_t, __kh_cached_hash, prefix##_cm_eq) \
+	SCOPE HType *prefix##_init(void) { return prefix##_cm_init(); } \
+	SCOPE void prefix##_destroy(HType *h) { prefix##_cm_destroy(h); } \
+	SCOPE khint_t prefix##_get(const HType *h, khkey_t key) { HType##_cm_bucket_t t; t.key = key; t.hash = __hash_fn(key); return prefix##_cm_getp(h, &t); } \
+	SCOPE int prefix##_del(HType *h, khint_t k) { return prefix##_cm_del(h, k); } \
+	SCOPE khint_t prefix##_put(HType *h, khkey_t key, int *absent) { HType##_cm_bucket_t t; t.key = key, t.hash = __hash_fn(key); return prefix##_cm_putp(h, &t, absent); }
+
+#define KHASHE_SET_INIT(SCOPE, HType, prefix, khkey_t, __hash_fn, __hash_eq) \
+	KHASHE_INIT(SCOPE, HType, prefix, khkey_t, __hash_fn, __hash_eq)
+
+#define KHASHE_MAP_INIT(SCOPE, HType, prefix, khkey_t, kh_val_t, __hash_fn, __hash_eq) \
+	typedef struct { khkey_t key; kh_val_t val; } kh_packed HType##_m_bucket_t; \
+	static kh_inline khint_t prefix##_m_hash(HType##_m_bucket_t x) { return __hash_fn(x.key); } \
+	static kh_inline int prefix##_m_eq(HType##_m_bucket_t x, HType##_m_bucket_t y) { return __hash_eq(x.key, y.key); } \
+	KHASHE_INIT(KH_LOCAL, HType, prefix##_m, HType##_m_bucket_t, prefix##_m_hash, prefix##_m_eq) \
+	SCOPE HType *prefix##_init(int bits) { return prefix##_m_init(bits); } \
+	SCOPE void prefix##_destroy(HType *h) { prefix##_m_destroy(h); } \
+	SCOPE kh_ensitr_t prefix##_get(const HType *h, khkey_t key) { HType##_m_bucket_t t; t.key = key; return prefix##_m_getp(h, &t); } \
+	SCOPE int prefix##_del(HType *h, kh_ensitr_t k) { return prefix##_m_del(h, k); } \
+	SCOPE kh_ensitr_t prefix##_put(HType *h, khkey_t key, int *absent) { HType##_m_bucket_t t; t.key = key; return prefix##_m_putp(h, &t, absent); }
+
+/**************************
+ * Public macro functions *
+ **************************/
+
+#define kh_bucket(h, x) ((h)->keys[x])
+#define kh_size(h) ((h)->count)
+#define kh_capacity(h) ((h)->keys? 1U<<(h)->bits : 0U)
+#define kh_end(h) kh_capacity(h)
+
+#define kh_key(h, x) ((h)->keys[x].key)
+#define kh_val(h, x) ((h)->keys[x].val)
+#define kh_exist(h, x) __kh_used((h)->used, (x))
+
+#define kh_foreach(h, x) for ((x) = 0; (x) != kh_end(h); ++(x)) if (kh_exist((h), (x)))
+
+#define kh_ens_key(g, x) kh_key(&(g)->sub[(x).sub], (x).pos)
+#define kh_ens_val(g, x) kh_val(&(g)->sub[(x).sub], (x).pos)
+#define kh_ens_exist(g, x) kh_exist(&(g)->sub[(x).sub], (x).pos)
+#define kh_ens_is_end(x) ((x).pos == (khint_t)-1)
+#define kh_ens_size(g) ((g)->count)
+
+#define kh_ens_foreach(g, x) for ((x).sub = 0; (x).sub != 1<<(g)->bits; ++(x).sub) for ((x).pos = 0; (x).pos != kh_end(&(g)->sub[(x).sub]); ++(x).pos) if (kh_ens_exist((g), (x)))
+
+/**************************************
+ * Common hash and equality functions *
+ **************************************/
+
+#define kh_eq_generic(a, b) ((a) == (b))
+#define kh_eq_str(a, b) (strcmp((a), (b)) == 0)
+#define kh_hash_dummy(x) ((khint_t)(x))
+
+static kh_inline khint_t kh_hash_uint32(khint_t x) { /* murmur finishing */
+	x ^= x >> 16;
+	x *= 0x85ebca6bU;
+	x ^= x >> 13;
+	x *= 0xc2b2ae35U;
+	x ^= x >> 16;
+	return x;
+}
+
+static kh_inline khint_t kh_hash_uint64(khint64_t x) { /* splitmix64; see https://nullprogram.com/blog/2018/07/31/ for inversion */
+	x ^= x >> 30;
+	x *= 0xbf58476d1ce4e5b9ULL;
+	x ^= x >> 27;
+	x *= 0x94d049bb133111ebULL;
+	x ^= x >> 31;
+	return (khint_t)x;
+}
+
+#define KH_FNV_SEED 11
+
+static kh_inline khint_t kh_hash_str(kh_cstr_t s) { /* FNV1a */
+	khint_t h = KH_FNV_SEED ^ 2166136261U;
+	const unsigned char *t = (const unsigned char*)s;
+	for (; *t; ++t)
+		h ^= *t, h *= 16777619;
+	return h;
+}
+
+static kh_inline khint_t kh_hash_bytes(int len, const unsigned char *s) {
+	khint_t h = KH_FNV_SEED ^ 2166136261U;
+	int i;
+	for (i = 0; i < len; ++i)
+		h ^= s[i], h *= 16777619;
+	return h;
+}
+
+#endif /* __AC_KHASHL_H */
diff --git a/ext/klib/khmm.c b/ext/klib/khmm.c
new file mode 100644
index 0000000..711ade5
--- /dev/null
+++ b/ext/klib/khmm.c
@@ -0,0 +1,423 @@
+#include <math.h>
+#include <stdio.h>
+#include <assert.h>
+#include <string.h>
+#include <stdlib.h>
+#include "khmm.h"
+
+// new/delete hmm_par_t
+
+hmm_par_t *hmm_new_par(int m, int n)
+{
+	hmm_par_t *hp;
+	int i;
+	assert(m > 0 && n > 0);
+	hp = (hmm_par_t*)calloc(1, sizeof(hmm_par_t));
+	hp->m = m; hp->n = n;
+	hp->a0 = (FLOAT*)calloc(n, sizeof(FLOAT));
+	hp->a = (FLOAT**)calloc2(n, n, sizeof(FLOAT));
+	hp->e = (FLOAT**)calloc2(m + 1, n, sizeof(FLOAT));
+	hp->ae = (FLOAT**)calloc2((m + 1) * n, n, sizeof(FLOAT));
+	for (i = 0; i != n; ++i) hp->e[m][i] = 1.0;
+	return hp;
+}
+void hmm_delete_par(hmm_par_t *hp)
+{
+	int i;
+	if (hp == 0) return;
+	for (i = 0; i != hp->n; ++i) free(hp->a[i]);
+	for (i = 0; i <= hp->m; ++i) free(hp->e[i]);
+	for (i = 0; i < (hp->m + 1) * hp->n; ++i) free(hp->ae[i]);
+	free(hp->a); free(hp->e); free(hp->a0); free(hp->ae);
+	free(hp);
+}
+
+// new/delete hmm_data_t
+
+hmm_data_t *hmm_new_data(int L, const char *seq, const hmm_par_t *hp)
+{
+	hmm_data_t *hd;
+	hd = (hmm_data_t*)calloc(1, sizeof(hmm_data_t));
+	hd->L = L;
+	hd->seq = (char*)malloc(L + 1);
+	memcpy(hd->seq + 1, seq, L);
+	return hd;
+}
+void hmm_delete_data(hmm_data_t *hd)
+{
+	int i;
+	if (hd == 0) return;
+	for (i = 0; i <= hd->L; ++i) {
+		if (hd->f) free(hd->f[i]);
+		if (hd->b) free(hd->b[i]);
+	}
+	free(hd->f); free(hd->b); free(hd->s); free(hd->v); free(hd->p); free(hd->seq);
+	free(hd);
+}
+
+// new/delete hmm_exp_t
+
+hmm_exp_t *hmm_new_exp(const hmm_par_t *hp)
+{
+	hmm_exp_t *he;
+	assert(hp);
+	he = (hmm_exp_t*)calloc(1, sizeof(hmm_exp_t));
+	he->m = hp->m; he->n = hp->n;
+	he->A0 = (FLOAT*)calloc(hp->n, sizeof(FLOAT));
+	he->A = (FLOAT**)calloc2(hp->n, hp->n, sizeof(FLOAT));
+	he->E = (FLOAT**)calloc2(hp->m + 1, hp->n, sizeof(FLOAT));
+	return he;
+}
+void hmm_delete_exp(hmm_exp_t *he)
+{
+	int i;
+	if (he == 0) return;
+	for (i = 0; i != he->n; ++i) free(he->A[i]);
+	for (i = 0; i <= he->m; ++i) free(he->E[i]);
+	free(he->A); free(he->E); free(he->A0);
+	free(he);
+}
+
+// Viterbi algorithm
+
+FLOAT hmm_Viterbi(const hmm_par_t *hp, hmm_data_t *hd)
+{
+	FLOAT **la, **le, *preV, *curV, max;
+	int **Vmax, max_l; // backtrace matrix
+	int k, l, b, u;
+	
+	if (hd->v) free(hd->v);
+	hd->v = (int*)calloc(hd->L+1, sizeof(int));
+	la = (FLOAT**)calloc2(hp->n, hp->n, sizeof(FLOAT));
+	le = (FLOAT**)calloc2(hp->m + 1, hp->n, sizeof(FLOAT));
+	Vmax = (int**)calloc2(hd->L+1, hp->n, sizeof(int));
+	preV = (FLOAT*)malloc(sizeof(FLOAT) * hp->n);
+	curV = (FLOAT*)malloc(sizeof(FLOAT) * hp->n);
+	for (k = 0; k != hp->n; ++k)
+		for (l = 0; l != hp->n; ++l)
+			la[k][l] = log(hp->a[l][k]); // this is not a bug
+	for (b = 0; b != hp->m; ++b)
+		for (k = 0; k != hp->n; ++k)
+			le[b][k] = log(hp->e[b][k]);
+	for (k = 0; k != hp->n; ++k) le[hp->m][k] = 0.0;
+	// V_k(1)
+	for (k = 0; k != hp->n; ++k) {
+		preV[k] = le[(int)hd->seq[1]][k] + log(hp->a0[k]);
+		Vmax[1][k] = 0;
+	}
+	// all the rest
+	for (u = 2; u <= hd->L; ++u) {
+		FLOAT *tmp, *leu = le[(int)hd->seq[u]];
+		for (k = 0; k != hp->n; ++k) {
+			FLOAT *laa = la[k];
+			for (l = 0, max = -HMM_INF, max_l = -1; l != hp->n; ++l) {
+				if (max < preV[l] + laa[l]) {
+					max = preV[l] + laa[l];
+					max_l = l;
+				}
+			}
+			assert(max_l >= 0); // cannot be zero
+			curV[k] = leu[k] + max;
+			Vmax[u][k] = max_l;
+		}
+		tmp = curV; curV = preV; preV = tmp; // swap
+	}
+	// backtrace
+	for (k = 0, max_l = -1, max = -HMM_INF; k != hp->n; ++k) {
+		if (max < preV[k]) {
+			max = preV[k]; max_l = k;
+		}
+	}
+	assert(max_l >= 0); // cannot be zero
+	hd->v[hd->L] = max_l;
+	for (u = hd->L; u >= 1; --u)
+		hd->v[u-1] = Vmax[u][hd->v[u]];
+	for (k = 0; k != hp->n; ++k) free(la[k]);
+	for (b = 0; b < hp->m; ++b) free(le[b]);
+	for (u = 0; u <= hd->L; ++u) free(Vmax[u]);
+	free(la); free(le); free(Vmax); free(preV); free(curV);
+	hd->status |= HMM_VITERBI;
+	return max;
+}
+
+// forward algorithm
+
+void hmm_forward(const hmm_par_t *hp, hmm_data_t *hd)
+{
+	FLOAT sum, tmp, **at;
+	int u, k, l;
+	int n, m, L;
+	assert(hp && hd);
+	// allocate memory for hd->f and hd->s
+	n = hp->n; m = hp->m; L = hd->L;
+	if (hd->s) free(hd->s);
+	if (hd->f) { 
+		for (k = 0; k <= hd->L; ++k) free(hd->f[k]);
+		free(hd->f);
+	}
+	hd->f = (FLOAT**)calloc2(hd->L+1, hp->n, sizeof(FLOAT));
+	hd->s = (FLOAT*)calloc(hd->L+1, sizeof(FLOAT));
+	hd->status &= ~(unsigned)HMM_FORWARD;
+	// at[][] array helps to improve the cache efficiency
+	at = (FLOAT**)calloc2(n, n, sizeof(FLOAT));
+	// transpose a[][]
+	for (k = 0; k != n; ++k)
+		for (l = 0; l != n; ++l)
+			at[k][l] = hp->a[l][k];
+	// f[0], but it should never be used
+	hd->s[0] = 1.0;
+	for (k = 0; k != n; ++k) hd->f[0][k] = 0.0;
+	// f[1]
+	for (k = 0, sum = 0.0; k != n; ++k)
+		sum += (hd->f[1][k] = hp->a0[k] * hp->e[(int)hd->seq[1]][k]);
+	for (k = 0; k != n; ++k) hd->f[1][k] /= sum;
+	hd->s[1] = sum;
+	// f[2..hmmL], the core loop
+	for (u = 2; u <= L; ++u) {
+		FLOAT *fu = hd->f[u], *fu1 = hd->f[u-1], *eu = hp->e[(int)hd->seq[u]];
+		for (k = 0, sum = 0.0; k != n; ++k) {
+			FLOAT *aa = at[k];
+			for (l = 0, tmp = 0.0; l != n; ++l) tmp += fu1[l] * aa[l];
+			sum += (fu[k] = eu[k] * tmp);
+		}
+		for (k = 0; k != n; ++k) fu[k] /= sum;
+		hd->s[u] = sum;
+	}
+	// free at array
+	for (k = 0; k != hp->n; ++k) free(at[k]);
+	free(at);
+	hd->status |= HMM_FORWARD;
+}
+
+//  precalculate hp->ae
+
+void hmm_pre_backward(hmm_par_t *hp)
+{
+	int m, n, b, k, l;
+	assert(hp);
+	m = hp->m; n = hp->n;
+	for (b = 0; b <= m; ++b) {
+		for (k = 0; k != n; ++k) {
+			FLOAT *p = hp->ae[b * hp->n + k];
+			for (l = 0; l != n; ++l)
+				p[l] = hp->e[b][l] * hp->a[k][l];
+		}
+	}
+}
+
+// backward algorithm
+
+void hmm_backward(const hmm_par_t *hp, hmm_data_t *hd)
+{
+	FLOAT tmp;
+	int k, l, u;
+	int m, n, L;
+	assert(hp && hd);
+	assert(hd->status & HMM_FORWARD);
+	// allocate memory for hd->b
+	m = hp->m; n = hp->n; L = hd->L;
+	if (hd->b) { 
+		for (k = 0; k <= hd->L; ++k) free(hd->b[k]);
+		free(hd->b);
+	}
+	hd->status &= ~(unsigned)HMM_BACKWARD;
+	hd->b = (FLOAT**)calloc2(L+1, hp->n, sizeof(FLOAT));
+	// b[L]
+	for (k = 0; k != hp->n; ++k) hd->b[L][k] = 1.0 / hd->s[L];
+	// b[1..L-1], the core loop
+	for (u = L-1; u >= 1; --u) {
+		FLOAT *bu1 = hd->b[u+1], **p = hp->ae + (int)hd->seq[u+1] * n;
+		for (k = 0; k != n; ++k) {
+			FLOAT *q = p[k];
+			for (l = 0, tmp = 0.0; l != n; ++l) tmp += q[l] * bu1[l];
+			hd->b[u][k] = tmp / hd->s[u];
+		}
+	}
+	hd->status |= HMM_BACKWARD;
+	for (l = 0, tmp = 0.0; l != n; ++l)
+		tmp += hp->a0[l] * hd->b[1][l] * hp->e[(int)hd->seq[1]][l];
+	if (tmp > 1.0 + 1e-6 || tmp < 1.0 - 1e-6) // in theory, tmp should always equal to 1
+		fprintf(stderr, "++ Underflow may have happened (%lg).\n", tmp);
+}
+
+// log-likelihood of the observation
+
+FLOAT hmm_lk(const hmm_data_t *hd)
+{
+    FLOAT sum = 0.0, prod = 1.0;
+	int u, L;
+	L = hd->L;
+	assert(hd->status & HMM_FORWARD);
+	for (u = 1; u <= L; ++u) {
+		prod *= hd->s[u];
+		if (prod < HMM_TINY || prod >= 1.0/HMM_TINY) { // reset
+			sum += log(prod);
+			prod = 1.0;
+		}
+	}
+	sum += log(prod);
+	return sum;
+}
+
+// posterior decoding
+
+void hmm_post_decode(const hmm_par_t *hp, hmm_data_t *hd)
+{
+	int u, k;
+	assert(hd->status && HMM_BACKWARD);
+	if (hd->p) free(hd->p);
+	hd->p = (int*)calloc(hd->L + 1, sizeof(int));
+	for (u = 1; u <= hd->L; ++u) {
+		FLOAT prob, max, *fu = hd->f[u], *bu = hd->b[u], su = hd->s[u];
+		int max_k;
+		for (k = 0, max = -1.0, max_k = -1; k != hp->n; ++k) {
+			if (max < (prob = fu[k] * bu[k] * su)) {
+				max = prob; max_k = k;
+			}
+		}
+		assert(max_k >= 0);
+		hd->p[u] = max_k;
+	}
+	hd->status |= HMM_POSTDEC;
+}
+
+// posterior probability of states
+
+FLOAT hmm_post_state(const hmm_par_t *hp, const hmm_data_t *hd, int u, FLOAT *prob)
+{
+	FLOAT sum = 0.0, ss = hd->s[u], *fu = hd->f[u], *bu = hd->b[u];
+	int k;
+	for (k = 0; k != hp->n; ++k)
+		sum += (prob[k] = fu[k] * bu[k] * ss);
+	return sum; // in theory, this should always equal to 1.0
+}
+
+// expected counts
+
+hmm_exp_t *hmm_expect(const hmm_par_t *hp, const hmm_data_t *hd)
+{
+	int k, l, u, b, m, n;
+	hmm_exp_t *he;
+	assert(hd->status & HMM_BACKWARD);
+	he = hmm_new_exp(hp);
+	// initialization
+	m = hp->m; n = hp->n;
+	for (k = 0; k != n; ++k)
+		for (l = 0; l != n; ++l) he->A[k][l] = HMM_TINY;
+	for (b = 0; b <= m; ++b)
+		for (l = 0; l != n; ++l) he->E[b][l] = HMM_TINY;
+	// calculate A_{kl} and E_k(b), k,l\in[0,n)
+	for (u = 1; u < hd->L; ++u) {
+		FLOAT *fu = hd->f[u], *bu = hd->b[u], *bu1 = hd->b[u+1], ss = hd->s[u];
+		FLOAT *Ec = he->E[(int)hd->seq[u]], **p = hp->ae + (int)hd->seq[u+1] * n;
+		for (k = 0; k != n; ++k) {
+			FLOAT *q = p[k], *AA = he->A[k], fuk = fu[k];
+			for (l = 0; l != n; ++l) // this is cache-efficient
+				AA[l] += fuk * q[l] * bu1[l];
+			Ec[k] += fuk * bu[k] * ss;
+		}
+	}
+	// calculate A0_l
+	for (l = 0; l != n; ++l)
+		he->A0[l] += hp->a0[l] * hp->e[(int)hd->seq[1]][l] * hd->b[1][l];
+	return he;
+}
+
+FLOAT hmm_Q0(const hmm_par_t *hp, hmm_exp_t *he)
+{
+	int k, l, b;
+	FLOAT sum = 0.0;
+	for (k = 0; k != hp->n; ++k) {
+		FLOAT tmp;
+		for (b = 0, tmp = 0.0; b != hp->m; ++b) tmp += he->E[b][k];
+		for (b = 0; b != hp->m; ++b)
+			sum += he->E[b][k] * log(he->E[b][k] / tmp);
+	}
+	for (k = 0; k != hp->n; ++k) {
+		FLOAT tmp, *A = he->A[k];
+		for (l = 0, tmp = 0.0; l != hp->n; ++l) tmp += A[l];
+		for (l = 0; l != hp->n; ++l) sum += A[l] * log(A[l] / tmp);
+	}
+	return (he->Q0 = sum);
+}
+
+// add he0 to he1
+
+void hmm_add_expect(const hmm_exp_t *he0, hmm_exp_t *he1)
+{
+	int b, k, l;
+	assert(he0->m == he1->m && he0->n == he1->n);
+	for (k = 0; k != he1->n; ++k) {
+		he1->A0[k] += he0->A0[k];
+		for (l = 0; l != he1->n; ++l)
+			he1->A[k][l] += he0->A[k][l];
+	}
+	for (b = 0; b != he1->m; ++b) {
+		for (l = 0; l != he1->n; ++l)
+			he1->E[b][l] += he0->E[b][l];
+	}
+}
+
+// the EM-Q function
+
+FLOAT hmm_Q(const hmm_par_t *hp, const hmm_exp_t *he)
+{
+	FLOAT sum = 0.0;
+	int bb, k, l;
+	for (bb = 0; bb != he->m; ++bb) {
+		FLOAT *eb = hp->e[bb], *Eb = he->E[bb];
+		for (k = 0; k != hp->n; ++k) {
+			if (eb[k] <= 0.0) return -HMM_INF;
+			sum += Eb[k] * log(eb[k]);
+		}
+	}
+	for (k = 0; k != he->n; ++k) {
+		FLOAT *Ak = he->A[k], *ak = hp->a[k];
+		for (l = 0; l != he->n; ++l) {
+			if (ak[l] <= 0.0) return -HMM_INF;
+			sum += Ak[l] * log(ak[l]);
+		}
+	}
+	return (sum -= he->Q0);
+}
+
+// simulate sequence
+
+char *hmm_simulate(const hmm_par_t *hp, int L)
+{
+	int i, k, l, b;
+	FLOAT x, y, **et;
+	char *seq;
+	seq = (char*)calloc(L+1, 1);
+	// calculate the transpose of hp->e[][]
+	et = (FLOAT**)calloc2(hp->n, hp->m, sizeof(FLOAT));
+	for (k = 0; k != hp->n; ++k)
+		for (b = 0; b != hp->m; ++b)
+			et[k][b] = hp->e[b][k];
+	// the initial state, drawn from a0[]
+	x = drand48();
+	for (k = 0, y = 0.0; k != hp->n; ++k) {
+		y += hp->a0[k];
+		if (y >= x) break;
+	}
+	// main loop
+	for (i = 0; i != L; ++i) {
+		FLOAT *el, *ak = hp->a[k];
+		x = drand48();
+		for (l = 0, y = 0.0; l != hp->n; ++l) {
+			y += ak[l];
+			if (y >= x) break;
+		}
+		el = et[l];
+		x = drand48();
+		for (b = 0, y = 0.0; b != hp->m; ++b) {
+			y += el[b];
+			if (y >= x) break;
+		} 
+		seq[i] = b;
+		k = l;
+	}
+	for (k = 0; k != hp->n; ++k) free(et[k]);
+	free(et);
+	return seq;
+}
diff --git a/ext/klib/khmm.h b/ext/klib/khmm.h
new file mode 100644
index 0000000..d87673b
--- /dev/null
+++ b/ext/klib/khmm.h
@@ -0,0 +1,107 @@
+#ifndef AC_SCHMM_H_
+#define AC_SCHMM_H_
+
+/*
+ * Last Modified: 2008-03-10
+ * Version: 0.1.0-8
+ *
+ * 2008-03-10, 0.1.0-8: make icc report two more "VECTORIZED"
+ * 2008-03-10, 0.1.0-7: accelerate for some CPU
+ * 2008-02-07, 0.1.0-6: simulate sequences
+ * 2008-01-15, 0.1.0-5: goodness of fit
+ * 2007-11-20, 0.1.0-4: add function declaration of hmm_post_decode()
+ * 2007-11-09: fix a memory leak
+ */
+
+#include <stdlib.h>
+
+#define HMM_VERSION "0.1.0-7"
+
+#define HMM_FORWARD  0x02
+#define HMM_BACKWARD 0x04
+#define HMM_VITERBI  0x40
+#define HMM_POSTDEC  0x80
+
+#ifndef FLOAT
+#define FLOAT double
+#endif
+#define HMM_TINY     1e-25
+#define HMM_INF      1e300
+
+typedef struct
+{
+	int m, n; // number of symbols, number of states
+	FLOAT **a, **e; // transition matrix and emitting probilities
+	FLOAT **ae; // auxiliary array for acceleration, should be calculated by hmm_pre_backward()
+	FLOAT *a0; // trasition matrix from the start state
+} hmm_par_t;
+
+typedef struct
+{
+	int L;
+	unsigned status;
+	char *seq;
+	FLOAT **f, **b, *s;
+	int *v; // Viterbi path
+	int *p; // posterior decoding
+} hmm_data_t;
+
+typedef struct
+{
+	int m, n;
+	FLOAT Q0, **A, **E, *A0;
+} hmm_exp_t;
+
+typedef struct
+{
+	int l, *obs;
+	FLOAT *thr;
+} hmm_gof_t;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+	/* initialize and destroy hmm_par_t */
+	hmm_par_t *hmm_new_par(int m, int n);
+	void hmm_delete_par(hmm_par_t *hp);
+	/* initialize and destroy hmm_data_t */
+	hmm_data_t *hmm_new_data(int L, const char *seq, const hmm_par_t *hp);
+	void hmm_delete_data(hmm_data_t *hd);
+	/* initialize and destroy hmm_exp_t */
+	hmm_exp_t *hmm_new_exp(const hmm_par_t *hp);
+	void hmm_delete_exp(hmm_exp_t *he);
+	/* Viterbi, forward and backward algorithms */
+	FLOAT hmm_Viterbi(const hmm_par_t *hp, hmm_data_t *hd);
+	void hmm_pre_backward(hmm_par_t *hp);
+	void hmm_forward(const hmm_par_t *hp, hmm_data_t *hd);
+	void hmm_backward(const hmm_par_t *hp, hmm_data_t *hd);
+	/* log-likelihood of the observations (natural based) */
+	FLOAT hmm_lk(const hmm_data_t *hd);
+	/* posterior probability at the position on the sequence */
+	FLOAT hmm_post_state(const hmm_par_t *hp, const hmm_data_t *hd, int u, FLOAT *prob);
+	/* posterior decoding */
+	void hmm_post_decode(const hmm_par_t *hp, hmm_data_t *hd);
+	/* expected counts of transitions and emissions */
+	hmm_exp_t *hmm_expect(const hmm_par_t *hp, const hmm_data_t *hd);
+	/* add he0 counts to he1 counts*/
+	void hmm_add_expect(const hmm_exp_t *he0, hmm_exp_t *he1);
+	/* the Q function that should be maximized in EM */
+	FLOAT hmm_Q(const hmm_par_t *hp, const hmm_exp_t *he);
+	FLOAT hmm_Q0(const hmm_par_t *hp, hmm_exp_t *he);
+	/* simulate sequences */
+	char *hmm_simulate(const hmm_par_t *hp, int L);
+#ifdef __cplusplus
+}
+#endif
+
+static inline void **calloc2(int n_row, int n_col, int size)
+{
+	char **p;
+	int k;
+	p = (char**)malloc(sizeof(char*) * n_row);
+	for (k = 0; k != n_row; ++k)
+		p[k] = (char*)calloc(n_col, size);
+	return (void**)p;
+}
+
+#endif
diff --git a/ext/klib/klist.h b/ext/klib/klist.h
new file mode 100644
index 0000000..adc3db1
--- /dev/null
+++ b/ext/klib/klist.h
@@ -0,0 +1,135 @@
+/* The MIT License
+
+   Copyright (c) 2008-2009, by Attractive Chaos <attractor@live.co.uk>
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be
+   included in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+*/
+
+#ifndef _AC_KLIST_H
+#define _AC_KLIST_H
+
+#include <stdlib.h>
+
+#ifndef klib_unused
+#if (defined __clang__ && __clang_major__ >= 3) || (defined __GNUC__ && __GNUC__ >= 3)
+#define klib_unused __attribute__ ((__unused__))
+#else
+#define klib_unused
+#endif
+#endif /* klib_unused */
+
+#define KMEMPOOL_INIT2(SCOPE, name, kmptype_t, kmpfree_f)				\
+	typedef struct {													\
+		size_t cnt, n, max;												\
+		kmptype_t **buf;												\
+	} kmp_##name##_t;													\
+	SCOPE kmp_##name##_t *kmp_init_##name(void) {						\
+		return calloc(1, sizeof(kmp_##name##_t));						\
+	}																	\
+	SCOPE void kmp_destroy_##name(kmp_##name##_t *mp) {					\
+		size_t k;														\
+		for (k = 0; k < mp->n; ++k) {									\
+			kmpfree_f(mp->buf[k]); free(mp->buf[k]);					\
+		}																\
+		free(mp->buf); free(mp);										\
+	}																	\
+	SCOPE kmptype_t *kmp_alloc_##name(kmp_##name##_t *mp) {				\
+		++mp->cnt;														\
+		if (mp->n == 0) return calloc(1, sizeof(kmptype_t));			\
+		return mp->buf[--mp->n];										\
+	}																	\
+	SCOPE void kmp_free_##name(kmp_##name##_t *mp, kmptype_t *p) {		\
+		--mp->cnt;														\
+		if (mp->n == mp->max) {											\
+			mp->max = mp->max? mp->max<<1 : 16;							\
+			mp->buf = realloc(mp->buf, sizeof(kmptype_t *) * mp->max);	\
+		}																\
+		mp->buf[mp->n++] = p;											\
+	}
+
+#define KMEMPOOL_INIT(name, kmptype_t, kmpfree_f)						\
+	KMEMPOOL_INIT2(static inline klib_unused, name, kmptype_t, kmpfree_f)
+
+#define kmempool_t(name) kmp_##name##_t
+#define kmp_init(name) kmp_init_##name()
+#define kmp_destroy(name, mp) kmp_destroy_##name(mp)
+#define kmp_alloc(name, mp) kmp_alloc_##name(mp)
+#define kmp_free(name, mp, p) kmp_free_##name(mp, p)
+
+#define KLIST_INIT2(SCOPE, name, kltype_t, kmpfree_t)					\
+	struct __kl1_##name {												\
+		kltype_t data;													\
+		struct __kl1_##name *next;										\
+	};																	\
+	typedef struct __kl1_##name kl1_##name;								\
+	KMEMPOOL_INIT2(SCOPE, name, kl1_##name, kmpfree_t)					\
+	typedef struct {													\
+		kl1_##name *head, *tail;										\
+		kmp_##name##_t *mp;												\
+		size_t size;													\
+	} kl_##name##_t;													\
+	SCOPE kl_##name##_t *kl_init_##name(void) {							\
+		kl_##name##_t *kl = calloc(1, sizeof(kl_##name##_t));			\
+		kl->mp = kmp_init(name);										\
+		kl->head = kl->tail = kmp_alloc(name, kl->mp);					\
+		kl->head->next = 0;												\
+		return kl;														\
+	}																	\
+	SCOPE void kl_destroy_##name(kl_##name##_t *kl) {					\
+		kl1_##name *p;													\
+		for (p = kl->head; p != kl->tail; p = p->next)					\
+			kmp_free(name, kl->mp, p);									\
+		kmp_free(name, kl->mp, p);										\
+		kmp_destroy(name, kl->mp);										\
+		free(kl);														\
+	}																	\
+	SCOPE kltype_t *kl_pushp_##name(kl_##name##_t *kl) {				\
+		kl1_##name *q, *p = kmp_alloc(name, kl->mp);					\
+		q = kl->tail; p->next = 0; kl->tail->next = p; kl->tail = p;	\
+		++kl->size;														\
+		return &q->data;												\
+	}																	\
+	SCOPE int kl_shift_##name(kl_##name##_t *kl, kltype_t *d) {			\
+		kl1_##name *p;													\
+		if (kl->head->next == 0) return -1;								\
+		--kl->size;														\
+		p = kl->head; kl->head = kl->head->next;						\
+		if (d) *d = p->data;											\
+		kmp_free(name, kl->mp, p);										\
+		return 0;														\
+	}
+
+#define KLIST_INIT(name, kltype_t, kmpfree_t)							\
+	KLIST_INIT2(static inline klib_unused, name, kltype_t, kmpfree_t)
+
+#define kliter_t(name) kl1_##name
+#define klist_t(name) kl_##name##_t
+#define kl_val(iter) ((iter)->data)
+#define kl_next(iter) ((iter)->next)
+#define kl_begin(kl) ((kl)->head)
+#define kl_end(kl) ((kl)->tail)
+
+#define kl_init(name) kl_init_##name()
+#define kl_destroy(name, kl) kl_destroy_##name(kl)
+#define kl_pushp(name, kl) kl_pushp_##name(kl)
+#define kl_shift(name, kl, d) kl_shift_##name(kl, d)
+
+#endif
diff --git a/ext/klib/kmath.c b/ext/klib/kmath.c
new file mode 100644
index 0000000..9368b2c
--- /dev/null
+++ b/ext/klib/kmath.c
@@ -0,0 +1,447 @@
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include "kmath.h"
+
+/******************************
+ *** Non-linear programming ***
+ ******************************/
+
+/* Hooke-Jeeves algorithm for nonlinear minimization
+ 
+   Based on the pseudocodes by Bell and Pike (CACM 9(9):684-685), and
+   the revision by Tomlin and Smith (CACM 12(11):637-638). Both of the
+   papers are comments on Kaupe's Algorithm 178 "Direct Search" (ACM
+   6(6):313-314). The original algorithm was designed by Hooke and
+   Jeeves (ACM 8:212-229). This program is further revised according to
+   Johnson's implementation at Netlib (opt/hooke.c).
+ 
+   Hooke-Jeeves algorithm is very simple and it works quite well on a
+   few examples. However, it might fail to converge due to its heuristic
+   nature. A possible improvement, as is suggested by Johnson, may be to
+   choose a small r at the beginning to quickly approach to the minimum
+   and a large r at later step to hit the minimum.
+ */
+
+static double __kmin_hj_aux(kmin_f func, int n, double *x1, void *data, double fx1, double *dx, int *n_calls)
+{
+	int k, j = *n_calls;
+	double ftmp;
+	for (k = 0; k != n; ++k) {
+		x1[k] += dx[k];
+		ftmp = func(n, x1, data); ++j;
+		if (ftmp < fx1) fx1 = ftmp;
+		else { /* search the opposite direction */
+			dx[k] = 0.0 - dx[k];
+			x1[k] += dx[k] + dx[k];
+			ftmp = func(n, x1, data); ++j;
+			if (ftmp < fx1) fx1 = ftmp;
+			else x1[k] -= dx[k]; /* back to the original x[k] */
+		}
+	}
+	*n_calls = j;
+	return fx1; /* here: fx1=f(n,x1) */
+}
+
+double kmin_hj(kmin_f func, int n, double *x, void *data, double r, double eps, int max_calls)
+{
+	double fx, fx1, *x1, *dx, radius;
+	int k, n_calls = 0;
+	x1 = (double*)calloc(n, sizeof(double));
+	dx = (double*)calloc(n, sizeof(double));
+	for (k = 0; k != n; ++k) { /* initial directions, based on MGJ */
+		dx[k] = fabs(x[k]) * r;
+		if (dx[k] == 0) dx[k] = r;
+	}
+	radius = r;
+	fx1 = fx = func(n, x, data); ++n_calls;
+	for (;;) {
+		memcpy(x1, x, n * sizeof(double)); /* x1 = x */
+		fx1 = __kmin_hj_aux(func, n, x1, data, fx, dx, &n_calls);
+		while (fx1 < fx) {
+			for (k = 0; k != n; ++k) {
+				double t = x[k];
+				dx[k] = x1[k] > x[k]? fabs(dx[k]) : 0.0 - fabs(dx[k]);
+				x[k] = x1[k];
+				x1[k] = x1[k] + x1[k] - t;
+			}
+			fx = fx1;
+			if (n_calls >= max_calls) break;
+			fx1 = func(n, x1, data); ++n_calls;
+			fx1 = __kmin_hj_aux(func, n, x1, data, fx1, dx, &n_calls);
+			if (fx1 >= fx) break;
+			for (k = 0; k != n; ++k)
+				if (fabs(x1[k] - x[k]) > .5 * fabs(dx[k])) break;
+			if (k == n) break;
+		}
+		if (radius >= eps) {
+			if (n_calls >= max_calls) break;
+			radius *= r;
+			for (k = 0; k != n; ++k) dx[k] *= r;
+		} else break; /* converge */
+	}
+	free(x1); free(dx);
+	return fx1;
+}
+
+// I copied this function somewhere several years ago with some of my modifications, but I forgot the source.
+double kmin_brent(kmin1_f func, double a, double b, void *data, double tol, double *xmin)
+{
+	double bound, u, r, q, fu, tmp, fa, fb, fc, c;
+	const double gold1 = 1.6180339887;
+	const double gold2 = 0.3819660113;
+	const double tiny = 1e-20;
+	const int max_iter = 100;
+
+	double e, d, w, v, mid, tol1, tol2, p, eold, fv, fw;
+	int iter;
+
+	fa = func(a, data); fb = func(b, data);
+	if (fb > fa) { // swap, such that f(a) > f(b)
+		tmp = a; a = b; b = tmp;
+		tmp = fa; fa = fb; fb = tmp;
+	}
+	c = b + gold1 * (b - a), fc = func(c, data); // golden section extrapolation
+	while (fb > fc) {
+		bound = b + 100.0 * (c - b); // the farthest point where we want to go
+		r = (b - a) * (fb - fc);
+		q = (b - c) * (fb - fa);
+		if (fabs(q - r) < tiny) { // avoid 0 denominator
+			tmp = q > r? tiny : 0.0 - tiny;
+		} else tmp = q - r;
+		u = b - ((b - c) * q - (b - a) * r) / (2.0 * tmp); // u is the parabolic extrapolation point
+		if ((b > u && u > c) || (b < u && u < c)) { // u lies between b and c
+			fu = func(u, data);
+			if (fu < fc) { // (b,u,c) bracket the minimum
+				a = b; b = u; fa = fb; fb = fu;
+				break;
+			} else if (fu > fb) { // (a,b,u) bracket the minimum
+				c = u; fc = fu;
+				break;
+			}
+			u = c + gold1 * (c - b); fu = func(u, data); // golden section extrapolation
+		} else if ((c > u && u > bound) || (c < u && u < bound)) { // u lies between c and bound
+			fu = func(u, data);
+			if (fu < fc) { // fb > fc > fu
+				b = c; c = u; u = c + gold1 * (c - b);
+				fb = fc; fc = fu; fu = func(u, data);
+			} else { // (b,c,u) bracket the minimum
+				a = b; b = c; c = u;
+				fa = fb; fb = fc; fc = fu;
+				break;
+			}
+		} else if ((u > bound && bound > c) || (u < bound && bound < c)) { // u goes beyond the bound
+			u = bound; fu = func(u, data);
+		} else { // u goes the other way around, use golden section extrapolation
+			u = c + gold1 * (c - b); fu = func(u, data);
+		}
+		a = b; b = c; c = u;
+		fa = fb; fb = fc; fc = fu;
+	}
+	if (a > c) u = a, a = c, c = u; // swap
+
+	// now, a<b<c, fa>fb and fb<fc, move on to Brent's algorithm
+	e = d = 0.0;
+	w = v = b; fv = fw = fb;
+	for (iter = 0; iter != max_iter; ++iter) {
+		mid = 0.5 * (a + c);
+		tol2 = 2.0 * (tol1 = tol * fabs(b) + tiny);
+		if (fabs(b - mid) <= (tol2 - 0.5 * (c - a))) {
+			*xmin = b; return fb; // found
+		}
+		if (fabs(e) > tol1) {
+			// related to parabolic interpolation
+			r = (b - w) * (fb - fv);
+			q = (b - v) * (fb - fw);
+			p = (b - v) * q - (b - w) * r;
+			q = 2.0 * (q - r);
+			if (q > 0.0) p = 0.0 - p;
+			else q = 0.0 - q;
+			eold = e; e = d;
+			if (fabs(p) >= fabs(0.5 * q * eold) || p <= q * (a - b) || p >= q * (c - b)) {
+				d = gold2 * (e = (b >= mid ? a - b : c - b));
+			} else {
+				d = p / q; u = b + d; // actual parabolic interpolation happens here
+				if (u - a < tol2 || c - u < tol2)
+					d = (mid > b)? tol1 : 0.0 - tol1;
+			}
+		} else d = gold2 * (e = (b >= mid ? a - b : c - b)); // golden section interpolation
+		u = fabs(d) >= tol1 ? b + d : b + (d > 0.0? tol1 : -tol1);
+		fu = func(u, data);
+		if (fu <= fb) { // u is the minimum point so far
+			if (u >= b) a = b;
+			else c = b;
+			v = w; w = b; b = u; fv = fw; fw = fb; fb = fu;
+		} else { // adjust (a,c) and (u,v,w)
+			if (u < b) a = u;
+			else c = u;
+			if (fu <= fw || w == b) {
+				v = w; w = u;
+				fv = fw; fw = fu;
+			} else if (fu <= fv || v == b || v == w) {
+				v = u; fv = fu;
+			}
+		}
+	}
+	*xmin = b;
+	return fb;
+}
+
+static inline float SIGN(float a, float b)
+{
+	return b >= 0 ? (a >= 0 ? a : -a) : (a >= 0 ? -a : a);
+}
+
+double krf_brent(double x1, double x2, double tol, double (*func)(double, void*), void *data, int *err)
+{
+	const int max_iter = 100;
+	const double eps = 3e-8f;
+	int i;
+	double a = x1, b = x2, c = x2, d, e, min1, min2;
+	double fa, fb, fc, p, q, r, s, tol1, xm;
+
+	*err = 0;
+	fa = func(a, data), fb = func(b, data);
+	if ((fa > 0.0f && fb > 0.0f) || (fa < 0.0f && fb < 0.0f)) {
+		*err = -1;
+		return 0.0f;
+	}
+	fc = fb;
+	for (i = 0; i < max_iter; ++i) {
+		if ((fb > 0.0f && fc > 0.0f) || (fb < 0.0f && fc < 0.0f)) {
+			c = a;
+			fc = fa;
+			e = d = b - a;
+		}
+		if (fabs(fc) < fabs(fb)) {
+			a = b, b = c, c = a;
+			fa = fb, fb = fc, fc = fa;
+		}
+		tol1 = 2.0f * eps * fabs(b) + 0.5f * tol;
+		xm = 0.5f * (c - b);
+		if (fabs(xm) <= tol1 || fb == 0.0f)
+			return b;
+		if (fabs(e) >= tol1 && fabs(fa) > fabs(fb)) {
+			s = fb / fa;
+			if (a == c) {
+				p = 2.0f * xm * s;
+				q = 1.0f - s;
+			} else {
+				q = fa / fc;
+				r = fb / fc;
+				p = s * (2.0f * xm * q * (q - r) - (b - a) * (r - 1.0f));
+				q = (q - 1.0f) * (r - 1.0f) * (s - 1.0f);
+			}
+			if (p > 0.0f) q = -q;
+			p = fabs(p);
+			min1 = 3.0f * xm * q - fabs(tol1 * q);
+			min2 = fabs(e * q);
+			if (2.0f * p < (min1 < min2 ? min1 : min2)) {
+				e = d;
+				d = p / q;
+			} else {
+				d = xm;
+				e = d;
+			}
+		} else {
+			d = xm;
+			e = d;
+		}
+		a = b;
+		fa = fb;
+		if (fabs(d) > tol1) b += d;
+		else b += SIGN(tol1, xm);
+		fb = func(b, data);
+	}
+	*err = -2;
+	return 0.0;
+}
+
+/*************************
+ *** Special functions ***
+ *************************/
+
+/* Log gamma function
+ * \log{\Gamma(z)}
+ * AS245, 2nd algorithm, http://lib.stat.cmu.edu/apstat/245
+ */
+double kf_lgamma(double z)
+{
+	double x = 0;
+	x += 0.1659470187408462e-06 / (z+7);
+	x += 0.9934937113930748e-05 / (z+6);
+	x -= 0.1385710331296526     / (z+5);
+	x += 12.50734324009056      / (z+4);
+	x -= 176.6150291498386      / (z+3);
+	x += 771.3234287757674      / (z+2);
+	x -= 1259.139216722289      / (z+1);
+	x += 676.5203681218835      / z;
+	x += 0.9999999999995183;
+	return log(x) - 5.58106146679532777 - z + (z-0.5) * log(z+6.5);
+}
+
+/* complementary error function
+ * \frac{2}{\sqrt{\pi}} \int_x^{\infty} e^{-t^2} dt
+ * AS66, 2nd algorithm, http://lib.stat.cmu.edu/apstat/66
+ */
+double kf_erfc(double x)
+{
+	const double p0 = 220.2068679123761;
+	const double p1 = 221.2135961699311;
+	const double p2 = 112.0792914978709;
+	const double p3 = 33.912866078383;
+	const double p4 = 6.37396220353165;
+	const double p5 = .7003830644436881;
+	const double p6 = .03526249659989109;
+	const double q0 = 440.4137358247522;
+	const double q1 = 793.8265125199484;
+	const double q2 = 637.3336333788311;
+	const double q3 = 296.5642487796737;
+	const double q4 = 86.78073220294608;
+	const double q5 = 16.06417757920695;
+	const double q6 = 1.755667163182642;
+	const double q7 = .08838834764831844;
+	double expntl, z, p;
+	z = fabs(x) * M_SQRT2;
+	if (z > 37.) return x > 0.? 0. : 2.;
+	expntl = exp(z * z * - .5);
+	if (z < 10. / M_SQRT2) // for small z
+	    p = expntl * ((((((p6 * z + p5) * z + p4) * z + p3) * z + p2) * z + p1) * z + p0)
+			/ (((((((q7 * z + q6) * z + q5) * z + q4) * z + q3) * z + q2) * z + q1) * z + q0);
+	else p = expntl / 2.506628274631001 / (z + 1. / (z + 2. / (z + 3. / (z + 4. / (z + .65)))));
+	return x > 0.? 2. * p : 2. * (1. - p);
+}
+
+/* The following computes regularized incomplete gamma functions.
+ * Formulas are taken from Wiki, with additional input from Numerical
+ * Recipes in C (for modified Lentz's algorithm) and AS245
+ * (http://lib.stat.cmu.edu/apstat/245).
+ *
+ * A good online calculator is available at:
+ *
+ *   http://www.danielsoper.com/statcalc/calc23.aspx
+ *
+ * It calculates upper incomplete gamma function, which equals
+ * kf_gammaq(s,z)*tgamma(s).
+ */
+
+#define KF_GAMMA_EPS 1e-14
+#define KF_TINY 1e-290
+
+// regularized lower incomplete gamma function, by series expansion
+static double _kf_gammap(double s, double z)
+{
+	double sum, x;
+	int k;
+	for (k = 1, sum = x = 1.; k < 100; ++k) {
+		sum += (x *= z / (s + k));
+		if (x / sum < KF_GAMMA_EPS) break;
+	}
+	return exp(s * log(z) - z - kf_lgamma(s + 1.) + log(sum));
+}
+// regularized upper incomplete gamma function, by continued fraction
+static double _kf_gammaq(double s, double z)
+{
+	int j;
+	double C, D, f;
+	f = 1. + z - s; C = f; D = 0.;
+	// Modified Lentz's algorithm for computing continued fraction
+	// See Numerical Recipes in C, 2nd edition, section 5.2
+	for (j = 1; j < 100; ++j) {
+		double a = j * (s - j), b = (j<<1) + 1 + z - s, d;
+		D = b + a * D;
+		if (D < KF_TINY) D = KF_TINY;
+		C = b + a / C;
+		if (C < KF_TINY) C = KF_TINY;
+		D = 1. / D;
+		d = C * D;
+		f *= d;
+		if (fabs(d - 1.) < KF_GAMMA_EPS) break;
+	}
+	return exp(s * log(z) - z - kf_lgamma(s) - log(f));
+}
+
+double kf_gammap(double s, double z)
+{
+	return z <= 1. || z < s? _kf_gammap(s, z) : 1. - _kf_gammaq(s, z);
+}
+
+double kf_gammaq(double s, double z)
+{
+	return z <= 1. || z < s? 1. - _kf_gammap(s, z) : _kf_gammaq(s, z);
+}
+
+/* Regularized incomplete beta function. The method is taken from
+ * Numerical Recipe in C, 2nd edition, section 6.4. The following web
+ * page calculates the incomplete beta function, which equals
+ * kf_betai(a,b,x) * gamma(a) * gamma(b) / gamma(a+b):
+ *
+ *   http://www.danielsoper.com/statcalc/calc36.aspx
+ */
+static double kf_betai_aux(double a, double b, double x)
+{
+	double C, D, f;
+	int j;
+	if (x == 0.) return 0.;
+	if (x == 1.) return 1.;
+	f = 1.; C = f; D = 0.;
+	// Modified Lentz's algorithm for computing continued fraction
+	for (j = 1; j < 200; ++j) {
+		double aa, d;
+		int m = j>>1;
+		aa = (j&1)? -(a + m) * (a + b + m) * x / ((a + 2*m) * (a + 2*m + 1))
+			: m * (b - m) * x / ((a + 2*m - 1) * (a + 2*m));
+		D = 1. + aa * D;
+		if (D < KF_TINY) D = KF_TINY;
+		C = 1. + aa / C;
+		if (C < KF_TINY) C = KF_TINY;
+		D = 1. / D;
+		d = C * D;
+		f *= d;
+		if (fabs(d - 1.) < KF_GAMMA_EPS) break;
+	}
+	return exp(kf_lgamma(a+b) - kf_lgamma(a) - kf_lgamma(b) + a * log(x) + b * log(1.-x)) / a / f;
+}
+double kf_betai(double a, double b, double x)
+{
+	return x < (a + 1.) / (a + b + 2.)? kf_betai_aux(a, b, x) : 1. - kf_betai_aux(b, a, 1. - x);
+}
+
+/******************
+ *** Statistics ***
+ ******************/
+
+double km_ks_dist(int na, const double a[], int nb, const double b[]) // a[] and b[] MUST BE sorted
+{
+	int ia = 0, ib = 0;
+	double fa = 0, fb = 0, sup = 0, na1 = 1. / na, nb1 = 1. / nb;
+	while (ia < na || ib < nb) {
+		if (ia == na) fb += nb1, ++ib;
+		else if (ib == nb) fa += na1, ++ia;
+		else if (a[ia] < b[ib]) fa += na1, ++ia;
+		else if (a[ia] > b[ib]) fb += nb1, ++ib;
+		else fa += na1, fb += nb1, ++ia, ++ib;
+		if (sup < fabs(fa - fb)) sup = fabs(fa - fb);
+	}
+	return sup;
+}
+
+#ifdef KF_MAIN
+#include <stdio.h>
+#include "ksort.h"
+KSORT_INIT_GENERIC(double)
+int main(int argc, char *argv[])
+{
+	double x = 5.5, y = 3;
+	double a, b;
+	double xx[] = {0.22, -0.87, -2.39, -1.79, 0.37, -1.54, 1.28, -0.31, -0.74, 1.72, 0.38, -0.17, -0.62, -1.10, 0.30, 0.15, 2.30, 0.19, -0.50, -0.09};
+	double yy[] = {-5.13, -2.19, -2.43, -3.83, 0.50, -3.25, 4.32, 1.63, 5.18, -0.43, 7.11, 4.87, -3.10, -5.81, 3.76, 6.31, 2.58, 0.07, 5.76, 3.50};
+	ks_introsort(double, 20, xx); ks_introsort(double, 20, yy);
+	printf("K-S distance: %f\n", km_ks_dist(20, xx, 20, yy));
+	printf("erfc(%lg): %lg, %lg\n", x, erfc(x), kf_erfc(x));
+	printf("upper-gamma(%lg,%lg): %lg\n", x, y, kf_gammaq(y, x)*tgamma(y));
+	a = 2; b = 2; x = 0.5;
+	printf("incomplete-beta(%lg,%lg,%lg): %lg\n", a, b, x, kf_betai(a, b, x) / exp(kf_lgamma(a+b) - kf_lgamma(a) - kf_lgamma(b)));
+	return 0;
+}
+#endif
diff --git a/ext/klib/kmath.h b/ext/klib/kmath.h
new file mode 100644
index 0000000..1815a14
--- /dev/null
+++ b/ext/klib/kmath.h
@@ -0,0 +1,38 @@
+#ifndef AC_KMATH_H
+#define AC_KMATH_H
+
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+	/**************************
+	 * Non-linear programming *
+	 **************************/
+
+	#define KMIN_RADIUS  0.5
+	#define KMIN_EPS     1e-7
+	#define KMIN_MAXCALL 50000
+
+	typedef double (*kmin_f)(int, double*, void*);
+	typedef double (*kmin1_f)(double, void*);
+
+	double kmin_hj(kmin_f func, int n, double *x, void *data, double r, double eps, int max_calls); // Hooke-Jeeves'
+	double kmin_brent(kmin1_f func, double a, double b, void *data, double tol, double *xmin); // Brent's 1-dimenssion
+
+	/*********************
+	 * Special functions *
+	 *********************/
+
+	double kf_lgamma(double z); // log gamma function
+	double kf_erfc(double x); // complementary error function
+	double kf_gammap(double s, double z); // regularized lower incomplete gamma function
+	double kf_gammaq(double s, double z); // regularized upper incomplete gamma function
+	double kf_betai(double a, double b, double x); // regularized incomplete beta function
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/ext/klib/knetfile.c b/ext/klib/knetfile.c
new file mode 100644
index 0000000..43ccb77
--- /dev/null
+++ b/ext/klib/knetfile.c
@@ -0,0 +1,628 @@
+/* The MIT License
+
+   Copyright (c) 2008 by Genome Research Ltd (GRL).
+                 2010 by Attractive Chaos <attractor@live.co.uk>
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be
+   included in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+*/
+
+/* Probably I will not do socket programming in the next few years and
+   therefore I decide to heavily annotate this file, for Linux and
+   Windows as well.  -ac */
+
+#include <time.h>
+#include <stdio.h>
+#include <ctype.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <unistd.h>
+#include <sys/types.h>
+
+#ifndef _WIN32
+#include <netdb.h>
+#include <arpa/inet.h>
+#include <sys/socket.h>
+#endif
+
+#include "knetfile.h"
+
+/* In winsock.h, the type of a socket is SOCKET, which is: "typedef
+ * u_int SOCKET". An invalid SOCKET is: "(SOCKET)(~0)", or signed
+ * integer -1. In knetfile.c, I use "int" for socket type
+ * throughout. This should be improved to avoid confusion.
+ *
+ * In Linux/Mac, recv() and read() do almost the same thing. You can see
+ * in the header file that netread() is simply an alias of read(). In
+ * Windows, however, they are different and using recv() is mandatory.
+ */
+
+/* This function tests if the file handler is ready for reading (or
+ * writing if is_read==0). */
+static int socket_wait(int fd, int is_read)
+{
+	fd_set fds, *fdr = 0, *fdw = 0;
+	struct timeval tv;
+	int ret;
+	tv.tv_sec = 5; tv.tv_usec = 0; // 5 seconds time out
+	FD_ZERO(&fds);
+	FD_SET(fd, &fds);
+	if (is_read) fdr = &fds;
+	else fdw = &fds;
+	ret = select(fd+1, fdr, fdw, 0, &tv);
+#ifndef _WIN32
+	if (ret == -1) perror("select");
+#else
+	if (ret == 0)
+		fprintf(stderr, "select time-out\n");
+	else if (ret == SOCKET_ERROR)
+		fprintf(stderr, "select: %d\n", WSAGetLastError());
+#endif
+	return ret;
+}
+
+#ifndef _WIN32
+/* This function does not work with Windows due to the lack of
+ * getaddrinfo() in winsock. It is addapted from an example in "Beej's
+ * Guide to Network Programming" (http://beej.us/guide/bgnet/). */
+static int socket_connect(const char *host, const char *port)
+{
+#define __err_connect(func) do { perror(func); freeaddrinfo(res); return -1; } while (0)
+
+	int ai_err, on = 1, fd;
+	struct linger lng = { 0, 0 };
+	struct addrinfo hints, *res = 0;
+	memset(&hints, 0, sizeof(struct addrinfo));
+	hints.ai_family = AF_UNSPEC;
+	hints.ai_socktype = SOCK_STREAM;
+	/* In Unix/Mac, getaddrinfo() is the most convenient way to get
+	 * server information. */
+	if ((ai_err = getaddrinfo(host, port, &hints, &res)) != 0) { fprintf(stderr, "can't resolve %s:%s: %s\n", host, port, gai_strerror(ai_err)); return -1; }
+	if ((fd = socket(res->ai_family, res->ai_socktype, res->ai_protocol)) == -1) __err_connect("socket");
+	/* The following two setsockopt() are used by ftplib
+	 * (http://nbpfaus.net/~pfau/ftplib/). I am not sure if they
+	 * necessary. */
+	if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) == -1) __err_connect("setsockopt");
+	if (setsockopt(fd, SOL_SOCKET, SO_LINGER, &lng, sizeof(lng)) == -1) __err_connect("setsockopt");
+	if (connect(fd, res->ai_addr, res->ai_addrlen) != 0) __err_connect("connect");
+	freeaddrinfo(res);
+	return fd;
+}
+#else
+/* MinGW's printf has problem with "%lld" */
+char *int64tostr(char *buf, int64_t x)
+{
+	int cnt;
+	int i = 0;
+	do {
+		buf[i++] = '0' + x % 10;
+		x /= 10;
+	} while (x);
+	buf[i] = 0;
+	for (cnt = i, i = 0; i < cnt/2; ++i) {
+		int c = buf[i]; buf[i] = buf[cnt-i-1]; buf[cnt-i-1] = c;
+	}
+	return buf;
+}
+
+int64_t strtoint64(const char *buf)
+{
+	int64_t x;
+	for (x = 0; *buf != '\0'; ++buf)
+		x = x * 10 + ((int64_t) *buf - 48);
+	return x;
+}
+/* In windows, the first thing is to establish the TCP connection. */
+int knet_win32_init()
+{
+	WSADATA wsaData;
+	return WSAStartup(MAKEWORD(2, 2), &wsaData);
+}
+void knet_win32_destroy()
+{
+	WSACleanup();
+}
+/* A slightly modfied version of the following function also works on
+ * Mac (and presummably Linux). However, this function is not stable on
+ * my Mac. It sometimes works fine but sometimes does not. Therefore for
+ * non-Windows OS, I do not use this one. */
+static SOCKET socket_connect(const char *host, const char *port)
+{
+#define __err_connect(func)										\
+	do {														\
+		fprintf(stderr, "%s: %d\n", func, WSAGetLastError());	\
+		return -1;												\
+	} while (0)
+
+	int on = 1;
+	SOCKET fd;
+	struct linger lng = { 0, 0 };
+	struct sockaddr_in server;
+	struct hostent *hp = 0;
+	// open socket
+	if ((fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) == INVALID_SOCKET) __err_connect("socket");
+	if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (char*)&on, sizeof(on)) == -1) __err_connect("setsockopt");
+	if (setsockopt(fd, SOL_SOCKET, SO_LINGER, (char*)&lng, sizeof(lng)) == -1) __err_connect("setsockopt");
+	// get host info
+	if (isalpha(host[0])) hp = gethostbyname(host);
+	else {
+		struct in_addr addr;
+		addr.s_addr = inet_addr(host);
+		hp = gethostbyaddr((char*)&addr, 4, AF_INET);
+	}
+	if (hp == 0) __err_connect("gethost");
+	// connect
+	server.sin_addr.s_addr = *((unsigned long*)hp->h_addr);
+	server.sin_family= AF_INET;
+	server.sin_port = htons(atoi(port));
+	if (connect(fd, (struct sockaddr*)&server, sizeof(server)) != 0) __err_connect("connect");
+	// freehostent(hp); // strangely in MSDN, hp is NOT freed (memory leak?!)
+	return fd;
+}
+#endif
+
+static off_t my_netread(int fd, void *buf, off_t len)
+{
+	off_t rest = len, curr, l = 0;
+	/* recv() and read() may not read the required length of data with
+	 * one call. They have to be called repeatedly. */
+	while (rest) {
+		if (socket_wait(fd, 1) <= 0) break; // socket is not ready for reading
+		curr = netread(fd, buf + l, rest);
+		/* According to the glibc manual, section 13.2, a zero returned
+		 * value indicates end-of-file (EOF), which should mean that
+		 * read() will not return zero if EOF has not been met but data
+		 * are not immediately available. */
+		if (curr == 0) break;
+		l += curr; rest -= curr;
+	}
+	return l;
+}
+
+/*************************
+ * FTP specific routines *
+ *************************/
+
+static int kftp_get_response(knetFile *ftp)
+{
+#ifndef _WIN32
+	unsigned char c;
+#else
+	char c;
+#endif
+	int n = 0;
+	char *p;
+	if (socket_wait(ftp->ctrl_fd, 1) <= 0) return 0;
+	while (netread(ftp->ctrl_fd, &c, 1)) { // FIXME: this is *VERY BAD* for unbuffered I/O
+		//fputc(c, stderr);
+		if (n >= ftp->max_response) {
+			ftp->max_response = ftp->max_response? ftp->max_response<<1 : 256;
+			ftp->response = (char*)realloc(ftp->response, ftp->max_response);
+		}
+		ftp->response[n++] = c;
+		if (c == '\n') {
+			if (n >= 4 && isdigit(ftp->response[0]) && isdigit(ftp->response[1]) && isdigit(ftp->response[2])
+				&& ftp->response[3] != '-') break;
+			n = 0;
+			continue;
+		}
+	}
+	if (n < 2) return -1;
+	ftp->response[n-2] = 0;
+	return strtol(ftp->response, &p, 0);
+}
+
+static int kftp_send_cmd(knetFile *ftp, const char *cmd, int is_get)
+{
+	if (socket_wait(ftp->ctrl_fd, 0) <= 0) return -1; // socket is not ready for writing
+	netwrite(ftp->ctrl_fd, cmd, strlen(cmd));
+	return is_get? kftp_get_response(ftp) : 0;
+}
+
+static int kftp_pasv_prep(knetFile *ftp)
+{
+	char *p;
+	int v[6];
+	kftp_send_cmd(ftp, "PASV\r\n", 1);
+	for (p = ftp->response; *p && *p != '('; ++p);
+	if (*p != '(') return -1;
+	++p;
+	sscanf(p, "%d,%d,%d,%d,%d,%d", &v[0], &v[1], &v[2], &v[3], &v[4], &v[5]);
+	memcpy(ftp->pasv_ip, v, 4 * sizeof(int));
+	ftp->pasv_port = (v[4]<<8&0xff00) + v[5];
+	return 0;
+}
+
+
+static int kftp_pasv_connect(knetFile *ftp)
+{
+	char host[80], port[10];
+	if (ftp->pasv_port == 0) {
+		fprintf(stderr, "[kftp_pasv_connect] kftp_pasv_prep() is not called before hand.\n");
+		return -1;
+	}
+	sprintf(host, "%d.%d.%d.%d", ftp->pasv_ip[0], ftp->pasv_ip[1], ftp->pasv_ip[2], ftp->pasv_ip[3]);
+	sprintf(port, "%d", ftp->pasv_port);
+	ftp->fd = socket_connect(host, port);
+	if (ftp->fd == -1) return -1;
+	return 0;
+}
+
+int kftp_connect(knetFile *ftp)
+{
+	ftp->ctrl_fd = socket_connect(ftp->host, ftp->port);
+	if (ftp->ctrl_fd == -1) return -1;
+	kftp_get_response(ftp);
+	kftp_send_cmd(ftp, "USER anonymous\r\n", 1);
+	kftp_send_cmd(ftp, "PASS kftp@\r\n", 1);
+	kftp_send_cmd(ftp, "TYPE I\r\n", 1);
+	return 0;
+}
+
+int kftp_reconnect(knetFile *ftp)
+{
+	if (ftp->ctrl_fd != -1) {
+		netclose(ftp->ctrl_fd);
+		ftp->ctrl_fd = -1;
+	}
+	netclose(ftp->fd);
+	ftp->fd = -1;
+	return kftp_connect(ftp);
+}
+
+// initialize ->type, ->host, ->retr and ->size
+knetFile *kftp_parse_url(const char *fn, const char *mode)
+{
+	knetFile *fp;
+	char *p;
+	int l;
+	if (strstr(fn, "ftp://") != fn) return 0;
+	for (p = (char*)fn + 6; *p && *p != '/'; ++p);
+	if (*p != '/') return 0;
+	l = p - fn - 6;
+	fp = (knetFile*)calloc(1, sizeof(knetFile));
+	fp->type = KNF_TYPE_FTP;
+	fp->fd = -1;
+	/* the Linux/Mac version of socket_connect() also recognizes a port
+	 * like "ftp", but the Windows version does not. */
+	fp->port = strdup("21");
+	fp->host = (char*)calloc(l + 1, 1);
+	if (strchr(mode, 'c')) fp->no_reconnect = 1;
+	strncpy(fp->host, fn + 6, l);
+	fp->retr = (char*)calloc(strlen(p) + 8, 1);
+	sprintf(fp->retr, "RETR %s\r\n", p);
+    fp->size_cmd = (char*)calloc(strlen(p) + 8, 1);
+    sprintf(fp->size_cmd, "SIZE %s\r\n", p);
+	fp->seek_offset = 0;
+	return fp;
+}
+// place ->fd at offset off
+int kftp_connect_file(knetFile *fp)
+{
+	int ret;
+	long long file_size;
+	if (fp->fd != -1) {
+		netclose(fp->fd);
+		if (fp->no_reconnect) kftp_get_response(fp);
+	}
+	kftp_pasv_prep(fp);
+    kftp_send_cmd(fp, fp->size_cmd, 1);
+#ifndef _WIN32
+    if ( sscanf(fp->response,"%*d %lld", &file_size) != 1 )
+    {
+        fprintf(stderr,"[kftp_connect_file] %s\n", fp->response);
+        return -1;
+    }
+#else
+	const char *p = fp->response;
+	while (*p != ' ') ++p;
+	while (*p < '0' || *p > '9') ++p;
+	file_size = strtoint64(p);
+#endif
+	fp->file_size = file_size;
+	if (fp->offset>=0) {
+		char tmp[32];
+#ifndef _WIN32
+		sprintf(tmp, "REST %lld\r\n", (long long)fp->offset);
+#else
+		strcpy(tmp, "REST ");
+		int64tostr(tmp + 5, fp->offset);
+		strcat(tmp, "\r\n");
+#endif
+		kftp_send_cmd(fp, tmp, 1);
+	}
+	kftp_send_cmd(fp, fp->retr, 0);
+	kftp_pasv_connect(fp);
+	ret = kftp_get_response(fp);
+	if (ret != 150) {
+		fprintf(stderr, "[kftp_connect_file] %s\n", fp->response);
+		netclose(fp->fd);
+		fp->fd = -1;
+		return -1;
+	}
+	fp->is_ready = 1;
+	return 0;
+}
+
+
+/**************************
+ * HTTP specific routines *
+ **************************/
+
+knetFile *khttp_parse_url(const char *fn, const char *mode)
+{
+	knetFile *fp;
+	char *p, *proxy, *q;
+	int l;
+	if (strstr(fn, "http://") != fn) return 0;
+	// set ->http_host
+	for (p = (char*)fn + 7; *p && *p != '/'; ++p);
+	l = p - fn - 7;
+	fp = (knetFile*)calloc(1, sizeof(knetFile));
+	fp->http_host = (char*)calloc(l + 1, 1);
+	strncpy(fp->http_host, fn + 7, l);
+	fp->http_host[l] = 0;
+	for (q = fp->http_host; *q && *q != ':'; ++q);
+	if (*q == ':') *q++ = 0;
+	// get http_proxy
+	proxy = getenv("http_proxy");
+	// set ->host, ->port and ->path
+	if (proxy == 0) {
+		fp->host = strdup(fp->http_host); // when there is no proxy, server name is identical to http_host name.
+		fp->port = strdup(*q? q : "80");
+		fp->path = strdup(*p? p : "/");
+	} else {
+		fp->host = (strstr(proxy, "http://") == proxy)? strdup(proxy + 7) : strdup(proxy);
+		for (q = fp->host; *q && *q != ':'; ++q);
+		if (*q == ':') *q++ = 0; 
+		fp->port = strdup(*q? q : "80");
+		fp->path = strdup(fn);
+	}
+	fp->type = KNF_TYPE_HTTP;
+	fp->ctrl_fd = fp->fd = -1;
+	fp->seek_offset = 0;
+	return fp;
+}
+
+int khttp_connect_file(knetFile *fp)
+{
+	int ret, l = 0;
+	char *buf, *p;
+	if (fp->fd != -1) netclose(fp->fd);
+	fp->fd = socket_connect(fp->host, fp->port);
+	buf = (char*)calloc(0x10000, 1); // FIXME: I am lazy... But in principle, 64KB should be large enough.
+	l += sprintf(buf + l, "GET %s HTTP/1.0\r\nHost: %s\r\n", fp->path, fp->http_host);
+    l += sprintf(buf + l, "Range: bytes=%lld-\r\n", (long long)fp->offset);
+	l += sprintf(buf + l, "\r\n");
+	netwrite(fp->fd, buf, l);
+	l = 0;
+	while (netread(fp->fd, buf + l, 1)) { // read HTTP header; FIXME: bad efficiency
+		if (buf[l] == '\n' && l >= 3)
+			if (strncmp(buf + l - 3, "\r\n\r\n", 4) == 0) break;
+		++l;
+	}
+	buf[l] = 0;
+	if (l < 14) { // prematured header
+		netclose(fp->fd);
+		fp->fd = -1;
+		return -1;
+	}
+	ret = strtol(buf + 8, &p, 0); // HTTP return code
+	if (ret == 200 && fp->offset>0) { // 200 (complete result); then skip beginning of the file
+		off_t rest = fp->offset;
+		while (rest) {
+			off_t l = rest < 0x10000? rest : 0x10000;
+			rest -= my_netread(fp->fd, buf, l);
+		}
+	} else if (ret != 206 && ret != 200) {
+		free(buf);
+		fprintf(stderr, "[khttp_connect_file] fail to open file (HTTP code: %d).\n", ret);
+		netclose(fp->fd);
+		fp->fd = -1;
+		return -1;
+	}
+	free(buf);
+	fp->is_ready = 1;
+	return 0;
+}
+
+/********************
+ * Generic routines *
+ ********************/
+
+knetFile *knet_open(const char *fn, const char *mode)
+{
+	knetFile *fp = 0;
+	if (mode[0] != 'r') {
+		fprintf(stderr, "[kftp_open] only mode \"r\" is supported.\n");
+		return 0;
+	}
+	if (strstr(fn, "ftp://") == fn) {
+		fp = kftp_parse_url(fn, mode);
+		if (fp == 0) return 0;
+		if (kftp_connect(fp) == -1) {
+			knet_close(fp);
+			return 0;
+		}
+		kftp_connect_file(fp);
+	} else if (strstr(fn, "http://") == fn) {
+		fp = khttp_parse_url(fn, mode);
+		if (fp == 0) return 0;
+		khttp_connect_file(fp);
+	} else { // local file
+#ifdef _WIN32
+		/* In windows, O_BINARY is necessary. In Linux/Mac, O_BINARY may
+		 * be undefined on some systems, although it is defined on my
+		 * Mac and the Linux I have tested on. */
+		int fd = open(fn, O_RDONLY | O_BINARY);
+#else		
+		int fd = open(fn, O_RDONLY);
+#endif
+		if (fd == -1) {
+			perror("open");
+			return 0;
+		}
+		fp = (knetFile*)calloc(1, sizeof(knetFile));
+		fp->type = KNF_TYPE_LOCAL;
+		fp->fd = fd;
+		fp->ctrl_fd = -1;
+	}
+	if (fp && fp->fd == -1) {
+		knet_close(fp);
+		return 0;
+	}
+	return fp;
+}
+
+knetFile *knet_dopen(int fd, const char *mode)
+{
+	knetFile *fp = (knetFile*)calloc(1, sizeof(knetFile));
+	fp->type = KNF_TYPE_LOCAL;
+	fp->fd = fd;
+	return fp;
+}
+
+off_t knet_read(knetFile *fp, void *buf, off_t len)
+{
+	off_t l = 0;
+	if (fp->fd == -1) return 0;
+	if (fp->type == KNF_TYPE_FTP) {
+		if (fp->is_ready == 0) {
+			if (!fp->no_reconnect) kftp_reconnect(fp);
+			kftp_connect_file(fp);
+		}
+	} else if (fp->type == KNF_TYPE_HTTP) {
+		if (fp->is_ready == 0)
+			khttp_connect_file(fp);
+	}
+	if (fp->type == KNF_TYPE_LOCAL) { // on Windows, the following block is necessary; not on UNIX
+		off_t rest = len, curr;
+		while (rest) {
+			do {
+				curr = read(fp->fd, buf + l, rest);
+			} while (curr < 0 && EINTR == errno);
+			if (curr < 0) return -1;
+			if (curr == 0) break;
+			l += curr; rest -= curr;
+		}
+	} else l = my_netread(fp->fd, buf, len);
+	fp->offset += l;
+	return l;
+}
+
+off_t knet_seek(knetFile *fp, int64_t off, int whence)
+{
+	if (whence == SEEK_SET && off == fp->offset) return 0;
+	if (fp->type == KNF_TYPE_LOCAL) {
+		/* Be aware that lseek() returns the offset after seeking,
+		 * while fseek() returns zero on success. */
+		off_t offset = lseek(fp->fd, off, whence);
+		if (offset == -1) {
+            // Be silent, it is OK for knet_seek to fail when the file is streamed
+            // fprintf(stderr,"[knet_seek] %s\n", strerror(errno));
+			return -1;
+		}
+		fp->offset = offset;
+		return off;
+	} else if (fp->type == KNF_TYPE_FTP) {
+        if (whence==SEEK_CUR)
+            fp->offset += off;
+        else if (whence==SEEK_SET)
+            fp->offset = off;
+        else if ( whence==SEEK_END)
+            fp->offset = fp->file_size+off;
+		fp->is_ready = 0;
+		return off;
+	} else if (fp->type == KNF_TYPE_HTTP) {
+		if (whence == SEEK_END) { // FIXME: can we allow SEEK_END in future?
+			fprintf(stderr, "[knet_seek] SEEK_END is not supported for HTTP. Offset is unchanged.\n");
+			errno = ESPIPE;
+			return -1;
+		}
+        if (whence==SEEK_CUR)
+            fp->offset += off;
+        else if (whence==SEEK_SET)
+            fp->offset = off;
+		fp->is_ready = 0;
+		return off;
+	}
+	errno = EINVAL;
+    fprintf(stderr,"[knet_seek] %s\n", strerror(errno));
+	return -1;
+}
+
+int knet_close(knetFile *fp)
+{
+	if (fp == 0) return 0;
+	if (fp->ctrl_fd != -1) netclose(fp->ctrl_fd); // FTP specific
+	if (fp->fd != -1) {
+		/* On Linux/Mac, netclose() is an alias of close(), but on
+		 * Windows, it is an alias of closesocket(). */
+		if (fp->type == KNF_TYPE_LOCAL) close(fp->fd);
+		else netclose(fp->fd);
+	}
+	free(fp->host); free(fp->port);
+	free(fp->response); free(fp->retr); // FTP specific
+	free(fp->path); free(fp->http_host); // HTTP specific
+	free(fp);
+	return 0;
+}
+
+#ifdef KNETFILE_MAIN
+int main(void)
+{
+	char *buf;
+	knetFile *fp;
+	int type = 4, l;
+#ifdef _WIN32
+	knet_win32_init();
+#endif
+	buf = calloc(0x100000, 1);
+	if (type == 0) {
+		fp = knet_open("knetfile.c", "r");
+		knet_seek(fp, 1000, SEEK_SET);
+	} else if (type == 1) { // NCBI FTP, large file
+		fp = knet_open("ftp://ftp.ncbi.nih.gov/1000genomes/ftp/data/NA12878/alignment/NA12878.chrom6.SLX.SRP000032.2009_06.bam", "r");
+		knet_seek(fp, 2500000000ll, SEEK_SET);
+		l = knet_read(fp, buf, 255);
+	} else if (type == 2) {
+		fp = knet_open("ftp://ftp.sanger.ac.uk/pub4/treefam/tmp/index.shtml", "r");
+		knet_seek(fp, 1000, SEEK_SET);
+	} else if (type == 3) {
+		fp = knet_open("http://www.sanger.ac.uk/Users/lh3/index.shtml", "r");
+		knet_seek(fp, 1000, SEEK_SET);
+	} else if (type == 4) {
+		fp = knet_open("http://www.sanger.ac.uk/Users/lh3/ex1.bam", "r");
+		knet_read(fp, buf, 10000);
+		knet_seek(fp, 20000, SEEK_SET);
+		knet_seek(fp, 10000, SEEK_SET);
+		l = knet_read(fp, buf+10000, 10000000) + 10000;
+	}
+	if (type != 4 && type != 1) {
+		knet_read(fp, buf, 255);
+		buf[255] = 0;
+		printf("%s\n", buf);
+	} else write(fileno(stdout), buf, l);
+	knet_close(fp);
+	free(buf);
+	return 0;
+}
+#endif
diff --git a/ext/klib/knetfile.h b/ext/klib/knetfile.h
new file mode 100644
index 0000000..0a0e66f
--- /dev/null
+++ b/ext/klib/knetfile.h
@@ -0,0 +1,75 @@
+#ifndef KNETFILE_H
+#define KNETFILE_H
+
+#include <stdint.h>
+#include <fcntl.h>
+
+#ifndef _WIN32
+#define netread(fd, ptr, len) read(fd, ptr, len)
+#define netwrite(fd, ptr, len) write(fd, ptr, len)
+#define netclose(fd) close(fd)
+#else
+#include <winsock2.h>
+#define netread(fd, ptr, len) recv(fd, ptr, len, 0)
+#define netwrite(fd, ptr, len) send(fd, ptr, len, 0)
+#define netclose(fd) closesocket(fd)
+#endif
+
+// FIXME: currently I/O is unbuffered
+
+#define KNF_TYPE_LOCAL 1
+#define KNF_TYPE_FTP   2
+#define KNF_TYPE_HTTP  3
+
+typedef struct knetFile_s {
+	int type, fd;
+	int64_t offset;
+	char *host, *port;
+
+	// the following are for FTP only
+	int ctrl_fd, pasv_ip[4], pasv_port, max_response, no_reconnect, is_ready;
+	char *response, *retr, *size_cmd;
+	int64_t seek_offset; // for lazy seek
+    int64_t file_size;
+
+	// the following are for HTTP only
+	char *path, *http_host;
+} knetFile;
+
+#define knet_tell(fp) ((fp)->offset)
+#define knet_fileno(fp) ((fp)->fd)
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef _WIN32
+	int knet_win32_init();
+	void knet_win32_destroy();
+#endif
+
+	knetFile *knet_open(const char *fn, const char *mode);
+
+	/* 
+	   This only works with local files.
+	 */
+	knetFile *knet_dopen(int fd, const char *mode);
+
+	/*
+	  If ->is_ready==0, this routine updates ->fd; otherwise, it simply
+	  reads from ->fd.
+	 */
+	off_t knet_read(knetFile *fp, void *buf, off_t len);
+
+	/*
+	  This routine only sets ->offset and ->is_ready=0. It does not
+	  communicate with the FTP server.
+	 */
+	off_t knet_seek(knetFile *fp, int64_t off, int whence);
+	int knet_close(knetFile *fp);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/ext/klib/knhx.c b/ext/klib/knhx.c
new file mode 100644
index 0000000..7448e46
--- /dev/null
+++ b/ext/klib/knhx.c
@@ -0,0 +1,172 @@
+#include <ctype.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "knhx.h"
+
+typedef struct {
+	int error, n, max;
+	knhx1_t *node;
+} knaux_t;
+
+static inline char *add_node(const char *s, knaux_t *aux, int x)
+{
+	char *p, *nbeg, *nend = 0;
+	knhx1_t *r;
+	if (aux->n == aux->max) {
+		aux->max = aux->max? aux->max<<1 : 8;
+		aux->node = (knhx1_t*)realloc(aux->node, sizeof(knhx1_t) * aux->max);
+	}
+	r = aux->node + (aux->n++);
+	r->n = x; r->parent = -1;
+	for (p = (char*)s, nbeg = p, r->d = -1.0; *p && *p != ',' && *p != ')'; ++p) {
+		if (*p == '[') {
+			if (nend == 0) nend = p;
+			do ++p; while (*p && *p != ']');
+			if (*p == 0) {
+				aux->error |= KNERR_BRACKET;
+				break;
+			}
+		} else if (*p == ':') {
+			if (nend == 0) nend = p;
+			r->d = strtod(p + 1, &p);
+			--p;
+		} else if (!isgraph(*p)) if (nend == 0) nend = p;
+	}
+	if (nend == 0) nend = p;
+	if (nend != nbeg) {
+		r->name = (char*)calloc(nend - nbeg + 1, 1);
+		strncpy(r->name, nbeg, nend - nbeg);
+	} else r->name = strdup("");
+	return p;
+}
+
+knhx1_t *kn_parse(const char *nhx, int *_n, int *_error)
+{
+	char *p;
+	int *stack, top, max;
+	knaux_t *aux;
+	knhx1_t *ret;
+
+#define __push_back(y) do {										\
+		if (top == max) {										\
+			max = max? max<<1 : 16;								\
+			stack = (int*)realloc(stack, sizeof(int) * max);	\
+		}														\
+		stack[top++] = (y);										\
+	} while (0)													\
+
+	stack = 0; top = max = 0;
+	p = (char*)nhx;
+	aux = (knaux_t*)calloc(1, sizeof(knaux_t));
+	while (*p) {
+		while (*p && !isgraph(*p)) ++p;
+		if (*p == 0) break;
+		if (*p == ',') ++p;
+		else if (*p == '(') {
+			__push_back(-1);
+			++p;
+		} else if (*p == ')') {
+			int x = aux->n, m, i;
+			for (i = top - 1; i >= 0; --i)
+				if (stack[i] < 0) break;
+			m = top - 1 - i;
+			p = add_node(p + 1, aux, m);
+			aux->node[x].child = (int*)calloc(m, sizeof(int));
+			for (i = top - 1, m = m - 1; m >= 0; --m, --i) {
+				aux->node[x].child[m] = stack[i];
+				aux->node[stack[i]].parent = x;
+			}
+			top = i;
+			__push_back(x);
+		} else {
+			__push_back(aux->n);
+			p = add_node(p, aux, 0);
+		}
+	}
+	*_n = aux->n;
+	*_error = aux->error;
+	ret = aux->node;
+	free(aux); free(stack);
+	return ret;
+}
+
+#ifndef kroundup32
+#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
+#endif
+
+static inline int kputsn(const char *p, int l, kstring_t *s)
+{
+	if (s->l + l + 1 >= s->m) {
+		s->m = s->l + l + 2;
+		kroundup32(s->m);
+		s->s = (char*)realloc(s->s, s->m);
+	}
+	memcpy(s->s + s->l, p, l);
+	s->l += l; s->s[s->l] = 0;
+	return l;
+}
+
+static inline int kputc(int c, kstring_t *s)
+{
+	if (s->l + 1 >= s->m) {
+		s->m = s->l + 2;
+		kroundup32(s->m);
+		s->s = (char*)realloc(s->s, s->m);
+	}
+	s->s[s->l++] = c; s->s[s->l] = 0;
+	return c;
+}
+
+static void format_node_recur(const knhx1_t *node, const knhx1_t *p, kstring_t *s, char *numbuf)
+{
+	if (p->n) {
+		int i;
+		kputc('(', s);
+		for (i = 0; i < p->n; ++i) {
+			if (i) kputc(',', s);
+			format_node_recur(node, &node[p->child[i]], s, numbuf);
+		}
+		kputc(')', s);
+		if (p->name) kputsn(p->name, strlen(p->name), s);
+		if (p->d >= 0) {
+			sprintf(numbuf, ":%g", p->d);
+			kputsn(numbuf, strlen(numbuf), s);
+		}
+	} else {
+	  kputsn(p->name, strlen(p->name), s);
+	  if (p->d >= 0) {
+	    sprintf(numbuf, ":%g", p->d);
+	    kputsn(numbuf, strlen(numbuf), s);
+	  }
+	}
+}
+
+void kn_format(const knhx1_t *node, int root, kstring_t *s) // TODO: get rid of recursion
+{
+	char numbuf[128];
+	format_node_recur(node, &node[root], s, numbuf);
+}
+
+#ifdef KNHX_MAIN
+int main(int argc, char *argv[])
+{
+	char *s = "((a[abc],d1)x:0.5,((b[&&NHX:S=MOUSE],h2)[&&NHX:S=HUMAN:B=99][blabla][&&NHX:K=foo],c))";
+	knhx1_t *node;
+	int i, j, n, error;
+	kstring_t str;
+	node = kn_parse(s, &n, &error);
+	for (i = 0; i < n; ++i) {
+		knhx1_t *p = node + i;
+		printf("[%d] %s\t%d\t%d\t%g", i, p->name, p->parent, p->n, p->d);
+		for (j = 0; j < p->n; ++j)
+			printf("\t%d", p->child[j]);
+		putchar('\n');
+	}
+	str.l = str.m = 0; str.s = 0;
+	kn_format(node, n-1, &str);
+	puts(str.s);
+	free(str.s);
+	return 0;
+}
+#endif
diff --git a/ext/klib/knhx.h b/ext/klib/knhx.h
new file mode 100644
index 0000000..dbad7dd
--- /dev/null
+++ b/ext/klib/knhx.h
@@ -0,0 +1,35 @@
+#ifndef KNHX_H_
+#define KNHX_H_
+
+#define KNERR_MISSING_LEFT   0x01
+#define KNERR_MISSING_RGHT   0x02
+#define KNERR_BRACKET        0x04
+#define KNERR_COLON          0x08
+
+typedef struct {
+	int parent, n;
+	int *child;
+	char *name;
+	double d;
+} knhx1_t;
+
+#ifndef KSTRING_T
+#define KSTRING_T kstring_t
+typedef struct __kstring_t {
+	size_t l, m;
+	char *s;
+} kstring_t;
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+	knhx1_t *kn_parse(const char *nhx, int *_n, int *_error);
+	void kn_format(const knhx1_t *node, int root, kstring_t *s);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/ext/klib/kopen.c b/ext/klib/kopen.c
new file mode 100644
index 0000000..ad825da
--- /dev/null
+++ b/ext/klib/kopen.c
@@ -0,0 +1,343 @@
+#include <stdio.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <ctype.h>
+#include <unistd.h>
+#include <string.h>
+#include <stdlib.h>
+#include <signal.h>
+#include <sys/types.h>
+#ifndef _WIN32
+#include <netdb.h>
+#include <arpa/inet.h>
+#include <sys/socket.h>
+#endif
+
+#ifdef _WIN32
+#define _KO_NO_NET
+#endif
+
+#ifndef _KO_NO_NET
+static int socket_wait(int fd, int is_read)
+{
+	fd_set fds, *fdr = 0, *fdw = 0;
+	struct timeval tv;
+	int ret;
+	tv.tv_sec = 5; tv.tv_usec = 0; // 5 seconds time out
+	FD_ZERO(&fds);
+	FD_SET(fd, &fds);
+	if (is_read) fdr = &fds;
+	else fdw = &fds;
+	ret = select(fd+1, fdr, fdw, 0, &tv);
+	if (ret == -1) perror("select");
+	return ret;
+}
+
+static int socket_connect(const char *host, const char *port)
+{
+#define __err_connect(func) do { perror(func); freeaddrinfo(res); return -1; } while (0)
+
+	int ai_err, on = 1, fd;
+	struct linger lng = { 0, 0 };
+	struct addrinfo hints, *res = 0;
+	memset(&hints, 0, sizeof(struct addrinfo));
+	hints.ai_family = AF_UNSPEC;
+	hints.ai_socktype = SOCK_STREAM;
+	if ((ai_err = getaddrinfo(host, port, &hints, &res)) != 0) { fprintf(stderr, "can't resolve %s:%s: %s\n", host, port, gai_strerror(ai_err)); return -1; }
+	if ((fd = socket(res->ai_family, res->ai_socktype, res->ai_protocol)) == -1) __err_connect("socket");
+	if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) == -1) __err_connect("setsockopt");
+	if (setsockopt(fd, SOL_SOCKET, SO_LINGER, &lng, sizeof(lng)) == -1) __err_connect("setsockopt");
+	if (connect(fd, res->ai_addr, res->ai_addrlen) != 0) __err_connect("connect");
+	freeaddrinfo(res);
+	return fd;
+#undef __err_connect
+}
+
+static int http_open(const char *fn)
+{
+	char *p, *proxy, *q, *http_host, *host, *port, *path, *buf;
+	int fd, ret, l;
+
+	/* parse URL; adapted from khttp_parse_url() in knetfile.c */
+	if (strstr(fn, "http://") != fn) return 0;
+	// set ->http_host
+	for (p = (char*)fn + 7; *p && *p != '/'; ++p);
+	l = p - fn - 7;
+	http_host = calloc(l + 1, 1);
+	strncpy(http_host, fn + 7, l);
+	http_host[l] = 0;
+	for (q = http_host; *q && *q != ':'; ++q);
+	if (*q == ':') *q++ = 0;
+	// get http_proxy
+	proxy = getenv("http_proxy");
+	// set host, port and path
+	if (proxy == 0) {
+		host = strdup(http_host); // when there is no proxy, server name is identical to http_host name.
+		port = strdup(*q? q : "80");
+		path = strdup(*p? p : "/");
+	} else {
+		host = (strstr(proxy, "http://") == proxy)? strdup(proxy + 7) : strdup(proxy);
+		for (q = host; *q && *q != ':'; ++q);
+		if (*q == ':') *q++ = 0; 
+		port = strdup(*q? q : "80");
+		path = strdup(fn);
+	}
+
+	/* connect; adapted from khttp_connect() in knetfile.c */
+	l = 0;
+	fd = socket_connect(host, port);
+	buf = calloc(0x10000, 1); // FIXME: I am lazy... But in principle, 64KB should be large enough.
+	l += sprintf(buf + l, "GET %s HTTP/1.0\r\nHost: %s\r\n", path, http_host);
+	l += sprintf(buf + l, "\r\n");
+	write(fd, buf, l);
+	l = 0;
+	while (read(fd, buf + l, 1)) { // read HTTP header; FIXME: bad efficiency
+		if (buf[l] == '\n' && l >= 3)
+			if (strncmp(buf + l - 3, "\r\n\r\n", 4) == 0) break;
+		++l;
+	}
+	buf[l] = 0;
+	if (l < 14) { // prematured header
+		close(fd);
+		fd = -1;
+	}
+	ret = strtol(buf + 8, &p, 0); // HTTP return code
+	if (ret != 200) {
+		close(fd);
+		fd = -1;
+	}
+	free(buf); free(http_host); free(host); free(port); free(path);
+	return fd;
+}
+
+typedef struct {
+	int max_response, ctrl_fd;
+	char *response;
+} ftpaux_t;
+
+static int kftp_get_response(ftpaux_t *aux)
+{
+	unsigned char c;
+	int n = 0;
+	char *p;
+	if (socket_wait(aux->ctrl_fd, 1) <= 0) return 0;
+	while (read(aux->ctrl_fd, &c, 1)) { // FIXME: this is *VERY BAD* for unbuffered I/O
+		if (n >= aux->max_response) {
+			aux->max_response = aux->max_response? aux->max_response<<1 : 256;
+			aux->response = realloc(aux->response, aux->max_response);
+		}
+		aux->response[n++] = c;
+		if (c == '\n') {
+			if (n >= 4 && isdigit(aux->response[0]) && isdigit(aux->response[1]) && isdigit(aux->response[2])
+				&& aux->response[3] != '-') break;
+			n = 0;
+			continue;
+		}
+	}
+	if (n < 2) return -1;
+	aux->response[n-2] = 0;
+	return strtol(aux->response, &p, 0);
+}
+
+static int kftp_send_cmd(ftpaux_t *aux, const char *cmd, int is_get)
+{
+	if (socket_wait(aux->ctrl_fd, 0) <= 0) return -1; // socket is not ready for writing
+	write(aux->ctrl_fd, cmd, strlen(cmd));
+	return is_get? kftp_get_response(aux) : 0;
+}
+
+static int ftp_open(const char *fn)
+{
+	char *p, *host = 0, *port = 0, *retr = 0;
+	char host2[80], port2[10];
+	int v[6], l, fd = -1, ret, pasv_port, pasv_ip[4];
+	ftpaux_t aux;
+	
+	/* parse URL */
+	if (strstr(fn, "ftp://") != fn) return 0;
+	for (p = (char*)fn + 6; *p && *p != '/'; ++p);
+	if (*p != '/') return 0;
+	l = p - fn - 6;
+	port = strdup("21");
+	host = calloc(l + 1, 1);
+	strncpy(host, fn + 6, l);
+	retr = calloc(strlen(p) + 8, 1);
+	sprintf(retr, "RETR %s\r\n", p);
+	
+	/* connect to ctrl */
+	memset(&aux, 0, sizeof(ftpaux_t));
+	aux.ctrl_fd = socket_connect(host, port);
+	if (aux.ctrl_fd == -1) goto ftp_open_end; /* fail to connect ctrl */
+
+	/* connect to the data stream */
+	kftp_get_response(&aux);
+	kftp_send_cmd(&aux, "USER anonymous\r\n", 1);
+	kftp_send_cmd(&aux, "PASS kopen@\r\n", 1);
+	kftp_send_cmd(&aux, "TYPE I\r\n", 1);
+	kftp_send_cmd(&aux, "PASV\r\n", 1);
+	for (p = aux.response; *p && *p != '('; ++p);
+	if (*p != '(') goto ftp_open_end;
+	++p;
+	sscanf(p, "%d,%d,%d,%d,%d,%d", &v[0], &v[1], &v[2], &v[3], &v[4], &v[5]);
+	memcpy(pasv_ip, v, 4 * sizeof(int));
+	pasv_port = (v[4]<<8&0xff00) + v[5];
+	kftp_send_cmd(&aux, retr, 0);
+	sprintf(host2, "%d.%d.%d.%d", pasv_ip[0], pasv_ip[1], pasv_ip[2], pasv_ip[3]);
+	sprintf(port2, "%d", pasv_port);
+	fd = socket_connect(host2, port2);
+	if (fd == -1) goto ftp_open_end;
+	ret = kftp_get_response(&aux);
+	if (ret != 150) {
+		close(fd);
+		fd = -1;
+	}
+	close(aux.ctrl_fd);
+
+ftp_open_end:
+	free(host); free(port); free(retr); free(aux.response);
+	return fd;
+}
+#endif /* !defined(_KO_NO_NET) */
+
+static char **cmd2argv(const char *cmd)
+{
+	int i, beg, end, argc;
+	char **argv, *p, *q, *str;
+	end = strlen(cmd);
+	for (i = end - 1; i >= 0; --i)
+		if (!isspace(cmd[i])) break;
+	end = i + 1;
+	for (beg = 0; beg < end; ++beg)
+		if (!isspace(cmd[beg])) break;
+	if (beg == end) return 0;
+	for (i = beg + 1, argc = 0; i < end; ++i)
+		if (isspace(cmd[i]) && !isspace(cmd[i-1]))
+			++argc;
+	argv = (char**)calloc(argc + 2, sizeof(void*));
+	argv[0] = str = (char*)calloc(end - beg + 1, 1);
+	strncpy(argv[0], cmd + beg, end - beg);
+	for (i = argc = 1, q = p = str; i < end - beg; ++i)
+		if (isspace(str[i])) str[i] = 0;
+		else if (str[i] && str[i-1] == 0) argv[argc++] = &str[i];
+	return argv;
+}
+
+#define KO_STDIN    1
+#define KO_FILE     2
+#define KO_PIPE     3
+#define KO_HTTP     4
+#define KO_FTP      5
+
+typedef struct {
+	int type, fd;
+	pid_t pid;
+} koaux_t;
+
+void *kopen(const char *fn, int *_fd)
+{
+	koaux_t *aux = 0;
+	*_fd = -1;
+	if (strstr(fn, "http://") == fn) {
+		aux = calloc(1, sizeof(koaux_t));
+		aux->type = KO_HTTP;
+		aux->fd = http_open(fn);
+	} else if (strstr(fn, "ftp://") == fn) {
+		aux = calloc(1, sizeof(koaux_t));
+		aux->type = KO_FTP;
+		aux->fd = ftp_open(fn);
+	} else if (strcmp(fn, "-") == 0) {
+		aux = calloc(1, sizeof(koaux_t));
+		aux->type = KO_STDIN;
+		aux->fd = STDIN_FILENO;
+	} else {
+		const char *p, *q;
+		for (p = fn; *p; ++p)
+			if (!isspace(*p)) break;
+		if (*p == '<') { // pipe open
+			int need_shell, pfd[2];
+			pid_t pid;
+			// a simple check to see if we need to invoke a shell; not always working
+			for (q = p + 1; *q; ++q)
+				if (ispunct(*q) && *q != '.' && *q != '_' && *q != '-' && *q != ':')
+					break;
+			need_shell = (*q != 0);
+			pipe(pfd);
+			pid = vfork();
+			if (pid == -1) { /* vfork() error */
+				close(pfd[0]); close(pfd[1]);
+				return 0;
+			}
+			if (pid == 0) { /* the child process */
+				char **argv; /* FIXME: I do not know if this will lead to a memory leak */
+				close(pfd[0]);
+				dup2(pfd[1], STDOUT_FILENO);
+				close(pfd[1]);
+				if (!need_shell) {
+					argv = cmd2argv(p + 1);
+					execvp(argv[0], argv);
+					free(argv[0]); free(argv);
+				} else execl("/bin/sh", "sh", "-c", p + 1, NULL);
+				exit(1);
+			} else { /* parent process */
+				close(pfd[1]);
+				aux = calloc(1, sizeof(koaux_t));
+				aux->type = KO_PIPE;
+				aux->fd = pfd[0];
+				aux->pid = pid;
+			}
+		} else {
+#ifdef _WIN32
+			*_fd = open(fn, O_RDONLY | O_BINARY);
+#else
+			*_fd = open(fn, O_RDONLY);
+#endif
+			if (*_fd) {
+				aux = calloc(1, sizeof(koaux_t));
+				aux->type = KO_FILE;
+				aux->fd = *_fd;
+			}
+		}
+	}
+	*_fd = aux->fd;
+	return aux;
+}
+
+int kclose(void *a)
+{
+	koaux_t *aux = (koaux_t*)a;
+	if (aux->type == KO_PIPE) {
+		int status;
+		pid_t pid;
+		pid = waitpid(aux->pid, &status, WNOHANG);
+		if (pid != aux->pid) kill(aux->pid, 15);
+	}
+	return 0;
+}
+
+#ifdef _KO_MAIN
+#define BUF_SIZE 0x10000
+int main(int argc, char *argv[])
+{
+	void *x;
+	int l, fd;
+	unsigned char buf[BUF_SIZE];
+	FILE *fp;
+	if (argc == 1) {
+		fprintf(stderr, "Usage: kopen <file>\n");
+		return 1;
+	}
+	x = kopen(argv[1], &fd);
+	fp = fdopen(fd, "r");
+	if (fp == 0) {
+		fprintf(stderr, "ERROR: fail to open the input\n");
+		return 1;
+	}
+	do {
+		if ((l = fread(buf, 1, BUF_SIZE, fp)) != 0)
+			fwrite(buf, 1, l, stdout);
+	} while (l == BUF_SIZE);
+	fclose(fp);
+	kclose(x);
+	return 0;
+}
+#endif
diff --git a/ext/klib/krmq.h b/ext/klib/krmq.h
new file mode 100644
index 0000000..4f4ea75
--- /dev/null
+++ b/ext/klib/krmq.h
@@ -0,0 +1,474 @@
+/* The MIT License
+
+   Copyright (c) 2019 by Attractive Chaos <attractor@live.co.uk>
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be
+   included in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+*/
+
+/* An example:
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include "krmq.h"
+
+struct my_node {
+  char key;
+  KRMQ_HEAD(struct my_node) head;
+};
+#define my_cmp(p, q) (((q)->key < (p)->key) - ((p)->key < (q)->key))
+KRMQ_INIT(my, struct my_node, head, my_cmp)
+
+int main(void) {
+  const char *str = "MNOLKQOPHIA"; // from wiki, except a duplicate
+  struct my_node *root = 0;
+  int i, l = strlen(str);
+  for (i = 0; i < l; ++i) {        // insert in the input order
+    struct my_node *q, *p = malloc(sizeof(*p));
+    p->key = str[i];
+    q = krmq_insert(my, &root, p, 0);
+    if (p != q) free(p);           // if already present, free
+  }
+  krmq_itr_t(my) itr;
+  krmq_itr_first(my, root, &itr);  // place at first
+  do {                             // traverse
+    const struct my_node *p = krmq_at(&itr);
+    putchar(p->key);
+    free((void*)p);                // free node
+  } while (krmq_itr_next(my, &itr));
+  putchar('\n');
+  return 0;
+}
+*/
+
+#ifndef KRMQ_H
+#define KRMQ_H
+
+#ifdef __STRICT_ANSI__
+#define inline __inline__
+#endif
+
+#define KRMQ_MAX_DEPTH 64
+
+#define krmq_size(head, p) ((p)? (p)->head.size : 0)
+#define krmq_size_child(head, q, i) ((q)->head.p[(i)]? (q)->head.p[(i)]->head.size : 0)
+
+#define KRMQ_HEAD(__type) \
+	struct { \
+		__type *p[2], *s; \
+		signed char balance; /* balance factor */ \
+		unsigned size; /* #elements in subtree */ \
+	}
+
+#define __KRMQ_FIND(suf, __scope, __type, __head,  __cmp) \
+	__scope __type *krmq_find_##suf(const __type *root, const __type *x, unsigned *cnt_) { \
+		const __type *p = root; \
+		unsigned cnt = 0; \
+		while (p != 0) { \
+			int cmp; \
+			cmp = __cmp(x, p); \
+			if (cmp >= 0) cnt += krmq_size_child(__head, p, 0) + 1; \
+			if (cmp < 0) p = p->__head.p[0]; \
+			else if (cmp > 0) p = p->__head.p[1]; \
+			else break; \
+		} \
+		if (cnt_) *cnt_ = cnt; \
+		return (__type*)p; \
+	} \
+	__scope __type *krmq_interval_##suf(const __type *root, const __type *x, __type **lower, __type **upper) { \
+		const __type *p = root, *l = 0, *u = 0; \
+		while (p != 0) { \
+			int cmp; \
+			cmp = __cmp(x, p); \
+			if (cmp < 0) u = p, p = p->__head.p[0]; \
+			else if (cmp > 0) l = p, p = p->__head.p[1]; \
+			else { l = u = p; break; } \
+		} \
+		if (lower) *lower = (__type*)l; \
+		if (upper) *upper = (__type*)u; \
+		return (__type*)p; \
+	}
+
+#define __KRMQ_RMQ(suf, __scope, __type, __head,  __cmp, __lt2) \
+	__scope __type *krmq_rmq_##suf(const __type *root, const __type *lo, const __type *up) { /* CLOSED interval */ \
+		const __type *p = root, *path[2][KRMQ_MAX_DEPTH], *min; \
+		int plen[2] = {0, 0}, pcmp[2][KRMQ_MAX_DEPTH], i, cmp, lca; \
+		if (root == 0) return 0; \
+		while (p) { \
+			cmp = __cmp(lo, p); \
+			path[0][plen[0]] = p, pcmp[0][plen[0]++] = cmp; \
+			if (cmp < 0) p = p->__head.p[0]; \
+			else if (cmp > 0) p = p->__head.p[1]; \
+			else break; \
+		} \
+		p = root; \
+		while (p) { \
+			cmp = __cmp(up, p); \
+			path[1][plen[1]] = p, pcmp[1][plen[1]++] = cmp; \
+			if (cmp < 0) p = p->__head.p[0]; \
+			else if (cmp > 0) p = p->__head.p[1]; \
+			else break; \
+		} \
+		for (i = 0; i < plen[0] && i < plen[1]; ++i) /* find the LCA */ \
+			if (path[0][i] == path[1][i] && pcmp[0][i] <= 0 && pcmp[1][i] >= 0) \
+				break; \
+		if (i == plen[0] || i == plen[1]) return 0; /* no elements in the closed interval */ \
+		lca = i, min = path[0][lca]; \
+		for (i = lca + 1; i < plen[0]; ++i) { \
+			if (pcmp[0][i] <= 0) { \
+				if (__lt2(path[0][i], min)) min = path[0][i]; \
+				if (path[0][i]->__head.p[1] && __lt2(path[0][i]->__head.p[1]->__head.s, min)) \
+					min = path[0][i]->__head.p[1]->__head.s; \
+			} \
+		} \
+		for (i = lca + 1; i < plen[1]; ++i) { \
+			if (pcmp[1][i] >= 0) { \
+				if (__lt2(path[1][i], min)) min = path[1][i]; \
+				if (path[1][i]->__head.p[0] && __lt2(path[1][i]->__head.p[0]->__head.s, min)) \
+					min = path[1][i]->__head.p[0]->__head.s; \
+			} \
+		} \
+		return (__type*)min; \
+	}
+
+#define __KRMQ_ROTATE(suf, __type, __head, __lt2) \
+	/* */ \
+	static inline void krmq_update_min_##suf(__type *p, const __type *q, const __type *r) { \
+		p->__head.s = !q || __lt2(p, q->__head.s)? p : q->__head.s; \
+		p->__head.s = !r || __lt2(p->__head.s, r->__head.s)? p->__head.s : r->__head.s; \
+	} \
+	/* one rotation: (a,(b,c)q)p => ((a,b)p,c)q */ \
+	static inline __type *krmq_rotate1_##suf(__type *p, int dir) { /* dir=0 to left; dir=1 to right */ \
+		int opp = 1 - dir; /* opposite direction */ \
+		__type *q = p->__head.p[opp], *s = p->__head.s; \
+		unsigned size_p = p->__head.size; \
+		p->__head.size -= q->__head.size - krmq_size_child(__head, q, dir); \
+		q->__head.size = size_p; \
+		krmq_update_min_##suf(p, p->__head.p[dir], q->__head.p[dir]); \
+		q->__head.s = s; \
+		p->__head.p[opp] = q->__head.p[dir]; \
+		q->__head.p[dir] = p; \
+		return q; \
+	} \
+	/* two consecutive rotations: (a,((b,c)r,d)q)p => ((a,b)p,(c,d)q)r */ \
+	static inline __type *krmq_rotate2_##suf(__type *p, int dir) { \
+		int b1, opp = 1 - dir; \
+		__type *q = p->__head.p[opp], *r = q->__head.p[dir], *s = p->__head.s; \
+		unsigned size_x_dir = krmq_size_child(__head, r, dir); \
+		r->__head.size = p->__head.size; \
+		p->__head.size -= q->__head.size - size_x_dir; \
+		q->__head.size -= size_x_dir + 1; \
+		krmq_update_min_##suf(p, p->__head.p[dir], r->__head.p[dir]); \
+		krmq_update_min_##suf(q, q->__head.p[opp], r->__head.p[opp]); \
+		r->__head.s = s; \
+		p->__head.p[opp] = r->__head.p[dir]; \
+		r->__head.p[dir] = p; \
+		q->__head.p[dir] = r->__head.p[opp]; \
+		r->__head.p[opp] = q; \
+		b1 = dir == 0? +1 : -1; \
+		if (r->__head.balance == b1) q->__head.balance = 0, p->__head.balance = -b1; \
+		else if (r->__head.balance == 0) q->__head.balance = p->__head.balance = 0; \
+		else q->__head.balance = b1, p->__head.balance = 0; \
+		r->__head.balance = 0; \
+		return r; \
+	}
+
+#define __KRMQ_INSERT(suf, __scope, __type, __head, __cmp, __lt2) \
+	__scope __type *krmq_insert_##suf(__type **root_, __type *x, unsigned *cnt_) { \
+		unsigned char stack[KRMQ_MAX_DEPTH]; \
+		__type *path[KRMQ_MAX_DEPTH]; \
+		__type *bp, *bq; \
+		__type *p, *q, *r = 0; /* _r_ is potentially the new root */ \
+		int i, which = 0, top, b1, path_len; \
+		unsigned cnt = 0; \
+		bp = *root_, bq = 0; \
+		/* find the insertion location */ \
+		for (p = bp, q = bq, top = path_len = 0; p; q = p, p = p->__head.p[which]) { \
+			int cmp; \
+			cmp = __cmp(x, p); \
+			if (cmp >= 0) cnt += krmq_size_child(__head, p, 0) + 1; \
+			if (cmp == 0) { \
+				if (cnt_) *cnt_ = cnt; \
+				return p; \
+			} \
+			if (p->__head.balance != 0) \
+				bq = q, bp = p, top = 0; \
+			stack[top++] = which = (cmp > 0); \
+			path[path_len++] = p; \
+		} \
+		if (cnt_) *cnt_ = cnt; \
+		x->__head.balance = 0, x->__head.size = 1, x->__head.p[0] = x->__head.p[1] = 0, x->__head.s = x; \
+		if (q == 0) *root_ = x; \
+		else q->__head.p[which] = x; \
+		if (bp == 0) return x; \
+		for (i = 0; i < path_len; ++i) ++path[i]->__head.size; \
+		for (i = path_len - 1; i >= 0; --i) { \
+			krmq_update_min_##suf(path[i], path[i]->__head.p[0], path[i]->__head.p[1]); \
+			if (path[i]->__head.s != x) break; \
+		} \
+		for (p = bp, top = 0; p != x; p = p->__head.p[stack[top]], ++top) /* update balance factors */ \
+			if (stack[top] == 0) --p->__head.balance; \
+			else ++p->__head.balance; \
+		if (bp->__head.balance > -2 && bp->__head.balance < 2) return x; /* no re-balance needed */ \
+		/* re-balance */ \
+		which = (bp->__head.balance < 0); \
+		b1 = which == 0? +1 : -1; \
+		q = bp->__head.p[1 - which]; \
+		if (q->__head.balance == b1) { \
+			r = krmq_rotate1_##suf(bp, which); \
+			q->__head.balance = bp->__head.balance = 0; \
+		} else r = krmq_rotate2_##suf(bp, which); \
+		if (bq == 0) *root_ = r; \
+		else bq->__head.p[bp != bq->__head.p[0]] = r; \
+		return x; \
+	}
+
+#define __KRMQ_ERASE(suf, __scope, __type, __head, __cmp, __lt2) \
+	__scope __type *krmq_erase_##suf(__type **root_, const __type *x, unsigned *cnt_) { \
+		__type *p, *path[KRMQ_MAX_DEPTH], fake; \
+		unsigned char dir[KRMQ_MAX_DEPTH]; \
+		int i, d = 0, cmp; \
+		unsigned cnt = 0; \
+		fake.__head.p[0] = *root_, fake.__head.p[1] = 0; \
+		if (cnt_) *cnt_ = 0; \
+		if (x) { \
+			for (cmp = -1, p = &fake; cmp; cmp = __cmp(x, p)) { \
+				int which = (cmp > 0); \
+				if (cmp > 0) cnt += krmq_size_child(__head, p, 0) + 1; \
+				dir[d] = which; \
+				path[d++] = p; \
+				p = p->__head.p[which]; \
+				if (p == 0) { \
+					if (cnt_) *cnt_ = 0; \
+					return 0; \
+				} \
+			} \
+			cnt += krmq_size_child(__head, p, 0) + 1; /* because p==x is not counted */ \
+		} else { \
+			for (p = &fake, cnt = 1; p; p = p->__head.p[0]) \
+				dir[d] = 0, path[d++] = p; \
+			p = path[--d]; \
+		} \
+		if (cnt_) *cnt_ = cnt; \
+		for (i = 1; i < d; ++i) --path[i]->__head.size; \
+		if (p->__head.p[1] == 0) { /* ((1,.)2,3)4 => (1,3)4; p=2 */ \
+			path[d-1]->__head.p[dir[d-1]] = p->__head.p[0]; \
+		} else { \
+			__type *q = p->__head.p[1]; \
+			if (q->__head.p[0] == 0) { /* ((1,2)3,4)5 => ((1)2,4)5; p=3,q=2 */ \
+				q->__head.p[0] = p->__head.p[0]; \
+				q->__head.balance = p->__head.balance; \
+				path[d-1]->__head.p[dir[d-1]] = q; \
+				path[d] = q, dir[d++] = 1; \
+				q->__head.size = p->__head.size - 1; \
+			} else { /* ((1,((.,2)3,4)5)6,7)8 => ((1,(2,4)5)3,7)8; p=6 */ \
+				__type *r; \
+				int e = d++; /* backup _d_ */\
+				for (;;) { \
+					dir[d] = 0; \
+					path[d++] = q; \
+					r = q->__head.p[0]; \
+					if (r->__head.p[0] == 0) break; \
+					q = r; \
+				} \
+				r->__head.p[0] = p->__head.p[0]; \
+				q->__head.p[0] = r->__head.p[1]; \
+				r->__head.p[1] = p->__head.p[1]; \
+				r->__head.balance = p->__head.balance; \
+				path[e-1]->__head.p[dir[e-1]] = r; \
+				path[e] = r, dir[e] = 1; \
+				for (i = e + 1; i < d; ++i) --path[i]->__head.size; \
+				r->__head.size = p->__head.size - 1; \
+			} \
+		} \
+		for (i = d - 1; i >= 0; --i) /* not sure why adding condition "path[i]->__head.s==p" doesn't work */ \
+			krmq_update_min_##suf(path[i], path[i]->__head.p[0], path[i]->__head.p[1]); \
+		while (--d > 0) { \
+			__type *q = path[d]; \
+			int which, other, b1 = 1, b2 = 2; \
+			which = dir[d], other = 1 - which; \
+			if (which) b1 = -b1, b2 = -b2; \
+			q->__head.balance += b1; \
+			if (q->__head.balance == b1) break; \
+			else if (q->__head.balance == b2) { \
+				__type *r = q->__head.p[other]; \
+				if (r->__head.balance == -b1) { \
+					path[d-1]->__head.p[dir[d-1]] = krmq_rotate2_##suf(q, which); \
+				} else { \
+					path[d-1]->__head.p[dir[d-1]] = krmq_rotate1_##suf(q, which); \
+					if (r->__head.balance == 0) { \
+						r->__head.balance = -b1; \
+						q->__head.balance = b1; \
+						break; \
+					} else r->__head.balance = q->__head.balance = 0; \
+				} \
+			} \
+		} \
+		*root_ = fake.__head.p[0]; \
+		return p; \
+	}
+
+#define krmq_free(__type, __head, __root, __free) do { \
+		__type *_p, *_q; \
+		for (_p = __root; _p; _p = _q) { \
+			if (_p->__head.p[0] == 0) { \
+				_q = _p->__head.p[1]; \
+				__free(_p); \
+			} else { \
+				_q = _p->__head.p[0]; \
+				_p->__head.p[0] = _q->__head.p[1]; \
+				_q->__head.p[1] = _p; \
+			} \
+		} \
+	} while (0)
+
+#define __KRMQ_ITR(suf, __scope, __type, __head, __cmp) \
+	struct krmq_itr_##suf { \
+		const __type *stack[KRMQ_MAX_DEPTH], **top; \
+	}; \
+	__scope void krmq_itr_first_##suf(const __type *root, struct krmq_itr_##suf *itr) { \
+		const __type *p; \
+		for (itr->top = itr->stack - 1, p = root; p; p = p->__head.p[0]) \
+			*++itr->top = p; \
+	} \
+	__scope int krmq_itr_find_##suf(const __type *root, const __type *x, struct krmq_itr_##suf *itr) { \
+		const __type *p = root; \
+		itr->top = itr->stack - 1; \
+		while (p != 0) { \
+			int cmp; \
+			*++itr->top = p; \
+			cmp = __cmp(x, p); \
+			if (cmp < 0) p = p->__head.p[0]; \
+			else if (cmp > 0) p = p->__head.p[1]; \
+			else break; \
+		} \
+		return p? 1 : 0; \
+	} \
+	__scope int krmq_itr_next_bidir_##suf(struct krmq_itr_##suf *itr, int dir) { \
+		const __type *p; \
+		if (itr->top < itr->stack) return 0; \
+		dir = !!dir; \
+		p = (*itr->top)->__head.p[dir]; \
+		if (p) { /* go down */ \
+			for (; p; p = p->__head.p[!dir]) \
+				*++itr->top = p; \
+			return 1; \
+		} else { /* go up */ \
+			do { \
+				p = *itr->top--; \
+			} while (itr->top >= itr->stack && p == (*itr->top)->__head.p[dir]); \
+			return itr->top < itr->stack? 0 : 1; \
+		} \
+	} \
+
+/**
+ * Insert a node to the tree
+ *
+ * @param suf     name suffix used in KRMQ_INIT()
+ * @param proot   pointer to the root of the tree (in/out: root may change)
+ * @param x       node to insert (in)
+ * @param cnt     number of nodes smaller than or equal to _x_; can be NULL (out)
+ *
+ * @return _x_ if not present in the tree, or the node equal to x.
+ */
+#define krmq_insert(suf, proot, x, cnt) krmq_insert_##suf(proot, x, cnt)
+
+/**
+ * Find a node in the tree
+ *
+ * @param suf     name suffix used in KRMQ_INIT()
+ * @param root    root of the tree
+ * @param x       node value to find (in)
+ * @param cnt     number of nodes smaller than or equal to _x_; can be NULL (out)
+ *
+ * @return node equal to _x_ if present, or NULL if absent
+ */
+#define krmq_find(suf, root, x, cnt) krmq_find_##suf(root, x, cnt)
+#define krmq_interval(suf, root, x, lower, upper) krmq_interval_##suf(root, x, lower, upper)
+#define krmq_rmq(suf, root, lo, up) krmq_rmq_##suf(root, lo, up)
+
+/**
+ * Delete a node from the tree
+ *
+ * @param suf     name suffix used in KRMQ_INIT()
+ * @param proot   pointer to the root of the tree (in/out: root may change)
+ * @param x       node value to delete; if NULL, delete the first node (in)
+ *
+ * @return node removed from the tree if present, or NULL if absent
+ */
+#define krmq_erase(suf, proot, x, cnt) krmq_erase_##suf(proot, x, cnt)
+#define krmq_erase_first(suf, proot) krmq_erase_##suf(proot, 0, 0)
+
+#define krmq_itr_t(suf) struct krmq_itr_##suf
+
+/**
+ * Place the iterator at the smallest object
+ *
+ * @param suf     name suffix used in KRMQ_INIT()
+ * @param root    root of the tree
+ * @param itr     iterator
+ */
+#define krmq_itr_first(suf, root, itr) krmq_itr_first_##suf(root, itr)
+
+/**
+ * Place the iterator at the object equal to or greater than the query
+ *
+ * @param suf     name suffix used in KRMQ_INIT()
+ * @param root    root of the tree
+ * @param x       query (in)
+ * @param itr     iterator (out)
+ *
+ * @return 1 if find; 0 otherwise. krmq_at(itr) is NULL if and only if query is
+ *         larger than all objects in the tree
+ */
+#define krmq_itr_find(suf, root, x, itr) krmq_itr_find_##suf(root, x, itr)
+
+/**
+ * Move to the next object in order
+ *
+ * @param itr     iterator (modified)
+ *
+ * @return 1 if there is a next object; 0 otherwise
+ */
+#define krmq_itr_next(suf, itr) krmq_itr_next_bidir_##suf(itr, 1)
+#define krmq_itr_prev(suf, itr) krmq_itr_next_bidir_##suf(itr, 0)
+
+/**
+ * Return the pointer at the iterator
+ *
+ * @param itr     iterator
+ *
+ * @return pointer if present; NULL otherwise
+ */
+#define krmq_at(itr) ((itr)->top < (itr)->stack? 0 : *(itr)->top)
+
+#define KRMQ_INIT2(suf, __scope, __type, __head, __cmp, __lt2) \
+	__KRMQ_FIND(suf, __scope, __type, __head,  __cmp) \
+	__KRMQ_RMQ(suf, __scope, __type, __head,  __cmp, __lt2) \
+	__KRMQ_ROTATE(suf, __type, __head, __lt2) \
+	__KRMQ_INSERT(suf, __scope, __type, __head, __cmp, __lt2) \
+	__KRMQ_ERASE(suf, __scope, __type, __head, __cmp, __lt2) \
+	__KRMQ_ITR(suf, __scope, __type, __head, __cmp)
+
+#define KRMQ_INIT(suf, __type, __head, __cmp, __lt2) \
+	KRMQ_INIT2(suf,, __type, __head, __cmp, __lt2)
+
+#endif
diff --git a/ext/klib/krng.h b/ext/klib/krng.h
new file mode 100644
index 0000000..7423a85
--- /dev/null
+++ b/ext/klib/krng.h
@@ -0,0 +1,54 @@
+#ifndef KRNG_H
+#define KRNG_H
+
+typedef struct {
+	uint64_t s[2];
+} krng_t;
+
+static inline uint64_t kr_splitmix64(uint64_t x)
+{
+	uint64_t z = (x += 0x9E3779B97F4A7C15ULL);
+	z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ULL;
+	z = (z ^ (z >> 27)) * 0x94D049BB133111EBULL;
+	return z ^ (z >> 31);
+}
+
+static inline uint64_t kr_rand_r(krng_t *r)
+{
+	const uint64_t s0 = r->s[0];
+	uint64_t s1 = r->s[1];
+	const uint64_t result = s0 + s1;
+	s1 ^= s0;
+	r->s[0] = (s0 << 55 | s0 >> 9) ^ s1 ^ (s1 << 14);
+	r->s[1] = s0 << 36 | s0 >> 28;
+	return result;
+}
+
+static inline void kr_jump_r(krng_t *r)
+{
+	static const uint64_t JUMP[] = { 0xbeac0467eba5facbULL, 0xd86b048b86aa9922ULL };
+	uint64_t s0 = 0, s1 = 0;
+	int i, b;
+	for (i = 0; i < 2; ++i)
+		for (b = 0; b < 64; b++) {
+			if (JUMP[i] & 1ULL << b)
+				s0 ^= r->s[0], s1 ^= r->s[1];
+			kr_rand_r(r);
+		}
+	r->s[0] = s0, r->s[1] = s1;
+}
+
+static inline void kr_srand_r(krng_t *r, uint64_t seed)
+{
+	r->s[0] = kr_splitmix64(seed);
+	r->s[1] = kr_splitmix64(r->s[0]);
+}
+
+static inline double kr_drand_r(krng_t *r)
+{
+	union { uint64_t i; double d; } u;
+	u.i = 0x3FFULL << 52 | kr_rand_r(r) >> 12;
+	return u.d - 1.0;
+}
+
+#endif
diff --git a/ext/klib/ksa.c b/ext/klib/ksa.c
new file mode 100644
index 0000000..18f686d
--- /dev/null
+++ b/ext/klib/ksa.c
@@ -0,0 +1,242 @@
+/*
+ * Copyright (c) 2008 Yuta Mori    All Rights Reserved.
+ *               2011 Attractive Chaos <attractor@live.co.uk>
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/* This is a library for constructing the suffix array for a string containing
+ * multiple sentinels with sentinels all represented by 0. The last symbol in
+ * the string must be a sentinel. The library is modified from an early version
+ * of Yuta Mori's SAIS library, but is slower than the lastest SAIS by about
+ * 30%, partly due to the recent optimization Yuta has applied and partly due
+ * to the extra comparisons between sentinels. This is not the first effort in
+ * supporting multi-sentinel strings, but is probably the easiest to use. */
+
+#include <stdlib.h>
+
+#ifdef _KSA64
+#include <stdint.h>
+typedef int64_t saint_t;
+#define SAINT_MAX INT64_MAX
+#define SAIS_CORE ksa_core64
+#define SAIS_BWT  ksa_bwt64
+#define SAIS_MAIN ksa_sa64
+#else
+#include <limits.h>
+typedef int saint_t;
+#define SAINT_MAX INT_MAX
+#define SAIS_CORE ksa_core
+#define SAIS_BWT  ksa_bwt
+#define SAIS_MAIN ksa_sa
+#endif
+
+/* T is of type "const unsigned char*". If T[i] is a sentinel, chr(i) takes a negative value */
+#define chr(i) (cs == sizeof(saint_t) ? ((const saint_t *)T)[i] : (T[i]? (saint_t)T[i] : i - SAINT_MAX))
+
+/** Count the occurrences of each symbol */
+static void getCounts(const unsigned char *T, saint_t *C, saint_t n, saint_t k, int cs)
+{
+	saint_t i;
+	for (i = 0; i < k; ++i) C[i] = 0;
+	for (i = 0; i < n; ++i) {
+		saint_t c = chr(i);
+		++C[c > 0? c : 0];
+	}
+}
+
+/**
+ * Find the end of each bucket
+ *
+ * @param C   occurrences computed by getCounts(); input
+ * @param B   start/end of each bucket; output
+ * @param k   size of alphabet
+ * @param end compute the end of bucket if true; otherwise compute the end
+ */
+static inline void getBuckets(const saint_t *C, saint_t *B, saint_t k, saint_t end)
+{
+	saint_t i, sum = 0;
+	if (end) for (i = 0; i < k; ++i) sum += C[i], B[i] = sum;
+	else for (i = 0; i < k; ++i) sum += C[i], B[i] = sum - C[i];
+}
+
+/** Induced sort */
+static void induceSA(const unsigned char *T, saint_t *SA, saint_t *C, saint_t *B, saint_t n, saint_t k, saint_t cs)
+{
+	saint_t *b, i, j;
+	saint_t  c0, c1;
+	/* left-to-right induced sort (for L-type) */
+	if (C == B) getCounts(T, C, n, k, cs);
+	getBuckets(C, B, k, 0);	/* find starts of buckets */
+	for (i = 0, b = 0, c1 = -1; i < n; ++i) {
+		j = SA[i], SA[i] = ~j;
+		if (0 < j) { /* >0 if j-1 is L-type; <0 if S-type; ==0 undefined */
+			--j;
+			if ((c0 = chr(j)) != c1) {
+				B[c1 > 0? c1 : 0] = b - SA;
+				c1 = c0;
+				b = SA + B[c1 > 0? c1 : 0];
+			}
+			*b++ = (0 < j && chr(j - 1) < c1) ? ~j : j;
+		}
+	}
+	/* right-to-left induced sort (for S-type) */
+	if (C == B) getCounts(T, C, n, k, cs);
+	getBuckets(C, B, k, 1);	/* find ends of buckets */
+	for (i = n - 1, b = 0, c1 = -1; 0 <= i; --i) {
+		if (0 < (j = SA[i])) { /* the prefix is S-type */
+			--j;
+			if ((c0 = chr(j)) != c1) {
+				B[c1 > 0? c1 : 0] = b - SA;
+				c1 = c0;
+				b = SA + B[c1 > 0? c1 : 0];
+			}
+			if (c0 > 0) *--b = (j == 0 || chr(j - 1) > c1) ? ~j : j;
+		} else SA[i] = ~j; /* if L-type, change the sign */
+	}
+}
+
+/**
+ * Recursively construct the suffix array for a string containing multiple
+ * sentinels. NULL is taken as the sentinel.
+ *
+ * @param T   NULL terminated input string (there can be multiple NULLs)
+ * @param SA  output suffix array
+ * @param fs  working space available in SA (typically 0 when first called)
+ * @param n   length of T, including the trailing NULL
+ * @param k   size of the alphabet (typically 256 when first called)
+ * @param cs  # bytes per element in T; 1 or sizeof(saint_t) (typically 1 when first called)
+ *
+ * @return    0 upon success
+ */
+int SAIS_CORE(const unsigned char *T, saint_t *SA, saint_t fs, saint_t n, saint_t k, int cs)
+{
+	saint_t *C, *B;
+	saint_t  i, j, c, m, q, qlen, name;
+	saint_t  c0, c1;
+
+	/* STAGE I: reduce the problem by at least 1/2 sort all the S-substrings */
+	if (k <= fs) C = SA + n, B = (k <= fs - k) ? C + k : C;
+	else {
+		if ((C = (saint_t*)malloc(k * (1 + (cs == 1)) * sizeof(saint_t))) == NULL) return -2;
+		B = cs == 1? C + k : C;
+	}
+	getCounts(T, C, n, k, cs);
+	getBuckets(C, B, k, 1);	/* find ends of buckets */
+	for (i = 0; i < n; ++i) SA[i] = 0;
+	/* mark L and S (the t array in Nong et al.), and keep the positions of LMS in the buckets */
+	for (i = n - 2, c = 1, c1 = chr(n - 1); 0 <= i; --i, c1 = c0) {
+		if ((c0 = chr(i)) < c1 + c) c = 1; /* c1 = chr(i+1); c==1 if in an S run */
+		else if (c) SA[--B[c1 > 0? c1 : 0]] = i + 1, c = 0;
+	}
+	induceSA(T, SA, C, B, n, k, cs);
+	if (fs < k) free(C);
+	/* pack all the sorted LMS into the first m items of SA 
+	   2*m must be not larger than n (see Nong et al. for the proof) */
+	for (i = 0, m = 0; i < n; ++i) {
+		saint_t p = SA[i];
+		if (p == n - 1) SA[m++] = p;
+		else if (0 < p && chr(p - 1) > (c0 = chr(p))) {
+			for (j = p + 1; j < n && c0 == (c1 = chr(j)); ++j);
+			if (j < n && c0 < c1) SA[m++] = p;
+		}
+	}
+	for (i = m; i < n; ++i) SA[i] = 0;	/* init the name array buffer */
+	/* store the length of all substrings */
+	for (i = n - 2, j = n, c = 1, c1 = chr(n - 1); 0 <= i; --i, c1 = c0) {
+		if ((c0 = chr(i)) < c1 + c) c = 1; /* c1 = chr(i+1) */
+		else if (c) SA[m + ((i + 1) >> 1)] = j - i - 1, j = i + 1, c = 0;
+	}
+	/* find the lexicographic names of all substrings */
+	for (i = 0, name = 0, q = n, qlen = 0; i < m; ++i) {
+		saint_t p = SA[i], plen = SA[m + (p >> 1)], diff = 1;
+		if (plen == qlen) {
+			for (j = 0; j < plen && chr(p + j) == chr(q + j); j++);
+			if (j == plen) diff = 0;
+		}
+		if (diff) ++name, q = p, qlen = plen;
+		SA[m + (p >> 1)] = name;
+	}
+
+	/* STAGE II: solve the reduced problem; recurse if names are not yet unique */
+	if (name < m) {
+		saint_t *RA = SA + n + fs - m - 1;
+		for (i = n - 1, j = m - 1; m <= i; --i)
+			if (SA[i] != 0) RA[j--] = SA[i];
+		RA[m] = 0; // add a sentinel; in the resulting SA, SA[0]==m always stands
+		if (SAIS_CORE((unsigned char *)RA, SA, fs + n - m * 2 - 2, m + 1, name + 1, sizeof(saint_t)) != 0) return -2;
+		for (i = n - 2, j = m - 1, c = 1, c1 = chr(n - 1); 0 <= i; --i, c1 = c0) {
+			if ((c0 = chr(i)) < c1 + c) c = 1;
+			else if (c) RA[j--] = i + 1, c = 0; /* get p1 */
+		}
+		for (i = 0; i < m; ++i) SA[i] = RA[SA[i+1]]; /* get index  */
+	}
+
+	/* STAGE III: induce the result for the original problem */
+	if (k <= fs) C = SA + n, B = (k <= fs - k) ? C + k : C;
+	else {
+		if ((C = (saint_t*)malloc(k * (1 + (cs == 1)) * sizeof(saint_t))) == NULL) return -2;
+		B = cs == 1? C + k : C;
+	}
+	/* put all LMS characters into their buckets */
+	getCounts(T, C, n, k, cs);
+	getBuckets(C, B, k, 1);	/* find ends of buckets */
+	for (i = m; i < n; ++i) SA[i] = 0; /* init SA[m..n-1] */
+	for (i = m - 1; 0 <= i; --i) {
+		j = SA[i], SA[i] = 0;
+		c = chr(j);
+		SA[--B[c > 0? c : 0]] = j;
+	}
+	induceSA(T, SA, C, B, n, k, cs);
+	if (fs < k) free(C);
+	return 0;
+}
+
+/**
+ * Construct the suffix array for a NULL terminated string possibly containing
+ * multiple sentinels (NULLs).
+ *
+ * @param T[0..n-1]  NULL terminated input string
+ * @param SA[0..n-1] output suffix array
+ * @param n          length of the given string, including NULL
+ * @param k          size of the alphabet including the sentinel; no more than 256
+ * @return           0 upon success
+ */
+int SAIS_MAIN(const unsigned char *T, saint_t *SA, saint_t n, int k)
+{
+	if (T == NULL || SA == NULL || T[n - 1] != '\0' || n <= 0) return -1;
+	if (k < 0 || k > 256) k = 256;
+	return SAIS_CORE(T, SA, 0, n, (saint_t)k, 1);
+}
+
+int SAIS_BWT(unsigned char *T, saint_t n, int k)
+{
+	saint_t *SA, i;
+	int ret;
+	if ((SA = malloc(n * sizeof(saint_t))) == 0) return -1;
+	if ((ret = SAIS_MAIN(T, SA, n, k)) != 0) return ret;
+	for (i = 0; i < n; ++i)
+		if (SA[i]) SA[i] = T[SA[i] - 1]; // if SA[i]==0, SA[i]=0
+	for (i = 0; i < n; ++i) T[i] = SA[i];
+	free(SA);
+	return 0;
+}
diff --git a/ext/klib/kseq.h b/ext/klib/kseq.h
new file mode 100644
index 0000000..2959622
--- /dev/null
+++ b/ext/klib/kseq.h
@@ -0,0 +1,242 @@
+/* The MIT License
+
+   Copyright (c) 2008, 2009, 2011 Attractive Chaos <attractor@live.co.uk>
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be
+   included in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+*/
+
+/* Last Modified: 05MAR2012 */
+
+#ifndef AC_KSEQ_H
+#define AC_KSEQ_H
+
+#include <ctype.h>
+#include <string.h>
+#include <stdlib.h>
+
+#define KS_SEP_SPACE 0 // isspace(): \t, \n, \v, \f, \r
+#define KS_SEP_TAB   1 // isspace() && !' '
+#define KS_SEP_LINE  2 // line separator: "\n" (Unix) or "\r\n" (Windows)
+#define KS_SEP_MAX   2
+
+#define __KS_TYPE(type_t)						\
+	typedef struct __kstream_t {				\
+		unsigned char *buf;						\
+		int begin, end, is_eof;					\
+		type_t f;								\
+	} kstream_t;
+
+#define ks_err(ks) ((ks)->end == -1)
+#define ks_eof(ks) ((ks)->is_eof && (ks)->begin >= (ks)->end)
+#define ks_rewind(ks) ((ks)->is_eof = (ks)->begin = (ks)->end = 0)
+
+#define __KS_BASIC(type_t, __bufsize)								\
+	static inline kstream_t *ks_init(type_t f)						\
+	{																\
+		kstream_t *ks = (kstream_t*)calloc(1, sizeof(kstream_t));	\
+		ks->f = f;													\
+		ks->buf = (unsigned char*)malloc(__bufsize);				\
+		return ks;													\
+	}																\
+	static inline void ks_destroy(kstream_t *ks)					\
+	{																\
+		if (ks) {													\
+			free(ks->buf);											\
+			free(ks);												\
+		}															\
+	}
+
+#define __KS_GETC(__read, __bufsize)						\
+	static inline int ks_getc(kstream_t *ks)				\
+	{														\
+		if (ks_err(ks)) return -3;							\
+		if (ks->is_eof && ks->begin >= ks->end) return -1;	\
+		if (ks->begin >= ks->end) {							\
+			ks->begin = 0;									\
+			ks->end = __read(ks->f, ks->buf, __bufsize);	\
+			if (ks->end == 0) { ks->is_eof = 1; return -1;}	\
+			if (ks->end == -1) { ks->is_eof = 1; return -3;}\
+		}													\
+		return (int)ks->buf[ks->begin++];					\
+	}
+
+#ifndef KSTRING_T
+#define KSTRING_T kstring_t
+typedef struct __kstring_t {
+	size_t l, m;
+	char *s;
+} kstring_t;
+#endif
+
+#ifndef kroundup32
+#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
+#endif
+
+#define __KS_GETUNTIL(__read, __bufsize)								\
+	static int ks_getuntil2(kstream_t *ks, int delimiter, kstring_t *str, int *dret, int append) \
+	{																	\
+		int gotany = 0;													\
+		if (dret) *dret = 0;											\
+		str->l = append? str->l : 0;									\
+		for (;;) {														\
+			int i;														\
+			if (ks_err(ks)) return -3;									\
+			if (ks->begin >= ks->end) {									\
+				if (!ks->is_eof) {										\
+					ks->begin = 0;										\
+					ks->end = __read(ks->f, ks->buf, __bufsize);		\
+					if (ks->end == 0) { ks->is_eof = 1; break; }		\
+					if (ks->end == -1) { ks->is_eof = 1; return -3; }	\
+				} else break;											\
+			}															\
+			if (delimiter == KS_SEP_LINE) {								\
+				unsigned char *sep = memchr(ks->buf + ks->begin, '\n', ks->end - ks->begin); \
+				i = sep != NULL ? sep - ks->buf : ks->end;				\
+			} else if (delimiter > KS_SEP_MAX) {						\
+				unsigned char *sep = memchr(ks->buf + ks->begin, delimiter, ks->end - ks->begin); \
+				i = sep != NULL ? sep - ks->buf : ks->end;				\
+			} else if (delimiter == KS_SEP_SPACE) {						\
+				for (i = ks->begin; i < ks->end; ++i)					\
+					if (isspace(ks->buf[i])) break;						\
+			} else if (delimiter == KS_SEP_TAB) {						\
+				for (i = ks->begin; i < ks->end; ++i)					\
+					if (isspace(ks->buf[i]) && ks->buf[i] != ' ') break; \
+			} else i = 0; /* never come to here! */						\
+			if (str->m - str->l < (size_t)(i - ks->begin + 1)) {		\
+				str->m = str->l + (i - ks->begin) + 1;					\
+				kroundup32(str->m);										\
+				str->s = (char*)realloc(str->s, str->m);				\
+			}															\
+			gotany = 1;													\
+			memcpy(str->s + str->l, ks->buf + ks->begin, i - ks->begin); \
+			str->l = str->l + (i - ks->begin);							\
+			ks->begin = i + 1;											\
+			if (i < ks->end) {											\
+				if (dret) *dret = ks->buf[i];							\
+				break;													\
+			}															\
+		}																\
+		if (!gotany && ks_eof(ks)) return -1;							\
+		if (str->s == 0) {												\
+			str->m = 1;													\
+			str->s = (char*)calloc(1, 1);								\
+		} else if (delimiter == KS_SEP_LINE && str->l > 1 && str->s[str->l-1] == '\r') --str->l; \
+		str->s[str->l] = '\0';											\
+		return str->l;													\
+	} \
+	static inline int ks_getuntil(kstream_t *ks, int delimiter, kstring_t *str, int *dret) \
+	{ return ks_getuntil2(ks, delimiter, str, dret, 0); }
+
+#define KSTREAM_INIT(type_t, __read, __bufsize) \
+	__KS_TYPE(type_t)							\
+	__KS_BASIC(type_t, __bufsize)				\
+	__KS_GETC(__read, __bufsize)				\
+	__KS_GETUNTIL(__read, __bufsize)
+
+#define kseq_rewind(ks) ((ks)->last_char = (ks)->f->is_eof = (ks)->f->begin = (ks)->f->end = 0)
+
+#define __KSEQ_BASIC(SCOPE, type_t)										\
+	SCOPE kseq_t *kseq_init(type_t fd)									\
+	{																	\
+		kseq_t *s = (kseq_t*)calloc(1, sizeof(kseq_t));					\
+		s->f = ks_init(fd);												\
+		return s;														\
+	}																	\
+	SCOPE void kseq_destroy(kseq_t *ks)									\
+	{																	\
+		if (!ks) return;												\
+		free(ks->name.s); free(ks->comment.s); free(ks->seq.s);	free(ks->qual.s); \
+		ks_destroy(ks->f);												\
+		free(ks);														\
+	}
+
+/* Return value:
+   >=0  length of the sequence (normal)
+   -1   end-of-file
+   -2   truncated quality string
+   -3   error reading stream
+ */
+#define __KSEQ_READ(SCOPE) \
+	SCOPE int kseq_read(kseq_t *seq) \
+	{ \
+		int c,r; \
+		kstream_t *ks = seq->f; \
+		if (seq->last_char == 0) { /* then jump to the next header line */ \
+			while ((c = ks_getc(ks)) >= 0 && c != '>' && c != '@'); \
+			if (c < 0) return c; /* end of file or error*/ \
+			seq->last_char = c; \
+		} /* else: the first header char has been read in the previous call */ \
+		seq->comment.l = seq->seq.l = seq->qual.l = 0; /* reset all members */ \
+		if ((r=ks_getuntil(ks, 0, &seq->name, &c)) < 0) return r;  /* normal exit: EOF or error */ \
+		if (c != '\n') ks_getuntil(ks, KS_SEP_LINE, &seq->comment, 0); /* read FASTA/Q comment */ \
+		if (seq->seq.s == 0) { /* we can do this in the loop below, but that is slower */ \
+			seq->seq.m = 256; \
+			seq->seq.s = (char*)malloc(seq->seq.m); \
+		} \
+		while ((c = ks_getc(ks)) >= 0 && c != '>' && c != '+' && c != '@') { \
+			if (c == '\n') continue; /* skip empty lines */ \
+			seq->seq.s[seq->seq.l++] = c; /* this is safe: we always have enough space for 1 char */ \
+			ks_getuntil2(ks, KS_SEP_LINE, &seq->seq, 0, 1); /* read the rest of the line */ \
+		} \
+		if (c == '>' || c == '@') seq->last_char = c; /* the first header char has been read */	\
+		if (seq->seq.l + 1 >= seq->seq.m) { /* seq->seq.s[seq->seq.l] below may be out of boundary */ \
+			seq->seq.m = seq->seq.l + 2; \
+			kroundup32(seq->seq.m); /* rounded to the next closest 2^k */ \
+			seq->seq.s = (char*)realloc(seq->seq.s, seq->seq.m); \
+		} \
+		seq->seq.s[seq->seq.l] = 0;	/* null terminated string */ \
+		if (c != '+') return seq->seq.l; /* FASTA */ \
+		if (seq->qual.m < seq->seq.m) {	/* allocate memory for qual in case insufficient */ \
+			seq->qual.m = seq->seq.m; \
+			seq->qual.s = (char*)realloc(seq->qual.s, seq->qual.m); \
+		} \
+		while ((c = ks_getc(ks)) >= 0 && c != '\n'); /* skip the rest of '+' line */ \
+		if (c == -1) return -2; /* error: no quality string */ \
+		while ((c = ks_getuntil2(ks, KS_SEP_LINE, &seq->qual, 0, 1) >= 0 && seq->qual.l < seq->seq.l)); \
+		if (c == -3) return -3; /* stream error */ \
+		seq->last_char = 0;	/* we have not come to the next header line */ \
+		if (seq->seq.l != seq->qual.l) return -2; /* error: qual string is of a different length */ \
+		return seq->seq.l; \
+	}
+
+#define __KSEQ_TYPE(type_t)						\
+	typedef struct {							\
+		kstring_t name, comment, seq, qual;		\
+		int last_char;							\
+		kstream_t *f;							\
+	} kseq_t;
+
+#define KSEQ_INIT2(SCOPE, type_t, __read)		\
+	KSTREAM_INIT(type_t, __read, 16384)			\
+	__KSEQ_TYPE(type_t)							\
+	__KSEQ_BASIC(SCOPE, type_t)					\
+	__KSEQ_READ(SCOPE)
+
+#define KSEQ_INIT(type_t, __read) KSEQ_INIT2(static, type_t, __read)
+
+#define KSEQ_DECLARE(type_t) \
+	__KS_TYPE(type_t) \
+	__KSEQ_TYPE(type_t) \
+	extern kseq_t *kseq_init(type_t fd); \
+	void kseq_destroy(kseq_t *ks); \
+	int kseq_read(kseq_t *seq);
+
+#endif
diff --git a/ext/klib/kson.c b/ext/klib/kson.c
new file mode 100644
index 0000000..a8bf160
--- /dev/null
+++ b/ext/klib/kson.c
@@ -0,0 +1,253 @@
+#include <string.h>
+#include <stdlib.h>
+#include <stdarg.h>
+#include <assert.h>
+#include <ctype.h>
+#include <stdio.h>
+#include "kson.h"
+
+/*************
+ *** Parse ***
+ *************/
+
+kson_node_t *kson_parse_core(const char *json, long *_n, int *error, long *parsed_len)
+{
+	long *stack = 0, top = 0, max = 0, n_a = 0, m_a = 0, i, j;
+	kson_node_t *a = 0, *u;
+	const char *p, *q;
+	size_t *tmp;
+
+#define __push_back(y) do { \
+		if (top == max) { \
+			max = max? max<<1 : 4; \
+			stack = (long*)realloc(stack, sizeof(long) * max); \
+		} \
+		stack[top++] = (y); \
+	} while (0)
+
+#define __new_node(z) do { \
+		if (n_a == m_a) { \
+			long old_m = m_a; \
+			m_a = m_a? m_a<<1 : 4; \
+			a = (kson_node_t*)realloc(a, sizeof(kson_node_t) * m_a); \
+			memset(a + old_m, 0, sizeof(kson_node_t) * (m_a - old_m)); \
+		} \
+		*(z) = &a[n_a++]; \
+	} while (0)
+
+	assert(sizeof(size_t) == sizeof(kson_node_t*));
+	*error = KSON_OK;
+	for (p = json; *p; ++p) {
+		while (*p && isspace(*p)) ++p;
+		if (*p == 0) break;
+		if (*p == ',') { // comma is somewhat redundant
+		} else if (*p == '[' || *p == '{') {
+			int t = *p == '['? -1 : -2;
+			if (top < 2 || stack[top-1] != -3) { // unnamed internal node
+				__push_back(n_a);
+				__new_node(&u);
+				__push_back(t);
+			} else stack[top-1] = t; // named internal node
+		} else if (*p == ']' || *p == '}') {
+			long i, start, t = *p == ']'? -1 : -2;
+			for (i = top - 1; i >= 0 && stack[i] != t; --i);
+			if (i < 0) { // error: an extra right bracket
+				*error = KSON_ERR_EXTRA_RIGHT;
+				break;
+			}
+			start = i;
+			u = &a[stack[start-1]];
+			u->key = u->v.str;
+			u->n = top - 1 - start;
+			u->v.child = (kson_node_t**)malloc(u->n * sizeof(kson_node_t*));
+			tmp = (size_t*)u->v.child;
+			for (i = start + 1; i < top; ++i)
+				tmp[i - start - 1] = stack[i];
+			u->type = *p == ']'? KSON_TYPE_BRACKET : KSON_TYPE_BRACE;
+			if ((top = start) == 1) break; // completed one object; remaining characters discarded
+		} else if (*p == ':') {
+			if (top == 0 || stack[top-1] == -3) {
+				*error = KSON_ERR_NO_KEY;
+				break;
+			}
+			__push_back(-3);
+		} else {
+			int c = *p;
+			// get the node to modify
+			if (top >= 2 && stack[top-1] == -3) { // we have a key:value pair here
+				--top;
+				u = &a[stack[top-1]];
+				u->key = u->v.str; // move old value to key
+			} else { // don't know if this is a bare value or a key:value pair; keep it as a value for now
+				__push_back(n_a);
+				__new_node(&u);
+			}
+			// parse string
+			if (c == '\'' || c == '"') {
+				for (q = ++p; *q && *q != c; ++q)
+					if (*q == '\\') ++q;
+			} else {
+				for (q = p; *q && *q != ']' && *q != '}' && *q != ',' && *q != ':' && *q != '\n'; ++q)
+					if (*q == '\\') ++q;
+			}
+			u->v.str = (char*)malloc(q - p + 1); strncpy(u->v.str, p, q - p); u->v.str[q-p] = 0; // equivalent to u->v.str=strndup(p, q-p)
+			u->type = c == '\''? KSON_TYPE_SGL_QUOTE : c == '"'? KSON_TYPE_DBL_QUOTE : KSON_TYPE_NO_QUOTE;
+			p = c == '\'' || c == '"'? q : q - 1;
+		}
+	}
+	while (*p && isspace(*p)) ++p; // skip trailing blanks
+	if (parsed_len) *parsed_len = p - json;
+	if (top != 1) *error = KSON_ERR_EXTRA_LEFT;
+
+	for (i = 0; i < n_a; ++i)
+		for (j = 0, u = &a[i], tmp = (size_t*)u->v.child; j < (long)u->n; ++j)
+			u->v.child[j] = &a[tmp[j]];
+
+	free(stack);
+	*_n = n_a;
+	return a;
+}
+
+void kson_destroy(kson_t *kson)
+{
+	long i;
+	if (kson == 0) return;
+	for (i = 0; i < kson->n_nodes; ++i) {
+		free(kson->root[i].key); free(kson->root[i].v.str);
+	}
+	free(kson->root); free(kson);
+}
+
+kson_t *kson_parse(const char *json)
+{
+	kson_t *kson;
+	int error;
+	kson = (kson_t*)calloc(1, sizeof(kson_t));
+	kson->root = kson_parse_core(json, &kson->n_nodes, &error, 0);
+	if (error) {
+		kson_destroy(kson);
+		return 0;
+	}
+	return kson;
+}
+
+/*************
+ *** Query ***
+ *************/
+
+const kson_node_t *kson_by_path(const kson_node_t *p, int depth, ...)
+{
+	va_list ap;
+	va_start(ap, depth);
+	while (p && depth > 0) {
+		if (p->type == KSON_TYPE_BRACE) {
+			p = kson_by_key(p, va_arg(ap, const char*));
+		} else if (p->type == KSON_TYPE_BRACKET) {
+			p = kson_by_index(p, va_arg(ap, long));
+		} else break;
+		--depth;
+	}
+	va_end(ap);
+	return p;
+}
+
+/**************
+ *** Fromat ***
+ **************/
+
+void kson_format_recur(const kson_node_t *p, int depth)
+{
+	long i;
+	if (p->key) printf("\"%s\":", p->key);
+	if (p->type == KSON_TYPE_BRACKET || p->type == KSON_TYPE_BRACE) {
+		putchar(p->type == KSON_TYPE_BRACKET? '[' : '{');
+		if (p->n) {
+			putchar('\n'); for (i = 0; i <= depth; ++i) fputs("  ", stdout);
+			for (i = 0; i < (long)p->n; ++i) {
+				if (i) {
+					int i;
+					putchar(',');
+					putchar('\n'); for (i = 0; i <= depth; ++i) fputs("  ", stdout);
+				}
+				kson_format_recur(p->v.child[i], depth + 1);
+			}
+			putchar('\n'); for (i = 0; i < depth; ++i) fputs("  ", stdout);
+		}
+		putchar(p->type == KSON_TYPE_BRACKET? ']' : '}');
+	} else {
+		if (p->type != KSON_TYPE_NO_QUOTE)
+			putchar(p->type == KSON_TYPE_SGL_QUOTE? '\'' : '"');
+		fputs(p->v.str, stdout);
+		if (p->type != KSON_TYPE_NO_QUOTE)
+			putchar(p->type == KSON_TYPE_SGL_QUOTE? '\'' : '"');
+	}
+}
+
+void kson_format(const kson_node_t *root)
+{
+	kson_format_recur(root, 0);
+	putchar('\n');
+}
+
+/*********************
+ *** Main function ***
+ *********************/
+
+#ifdef KSON_MAIN
+#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
+int main(int argc, char *argv[])
+{
+	kson_t *kson = 0;
+	if (argc > 1) {
+		FILE *fp;
+		int len = 0, max = 0, tmp, i;
+		char *json = 0, buf[0x10000];
+		if ((fp = fopen(argv[1], "rb")) != 0) {
+			// read the entire file into a string
+			while ((tmp = fread(buf, 1, 0x10000, fp)) != 0) {
+				if (len + tmp + 1 > max) {
+					max = len + tmp + 1;
+					kroundup32(max);
+					json = (char*)realloc(json, max);
+				}
+				memcpy(json + len, buf, tmp);
+				len += tmp;
+			}
+			fclose(fp);
+			// parse
+			kson = kson_parse(json);
+			free(json);
+			if (kson) {
+				kson_format(kson->root);
+				if (argc > 2) {
+					// path finding
+					const kson_node_t *p = kson->root;
+					for (i = 2; i < argc && p; ++i) {
+						if (p->type == KSON_TYPE_BRACKET)
+							p = kson_by_index(p, atoi(argv[i]));
+						else if (p->type == KSON_TYPE_BRACE)
+							p = kson_by_key(p, argv[i]);
+						else p = 0;
+					}
+					if (p) {
+						if (kson_is_internal(p)) printf("Reached an internal node\n");
+						else printf("Value: %s\n", p->v.str);
+					} else printf("Failed to find the slot\n");
+				}
+			} else printf("Failed to parse\n");
+		}
+	} else {
+		kson = kson_parse("{'a' : 1,'b':[0,'isn\\'t',true],'d':[{\n\n\n}]}");
+		if (kson) {
+			const kson_node_t *p = kson_by_path(kson->root, 2, "b", 1);
+			if (p) printf("*** %s\n", p->v.str);
+			else printf("!!! not found\n");
+			kson_format(kson->root);
+		} else {
+			printf("Failed to parse\n");
+		}
+	}
+	kson_destroy(kson);
+	return 0;
+}
+#endif
diff --git a/ext/klib/kson.h b/ext/klib/kson.h
new file mode 100644
index 0000000..a03eb52
--- /dev/null
+++ b/ext/klib/kson.h
@@ -0,0 +1,64 @@
+#ifndef KSON_H
+#define KSON_H
+
+#include <string.h>
+
+#define KSON_TYPE_NO_QUOTE  1
+#define KSON_TYPE_SGL_QUOTE 2
+#define KSON_TYPE_DBL_QUOTE 3
+#define KSON_TYPE_BRACKET   4
+#define KSON_TYPE_BRACE     5
+
+#define KSON_OK              0
+#define KSON_ERR_EXTRA_LEFT  1
+#define KSON_ERR_EXTRA_RIGHT 2
+#define KSON_ERR_NO_KEY      3
+
+typedef struct kson_node_s {
+	unsigned long long type:3, n:61;
+	char *key;
+	union {
+		struct kson_node_s **child;
+		char *str;
+	} v;
+} kson_node_t;
+
+typedef struct {
+	long n_nodes;
+	kson_node_t *root;
+} kson_t;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+	kson_t *kson_parse(const char *json);
+	void kson_destroy(kson_t *kson);
+	const kson_node_t *kson_by_path(const kson_node_t *root, int path_len, ...);
+	void kson_format(const kson_node_t *root);
+
+#ifdef __cplusplus
+}
+#endif
+
+#define kson_is_internal(p) ((p)->type == KSON_TYPE_BRACKET || (p)->type == KSON_TYPE_BRACE)
+
+static inline const kson_node_t *kson_by_key(const kson_node_t *p, const char *key)
+{
+	long i;
+	if (!kson_is_internal(p)) return 0;
+	for (i = 0; i < (long)p->n; ++i) {
+		const kson_node_t *q = p->v.child[i];
+		if (q->key && strcmp(q->key, key) == 0)
+			return q;
+	}
+	return 0;
+}
+
+static inline const kson_node_t *kson_by_index(const kson_node_t *p, long i)
+{
+	if (!kson_is_internal(p)) return 0;
+	return 0 <= i && i < (long)p->n? p->v.child[i] : 0;
+}
+
+#endif
diff --git a/ext/klib/ksort.h b/ext/klib/ksort.h
new file mode 100644
index 0000000..5a2a1a6
--- /dev/null
+++ b/ext/klib/ksort.h
@@ -0,0 +1,353 @@
+/* The MIT License
+
+   Copyright (c) 2008, 2011 Attractive Chaos <attractor@live.co.uk>
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be
+   included in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+*/
+
+/*
+  2011-04-10 (0.1.6):
+
+  	* Added sample
+
+  2011-03 (0.1.5):
+
+	* Added shuffle/permutation
+
+  2008-11-16 (0.1.4):
+
+    * Fixed a bug in introsort() that happens in rare cases.
+
+  2008-11-05 (0.1.3):
+
+    * Fixed a bug in introsort() for complex comparisons.
+
+	* Fixed a bug in mergesort(). The previous version is not stable.
+
+  2008-09-15 (0.1.2):
+
+	* Accelerated introsort. On my Mac (not on another Linux machine),
+	  my implementation is as fast as std::sort on random input.
+
+	* Added combsort and in introsort, switch to combsort if the
+	  recursion is too deep.
+
+  2008-09-13 (0.1.1):
+
+	* Added k-small algorithm
+
+  2008-09-05 (0.1.0):
+
+	* Initial version
+
+*/
+
+#ifndef AC_KSORT_H
+#define AC_KSORT_H
+
+#include <stdlib.h>
+#include <string.h>
+
+typedef struct {
+	void *left, *right;
+	int depth;
+} ks_isort_stack_t;
+
+#define KSORT_SWAP(type_t, a, b) { register type_t t=(a); (a)=(b); (b)=t; }
+
+#define KSORT_INIT(name, type_t, __sort_lt)								\
+	void ks_mergesort_##name(size_t n, type_t array[], type_t temp[])	\
+	{																	\
+		type_t *a2[2], *a, *b;											\
+		int curr, shift;												\
+																		\
+		a2[0] = array;													\
+		a2[1] = temp? temp : (type_t*)malloc(sizeof(type_t) * n);		\
+		for (curr = 0, shift = 0; (1ul<<shift) < n; ++shift) {			\
+			a = a2[curr]; b = a2[1-curr];								\
+			if (shift == 0) {											\
+				type_t *p = b, *i, *eb = a + n;							\
+				for (i = a; i < eb; i += 2) {							\
+					if (i == eb - 1) *p++ = *i;							\
+					else {												\
+						if (__sort_lt(*(i+1), *i)) {					\
+							*p++ = *(i+1); *p++ = *i;					\
+						} else {										\
+							*p++ = *i; *p++ = *(i+1);					\
+						}												\
+					}													\
+				}														\
+			} else {													\
+				size_t i, step = 1ul<<shift;							\
+				for (i = 0; i < n; i += step<<1) {						\
+					type_t *p, *j, *k, *ea, *eb;						\
+					if (n < i + step) {									\
+						ea = a + n; eb = a;								\
+					} else {											\
+						ea = a + i + step;								\
+						eb = a + (n < i + (step<<1)? n : i + (step<<1)); \
+					}													\
+					j = a + i; k = a + i + step; p = b + i;				\
+					while (j < ea && k < eb) {							\
+						if (__sort_lt(*k, *j)) *p++ = *k++;				\
+						else *p++ = *j++;								\
+					}													\
+					while (j < ea) *p++ = *j++;							\
+					while (k < eb) *p++ = *k++;							\
+				}														\
+			}															\
+			curr = 1 - curr;											\
+		}																\
+		if (curr == 1) {												\
+			type_t *p = a2[0], *i = a2[1], *eb = array + n;				\
+			for (; p < eb; ++i) *p++ = *i;								\
+		}																\
+		if (temp == 0) free(a2[1]);										\
+	}																	\
+	void ks_heapadjust_##name(size_t i, size_t n, type_t l[])			\
+	{																	\
+		size_t k = i;													\
+		type_t tmp = l[i];												\
+		while ((k = (k << 1) + 1) < n) {								\
+			if (k != n - 1 && __sort_lt(l[k], l[k+1])) ++k;				\
+			if (__sort_lt(l[k], tmp)) break;							\
+			l[i] = l[k]; i = k;											\
+		}																\
+		l[i] = tmp;														\
+	}																	\
+	void ks_heapmake_##name(size_t lsize, type_t l[])					\
+	{																	\
+		size_t i;														\
+		for (i = (lsize >> 1) - 1; i != (size_t)(-1); --i)				\
+			ks_heapadjust_##name(i, lsize, l);							\
+	}																	\
+	void ks_heapsort_##name(size_t lsize, type_t l[])					\
+	{																	\
+		size_t i;														\
+		for (i = lsize - 1; i > 0; --i) {								\
+			type_t tmp;													\
+			tmp = *l; *l = l[i]; l[i] = tmp; ks_heapadjust_##name(0, i, l); \
+		}																\
+	}																	\
+	static inline void __ks_insertsort_##name(type_t *s, type_t *t)			\
+	{																	\
+		type_t *i, *j, swap_tmp;										\
+		for (i = s + 1; i < t; ++i)										\
+			for (j = i; j > s && __sort_lt(*j, *(j-1)); --j) {			\
+				swap_tmp = *j; *j = *(j-1); *(j-1) = swap_tmp;			\
+			}															\
+	}																	\
+	void ks_combsort_##name(size_t n, type_t a[])						\
+	{																	\
+		const double shrink_factor = 1.2473309501039786540366528676643; \
+		int do_swap;													\
+		size_t gap = n;													\
+		type_t tmp, *i, *j;												\
+		do {															\
+			if (gap > 2) {												\
+				gap = (size_t)(gap / shrink_factor);					\
+				if (gap == 9 || gap == 10) gap = 11;					\
+			}															\
+			do_swap = 0;												\
+			for (i = a; i < a + n - gap; ++i) {							\
+				j = i + gap;											\
+				if (__sort_lt(*j, *i)) {								\
+					tmp = *i; *i = *j; *j = tmp;						\
+					do_swap = 1;										\
+				}														\
+			}															\
+		} while (do_swap || gap > 2);									\
+		if (gap != 1) __ks_insertsort_##name(a, a + n);					\
+	}																	\
+	void ks_introsort_##name(size_t n, type_t a[])						\
+	{																	\
+		int d;															\
+		ks_isort_stack_t *top, *stack;									\
+		type_t rp, swap_tmp;											\
+		type_t *s, *t, *i, *j, *k;										\
+																		\
+		if (n < 1) return;												\
+		else if (n == 2) {												\
+			if (__sort_lt(a[1], a[0])) { swap_tmp = a[0]; a[0] = a[1]; a[1] = swap_tmp; } \
+			return;														\
+		}																\
+		for (d = 2; 1ul<<d < n; ++d);									\
+		stack = (ks_isort_stack_t*)malloc(sizeof(ks_isort_stack_t) * ((sizeof(size_t)*d)+2)); \
+		top = stack; s = a; t = a + (n-1); d <<= 1;						\
+		while (1) {														\
+			if (s < t) {												\
+				if (--d == 0) {											\
+					ks_combsort_##name(t - s + 1, s);					\
+					t = s;												\
+					continue;											\
+				}														\
+				i = s; j = t; k = i + ((j-i)>>1) + 1;					\
+				if (__sort_lt(*k, *i)) {								\
+					if (__sort_lt(*k, *j)) k = j;						\
+				} else k = __sort_lt(*j, *i)? i : j;					\
+				rp = *k;												\
+				if (k != t) { swap_tmp = *k; *k = *t; *t = swap_tmp; }	\
+				for (;;) {												\
+					do ++i; while (__sort_lt(*i, rp));					\
+					do --j; while (i <= j && __sort_lt(rp, *j));		\
+					if (j <= i) break;									\
+					swap_tmp = *i; *i = *j; *j = swap_tmp;				\
+				}														\
+				swap_tmp = *i; *i = *t; *t = swap_tmp;					\
+				if (i-s > t-i) {										\
+					if (i-s > 16) { top->left = s; top->right = i-1; top->depth = d; ++top; } \
+					s = t-i > 16? i+1 : t;								\
+				} else {												\
+					if (t-i > 16) { top->left = i+1; top->right = t; top->depth = d; ++top; } \
+					t = i-s > 16? i-1 : s;								\
+				}														\
+			} else {													\
+				if (top == stack) {										\
+					free(stack);										\
+					__ks_insertsort_##name(a, a+n);						\
+					return;												\
+				} else { --top; s = (type_t*)top->left; t = (type_t*)top->right; d = top->depth; } \
+			}															\
+		}																\
+	}																	\
+	/* This function is adapted from: http://ndevilla.free.fr/median/ */ \
+	/* 0 <= kk < n */													\
+	type_t ks_ksmall_##name(size_t n, type_t arr[], size_t kk)			\
+	{																	\
+		type_t *low, *high, *k, *ll, *hh, *mid;							\
+		low = arr; high = arr + n - 1; k = arr + kk;					\
+		for (;;) {														\
+			if (high <= low) return *k;									\
+			if (high == low + 1) {										\
+				if (__sort_lt(*high, *low)) KSORT_SWAP(type_t, *low, *high); \
+				return *k;												\
+			}															\
+			mid = low + (high - low) / 2;								\
+			if (__sort_lt(*high, *mid)) KSORT_SWAP(type_t, *mid, *high); \
+			if (__sort_lt(*high, *low)) KSORT_SWAP(type_t, *low, *high); \
+			if (__sort_lt(*low, *mid)) KSORT_SWAP(type_t, *mid, *low);	\
+			KSORT_SWAP(type_t, *mid, *(low+1));							\
+			ll = low + 1; hh = high;									\
+			for (;;) {													\
+				do ++ll; while (__sort_lt(*ll, *low));					\
+				do --hh; while (__sort_lt(*low, *hh));					\
+				if (hh < ll) break;										\
+				KSORT_SWAP(type_t, *ll, *hh);							\
+			}															\
+			KSORT_SWAP(type_t, *low, *hh);								\
+			if (hh <= k) low = ll;										\
+			if (hh >= k) high = hh - 1;									\
+		}																\
+	}																	\
+	void ks_shuffle_##name(size_t n, type_t a[])						\
+	{																	\
+		int i, j;														\
+		for (i = n; i > 1; --i) {										\
+			type_t tmp;													\
+			j = (int)(drand48() * i);									\
+			tmp = a[j]; a[j] = a[i-1]; a[i-1] = tmp;					\
+		}																\
+	}																	\
+	void ks_sample_##name(size_t n, size_t r, type_t a[]) /* FIXME: NOT TESTED!!! */ \
+	{ /* reference: http://code.activestate.com/recipes/272884/ */ \
+		int i, k, pop = n; \
+		for (i = (int)r, k = 0; i >= 0; --i) { \
+			double z = 1., x = drand48(); \
+			type_t tmp; \
+			while (x < z) z -= z * i / (pop--); \
+			if (k != n - pop - 1) tmp = a[k], a[k] = a[n-pop-1], a[n-pop-1] = tmp; \
+			++k; \
+		} \
+	}
+
+#define ks_mergesort(name, n, a, t) ks_mergesort_##name(n, a, t)
+#define ks_introsort(name, n, a) ks_introsort_##name(n, a)
+#define ks_combsort(name, n, a) ks_combsort_##name(n, a)
+#define ks_heapsort(name, n, a) ks_heapsort_##name(n, a)
+#define ks_heapmake(name, n, a) ks_heapmake_##name(n, a)
+#define ks_heapadjust(name, i, n, a) ks_heapadjust_##name(i, n, a)
+#define ks_ksmall(name, n, a, k) ks_ksmall_##name(n, a, k)
+#define ks_shuffle(name, n, a) ks_shuffle_##name(n, a)
+
+#define ks_lt_generic(a, b) ((a) < (b))
+#define ks_lt_str(a, b) (strcmp((a), (b)) < 0)
+
+typedef const char *ksstr_t;
+
+#define KSORT_INIT_GENERIC(type_t) KSORT_INIT(type_t, type_t, ks_lt_generic)
+#define KSORT_INIT_STR KSORT_INIT(str, ksstr_t, ks_lt_str)
+
+#define RS_MIN_SIZE 64
+#define RS_MAX_BITS 8
+
+#define KRADIX_SORT_INIT(name, rstype_t, rskey, sizeof_key) \
+	typedef struct { \
+		rstype_t *b, *e; \
+	} rsbucket_##name##_t; \
+	void rs_insertsort_##name(rstype_t *beg, rstype_t *end) \
+	{ \
+		rstype_t *i; \
+		for (i = beg + 1; i < end; ++i) \
+			if (rskey(*i) < rskey(*(i - 1))) { \
+				rstype_t *j, tmp = *i; \
+				for (j = i; j > beg && rskey(tmp) < rskey(*(j-1)); --j) \
+					*j = *(j - 1); \
+				*j = tmp; \
+			} \
+	} \
+	void rs_sort_##name(rstype_t *beg, rstype_t *end, int n_bits, int s) \
+	{ \
+		rstype_t *i; \
+		int size = 1<<n_bits, m = size - 1; \
+		rsbucket_##name##_t *k, b[1<<RS_MAX_BITS], *be = b + size; \
+		assert(n_bits <= RS_MAX_BITS); \
+		for (k = b; k != be; ++k) k->b = k->e = beg; \
+		for (i = beg; i != end; ++i) ++b[rskey(*i)>>s&m].e; \
+		for (k = b + 1; k != be; ++k) \
+			k->e += (k-1)->e - beg, k->b = (k-1)->e; \
+		for (k = b; k != be;) { \
+			if (k->b != k->e) { \
+				rsbucket_##name##_t *l; \
+				if ((l = b + (rskey(*k->b)>>s&m)) != k) { \
+					rstype_t tmp = *k->b, swap; \
+					do { \
+						swap = tmp; tmp = *l->b; *l->b++ = swap; \
+						l = b + (rskey(tmp)>>s&m); \
+					} while (l != k); \
+					*k->b++ = tmp; \
+				} else ++k->b; \
+			} else ++k; \
+		} \
+		for (b->b = beg, k = b + 1; k != be; ++k) k->b = (k-1)->e; \
+		if (s) { \
+			s = s > n_bits? s - n_bits : 0; \
+			for (k = b; k != be; ++k) \
+				if (k->e - k->b > RS_MIN_SIZE) rs_sort_##name(k->b, k->e, n_bits, s); \
+				else if (k->e - k->b > 1) rs_insertsort_##name(k->b, k->e); \
+		} \
+	} \
+	void radix_sort_##name(rstype_t *beg, rstype_t *end) \
+	{ \
+		if (end - beg <= RS_MIN_SIZE) rs_insertsort_##name(beg, end); \
+		else rs_sort_##name(beg, end, RS_MAX_BITS, (sizeof_key - 1) * RS_MAX_BITS); \
+	}
+
+#endif
diff --git a/ext/klib/kstring.c b/ext/klib/kstring.c
new file mode 100644
index 0000000..aa6c2ee
--- /dev/null
+++ b/ext/klib/kstring.c
@@ -0,0 +1,250 @@
+#include <stdarg.h>
+#include <stdio.h>
+#include <ctype.h>
+#include <string.h>
+#include <stdint.h>
+#include "kstring.h"
+
+int kvsprintf(kstring_t *s, const char *fmt, va_list ap)
+{
+	va_list args;
+	int l;
+	va_copy(args, ap);
+	l = vsnprintf(s->s + s->l, s->m - s->l, fmt, args); // This line does not work with glibc 2.0. See `man snprintf'.
+	va_end(args);
+	if (l + 1 > s->m - s->l) {
+		s->m = s->l + l + 2;
+		kroundup32(s->m);
+		s->s = (char*)realloc(s->s, s->m);
+		va_copy(args, ap);
+		l = vsnprintf(s->s + s->l, s->m - s->l, fmt, args);
+		va_end(args);
+	}
+	s->l += l;
+	return l;
+}
+
+int ksprintf(kstring_t *s, const char *fmt, ...)
+{
+	va_list ap;
+	int l;
+	va_start(ap, fmt);
+	l = kvsprintf(s, fmt, ap);
+	va_end(ap);
+	return l;
+}
+
+char *kstrtok(const char *str, const char *sep_in, ks_tokaux_t *aux)
+{
+	const unsigned char *p, *start, *sep = (unsigned char *) sep_in;
+	if (sep) { // set up the table
+		if (str == 0 && aux->finished) return 0; // no need to set up if we have finished
+		aux->finished = 0;
+		if (sep[0] && sep[1]) {
+			aux->sep = -1;
+			aux->tab[0] = aux->tab[1] = aux->tab[2] = aux->tab[3] = 0;
+			for (p = sep; *p; ++p) aux->tab[*p>>6] |= 1ull<<(*p&0x3f);
+		} else aux->sep = sep[0];
+	}
+	if (aux->finished) return 0;
+	else if (str) start = (unsigned char *) str, aux->finished = 0;
+	else start = (unsigned char *) aux->p + 1;
+	if (aux->sep < 0) {
+		for (p = start; *p; ++p)
+			if (aux->tab[*p>>6]>>(*p&0x3f)&1) break;
+	} else {
+		for (p = start; *p; ++p)
+			if (*p == aux->sep) break;
+	}
+	aux->p = (const char *) p; // end of token
+	if (*p == 0) aux->finished = 1; // no more tokens
+	return (char*)start;
+}
+
+// s MUST BE a null terminated string; l = strlen(s)
+int ksplit_core(char *s, int delimiter, int *_max, int **_offsets)
+{
+	int i, n, max, last_char, last_start, *offsets, l;
+	n = 0; max = *_max; offsets = *_offsets;
+	l = strlen(s);
+	
+#define __ksplit_aux do {						\
+		if (_offsets) {						\
+			s[i] = 0;					\
+			if (n == max) {					\
+				int *tmp;				\
+				max = max? max<<1 : 2;			\
+				if ((tmp = (int*)realloc(offsets, sizeof(int) * max))) {  \
+					offsets = tmp;			\
+				} else	{				\
+					free(offsets);			\
+					*_offsets = NULL;		\
+					return 0;			\
+				}					\
+			}						\
+			offsets[n++] = last_start;			\
+		} else ++n;						\
+	} while (0)
+
+	for (i = 0, last_char = last_start = 0; i <= l; ++i) {
+		if (delimiter == 0) {
+			if (isspace(s[i]) || s[i] == 0) {
+				if (isgraph(last_char)) __ksplit_aux; // the end of a field
+			} else {
+				if (isspace(last_char) || last_char == 0) last_start = i;
+			}
+		} else {
+			if (s[i] == delimiter || s[i] == 0) {
+				if (last_char != 0 && last_char != delimiter) __ksplit_aux; // the end of a field
+			} else {
+				if (last_char == delimiter || last_char == 0) last_start = i;
+			}
+		}
+		last_char = s[i];
+	}
+	*_max = max; *_offsets = offsets;
+	return n;
+}
+
+int kgetline(kstring_t *s, kgets_func *fgets_fn, void *fp)
+{
+	size_t l0 = s->l;
+
+	while (s->l == l0 || s->s[s->l-1] != '\n') {
+		if (s->m - s->l < 200) ks_resize(s, s->m + 200);
+		if (fgets_fn(s->s + s->l, s->m - s->l, fp) == NULL) break;
+		s->l += strlen(s->s + s->l);
+	}
+
+	if (s->l == l0) return EOF;
+
+	if (s->l > l0 && s->s[s->l-1] == '\n') {
+		s->l--;
+		if (s->l > l0 && s->s[s->l-1] == '\r') s->l--;
+	}
+	s->s[s->l] = '\0';
+	return 0;
+}
+
+/**********************
+ * Boyer-Moore search *
+ **********************/
+
+typedef unsigned char ubyte_t;
+
+// reference: http://www-igm.univ-mlv.fr/~lecroq/string/node14.html
+static int *ksBM_prep(const ubyte_t *pat, int m)
+{
+	int i, *suff, *prep, *bmGs, *bmBc;
+	prep = (int*)calloc(m + 256, sizeof(int));
+	bmGs = prep; bmBc = prep + m;
+	{ // preBmBc()
+		for (i = 0; i < 256; ++i) bmBc[i] = m;
+		for (i = 0; i < m - 1; ++i) bmBc[pat[i]] = m - i - 1;
+	}
+	suff = (int*)calloc(m, sizeof(int));
+	{ // suffixes()
+		int f = 0, g;
+		suff[m - 1] = m;
+		g = m - 1;
+		for (i = m - 2; i >= 0; --i) {
+			if (i > g && suff[i + m - 1 - f] < i - g)
+				suff[i] = suff[i + m - 1 - f];
+			else {
+				if (i < g) g = i;
+				f = i;
+				while (g >= 0 && pat[g] == pat[g + m - 1 - f]) --g;
+				suff[i] = f - g;
+			}
+		}
+	}
+	{ // preBmGs()
+		int j = 0;
+		for (i = 0; i < m; ++i) bmGs[i] = m;
+		for (i = m - 1; i >= 0; --i)
+			if (suff[i] == i + 1)
+				for (; j < m - 1 - i; ++j)
+					if (bmGs[j] == m)
+						bmGs[j] = m - 1 - i;
+		for (i = 0; i <= m - 2; ++i)
+			bmGs[m - 1 - suff[i]] = m - 1 - i;
+	}
+	free(suff);
+	return prep;
+}
+
+void *kmemmem(const void *_str, int n, const void *_pat, int m, int **_prep)
+{
+	int i, j, *prep = 0, *bmGs, *bmBc;
+	const ubyte_t *str, *pat;
+	str = (const ubyte_t*)_str; pat = (const ubyte_t*)_pat;
+	prep = (_prep == 0 || *_prep == 0)? ksBM_prep(pat, m) : *_prep;
+	if (_prep && *_prep == 0) *_prep = prep;
+	bmGs = prep; bmBc = prep + m;
+	j = 0;
+	while (j <= n - m) {
+		for (i = m - 1; i >= 0 && pat[i] == str[i+j]; --i);
+		if (i >= 0) {
+			int max = bmBc[str[i+j]] - m + 1 + i;
+			if (max < bmGs[i]) max = bmGs[i];
+			j += max;
+		} else return (void*)(str + j);
+	}
+	if (_prep == 0) free(prep);
+	return 0;
+}
+
+char *kstrstr(const char *str, const char *pat, int **_prep)
+{
+	return (char*)kmemmem(str, strlen(str), pat, strlen(pat), _prep);
+}
+
+char *kstrnstr(const char *str, const char *pat, int n, int **_prep)
+{
+	return (char*)kmemmem(str, n, pat, strlen(pat), _prep);
+}
+
+/***********************
+ * The main() function *
+ ***********************/
+
+#ifdef KSTRING_MAIN
+#include <stdio.h>
+int main()
+{
+	kstring_t *s;
+	int *fields, n, i;
+	ks_tokaux_t aux;
+	char *p;
+	s = (kstring_t*)calloc(1, sizeof(kstring_t));
+	// test ksprintf()
+	ksprintf(s, " abcdefg:    %d ", 100);
+	printf("'%s'\n", s->s);
+	// test ksplit()
+	fields = ksplit(s, 0, &n);
+	for (i = 0; i < n; ++i)
+		printf("field[%d] = '%s'\n", i, s->s + fields[i]);
+	// test kstrtok()
+	s->l = 0;
+	for (p = kstrtok("ab:cde:fg/hij::k", ":/", &aux); p; p = kstrtok(0, 0, &aux)) {
+		kputsn(p, aux.p - p, s);
+		kputc('\n', s);
+	}
+	printf("%s", s->s);
+	// free
+	free(s->s); free(s); free(fields);
+
+	{
+		static char *str = "abcdefgcdgcagtcakcdcd";
+		static char *pat = "cd";
+		char *ret, *s = str;
+		int *prep = 0;
+		while ((ret = kstrstr(s, pat, &prep)) != 0) {
+			printf("match: %s\n", ret);
+			s = ret + prep[0];
+		}
+		free(prep);
+	}
+	return 0;
+}
+#endif
diff --git a/ext/klib/kstring.h b/ext/klib/kstring.h
new file mode 100644
index 0000000..f13fcd9
--- /dev/null
+++ b/ext/klib/kstring.h
@@ -0,0 +1,277 @@
+/* The MIT License
+
+   Copyright (c) by Attractive Chaos <attractor@live.co.uk> 
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be
+   included in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+*/
+
+#ifndef KSTRING_H
+#define KSTRING_H
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdarg.h>
+#include <stdint.h>
+#include <stdio.h>
+
+#ifndef kroundup32
+#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
+#endif
+
+#if __GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ > 4)
+#define KS_ATTR_PRINTF(fmt, arg) __attribute__((__format__ (__printf__, fmt, arg)))
+#else
+#define KS_ATTR_PRINTF(fmt, arg)
+#endif
+
+
+/* kstring_t is a simple non-opaque type whose fields are likely to be
+ * used directly by user code (but see also ks_str() and ks_len() below).
+ * A kstring_t object is initialised by either of
+ *       kstring_t str = { 0, 0, NULL };
+ *       kstring_t str; ...; str.l = str.m = 0; str.s = NULL;
+ * and either ownership of the underlying buffer should be given away before
+ * the object disappears (see ks_release() below) or the kstring_t should be
+ * destroyed with  free(str.s);  */
+#ifndef KSTRING_T
+#define KSTRING_T kstring_t
+typedef struct __kstring_t {
+	size_t l, m;
+	char *s;
+} kstring_t;
+#endif
+
+typedef struct {
+	uint64_t tab[4];
+	int sep, finished;
+	const char *p; // end of the current token
+} ks_tokaux_t;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+	int kvsprintf(kstring_t *s, const char *fmt, va_list ap) KS_ATTR_PRINTF(2,0);
+	int ksprintf(kstring_t *s, const char *fmt, ...) KS_ATTR_PRINTF(2,3);
+	int ksplit_core(char *s, int delimiter, int *_max, int **_offsets);
+	char *kstrstr(const char *str, const char *pat, int **_prep);
+	char *kstrnstr(const char *str, const char *pat, int n, int **_prep);
+	void *kmemmem(const void *_str, int n, const void *_pat, int m, int **_prep);
+
+	/* kstrtok() is similar to strtok_r() except that str is not
+	 * modified and both str and sep can be NULL. For efficiency, it is
+	 * actually recommended to set both to NULL in the subsequent calls
+	 * if sep is not changed. */
+	char *kstrtok(const char *str, const char *sep, ks_tokaux_t *aux);
+
+	/* kgetline() uses the supplied fgets()-like function to read a "\n"-
+	 * or "\r\n"-terminated line from fp.  The line read is appended to the
+	 * kstring without its terminator and 0 is returned; EOF is returned at
+	 * EOF or on error (determined by querying fp, as per fgets()). */
+	typedef char *kgets_func(char *, int, void *);
+	int kgetline(kstring_t *s, kgets_func *fgets, void *fp);
+
+#ifdef __cplusplus
+}
+#endif
+
+static inline int ks_resize(kstring_t *s, size_t size)
+{
+	if (s->m < size) {
+		char *tmp;
+		s->m = size;
+		kroundup32(s->m);
+		if ((tmp = (char*)realloc(s->s, s->m)))
+			s->s = tmp;
+		else
+			return -1;
+	}
+	return 0;
+}
+
+static inline char *ks_str(kstring_t *s)
+{
+	return s->s;
+}
+
+static inline size_t ks_len(kstring_t *s)
+{
+	return s->l;
+}
+
+// Give ownership of the underlying buffer away to something else (making
+// that something else responsible for freeing it), leaving the kstring_t
+// empty and ready to be used again, or ready to go out of scope without
+// needing  free(str.s)  to prevent a memory leak.
+static inline char *ks_release(kstring_t *s)
+{
+	char *ss = s->s;
+	s->l = s->m = 0;
+	s->s = NULL;
+	return ss;
+}
+
+static inline int kputsn(const char *p, int l, kstring_t *s)
+{
+	if (s->l + l + 1 >= s->m) {
+		char *tmp;
+		s->m = s->l + l + 2;
+		kroundup32(s->m);
+		if ((tmp = (char*)realloc(s->s, s->m)))
+			s->s = tmp;
+		else
+			return EOF;
+	}
+	memcpy(s->s + s->l, p, l);
+	s->l += l;
+	s->s[s->l] = 0;
+	return l;
+}
+
+static inline int kputs(const char *p, kstring_t *s)
+{
+	return kputsn(p, strlen(p), s);
+}
+
+static inline int kputc(int c, kstring_t *s)
+{
+	if (s->l + 1 >= s->m) {
+		char *tmp;
+		s->m = s->l + 2;
+		kroundup32(s->m);
+		if ((tmp = (char*)realloc(s->s, s->m)))
+			s->s = tmp;
+		else
+			return EOF;
+	}
+	s->s[s->l++] = c;
+	s->s[s->l] = 0;
+	return c;
+}
+
+static inline int kputc_(int c, kstring_t *s)
+{
+	if (s->l + 1 > s->m) {
+		char *tmp;
+		s->m = s->l + 1;
+		kroundup32(s->m);
+		if ((tmp = (char*)realloc(s->s, s->m)))
+			s->s = tmp;
+		else
+			return EOF;
+	}
+	s->s[s->l++] = c;
+	return 1;
+}
+
+static inline int kputsn_(const void *p, int l, kstring_t *s)
+{
+	if (s->l + l > s->m) {
+		char *tmp;
+		s->m = s->l + l;
+		kroundup32(s->m);
+		if ((tmp = (char*)realloc(s->s, s->m)))
+			s->s = tmp;
+		else
+			return EOF;
+	}
+	memcpy(s->s + s->l, p, l);
+	s->l += l;
+	return l;
+}
+
+static inline int kputw(int c, kstring_t *s)
+{
+	char buf[16];
+	int i, l = 0;
+	unsigned int x = c;
+	if (c < 0) x = -x;
+	do { buf[l++] = x%10 + '0'; x /= 10; } while (x > 0);
+	if (c < 0) buf[l++] = '-';
+	if (s->l + l + 1 >= s->m) {
+		char *tmp;
+		s->m = s->l + l + 2;
+		kroundup32(s->m);
+		if ((tmp = (char*)realloc(s->s, s->m)))
+			s->s = tmp;
+		else
+			return EOF;
+	}
+	for (i = l - 1; i >= 0; --i) s->s[s->l++] = buf[i];
+	s->s[s->l] = 0;
+	return 0;
+}
+
+static inline int kputuw(unsigned c, kstring_t *s)
+{
+	char buf[16];
+	int l, i;
+	unsigned x;
+	if (c == 0) return kputc('0', s);
+	for (l = 0, x = c; x > 0; x /= 10) buf[l++] = x%10 + '0';
+	if (s->l + l + 1 >= s->m) {
+		char *tmp;
+		s->m = s->l + l + 2;
+		kroundup32(s->m);
+		if ((tmp = (char*)realloc(s->s, s->m)))
+			s->s = tmp;
+		else
+			return EOF;
+	}
+	for (i = l - 1; i >= 0; --i) s->s[s->l++] = buf[i];
+	s->s[s->l] = 0;
+	return 0;
+}
+
+static inline int kputl(long c, kstring_t *s)
+{
+	char buf[32];
+	int i, l = 0;
+	unsigned long x = c;
+	if (c < 0) x = -x;
+	do { buf[l++] = x%10 + '0'; x /= 10; } while (x > 0);
+	if (c < 0) buf[l++] = '-';
+	if (s->l + l + 1 >= s->m) {
+		char *tmp;
+		s->m = s->l + l + 2;
+		kroundup32(s->m);
+		if ((tmp = (char*)realloc(s->s, s->m)))
+			s->s = tmp;
+		else
+			return EOF;
+	}
+	for (i = l - 1; i >= 0; --i) s->s[s->l++] = buf[i];
+	s->s[s->l] = 0;
+	return 0;
+}
+
+/*
+ * Returns 's' split by delimiter, with *n being the number of components;
+ *         NULL on failue.
+ */
+static inline int *ksplit(kstring_t *s, int delimiter, int *n)
+{
+	int max = 0, *offsets = 0;
+	*n = ksplit_core(s->s, delimiter, &max, &offsets);
+	return offsets;
+}
+
+#endif
diff --git a/ext/klib/ksw.c b/ext/klib/ksw.c
new file mode 100644
index 0000000..742fec9
--- /dev/null
+++ b/ext/klib/ksw.c
@@ -0,0 +1,633 @@
+/* The MIT License
+
+   Copyright (c) 2011 by Attractive Chaos <attractor@live.co.uk>
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be
+   included in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+*/
+
+#include <stdlib.h>
+#include <stdint.h>
+#include <emmintrin.h>
+#include "ksw.h"
+
+#ifdef __GNUC__
+#define LIKELY(x) __builtin_expect((x),1)
+#define UNLIKELY(x) __builtin_expect((x),0)
+#else
+#define LIKELY(x) (x)
+#define UNLIKELY(x) (x)
+#endif
+
+const kswr_t g_defr = { 0, -1, -1, -1, -1, -1, -1 };
+
+struct _kswq_t {
+	int qlen, slen;
+	uint8_t shift, mdiff, max, size;
+	__m128i *qp, *H0, *H1, *E, *Hmax;
+};
+
+/**
+ * Initialize the query data structure
+ *
+ * @param size   Number of bytes used to store a score; valid valures are 1 or 2
+ * @param qlen   Length of the query sequence
+ * @param query  Query sequence
+ * @param m      Size of the alphabet
+ * @param mat    Scoring matrix in a one-dimension array
+ *
+ * @return       Query data structure
+ */
+kswq_t *ksw_qinit(int size, int qlen, const uint8_t *query, int m, const int8_t *mat)
+{
+	kswq_t *q;
+	int slen, a, tmp, p;
+
+	size = size > 1? 2 : 1;
+	p = 8 * (3 - size); // # values per __m128i
+	slen = (qlen + p - 1) / p; // segmented length
+	q = (kswq_t*)malloc(sizeof(kswq_t) + 256 + 16 * slen * (m + 4)); // a single block of memory
+	q->qp = (__m128i*)(((size_t)q + sizeof(kswq_t) + 15) >> 4 << 4); // align memory
+	q->H0 = q->qp + slen * m;
+	q->H1 = q->H0 + slen;
+	q->E  = q->H1 + slen;
+	q->Hmax = q->E + slen;
+	q->slen = slen; q->qlen = qlen; q->size = size;
+	// compute shift
+	tmp = m * m;
+	for (a = 0, q->shift = 127, q->mdiff = 0; a < tmp; ++a) { // find the minimum and maximum score
+		if (mat[a] < (int8_t)q->shift) q->shift = mat[a];
+		if (mat[a] > (int8_t)q->mdiff) q->mdiff = mat[a];
+	}
+	q->max = q->mdiff;
+	q->shift = 256 - q->shift; // NB: q->shift is uint8_t
+	q->mdiff += q->shift; // this is the difference between the min and max scores
+	// An example: p=8, qlen=19, slen=3 and segmentation:
+	//  {{0,3,6,9,12,15,18,-1},{1,4,7,10,13,16,-1,-1},{2,5,8,11,14,17,-1,-1}}
+	if (size == 1) {
+		int8_t *t = (int8_t*)q->qp;
+		for (a = 0; a < m; ++a) {
+			int i, k, nlen = slen * p;
+			const int8_t *ma = mat + a * m;
+			for (i = 0; i < slen; ++i)
+				for (k = i; k < nlen; k += slen) // p iterations
+					*t++ = (k >= qlen? 0 : ma[query[k]]) + q->shift;
+		}
+	} else {
+		int16_t *t = (int16_t*)q->qp;
+		for (a = 0; a < m; ++a) {
+			int i, k, nlen = slen * p;
+			const int8_t *ma = mat + a * m;
+			for (i = 0; i < slen; ++i)
+				for (k = i; k < nlen; k += slen) // p iterations
+					*t++ = (k >= qlen? 0 : ma[query[k]]);
+		}
+	}
+	return q;
+}
+
+kswr_t ksw_u8(kswq_t *q, int tlen, const uint8_t *target, int _gapo, int _gape, int xtra) // the first gap costs -(_o+_e)
+{
+	int slen, i, m_b, n_b, te = -1, gmax = 0, minsc, endsc;
+	uint64_t *b;
+	__m128i zero, gapoe, gape, shift, *H0, *H1, *E, *Hmax;
+	kswr_t r;
+
+#define __max_16(ret, xx) do { \
+		(xx) = _mm_max_epu8((xx), _mm_srli_si128((xx), 8)); \
+		(xx) = _mm_max_epu8((xx), _mm_srli_si128((xx), 4)); \
+		(xx) = _mm_max_epu8((xx), _mm_srli_si128((xx), 2)); \
+		(xx) = _mm_max_epu8((xx), _mm_srli_si128((xx), 1)); \
+    	(ret) = _mm_extract_epi16((xx), 0) & 0x00ff; \
+	} while (0)
+
+	// initialization
+	r = g_defr;
+	minsc = (xtra&KSW_XSUBO)? xtra&0xffff : 0x10000;
+	endsc = (xtra&KSW_XSTOP)? xtra&0xffff : 0x10000;
+	m_b = n_b = 0; b = 0;
+	zero = _mm_set1_epi32(0);
+	gapoe = _mm_set1_epi8(_gapo + _gape);
+	gape = _mm_set1_epi8(_gape);
+	shift = _mm_set1_epi8(q->shift);
+	H0 = q->H0; H1 = q->H1; E = q->E; Hmax = q->Hmax;
+	slen = q->slen;
+	for (i = 0; i < slen; ++i) {
+		_mm_store_si128(E + i, zero);
+		_mm_store_si128(H0 + i, zero);
+		_mm_store_si128(Hmax + i, zero);
+	}
+	// the core loop
+	for (i = 0; i < tlen; ++i) {
+		int j, k, cmp, imax;
+		__m128i e, h, f = zero, max = zero, *S = q->qp + target[i] * slen; // s is the 1st score vector
+		h = _mm_load_si128(H0 + slen - 1); // h={2,5,8,11,14,17,-1,-1} in the above example
+		h = _mm_slli_si128(h, 1); // h=H(i-1,-1); << instead of >> because x64 is little-endian
+		for (j = 0; LIKELY(j < slen); ++j) {
+			/* SW cells are computed in the following order:
+			 *   H(i,j)   = max{H(i-1,j-1)+S(i,j), E(i,j), F(i,j)}
+			 *   E(i+1,j) = max{H(i,j)-q, E(i,j)-r}
+			 *   F(i,j+1) = max{H(i,j)-q, F(i,j)-r}
+			 */
+			// compute H'(i,j); note that at the beginning, h=H'(i-1,j-1)
+			h = _mm_adds_epu8(h, _mm_load_si128(S + j));
+			h = _mm_subs_epu8(h, shift); // h=H'(i-1,j-1)+S(i,j)
+			e = _mm_load_si128(E + j); // e=E'(i,j)
+			h = _mm_max_epu8(h, e);
+			h = _mm_max_epu8(h, f); // h=H'(i,j)
+			max = _mm_max_epu8(max, h); // set max
+			_mm_store_si128(H1 + j, h); // save to H'(i,j)
+			// now compute E'(i+1,j)
+			h = _mm_subs_epu8(h, gapoe); // h=H'(i,j)-gapo
+			e = _mm_subs_epu8(e, gape); // e=E'(i,j)-gape
+			e = _mm_max_epu8(e, h); // e=E'(i+1,j)
+			_mm_store_si128(E + j, e); // save to E'(i+1,j)
+			// now compute F'(i,j+1)
+			f = _mm_subs_epu8(f, gape);
+			f = _mm_max_epu8(f, h);
+			// get H'(i-1,j) and prepare for the next j
+			h = _mm_load_si128(H0 + j); // h=H'(i-1,j)
+		}
+		// NB: we do not need to set E(i,j) as we disallow adjecent insertion and then deletion
+		for (k = 0; LIKELY(k < 16); ++k) { // this block mimics SWPS3; NB: H(i,j) updated in the lazy-F loop cannot exceed max
+			f = _mm_slli_si128(f, 1);
+			for (j = 0; LIKELY(j < slen); ++j) {
+				h = _mm_load_si128(H1 + j);
+				h = _mm_max_epu8(h, f); // h=H'(i,j)
+				_mm_store_si128(H1 + j, h);
+				h = _mm_subs_epu8(h, gapoe);
+				f = _mm_subs_epu8(f, gape);
+				cmp = _mm_movemask_epi8(_mm_cmpeq_epi8(_mm_subs_epu8(f, h), zero));
+				if (UNLIKELY(cmp == 0xffff)) goto end_loop16;
+			}
+		}
+end_loop16:
+		//int k;for (k=0;k<16;++k)printf("%d ", ((uint8_t*)&max)[k]);printf("\n");
+		__max_16(imax, max); // imax is the maximum number in max
+		if (imax >= minsc) { // write the b array; this condition adds branching unfornately
+			if (n_b == 0 || (int32_t)b[n_b-1] + 1 != i) { // then append
+				if (n_b == m_b) {
+					m_b = m_b? m_b<<1 : 8;
+					b = (uint64_t*)realloc(b, 8 * m_b);
+				}
+				b[n_b++] = (uint64_t)imax<<32 | i;
+			} else if ((int)(b[n_b-1]>>32) < imax) b[n_b-1] = (uint64_t)imax<<32 | i; // modify the last
+		}
+		if (imax > gmax) {
+			gmax = imax; te = i; // te is the end position on the target
+			for (j = 0; LIKELY(j < slen); ++j) // keep the H1 vector
+				_mm_store_si128(Hmax + j, _mm_load_si128(H1 + j));
+			if (gmax + q->shift >= 255 || gmax >= endsc) break;
+		}
+		S = H1; H1 = H0; H0 = S; // swap H0 and H1
+	}
+	r.score = gmax + q->shift < 255? gmax : 255;
+	r.te = te;
+	if (r.score != 255) { // get a->qe, the end of query match; find the 2nd best score
+		int max = -1, low, high, qlen = slen * 16;
+		uint8_t *t = (uint8_t*)Hmax;
+		for (i = 0; i < qlen; ++i, ++t)
+			if ((int)*t > max) max = *t, r.qe = i / 16 + i % 16 * slen;
+		//printf("%d,%d\n", max, gmax);
+		if (b) {
+			i = (r.score + q->max - 1) / q->max;
+			low = te - i; high = te + i;
+			for (i = 0; i < n_b; ++i) {
+				int e = (int32_t)b[i];
+				if ((e < low || e > high) && (int)(b[i]>>32) > r.score2)
+					r.score2 = b[i]>>32, r.te2 = e;
+			}
+		}
+	}
+	free(b);
+	return r;
+}
+
+kswr_t ksw_i16(kswq_t *q, int tlen, const uint8_t *target, int _gapo, int _gape, int xtra) // the first gap costs -(_o+_e)
+{
+	int slen, i, m_b, n_b, te = -1, gmax = 0, minsc, endsc;
+	uint64_t *b;
+	__m128i zero, gapoe, gape, *H0, *H1, *E, *Hmax;
+	kswr_t r;
+
+#define __max_8(ret, xx) do { \
+		(xx) = _mm_max_epi16((xx), _mm_srli_si128((xx), 8)); \
+		(xx) = _mm_max_epi16((xx), _mm_srli_si128((xx), 4)); \
+		(xx) = _mm_max_epi16((xx), _mm_srli_si128((xx), 2)); \
+    	(ret) = _mm_extract_epi16((xx), 0); \
+	} while (0)
+
+	// initialization
+	r = g_defr;
+	minsc = (xtra&KSW_XSUBO)? xtra&0xffff : 0x10000;
+	endsc = (xtra&KSW_XSTOP)? xtra&0xffff : 0x10000;
+	m_b = n_b = 0; b = 0;
+	zero = _mm_set1_epi32(0);
+	gapoe = _mm_set1_epi16(_gapo + _gape);
+	gape = _mm_set1_epi16(_gape);
+	H0 = q->H0; H1 = q->H1; E = q->E; Hmax = q->Hmax;
+	slen = q->slen;
+	for (i = 0; i < slen; ++i) {
+		_mm_store_si128(E + i, zero);
+		_mm_store_si128(H0 + i, zero);
+		_mm_store_si128(Hmax + i, zero);
+	}
+	// the core loop
+	for (i = 0; i < tlen; ++i) {
+		int j, k, imax;
+		__m128i e, h, f = zero, max = zero, *S = q->qp + target[i] * slen; // s is the 1st score vector
+		h = _mm_load_si128(H0 + slen - 1); // h={2,5,8,11,14,17,-1,-1} in the above example
+		h = _mm_slli_si128(h, 2);
+		for (j = 0; LIKELY(j < slen); ++j) {
+			h = _mm_adds_epi16(h, *S++);
+			e = _mm_load_si128(E + j);
+			h = _mm_max_epi16(h, e);
+			h = _mm_max_epi16(h, f);
+			max = _mm_max_epi16(max, h);
+			_mm_store_si128(H1 + j, h);
+			h = _mm_subs_epu16(h, gapoe);
+			e = _mm_subs_epu16(e, gape);
+			e = _mm_max_epi16(e, h);
+			_mm_store_si128(E + j, e);
+			f = _mm_subs_epu16(f, gape);
+			f = _mm_max_epi16(f, h);
+			h = _mm_load_si128(H0 + j);
+		}
+		for (k = 0; LIKELY(k < 16); ++k) {
+			f = _mm_slli_si128(f, 2);
+			for (j = 0; LIKELY(j < slen); ++j) {
+				h = _mm_load_si128(H1 + j);
+				h = _mm_max_epi16(h, f);
+				_mm_store_si128(H1 + j, h);
+				h = _mm_subs_epu16(h, gapoe);
+				f = _mm_subs_epu16(f, gape);
+				if(UNLIKELY(!_mm_movemask_epi8(_mm_cmpgt_epi16(f, h)))) goto end_loop8;
+			}
+		}
+end_loop8:
+		__max_8(imax, max);
+		if (imax >= minsc) {
+			if (n_b == 0 || (int32_t)b[n_b-1] + 1 != i) {
+				if (n_b == m_b) {
+					m_b = m_b? m_b<<1 : 8;
+					b = (uint64_t*)realloc(b, 8 * m_b);
+				}
+				b[n_b++] = (uint64_t)imax<<32 | i;
+			} else if ((int)(b[n_b-1]>>32) < imax) b[n_b-1] = (uint64_t)imax<<32 | i; // modify the last
+		}
+		if (imax > gmax) {
+			gmax = imax; te = i;
+			for (j = 0; LIKELY(j < slen); ++j)
+				_mm_store_si128(Hmax + j, _mm_load_si128(H1 + j));
+			if (gmax >= endsc) break;
+		}
+		S = H1; H1 = H0; H0 = S;
+	}
+	r.score = gmax; r.te = te;
+	{
+		int max = -1, low, high, qlen = slen * 8;
+		uint16_t *t = (uint16_t*)Hmax;
+		for (i = 0, r.qe = -1; i < qlen; ++i, ++t)
+			if ((int)*t > max) max = *t, r.qe = i / 8 + i % 8 * slen;
+		if (b) {
+			i = (r.score + q->max - 1) / q->max;
+			low = te - i; high = te + i;
+			for (i = 0; i < n_b; ++i) {
+				int e = (int32_t)b[i];
+				if ((e < low || e > high) && (int)(b[i]>>32) > r.score2)
+					r.score2 = b[i]>>32, r.te2 = e;
+			}
+		}
+	}
+	free(b);
+	return r;
+}
+
+static void revseq(int l, uint8_t *s)
+{
+	int i, t;
+	for (i = 0; i < l>>1; ++i)
+		t = s[i], s[i] = s[l - 1 - i], s[l - 1 - i] = t;
+}
+
+kswr_t ksw_align(int qlen, uint8_t *query, int tlen, uint8_t *target, int m, const int8_t *mat, int gapo, int gape, int xtra, kswq_t **qry)
+{
+	int size;
+	kswq_t *q;
+	kswr_t r, rr;
+	kswr_t (*func)(kswq_t*, int, const uint8_t*, int, int, int);
+
+	q = (qry && *qry)? *qry : ksw_qinit((xtra&KSW_XBYTE)? 1 : 2, qlen, query, m, mat);
+	if (qry && *qry == 0) *qry = q;
+	func = q->size == 2? ksw_i16 : ksw_u8;
+	size = q->size;
+	r = func(q, tlen, target, gapo, gape, xtra);
+	if (qry == 0) free(q);
+	if ((xtra&KSW_XSTART) == 0 || ((xtra&KSW_XSUBO) && r.score < (xtra&0xffff))) return r;
+	revseq(r.qe + 1, query); revseq(r.te + 1, target); // +1 because qe/te points to the exact end, not the position after the end
+	q = ksw_qinit(size, r.qe + 1, query, m, mat);
+	rr = func(q, tlen, target, gapo, gape, KSW_XSTOP | r.score);
+	revseq(r.qe + 1, query); revseq(r.te + 1, target);
+	free(q);
+	if (r.score == rr.score)
+		r.tb = r.te - rr.te, r.qb = r.qe - rr.qe;
+	return r;
+}
+
+/********************
+ *** SW extension ***
+ ********************/
+
+typedef struct {
+	int32_t h, e;
+} eh_t;
+
+int ksw_extend(int qlen, const uint8_t *query, int tlen, const uint8_t *target, int m, const int8_t *mat, int gapo, int gape, int w, int h0, int *_qle, int *_tle)
+{
+	eh_t *eh; // score array
+	int8_t *qp; // query profile
+	int i, j, k, gapoe = gapo + gape, beg, end, max, max_i, max_j, max_gap;
+	if (h0 < 0) h0 = 0;
+	// allocate memory
+	qp = malloc(qlen * m);
+	eh = calloc(qlen + 1, 8);
+	// generate the query profile
+	for (k = i = 0; k < m; ++k) {
+		const int8_t *p = &mat[k * m];
+		for (j = 0; j < qlen; ++j) qp[i++] = p[query[j]];
+	}
+	// fill the first row
+	eh[0].h = h0; eh[1].h = h0 > gapoe? h0 - gapoe : 0;
+	for (j = 2; j <= qlen && eh[j-1].h > gape; ++j)
+		eh[j].h = eh[j-1].h - gape;
+	// adjust $w if it is too large
+	k = m * m;
+	for (i = 0, max = 0; i < k; ++i) // get the max score
+		max = max > mat[i]? max : mat[i];
+	max_gap = (int)((double)(qlen * max - gapo) / gape + 1.);
+	max_gap = max_gap > 1? max_gap : 1;
+	w = w < max_gap? w : max_gap;
+	// DP loop
+	max = h0, max_i = max_j = -1;
+	beg = 0, end = qlen;
+	for (i = 0; LIKELY(i < tlen); ++i) {
+		int f = 0, h1, m = 0, mj = -1;
+		int8_t *q = &qp[target[i] * qlen];
+		// compute the first column
+		h1 = h0 - (gapo + gape * (i + 1));
+		if (h1 < 0) h1 = 0;
+		// apply the band and the constraint (if provided)
+		if (beg < i - w) beg = i - w;
+		if (end > i + w + 1) end = i + w + 1;
+		if (end > qlen) end = qlen;
+		for (j = beg; LIKELY(j < end); ++j) {
+			// At the beginning of the loop: eh[j] = { H(i-1,j-1), E(i,j) }, f = F(i,j) and h1 = H(i,j-1)
+			// Similar to SSE2-SW, cells are computed in the following order:
+			//   H(i,j)   = max{H(i-1,j-1)+S(i,j), E(i,j), F(i,j)}
+			//   E(i+1,j) = max{H(i,j)-gapo, E(i,j)} - gape
+			//   F(i,j+1) = max{H(i,j)-gapo, F(i,j)} - gape
+			eh_t *p = &eh[j];
+			int h = p->h, e = p->e; // get H(i-1,j-1) and E(i-1,j)
+			p->h = h1;          // set H(i,j-1) for the next row
+			h += q[j];
+			h = h > e? h : e;
+			h = h > f? h : f;
+			h1 = h;             // save H(i,j) to h1 for the next column
+			mj = m > h? mj : j;
+			m = m > h? m : h;   // m is stored at eh[mj+1]
+			h -= gapoe;
+			h = h > 0? h : 0;
+			e -= gape;
+			e = e > h? e : h;   // computed E(i+1,j)
+			p->e = e;           // save E(i+1,j) for the next row
+			f -= gape;
+			f = f > h? f : h;   // computed F(i,j+1)
+		}
+		eh[end].h = h1; eh[end].e = 0;
+		if (m == 0) break;
+		if (m > max) max = m, max_i = i, max_j = mj;
+		// update beg and end for the next round
+		for (j = mj; j >= beg && eh[j].h; --j);
+		beg = j + 1;
+		for (j = mj + 2; j <= end && eh[j].h; ++j);
+		end = j;
+		//beg = 0; end = qlen; // uncomment this line for debugging
+	}
+	free(eh); free(qp);
+	if (_qle) *_qle = max_j + 1;
+	if (_tle) *_tle = max_i + 1;
+	return max;
+}
+
+/********************
+ * Global alignment *
+ ********************/
+
+#define MINUS_INF -0x40000000
+
+static inline uint32_t *push_cigar(int *n_cigar, int *m_cigar, uint32_t *cigar, int op, int len)
+{
+	if (*n_cigar == 0 || op != (cigar[(*n_cigar) - 1]&0xf)) {
+		if (*n_cigar == *m_cigar) {
+			*m_cigar = *m_cigar? (*m_cigar)<<1 : 4;
+			cigar = realloc(cigar, (*m_cigar) << 2);
+		}
+		cigar[(*n_cigar)++] = len<<4 | op;
+	} else cigar[(*n_cigar)-1] += len<<4;
+	return cigar;
+}
+
+int ksw_global(int qlen, const uint8_t *query, int tlen, const uint8_t *target, int m, const int8_t *mat, int gapo, int gape, int w, int *n_cigar_, uint32_t **cigar_)
+{
+	eh_t *eh;
+	int8_t *qp; // query profile
+	int i, j, k, gapoe = gapo + gape, score, n_col;
+	uint8_t *z; // backtrack matrix; in each cell: f<<4|e<<2|h; in principle, we can halve the memory, but backtrack will be a little more complex
+	if (n_cigar_) *n_cigar_ = 0;
+	// allocate memory
+	n_col = qlen < 2*w+1? qlen : 2*w+1; // maximum #columns of the backtrack matrix
+	z = malloc(n_col * tlen);
+	qp = malloc(qlen * m);
+	eh = calloc(qlen + 1, 8);
+	// generate the query profile
+	for (k = i = 0; k < m; ++k) {
+		const int8_t *p = &mat[k * m];
+		for (j = 0; j < qlen; ++j) qp[i++] = p[query[j]];
+	}
+	// fill the first row
+	eh[0].h = 0; eh[0].e = MINUS_INF;
+	for (j = 1; j <= qlen && j <= w; ++j)
+		eh[j].h = -(gapo + gape * j), eh[j].e = MINUS_INF;
+	for (; j <= qlen; ++j) eh[j].h = eh[j].e = MINUS_INF; // everything is -inf outside the band
+	// DP loop
+	for (i = 0; LIKELY(i < tlen); ++i) { // target sequence is in the outer loop
+		int32_t f = MINUS_INF, h1, beg, end;
+		int8_t *q = &qp[target[i] * qlen];
+		uint8_t *zi = &z[i * n_col];
+		beg = i > w? i - w : 0;
+		end = i + w + 1 < qlen? i + w + 1 : qlen; // only loop through [beg,end) of the query sequence
+		h1 = beg == 0? -(gapo + gape * (i + 1)) : MINUS_INF;
+		for (j = beg; LIKELY(j < end); ++j) {
+			// This loop is organized in a similar way to ksw_extend() and ksw_sse2(), except:
+			// 1) not checking h>0; 2) recording direction for backtracking
+			eh_t *p = &eh[j];
+			int32_t h = p->h, e = p->e;
+			uint8_t d; // direction
+			p->h = h1;
+			h += q[j];
+			d = h > e? 0 : 1;
+			h = h > e? h : e;
+			d = h > f? d : 2;
+			h = h > f? h : f;
+			h1 = h;
+			h -= gapoe;
+			e -= gape;
+			d |= e > h? 1<<2 : 0;
+			e = e > h? e : h;
+			p->e = e;
+			f -= gape;
+			d |= f > h? 2<<4 : 0; // if we want to halve the memory, use one bit only, instead of two
+			f = f > h? f : h;
+			zi[j - beg] = d; // z[i,j] keeps h for the current cell and e/f for the next cell
+		}
+		eh[end].h = h1; eh[end].e = MINUS_INF;
+	}
+	score = eh[qlen].h;
+	if (n_cigar_ && cigar_) { // backtrack
+		int n_cigar = 0, m_cigar = 0, which = 0;
+		uint32_t *cigar = 0, tmp;
+		i = tlen - 1; k = (i + w + 1 < qlen? i + w + 1 : qlen) - 1; // (i,k) points to the last cell
+		while (i >= 0 && k >= 0) {
+			which = z[i * n_col + (k - (i > w? i - w : 0))] >> (which<<1) & 3;
+			if (which == 0)      cigar = push_cigar(&n_cigar, &m_cigar, cigar, 0, 1), --i, --k;
+			else if (which == 1) cigar = push_cigar(&n_cigar, &m_cigar, cigar, 2, 1), --i;
+			else                 cigar = push_cigar(&n_cigar, &m_cigar, cigar, 1, 1), --k;
+		}
+		if (i >= 0) cigar = push_cigar(&n_cigar, &m_cigar, cigar, 2, i + 1);
+		if (k >= 0) cigar = push_cigar(&n_cigar, &m_cigar, cigar, 1, k + 1);
+		for (i = 0; i < n_cigar>>1; ++i) // reverse CIGAR
+			tmp = cigar[i], cigar[i] = cigar[n_cigar-1-i], cigar[n_cigar-1-i] = tmp;
+		*n_cigar_ = n_cigar, *cigar_ = cigar;
+	}
+	free(eh); free(qp); free(z);
+	return score;
+}
+
+/*******************************************
+ * Main function (not compiled by default) *
+ *******************************************/
+
+#ifdef _KSW_MAIN
+
+#include <unistd.h>
+#include <stdio.h>
+#include <zlib.h>
+#include "kseq.h"
+KSEQ_INIT(gzFile, gzread)
+
+unsigned char seq_nt4_table[256] = {
+	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
+	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
+	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,
+	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
+	4, 0, 4, 1,  4, 4, 4, 2,  4, 4, 4, 4,  4, 4, 4, 4, 
+	4, 4, 4, 4,  3, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
+	4, 0, 4, 1,  4, 4, 4, 2,  4, 4, 4, 4,  4, 4, 4, 4, 
+	4, 4, 4, 4,  3, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
+	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
+	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
+	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
+	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
+	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
+	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
+	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4, 
+	4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4,  4, 4, 4, 4
+};
+
+int main(int argc, char *argv[])
+{
+	int c, sa = 1, sb = 3, i, j, k, forward_only = 0, max_rseq = 0;
+	int8_t mat[25];
+	int gapo = 5, gape = 2, minsc = 0, xtra = KSW_XSTART;
+	uint8_t *rseq = 0;
+	gzFile fpt, fpq;
+	kseq_t *kst, *ksq;
+
+	// parse command line
+	while ((c = getopt(argc, argv, "a:b:q:r:ft:1")) >= 0) {
+		switch (c) {
+			case 'a': sa = atoi(optarg); break;
+			case 'b': sb = atoi(optarg); break;
+			case 'q': gapo = atoi(optarg); break;
+			case 'r': gape = atoi(optarg); break;
+			case 't': minsc = atoi(optarg); break;
+			case 'f': forward_only = 1; break;
+			case '1': xtra |= KSW_XBYTE; break;
+		}
+	}
+	if (optind + 2 > argc) {
+		fprintf(stderr, "Usage: ksw [-1] [-f] [-a%d] [-b%d] [-q%d] [-r%d] [-t%d] <target.fa> <query.fa>\n", sa, sb, gapo, gape, minsc);
+		return 1;
+	}
+	if (minsc > 0xffff) minsc = 0xffff;
+	xtra |= KSW_XSUBO | minsc;
+	// initialize scoring matrix
+	for (i = k = 0; i < 4; ++i) {
+		for (j = 0; j < 4; ++j)
+			mat[k++] = i == j? sa : -sb;
+		mat[k++] = 0; // ambiguous base
+	}
+	for (j = 0; j < 5; ++j) mat[k++] = 0;
+	// open file
+	fpt = gzopen(argv[optind],   "r"); kst = kseq_init(fpt);
+	fpq = gzopen(argv[optind+1], "r"); ksq = kseq_init(fpq);
+	// all-pair alignment
+	while (kseq_read(ksq) > 0) {
+		kswq_t *q[2] = {0, 0};
+		kswr_t r;
+		for (i = 0; i < (int)ksq->seq.l; ++i) ksq->seq.s[i] = seq_nt4_table[(int)ksq->seq.s[i]];
+		if (!forward_only) { // reverse
+			if ((int)ksq->seq.m > max_rseq) {
+				max_rseq = ksq->seq.m;
+				rseq = (uint8_t*)realloc(rseq, max_rseq);
+			}
+			for (i = 0, j = ksq->seq.l - 1; i < (int)ksq->seq.l; ++i, --j)
+				rseq[j] = ksq->seq.s[i] == 4? 4 : 3 - ksq->seq.s[i];
+		}
+		gzrewind(fpt); kseq_rewind(kst);
+		while (kseq_read(kst) > 0) {
+			for (i = 0; i < (int)kst->seq.l; ++i) kst->seq.s[i] = seq_nt4_table[(int)kst->seq.s[i]];
+			r = ksw_align(ksq->seq.l, (uint8_t*)ksq->seq.s, kst->seq.l, (uint8_t*)kst->seq.s, 5, mat, gapo, gape, xtra, &q[0]);
+			if (r.score >= minsc)
+				printf("%s\t%d\t%d\t%s\t%d\t%d\t%d\t%d\t%d\n", kst->name.s, r.tb, r.te+1, ksq->name.s, r.qb, r.qe+1, r.score, r.score2, r.te2);
+			if (rseq) {
+				r = ksw_align(ksq->seq.l, rseq, kst->seq.l, (uint8_t*)kst->seq.s, 5, mat, gapo, gape, xtra, &q[1]);
+				if (r.score >= minsc)
+					printf("%s\t%d\t%d\t%s\t%d\t%d\t%d\t%d\t%d\n", kst->name.s, r.tb, r.te+1, ksq->name.s, (int)ksq->seq.l - r.qb, (int)ksq->seq.l - 1 - r.qe, r.score, r.score2, r.te2);
+			}
+		}
+		free(q[0]); free(q[1]);
+	}
+	free(rseq);
+	kseq_destroy(kst); gzclose(fpt);
+	kseq_destroy(ksq); gzclose(fpq);
+	return 0;
+}
+#endif
diff --git a/ext/klib/ksw.h b/ext/klib/ksw.h
new file mode 100644
index 0000000..5162dc0
--- /dev/null
+++ b/ext/klib/ksw.h
@@ -0,0 +1,72 @@
+#ifndef __AC_KSW_H
+#define __AC_KSW_H
+
+#include <stdint.h>
+
+#define KSW_XBYTE  0x10000
+#define KSW_XSTOP  0x20000
+#define KSW_XSUBO  0x40000
+#define KSW_XSTART 0x80000
+
+struct _kswq_t;
+typedef struct _kswq_t kswq_t;
+
+typedef struct {
+	int score; // best score
+	int te, qe; // target end and query end
+	int score2, te2; // second best score and ending position on the target
+	int tb, qb; // target start and query start
+} kswr_t;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+	/**
+	 * Aligning two sequences
+	 *
+	 * @param qlen    length of the query sequence (typically <tlen)
+	 * @param query   query sequence with 0 <= query[i] < m
+	 * @param tlen    length of the target sequence
+	 * @param target  target sequence
+	 * @param m       number of residue types
+	 * @param mat     m*m scoring matrix in one-dimention array
+	 * @param gapo    gap open penalty; a gap of length l cost "-(gapo+l*gape)"
+	 * @param gape    gap extension penalty
+	 * @param xtra    extra information (see below)
+	 * @param qry     query profile (see below)
+	 *
+	 * @return        alignment information in a struct; unset values to -1
+	 *
+	 * When xtra==0, ksw_align() uses a signed two-byte integer to store a
+	 * score and only finds the best score and the end positions. The 2nd best
+	 * score or the start positions are not attempted. The default behavior can
+	 * be tuned by setting KSW_X* flags:
+	 *
+	 *   KSW_XBYTE:  use an unsigned byte to store a score. If overflow occurs,
+	 *               kswr_t::score will be set to 255
+	 *
+	 *   KSW_XSUBO:  track the 2nd best score and the ending position on the
+	 *               target if the 2nd best is higher than (xtra&0xffff)
+	 *
+	 *   KSW_XSTOP:  stop if the maximum score is above (xtra&0xffff)
+	 *
+	 *   KSW_XSTART: find the start positions
+	 *
+	 * When *qry==NULL, ksw_align() will compute and allocate the query profile
+	 * and when the function returns, *qry will point to the profile, which can
+	 * be deallocated simply by free(). If one query is aligned against multiple
+	 * target sequences, *qry should be set to NULL during the first call and
+	 * freed after the last call. Note that qry can equal 0. In this case, the
+	 * query profile will be deallocated in ksw_align().
+	 */
+	kswr_t ksw_align(int qlen, uint8_t *query, int tlen, uint8_t *target, int m, const int8_t *mat, int gapo, int gape, int xtra, kswq_t **qry);
+
+	int ksw_extend(int qlen, const uint8_t *query, int tlen, const uint8_t *target, int m, const int8_t *mat, int gapo, int gape, int w, int h0, int *_qle, int *_tle);
+	int ksw_global(int qlen, const uint8_t *query, int tlen, const uint8_t *target, int m, const int8_t *mat, int gapo, int gape, int w, int *_n_cigar, uint32_t **_cigar);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/ext/klib/kthread.c b/ext/klib/kthread.c
new file mode 100644
index 0000000..986f695
--- /dev/null
+++ b/ext/klib/kthread.c
@@ -0,0 +1,256 @@
+#include <pthread.h>
+#include <stdlib.h>
+#include <limits.h>
+
+/************
+ * kt_for() *
+ ************/
+
+struct kt_for_t;
+
+typedef struct {
+	struct kt_for_t *t;
+	long i;
+} ktf_worker_t;
+
+typedef struct kt_for_t {
+	int n_threads;
+	long n;
+	ktf_worker_t *w;
+	void (*func)(void*,long,int);
+	void *data;
+} kt_for_t;
+
+static inline long steal_work(kt_for_t *t)
+{
+	int i, min_i = -1;
+	long k, min = LONG_MAX;
+	for (i = 0; i < t->n_threads; ++i)
+		if (min > t->w[i].i) min = t->w[i].i, min_i = i;
+	k = __sync_fetch_and_add(&t->w[min_i].i, t->n_threads);
+	return k >= t->n? -1 : k;
+}
+
+static void *ktf_worker(void *data)
+{
+	ktf_worker_t *w = (ktf_worker_t*)data;
+	long i;
+	for (;;) {
+		i = __sync_fetch_and_add(&w->i, w->t->n_threads);
+		if (i >= w->t->n) break;
+		w->t->func(w->t->data, i, w - w->t->w);
+	}
+	while ((i = steal_work(w->t)) >= 0)
+		w->t->func(w->t->data, i, w - w->t->w);
+	pthread_exit(0);
+}
+
+void kt_for(int n_threads, void (*func)(void*,long,int), void *data, long n)
+{
+	if (n_threads > 1) {
+		int i;
+		kt_for_t t;
+		pthread_t *tid;
+		t.func = func, t.data = data, t.n_threads = n_threads, t.n = n;
+		t.w = (ktf_worker_t*)alloca(n_threads * sizeof(ktf_worker_t));
+		tid = (pthread_t*)alloca(n_threads * sizeof(pthread_t));
+		for (i = 0; i < n_threads; ++i)
+			t.w[i].t = &t, t.w[i].i = i;
+		for (i = 0; i < n_threads; ++i) pthread_create(&tid[i], 0, ktf_worker, &t.w[i]);
+		for (i = 0; i < n_threads; ++i) pthread_join(tid[i], 0);
+	} else {
+		long j;
+		for (j = 0; j < n; ++j) func(data, j, 0);
+	}
+}
+
+/***************************
+ * kt_for with thread pool *
+ ***************************/
+
+struct kt_forpool_t;
+
+typedef struct {
+	struct kt_forpool_t *t;
+	long i;
+	int action;
+} kto_worker_t;
+
+typedef struct kt_forpool_t {
+	int n_threads, n_pending;
+	long n;
+	pthread_t *tid;
+	kto_worker_t *w;
+	void (*func)(void*,long,int);
+	void *data;
+	pthread_mutex_t mutex;
+	pthread_cond_t cv_m, cv_s;
+} kt_forpool_t;
+
+static inline long kt_fp_steal_work(kt_forpool_t *t)
+{
+	int i, min_i = -1;
+	long k, min = LONG_MAX;
+	for (i = 0; i < t->n_threads; ++i)
+		if (min > t->w[i].i) min = t->w[i].i, min_i = i;
+	k = __sync_fetch_and_add(&t->w[min_i].i, t->n_threads);
+	return k >= t->n? -1 : k;
+}
+
+static void *kt_fp_worker(void *data)
+{
+	kto_worker_t *w = (kto_worker_t*)data;
+	kt_forpool_t *fp = w->t;
+	for (;;) {
+		long i;
+		int action;
+		pthread_mutex_lock(&fp->mutex);
+		if (--fp->n_pending == 0)
+			pthread_cond_signal(&fp->cv_m);
+		w->action = 0;
+		while (w->action == 0) pthread_cond_wait(&fp->cv_s, &fp->mutex);
+		action = w->action;
+		pthread_mutex_unlock(&fp->mutex);
+		if (action < 0) break;
+		for (;;) { // process jobs allocated to this worker
+			i = __sync_fetch_and_add(&w->i, fp->n_threads);
+			if (i >= fp->n) break;
+			fp->func(fp->data, i, w - fp->w);
+		}
+		while ((i = kt_fp_steal_work(fp)) >= 0) // steal jobs allocated to other workers
+			fp->func(fp->data, i, w - fp->w);
+	}
+	pthread_exit(0);
+}
+
+void *kt_forpool_init(int n_threads)
+{
+	kt_forpool_t *fp;
+	int i;
+	fp = (kt_forpool_t*)calloc(1, sizeof(kt_forpool_t));
+	fp->n_threads = fp->n_pending = n_threads;
+	fp->tid = (pthread_t*)calloc(fp->n_threads, sizeof(pthread_t));
+	fp->w = (kto_worker_t*)calloc(fp->n_threads, sizeof(kto_worker_t));
+	for (i = 0; i < fp->n_threads; ++i) fp->w[i].t = fp;
+	pthread_mutex_init(&fp->mutex, 0);
+	pthread_cond_init(&fp->cv_m, 0);
+	pthread_cond_init(&fp->cv_s, 0);
+	for (i = 0; i < fp->n_threads; ++i) pthread_create(&fp->tid[i], 0, kt_fp_worker, &fp->w[i]);
+	pthread_mutex_lock(&fp->mutex);
+	while (fp->n_pending) pthread_cond_wait(&fp->cv_m, &fp->mutex);
+	pthread_mutex_unlock(&fp->mutex);
+	return fp;
+}
+
+void kt_forpool_destroy(void *_fp)
+{
+	kt_forpool_t *fp = (kt_forpool_t*)_fp;
+	int i;
+	for (i = 0; i < fp->n_threads; ++i) fp->w[i].action = -1;
+	pthread_cond_broadcast(&fp->cv_s);
+	for (i = 0; i < fp->n_threads; ++i) pthread_join(fp->tid[i], 0);
+	pthread_cond_destroy(&fp->cv_s);
+	pthread_cond_destroy(&fp->cv_m);
+	pthread_mutex_destroy(&fp->mutex);
+	free(fp->w); free(fp->tid); free(fp);
+}
+
+void kt_forpool(void *_fp, void (*func)(void*,long,int), void *data, long n)
+{
+	kt_forpool_t *fp = (kt_forpool_t*)_fp;
+	long i;
+	if (fp && fp->n_threads > 1) {
+		fp->n = n, fp->func = func, fp->data = data, fp->n_pending = fp->n_threads;
+		for (i = 0; i < fp->n_threads; ++i) fp->w[i].i = i, fp->w[i].action = 1;
+		pthread_mutex_lock(&fp->mutex);
+		pthread_cond_broadcast(&fp->cv_s);
+		while (fp->n_pending) pthread_cond_wait(&fp->cv_m, &fp->mutex);
+		pthread_mutex_unlock(&fp->mutex);
+	} else for (i = 0; i < n; ++i) func(data, i, 0);
+}
+
+/*****************
+ * kt_pipeline() *
+ *****************/
+
+struct ktp_t;
+
+typedef struct {
+	struct ktp_t *pl;
+	int64_t index;
+	int step;
+	void *data;
+} ktp_worker_t;
+
+typedef struct ktp_t {
+	void *shared;
+	void *(*func)(void*, int, void*);
+	int64_t index;
+	int n_workers, n_steps;
+	ktp_worker_t *workers;
+	pthread_mutex_t mutex;
+	pthread_cond_t cv;
+} ktp_t;
+
+static void *ktp_worker(void *data)
+{
+	ktp_worker_t *w = (ktp_worker_t*)data;
+	ktp_t *p = w->pl;
+	while (w->step < p->n_steps) {
+		// test whether we can kick off the job with this worker
+		pthread_mutex_lock(&p->mutex);
+		for (;;) {
+			int i;
+			// test whether another worker is doing the same step
+			for (i = 0; i < p->n_workers; ++i) {
+				if (w == &p->workers[i]) continue; // ignore itself
+				if (p->workers[i].step <= w->step && p->workers[i].index < w->index)
+					break;
+			}
+			if (i == p->n_workers) break; // no workers with smaller indices are doing w->step or the previous steps
+			pthread_cond_wait(&p->cv, &p->mutex);
+		}
+		pthread_mutex_unlock(&p->mutex);
+
+		// working on w->step
+		w->data = p->func(p->shared, w->step, w->step? w->data : 0); // for the first step, input is NULL
+
+		// update step and let other workers know
+		pthread_mutex_lock(&p->mutex);
+		w->step = w->step == p->n_steps - 1 || w->data? (w->step + 1) % p->n_steps : p->n_steps;
+		if (w->step == 0) w->index = p->index++;
+		pthread_cond_broadcast(&p->cv);
+		pthread_mutex_unlock(&p->mutex);
+	}
+	pthread_exit(0);
+}
+
+void kt_pipeline(int n_threads, void *(*func)(void*, int, void*), void *shared_data, int n_steps)
+{
+	ktp_t aux;
+	pthread_t *tid;
+	int i;
+
+	if (n_threads < 1) n_threads = 1;
+	aux.n_workers = n_threads;
+	aux.n_steps = n_steps;
+	aux.func = func;
+	aux.shared = shared_data;
+	aux.index = 0;
+	pthread_mutex_init(&aux.mutex, 0);
+	pthread_cond_init(&aux.cv, 0);
+
+	aux.workers = (ktp_worker_t*)alloca(n_threads * sizeof(ktp_worker_t));
+	for (i = 0; i < n_threads; ++i) {
+		ktp_worker_t *w = &aux.workers[i];
+		w->step = 0; w->pl = &aux; w->data = 0;
+		w->index = aux.index++;
+	}
+
+	tid = (pthread_t*)alloca(n_threads * sizeof(pthread_t));
+	for (i = 0; i < n_threads; ++i) pthread_create(&tid[i], 0, ktp_worker, &aux.workers[i]);
+	for (i = 0; i < n_threads; ++i) pthread_join(tid[i], 0);
+
+	pthread_mutex_destroy(&aux.mutex);
+	pthread_cond_destroy(&aux.cv);
+}
diff --git a/ext/klib/kthread.h b/ext/klib/kthread.h
new file mode 100644
index 0000000..8325c3f
--- /dev/null
+++ b/ext/klib/kthread.h
@@ -0,0 +1,19 @@
+#ifndef KTHREAD_H
+#define KTHREAD_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void kt_for(int n_threads, void (*func)(void*,long,int), void *data, long n);
+void kt_pipeline(int n_threads, void *(*func)(void*, int, void*), void *shared_data, int n_steps);
+
+void *kt_forpool_init(int n_threads);
+void kt_forpool_destroy(void *_fp);
+void kt_forpool(void *_fp, void (*func)(void*,long,int), void *data, long n);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/ext/klib/kurl.c b/ext/klib/kurl.c
new file mode 100644
index 0000000..3bf9290
--- /dev/null
+++ b/ext/klib/kurl.c
@@ -0,0 +1,583 @@
+#include <stdio.h>
+#include <fcntl.h>
+#include <ctype.h>
+#include <assert.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <curl/curl.h>
+#include "kurl.h"
+
+/**********************
+ *** Core kurl APIs ***
+ **********************/
+
+#define KU_DEF_BUFLEN   0x8000
+#define KU_MAX_SKIP     (KU_DEF_BUFLEN<<1) // if seek step is smaller than this, skip
+
+#define kurl_isfile(u) ((u)->fd >= 0)
+
+#ifndef kroundup32
+#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
+#endif
+
+struct kurl_t {
+	CURLM *multi; // cURL multi handler
+	CURL *curl;   // cURL easy handle
+	uint8_t *buf; // buffer
+	off_t off0;   // offset of the first byte in the buffer; the actual file offset equals off0 + p_buf
+	int fd;       // file descriptor for a normal file; <0 for a remote file
+	int m_buf;    // max buffer size; for a remote file, CURL_MAX_WRITE_SIZE*2 is recommended
+	int l_buf;    // length of the buffer; l_buf == 0 iff the input read entirely; l_buf <= m_buf
+	int p_buf;    // file position in the buffer; p_buf <= l_buf
+	int done_reading; // true if we can read nothing from the file; buffer may not be empty even if done_reading is set
+	int err;      // error code
+	struct curl_slist *hdr;
+};
+
+typedef struct {
+	char *url, *date, *auth;
+} s3aux_t;
+
+int kurl_init(void) // required for SSL and win32 socket; NOT thread safe
+{
+	return curl_global_init(CURL_GLOBAL_DEFAULT);
+}
+
+void kurl_destroy(void)
+{
+	curl_global_cleanup();
+}
+
+static int prepare(kurl_t *ku, int do_seek)
+{
+	if (kurl_isfile(ku)) {
+		if (do_seek && lseek(ku->fd, ku->off0, SEEK_SET) != ku->off0)
+			return -1;
+	} else { // FIXME: for S3, we need to re-authorize
+		int rc;
+		rc = curl_multi_remove_handle(ku->multi, ku->curl);
+		rc = curl_easy_setopt(ku->curl, CURLOPT_RESUME_FROM, ku->off0);
+		rc = curl_multi_add_handle(ku->multi, ku->curl);
+	}
+	ku->p_buf = ku->l_buf = 0; // empty the buffer
+	return 0;
+}
+
+static size_t write_cb(char *ptr, size_t size, size_t nmemb, void *data) // callback required by cURL
+{
+	kurl_t *ku = (kurl_t*)data;
+	ssize_t nbytes = size * nmemb;
+	if (nbytes + ku->l_buf > ku->m_buf)
+		return CURL_WRITEFUNC_PAUSE;
+	memcpy(ku->buf + ku->l_buf, ptr, nbytes);
+	ku->l_buf += nbytes;
+	return nbytes;
+}
+
+static int fill_buffer(kurl_t *ku) // fill the buffer
+{
+	assert(ku->p_buf == ku->l_buf); // buffer is always used up when fill_buffer() is called; otherwise a bug
+	ku->off0 += ku->l_buf;
+	ku->p_buf = ku->l_buf = 0;
+	if (ku->done_reading) return 0;
+	if (kurl_isfile(ku)) {
+		// The following block is equivalent to "ku->l_buf = read(ku->fd, ku->buf, ku->m_buf)" on Mac.
+		// On Linux, the man page does not specify whether read() guarantees to read ku->m_buf bytes
+		// even if ->fd references a normal file with sufficient remaining bytes.
+		while (ku->l_buf < ku->m_buf) {
+			int l;
+			l = read(ku->fd, ku->buf + ku->l_buf, ku->m_buf - ku->l_buf);
+			if (l == 0) break;
+			ku->l_buf += l;
+		}
+		if (ku->l_buf < ku->m_buf) ku->done_reading = 1;
+	} else {
+		int n_running, rc;
+		fd_set fdr, fdw, fde;
+		do {
+			int maxfd = -1;
+			long curl_to = -1;
+			struct timeval to;
+			// the following is adaped from docs/examples/fopen.c 
+			to.tv_sec = 10, to.tv_usec = 0; // 10 seconds
+			curl_multi_timeout(ku->multi, &curl_to);
+			if (curl_to >= 0) {
+				to.tv_sec = curl_to / 1000;
+				if (to.tv_sec > 1) to.tv_sec = 1;
+				else to.tv_usec = (curl_to % 1000) * 1000;
+			}
+			FD_ZERO(&fdr); FD_ZERO(&fdw); FD_ZERO(&fde);
+			curl_multi_fdset(ku->multi, &fdr, &fdw, &fde, &maxfd); // FIXME: check return code
+			if (maxfd >= 0 && (rc = select(maxfd+1, &fdr, &fdw, &fde, &to)) < 0) break;
+			if (maxfd < 0) { // check curl_multi_fdset.3 about why we wait for 100ms here
+				struct timespec req, rem;
+				req.tv_sec = 0; req.tv_nsec = 100000000; // this is 100ms
+				nanosleep(&req, &rem);
+			}
+			curl_easy_pause(ku->curl, CURLPAUSE_CONT);
+			rc = curl_multi_perform(ku->multi, &n_running); // FIXME: check return code
+		} while (n_running && ku->l_buf < ku->m_buf - CURL_MAX_WRITE_SIZE);
+		if (ku->l_buf < ku->m_buf - CURL_MAX_WRITE_SIZE) ku->done_reading = 1;
+	}
+	return ku->l_buf;
+}
+
+int kurl_close(kurl_t *ku)
+{
+	if (ku == 0) return 0;
+	if (ku->fd < 0) {
+		curl_multi_remove_handle(ku->multi, ku->curl);
+		curl_easy_cleanup(ku->curl);
+		curl_multi_cleanup(ku->multi);
+		if (ku->hdr) curl_slist_free_all(ku->hdr);
+	} else close(ku->fd);
+	free(ku->buf);
+	free(ku);
+	return 0;
+}
+
+kurl_t *kurl_open(const char *url, kurl_opt_t *opt)
+{
+	extern s3aux_t s3_parse(const char *url, const char *_id, const char *_secret, const char *fn);
+	const char *p, *q;
+	kurl_t *ku;
+	int fd = -1, is_file = 1, failed = 0;
+
+	p = strstr(url, "://");
+	if (p && *p) {
+		for (q = url; q != p; ++q)
+			if (!isalnum(*q)) break;
+		if (q == p) is_file = 0;
+	}
+	if (is_file && (fd = open(url, O_RDONLY)) < 0) return 0;
+
+	ku = (kurl_t*)calloc(1, sizeof(kurl_t));
+	ku->fd = is_file? fd : -1;
+	if (!kurl_isfile(ku)) {
+		ku->multi = curl_multi_init();
+		ku->curl  = curl_easy_init();
+		if (strstr(url, "s3://") == url) {
+			s3aux_t a;
+			a = s3_parse(url, (opt? opt->s3keyid : 0), (opt? opt->s3secretkey : 0), (opt? opt->s3key_fn : 0));
+			if (a.url == 0 || a.date == 0 || a.auth == 0) {
+				kurl_close(ku);
+				return 0;
+			}
+			ku->hdr = curl_slist_append(ku->hdr, a.date);
+			ku->hdr = curl_slist_append(ku->hdr, a.auth);
+			curl_easy_setopt(ku->curl, CURLOPT_URL, a.url);
+			curl_easy_setopt(ku->curl, CURLOPT_HTTPHEADER, ku->hdr);
+			free(a.date); free(a.auth); free(a.url);
+		} else curl_easy_setopt(ku->curl, CURLOPT_URL, url);
+		curl_easy_setopt(ku->curl, CURLOPT_WRITEDATA, ku);
+		curl_easy_setopt(ku->curl, CURLOPT_VERBOSE, 0L);
+		curl_easy_setopt(ku->curl, CURLOPT_NOSIGNAL, 1L);
+		curl_easy_setopt(ku->curl, CURLOPT_WRITEFUNCTION, write_cb);
+		curl_easy_setopt(ku->curl, CURLOPT_SSL_VERIFYPEER, 0L);
+		curl_easy_setopt(ku->curl, CURLOPT_SSL_VERIFYHOST, 0L);
+		curl_easy_setopt(ku->curl, CURLOPT_FOLLOWLOCATION, 1L);
+	}
+	ku->m_buf = KU_DEF_BUFLEN;
+	if (!kurl_isfile(ku) && ku->m_buf < CURL_MAX_WRITE_SIZE * 2)
+		ku->m_buf = CURL_MAX_WRITE_SIZE * 2; // for remote files, the buffer set to 2*CURL_MAX_WRITE_SIZE
+	ku->buf = (uint8_t*)calloc(ku->m_buf, 1);
+	if (kurl_isfile(ku)) failed = (fill_buffer(ku) <= 0);
+	else failed = (prepare(ku, 0) < 0 || fill_buffer(ku) <= 0);
+	if (failed) {
+		kurl_close(ku);
+		return 0;
+	}
+	return ku;
+}
+
+kurl_t *kurl_dopen(int fd)
+{
+	kurl_t *ku;
+	ku = (kurl_t*)calloc(1, sizeof(kurl_t));
+	ku->fd = fd;
+	ku->m_buf = KU_DEF_BUFLEN;
+	ku->buf = (uint8_t*)calloc(ku->m_buf, 1);
+	if (prepare(ku, 0) < 0 || fill_buffer(ku) <= 0) {
+		kurl_close(ku);
+		return 0;
+	}
+	return ku;
+}
+
+int kurl_buflen(kurl_t *ku, int len)
+{
+	if (len <= 0 || len < ku->l_buf) return ku->m_buf;
+	if (!kurl_isfile(ku) && len < CURL_MAX_WRITE_SIZE * 2) return ku->m_buf;
+	ku->m_buf = len;
+	kroundup32(ku->m_buf);
+	ku->buf = (uint8_t*)realloc(ku->buf, ku->m_buf);
+	return ku->m_buf;
+}
+
+ssize_t kurl_read(kurl_t *ku, void *buf, size_t nbytes)
+{
+	ssize_t rest = nbytes;
+	if (ku->l_buf == 0) return 0; // end-of-file
+	while (rest) {
+		if (ku->l_buf - ku->p_buf >= rest) {
+			if (buf) memcpy((uint8_t*)buf + (nbytes - rest), ku->buf + ku->p_buf, rest);
+			ku->p_buf += rest;
+			rest = 0;
+		} else {
+			int ret;
+			if (buf && ku->l_buf > ku->p_buf)
+				memcpy((uint8_t*)buf + (nbytes - rest), ku->buf + ku->p_buf, ku->l_buf - ku->p_buf);
+			rest -= ku->l_buf - ku->p_buf;
+			ku->p_buf = ku->l_buf;
+			ret = fill_buffer(ku);
+			if (ret <= 0) break;
+		}
+	}
+	return nbytes - rest;
+}
+
+off_t kurl_seek(kurl_t *ku, off_t offset, int whence) // FIXME: sometimes when seek() fails, read() will fail as well.
+{
+	off_t new_off = -1, cur_off;
+	int failed = 0, seek_end = 0;
+	if (ku == 0) return -1;
+	cur_off = ku->off0 + ku->p_buf;
+	if (whence == SEEK_SET) new_off = offset;
+	else if (whence == SEEK_CUR) new_off += cur_off + offset;
+	else if (whence == SEEK_END && kurl_isfile(ku)) new_off = lseek(ku->fd, offset, SEEK_END), seek_end = 1;
+	else { // not supported whence
+		ku->err = KURL_INV_WHENCE;
+		return -1;
+	}
+	if (new_off < 0) { // negtive absolute offset
+		ku->err = KURL_SEEK_OUT;
+		return -1;
+	}
+	if (!seek_end && new_off >= cur_off && new_off - cur_off + ku->p_buf < ku->l_buf) {
+		ku->p_buf += new_off - cur_off;
+		return ku->off0 + ku->p_buf;
+	}
+	if (seek_end || new_off < cur_off || new_off - cur_off > KU_MAX_SKIP) { // if jump is large, do actual seek
+		ku->off0 = new_off;
+		ku->done_reading = 0;
+		if (prepare(ku, 1) < 0 || fill_buffer(ku) <= 0) failed = 1;
+	} else { // if jump is small, read through
+		off_t r;
+		r = kurl_read(ku, 0, new_off - cur_off);
+		if (r + cur_off != new_off) failed = 1; // out of range
+	}
+	if (failed) ku->err = KURL_SEEK_OUT, ku->l_buf = ku->p_buf = 0, new_off = -1;
+	return new_off;
+}
+
+off_t kurl_tell(const kurl_t *ku)
+{
+	if (ku == 0) return -1;
+	return ku->off0 + ku->p_buf;
+}
+
+int kurl_eof(const kurl_t *ku)
+{
+	if (ku == 0) return 1;
+	return (ku->l_buf == 0); // unless file end, buffer should never be empty
+}
+
+int kurl_fileno(const kurl_t *ku)
+{
+	if (ku == 0) return -1;
+	return ku->fd;
+}
+
+int kurl_error(const kurl_t *ku)
+{
+	if (ku == 0) return KURL_NULL;
+	return ku->err;
+}
+
+/*****************
+ *** HMAC-SHA1 ***
+ *****************/
+
+/* This code is public-domain - it is based on libcrypt placed in the public domain by Wei Dai and other contributors. */
+
+#define HASH_LENGTH 20
+#define BLOCK_LENGTH 64
+
+typedef struct sha1nfo {
+	union { uint8_t b[BLOCK_LENGTH]; uint32_t w[BLOCK_LENGTH/4]; } buf;
+	uint8_t bufOffset;
+	union { uint8_t b[HASH_LENGTH]; uint32_t w[HASH_LENGTH/4]; } state;
+	uint32_t byteCount;
+	uint8_t keyBuffer[BLOCK_LENGTH];
+	uint8_t innerHash[HASH_LENGTH];
+} sha1nfo;
+
+void sha1_init(sha1nfo *s)
+{
+	const uint8_t table[] = { 0x01,0x23,0x45,0x67, 0x89,0xab,0xcd,0xef, 0xfe,0xdc,0xba,0x98, 0x76,0x54,0x32,0x10, 0xf0,0xe1,0xd2,0xc3 };
+	memcpy(s->state.b, table, HASH_LENGTH);
+	s->byteCount = 0;
+	s->bufOffset = 0;
+}
+
+#define rol32(value, bits) (((value) << (bits)) | ((value) >> (32 - (bits))))
+
+static void sha1_hashBlock(sha1nfo *s)
+{
+	uint32_t i, t, a = s->state.w[0], b = s->state.w[1], c = s->state.w[2], d = s->state.w[3], e = s->state.w[4];
+	for (i = 0; i < 80; i++) {
+		if (i >= 16) {
+			t = s->buf.w[(i+13)&15] ^ s->buf.w[(i+8)&15] ^ s->buf.w[(i+2)&15] ^ s->buf.w[i&15];
+			s->buf.w[i&15] = rol32(t, 1);
+		}
+		if (i < 20)      t = 0x5a827999 + (d ^ (b & (c ^ d)));
+		else if (i < 40) t = 0x6ed9eba1 + (b ^ c ^ d);
+		else if (i < 60) t = 0x8f1bbcdc + ((b & c) | (d & (b | c)));
+		else             t = 0xca62c1d6 + (b ^ c ^ d);
+		t += rol32(a, 5) + e + s->buf.w[i&15];
+		e = d; d = c; c = rol32(b, 30); b = a; a = t;
+	}
+	s->state.w[0] += a; s->state.w[1] += b; s->state.w[2] += c; s->state.w[3] += d; s->state.w[4] += e;
+}
+
+static inline void sha1_add(sha1nfo *s, uint8_t data)
+{
+	s->buf.b[s->bufOffset ^ 3] = data;
+	if (++s->bufOffset == BLOCK_LENGTH) {
+		sha1_hashBlock(s);
+		s->bufOffset = 0;
+	}
+}
+
+void sha1_write1(sha1nfo *s, uint8_t data)
+{
+	++s->byteCount;
+	sha1_add(s, data);
+}
+
+void sha1_write(sha1nfo *s, const char *data, size_t len)
+{
+	while (len--) sha1_write1(s, (uint8_t)*data++);
+}
+
+const uint8_t *sha1_final(sha1nfo *s)
+{
+	int i;
+	sha1_add(s, 0x80);
+	while (s->bufOffset != 56) sha1_add(s, 0);
+	sha1_add(s, 0);
+	sha1_add(s, 0);
+	sha1_add(s, 0);
+	sha1_add(s, s->byteCount >> 29);
+	sha1_add(s, s->byteCount >> 21);
+	sha1_add(s, s->byteCount >> 13);
+	sha1_add(s, s->byteCount >> 5);
+	sha1_add(s, s->byteCount << 3);
+	for (i = 0; i < 5; ++i) {
+		uint32_t a = s->state.w[i];
+		s->state.w[i] = a<<24 | (a<<8&0x00ff0000) | (a>>8&0x0000ff00) | a>>24;
+	}
+	return s->state.b;
+}
+
+#define HMAC_IPAD 0x36
+#define HMAC_OPAD 0x5c
+
+void sha1_init_hmac(sha1nfo *s, const uint8_t* key, int l_key)
+{
+	uint8_t i;
+	memset(s->keyBuffer, 0, BLOCK_LENGTH);
+	if (l_key > BLOCK_LENGTH) {
+		sha1_init(s);
+		while (l_key--) sha1_write1(s, *key++);
+		memcpy(s->keyBuffer, sha1_final(s), HASH_LENGTH);
+	} else memcpy(s->keyBuffer, key, l_key);
+	sha1_init(s);
+	for (i = 0; i < BLOCK_LENGTH; ++i)
+		sha1_write1(s, s->keyBuffer[i] ^ HMAC_IPAD);
+}
+
+const uint8_t *sha1_final_hmac(sha1nfo *s)
+{
+	uint8_t i;
+	memcpy(s->innerHash, sha1_final(s), HASH_LENGTH);
+	sha1_init(s);
+	for (i = 0; i < BLOCK_LENGTH; ++i) sha1_write1(s, s->keyBuffer[i] ^ HMAC_OPAD);
+	for (i = 0; i < HASH_LENGTH; ++i) sha1_write1(s, s->innerHash[i]);
+	return sha1_final(s);
+}
+
+/*******************
+ *** S3 protocol ***
+ *******************/
+
+#include <time.h>
+#include <ctype.h>
+
+static void s3_sign(const char *key, const char *data, char out[29])
+{
+	const char *b64tab = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+	const uint8_t *digest;
+	int i, j, rest;
+	sha1nfo s;
+	sha1_init_hmac(&s, (uint8_t*)key, strlen(key));
+	sha1_write(&s, data, strlen(data));
+	digest = sha1_final_hmac(&s);
+	for (j = i = 0, rest = 8; i < 20; ++j) { // base64 encoding
+		if (rest <= 6) {
+			int next = i < 19? digest[i+1] : 0;
+			out[j] = b64tab[(int)(digest[i] << (6-rest) & 0x3f) | next >> (rest+2)], ++i, rest += 2;
+		} else out[j] = b64tab[(int)digest[i] >> (rest-6) & 0x3f], rest -= 6;
+	}
+	out[j++] = '='; out[j] = 0; // SHA1 digest always has 160 bits, or 20 bytes. We need one '=' at the end.
+}
+
+static char *s3_read_awssecret(const char *fn)
+{
+	char *p, *secret, buf[128], *path;
+	FILE *fp;
+	int l;
+	if (fn == 0) {
+		char *home;
+		home = getenv("HOME");
+		if (home == 0) return 0;
+		l = strlen(home) + 12;
+		path = (char*)malloc(strlen(home) + 12);
+		strcat(strcpy(path, home), "/.awssecret");
+	} else path = (char*)fn;
+	fp = fopen(path, "r");
+	if (path != fn) free(path);
+	if (fp == 0) return 0;
+	l = fread(buf, 1, 127, fp);
+	fclose(fp);
+	buf[l] = 0;
+	for (p = buf; *p != 0 && *p != '\n'; ++p);
+	if (*p == 0) return 0;
+	*p = 0; secret = p + 1;
+	for (++p; *p != 0 && *p != '\n'; ++p);
+	*p = 0;
+	l = p - buf + 1;
+	p = (char*)malloc(l);
+	memcpy(p, buf, l);
+	return p;
+}
+
+typedef struct { int l, m; char *s; } kstring_t;
+
+static inline int kputsn(const char *p, int l, kstring_t *s)
+{
+	if (s->l + l + 1 >= s->m) {
+		s->m = s->l + l + 2;
+		kroundup32(s->m);
+		s->s = (char*)realloc(s->s, s->m);
+	}
+	memcpy(s->s + s->l, p, l);
+	s->l += l;
+	s->s[s->l] = 0;
+	return l;
+}
+
+s3aux_t s3_parse(const char *url, const char *_id, const char *_secret, const char *fn_secret)
+{
+	const char *id, *secret, *bucket, *obj;
+	char *id_secret = 0, date[64], sig[29];
+	time_t t;
+	struct tm tmt;
+	s3aux_t a = {0,0};
+	kstring_t str = {0,0,0};
+	// parse URL
+	if (strstr(url, "s3://") != url) return a;
+	bucket = url + 5;
+	for (obj = bucket; *obj && *obj != '/'; ++obj);
+	if (*obj == 0) return a; // no object
+	// acquire AWS credential and time
+	if (_id == 0 || _secret == 0) {
+		id_secret = s3_read_awssecret(fn_secret);
+		if (id_secret == 0) return a; // fail to read the AWS credential
+		id = id_secret;
+		secret = id_secret + strlen(id) + 1;
+	} else id = _id, secret = _secret;
+	// compose URL for curl
+	kputsn("https://", 8, &str);
+	kputsn(bucket, obj - bucket, &str);
+	kputsn(".s3.amazonaws.com", 17, &str);
+	kputsn(obj, strlen(obj), &str);
+	a.url = str.s;
+	// compose the Date line
+	str.l = str.m = 0; str.s = 0;
+	t = time(0);
+	strftime(date, 64, "%a, %d %b %Y %H:%M:%S +0000", gmtime_r(&t, &tmt));
+	kputsn("Date: ", 6, &str);
+	kputsn(date, strlen(date), &str);
+	a.date = str.s;
+	// compose the string to sign and sign it
+	str.l = str.m = 0; str.s = 0;
+	kputsn("GET\n\n\n", 6, &str);
+	kputsn(date, strlen(date), &str);
+	kputsn("\n", 1, &str);
+	kputsn(bucket-1, strlen(bucket-1), &str);
+	s3_sign(secret, str.s, sig);
+	// compose the Authorization line
+	str.l = 0;
+	kputsn("Authorization: AWS ", 19, &str);
+	kputsn(id, strlen(id), &str);
+	kputsn(":", 1, &str);
+	kputsn(sig, strlen(sig), &str);
+	a.auth = str.s;
+//	printf("curl -H '%s' -H '%s' %s\n", a.date, a.auth, a.url);
+	return a;
+}
+
+/*********************
+ *** Main function ***
+ *********************/
+
+#ifdef KURL_MAIN
+int main(int argc, char *argv[])
+{
+	kurl_t *f;
+	int c, l, l_buf = 0x10000;
+	off_t start = 0, rest = -1;
+	uint8_t *buf;
+	char *p;
+	kurl_opt_t opt;
+
+	memset(&opt, 0, sizeof(kurl_opt_t));
+	while ((c = getopt(argc, argv, "c:l:a:")) >= 0) {
+		if (c == 'c') start = strtol(optarg, &p, 0);
+		else if (c == 'l') rest = strtol(optarg, &p, 0);
+		else if (c == 'a') opt.s3key_fn = optarg;
+	}
+	if (optind == argc) {
+		fprintf(stderr, "Usage: kurl [-c start] [-l length] <url>\n");
+		return 1;
+	}
+	kurl_init();
+	f = kurl_open(argv[optind], &opt);
+	if (f == 0) {
+		fprintf(stderr, "ERROR: fail to open URL\n");
+		return 2;
+	}
+	if (start > 0) {
+		if (kurl_seek(f, start, SEEK_SET) < 0) {
+			kurl_close(f);
+			fprintf(stderr, "ERROR: fail to seek\n");
+			return 3;
+		}
+	}
+	buf = (uint8_t*)calloc(l_buf, 1);
+	while (rest != 0) {
+		int to_read = rest > 0 && rest < l_buf? rest : l_buf;
+		l = kurl_read(f, buf, to_read);
+		if (l == 0) break;
+		fwrite(buf, 1, l, stdout);
+		rest -= l;
+	}
+	free(buf);
+	kurl_close(f);
+	kurl_destroy();
+	return 0;
+}
+#endif
diff --git a/ext/klib/kurl.h b/ext/klib/kurl.h
new file mode 100644
index 0000000..f07f641
--- /dev/null
+++ b/ext/klib/kurl.h
@@ -0,0 +1,57 @@
+#ifndef KURL_H
+#define KURL_H
+
+#include <sys/types.h>
+
+#define KURL_NULL       1
+#define KURL_INV_WHENCE 2
+#define KURL_SEEK_OUT   3
+#define KURL_NO_AUTH    4
+
+struct kurl_t;
+typedef struct kurl_t kurl_t;
+
+typedef struct {
+	const char *s3keyid;
+	const char *s3secretkey;
+	const char *s3key_fn;
+} kurl_opt_t;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int kurl_init(void);
+void kurl_destroy(void);
+
+kurl_t *kurl_open(const char *url, kurl_opt_t *opt);
+kurl_t *kurl_dopen(int fd);
+int kurl_close(kurl_t *ku);
+ssize_t kurl_read(kurl_t *ku, void *buf, size_t nbytes);
+off_t kurl_seek(kurl_t *ku, off_t offset, int whence);
+int kurl_buflen(kurl_t *ku, int len);
+
+off_t kurl_tell(const kurl_t *ku);
+int kurl_eof(const kurl_t *ku);
+int kurl_fileno(const kurl_t *ku);
+int kurl_error(const kurl_t *ku);
+
+#ifdef __cplusplus
+}
+#endif
+
+#ifndef KNETFILE_H
+#define KNETFILE_H
+typedef kurl_t knetFile;
+#define knet_open(fn, mode) kurl_open(fn, 0)
+#define knet_dopen(fd, mode) kurl_dopen(fd)
+#define knet_close(fp) kurl_close(fp)
+#define knet_read(fp, buf, len) kurl_read(fp, buf, len)
+#define knet_seek(fp, off, whence) kurl_seek(fp, off, whence)
+#define knet_tell(fp) kurl_tell(fp)
+#define knet_fileno(fp) kurl_fileno(fp)
+#define knet_win32_init() kurl_init()
+#define knet_win32_destroy() kurl_destroy()
+#endif
+
+#endif
diff --git a/ext/klib/kvec.h b/ext/klib/kvec.h
new file mode 100644
index 0000000..676be8b
--- /dev/null
+++ b/ext/klib/kvec.h
@@ -0,0 +1,90 @@
+/* The MIT License
+
+   Copyright (c) 2008, by Attractive Chaos <attractor@live.co.uk>
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be
+   included in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+*/
+
+/*
+  An example:
+
+#include "kvec.h"
+int main() {
+	kvec_t(int) array;
+	kv_init(array);
+	kv_push(int, array, 10); // append
+	kv_a(int, array, 20) = 5; // dynamic
+	kv_A(array, 20) = 4; // static
+	kv_destroy(array);
+	return 0;
+}
+*/
+
+/*
+  2008-09-22 (0.1.0):
+
+	* The initial version.
+
+*/
+
+#ifndef AC_KVEC_H
+#define AC_KVEC_H
+
+#include <stdlib.h>
+
+#define kv_roundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
+
+#define kvec_t(type) struct { size_t n, m; type *a; }
+#define kv_init(v) ((v).n = (v).m = 0, (v).a = 0)
+#define kv_destroy(v) free((v).a)
+#define kv_A(v, i) ((v).a[(i)])
+#define kv_pop(v) ((v).a[--(v).n])
+#define kv_size(v) ((v).n)
+#define kv_max(v) ((v).m)
+
+#define kv_resize(type, v, s)  ((v).m = (s), (v).a = (type*)realloc((v).a, sizeof(type) * (v).m))
+
+#define kv_copy(type, v1, v0) do {							\
+		if ((v1).m < (v0).n) kv_resize(type, v1, (v0).n);	\
+		(v1).n = (v0).n;									\
+		memcpy((v1).a, (v0).a, sizeof(type) * (v0).n);		\
+	} while (0)												\
+
+#define kv_push(type, v, x) do {									\
+		if ((v).n == (v).m) {										\
+			(v).m = (v).m? (v).m<<1 : 2;							\
+			(v).a = (type*)realloc((v).a, sizeof(type) * (v).m);	\
+		}															\
+		(v).a[(v).n++] = (x);										\
+	} while (0)
+
+#define kv_pushp(type, v) (((v).n == (v).m)?							\
+						   ((v).m = ((v).m? (v).m<<1 : 2),				\
+							(v).a = (type*)realloc((v).a, sizeof(type) * (v).m), 0)	\
+						   : 0), ((v).a + ((v).n++))
+
+#define kv_a(type, v, i) (((v).m <= (size_t)(i)? \
+						  ((v).m = (v).n = (i) + 1, kv_roundup32((v).m), \
+						   (v).a = (type*)realloc((v).a, sizeof(type) * (v).m), 0) \
+						  : (v).n <= (size_t)(i)? (v).n = (i) + 1 \
+						  : 0), (v).a[(i)])
+
+#endif
diff --git a/ext/klib/lua/bio.lua b/ext/klib/lua/bio.lua
new file mode 100644
index 0000000..c9f2200
--- /dev/null
+++ b/ext/klib/lua/bio.lua
@@ -0,0 +1,149 @@
+-- bioinformatics routines
+
+-- Description: read a fasta/fastq file
+local function readseq(fp)
+	local finished, last = false, nil;
+	return function()
+		local match;
+		if finished then return nil end
+		if (last == nil) then -- the first record or a record following a fastq
+			for l in fp:lines() do
+				if l:byte(1) == 62 or l:byte(1) == 64 then -- ">" || "@"
+					last = l;
+					break;
+				end
+			end
+			if last == nil then
+				finished = true;
+				return nil;
+			end
+		end
+		local tmp = last:find("%s");
+		name = (tmp and last:sub(2, tmp-1)) or last:sub(2); -- sequence name
+		local seqs = {};
+		local c; -- the first character of the last line
+		last = nil;
+		for l in fp:lines() do -- read sequence
+			c = l:byte(1);
+			if c == 62 or c == 64 or c == 43 then
+				last = l;
+				break;
+			end
+			table.insert(seqs, l);
+		end
+		if last == nil then finished = true end -- end of file
+		if c ~= 43 then return name, table.concat(seqs) end -- a fasta record
+		local seq, len = table.concat(seqs), 0; -- prepare to parse quality
+		seqs = {};
+		for l in fp:lines() do -- read quality
+			table.insert(seqs, l);
+			len = len + #l;
+			if len >= #seq then
+				last = nil;
+				return name, seq, table.concat(seqs);
+			end
+		end
+		finished = true;
+		return name, seq;
+	end
+end
+
+-- extract subsequence from a fasta file indexe by samtools faidx
+local function faidxsub(fn)
+	local fpidx = io.open(fn .. ".fai");
+	if fpidx == nil then
+		io.stderr:write("[faidxsub] fail to open the FASTA index file.\n");
+		return nil
+	end
+	local idx = {};
+	for l in fpidx:lines() do
+		local name, len, offset, line_blen, line_len = l:match("(%S+)%s(%d+)%s(%d+)%s(%d+)%s(%d+)");
+		if name then
+			idx[name] = {tonumber(len), offset, line_blen, line_len};
+		end
+	end
+	fpidx:close();
+	local fp = io.open(fn);
+	return function(name, beg_, end_) -- 0-based coordinate
+		if name == nil then fp:close(); return nil; end
+		if idx[name] then
+			local a = idx[name];
+			beg_ = beg_ or 0;
+			end_ = end_ or a[1];
+			end_ = (end_ <= a[1] and end_) or a[1];
+			local fb, fe = math.floor(beg_ / a[3]), math.floor(end_ / a[3]);
+			local qb, qe = beg_ - fb * a[3], end_ - fe * a[3];
+			fp:seek("set", a[2] + fb * a[4] + qb);
+			local s = fp:read((fe - fb) * a[4] + (qe - qb)):gsub("%s", "");
+			return s;
+		end
+	end
+end
+
+--Description: Index a list of intervals and test if a given interval overlaps with the list
+--Example: lua -lbio -e 'a={{100,201},{200,300},{400,600}};f=bio.intvovlp(a);print(f(600,700))'
+--[[
+  By default, we keep for each tiling 8192 window the interval overlaping the
+  window while having the smallest start position. This method may not work
+  well when most intervals are small but few intervals span a long distance.
+]]--
+local function intvovlp(intv, bits)
+	bits = bits or 13 -- the default bin size is 8192 = 1<<13
+	table.sort(intv, function(a,b) return a[1] < b[1] end) -- sort by the start
+	-- merge intervals; the step speeds up testing, but can be skipped
+	local b, e, k = -1, -1, 1
+	for i = 1, #intv do
+		if e < intv[i][1] then
+			if e >= 0 then intv[k], k = {b, e}, k + 1 end
+			b, e = intv[i][1], intv[i][2]
+		else e = intv[i][2] end
+	end
+	if e >= 0 then intv[k] = {b, e} end
+	while #a > k do table.remove(a) end -- truncate the interval list
+	-- build the index for the list of intervals
+	local idx, size, max = {}, math.pow(2, bits), 0
+	for i = 1, #a do
+		b = math.modf(intv[i][1] / size)
+		e = math.modf(intv[i][2] / size)
+		if b == e then idx[b] = idx[b] or i
+		else for j = b, e do idx[j] = idx[j] or i end end
+		max = (max > e and max) or e
+	end
+	-- return a function (closure)
+	return function(_beg, _end)
+		local x = math.modf(_beg / size)
+		if x > max then return false end
+		local off = idx[x]; -- the start bin
+		if off == nil then -- the following is not the best in efficiency
+			for i = x - 1, 0, -1 do -- find the minimum bin with a value
+				if idx[i] ~= nil then off = idx[i]; break; end
+			end
+			if off == nil then return false end
+		end
+		for i = off, #intv do -- start from off and search for overlaps
+			if intv[i][1] >= _end then return false
+			elseif intv[i][2] > _beg then return true end 
+		end
+		return false
+	end
+end
+
+bio = {
+	readseq = readseq,
+	faidxsub = faidxsub,
+	intvovlp = intvovlp
+}
+
+bio.nt16 = {
+[0]=15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,
+	15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,
+	15, 1,14, 2, 13,15,15, 4, 11,15,15,12, 15, 3,15,15, 15,15, 5, 6,  8,15, 7, 9,  0,10,15,15, 15,15,15,15,
+	15, 1,14, 2, 13,15,15, 4, 11,15,15,12, 15, 3,15,15, 15,15, 5, 6,  8,15, 7, 9,  0,10,15,15, 15,15,15,15,
+	15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,
+	15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,
+	15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15,
+	15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15, 15,15,15,15
+}
+bio.ntcnt  = { [0]=4, 1, 1,  2, 1,  2, 2,  3, 1, 2, 2,  3, 2,  3, 3,  4 }
+bio.ntcomp = { [0]=0, 8, 4, 12, 2, 10, 9, 14, 1, 6, 5, 13, 3, 11, 7, 15 }
+bio.ntrev  = 'XACMGRSVTWYHKDBN'
diff --git a/ext/klib/lua/klib.lua b/ext/klib/lua/klib.lua
new file mode 100644
index 0000000..bfe52f7
--- /dev/null
+++ b/ext/klib/lua/klib.lua
@@ -0,0 +1,677 @@
+--[[
+  The MIT License
+  
+  Copyright (c) 2011, Attractive Chaos <attractor@live.co.uk>
+  
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+  
+  The above copyright notice and this permission notice shall be included in
+  all copies or substantial portions of the Software.
+  
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
+]]--
+
+--[[
+  This is a Lua library, more exactly a collection of Lua snippets, covering
+  utilities (e.g. getopt), string operations (e.g. split), statistics (e.g.
+  Fisher's exact test), special functions (e.g. logarithm gamma) and matrix
+  operations (e.g. Gauss-Jordan elimination). The routines are designed to be
+  as independent as possible, such that one can copy-paste relevant pieces of
+  code without worrying about additional library dependencies.
+
+  If you use routines from this library, please include the licensing
+  information above where appropriate.
+]]--
+
+--[[
+  Library functions and dependencies. "a>b" means "a is required by b"; "b<a"
+  means "b depends on a".
+
+  os.getopt()
+  string:split()
+  io.xopen()
+  table.ksmall()
+  table.shuffle()
+  math.lgamma() >math.lbinom() >math.igamma()
+  math.igamma() <math.lgamma() >matrix.chi2()
+  math.erfc()
+  math.lbinom() <math.lgamma() >math.fisher_exact()
+  math.bernstein_poly() <math.lbinom()
+  math.fisher_exact() <math.lbinom()
+  math.jackknife()
+  math.pearson()
+  math.spearman()
+  math.fmin()
+  matrix
+  matrix.add()
+  matrix.T() >matrix.mul()
+  matrix.mul() <matrix.T()
+  matrix.tostring()
+  matrix.chi2() <math.igamma()
+  matrix.solve()
+]]--
+
+-- Description: getopt() translated from the BSD getopt(); compatible with the default Unix getopt()
+--[[ Example:
+	for o, a in os.getopt(arg, 'a:b') do
+		print(o, a)
+	end
+]]--
+function os.getopt(args, ostr)
+	local arg, place = nil, 0;
+	return function ()
+		if place == 0 then -- update scanning pointer
+			place = 1
+			if #args == 0 or args[1]:sub(1, 1) ~= '-' then place = 0; return nil end
+			if #args[1] >= 2 then
+				place = place + 1
+				if args[1]:sub(2, 2) == '-' then -- found "--"
+					place = 0
+					table.remove(args, 1);
+					return nil;
+				end
+			end
+		end
+		local optopt = args[1]:sub(place, place);
+		place = place + 1;
+		local oli = ostr:find(optopt);
+		if optopt == ':' or oli == nil then -- unknown option
+			if optopt == '-' then return nil end
+			if place > #args[1] then
+				table.remove(args, 1);
+				place = 0;
+			end
+			return '?';
+		end
+		oli = oli + 1;
+		if ostr:sub(oli, oli) ~= ':' then -- do not need argument
+			arg = nil;
+			if place > #args[1] then
+				table.remove(args, 1);
+				place = 0;
+			end
+		else -- need an argument
+			if place <= #args[1] then  -- no white space
+				arg = args[1]:sub(place);
+			else
+				table.remove(args, 1);
+				if #args == 0 then -- an option requiring argument is the last one
+					place = 0;
+					if ostr:sub(1, 1) == ':' then return ':' end
+					return '?';
+				else arg = args[1] end
+			end
+			table.remove(args, 1);
+			place = 0;
+		end
+		return optopt, arg;
+	end
+end
+
+-- Description: string split
+function string:split(sep, n)
+	local a, start = {}, 1;
+	sep = sep or "%s+";
+	repeat
+		local b, e = self:find(sep, start);
+		if b == nil then
+			table.insert(a, self:sub(start));
+			break
+		end
+		a[#a+1] = self:sub(start, b - 1);
+		start = e + 1;
+		if n and #a == n then
+			table.insert(a, self:sub(start));
+			break
+		end
+	until start > #self;
+	return a;
+end
+
+-- Description: smart file open
+function io.xopen(fn, mode)
+	mode = mode or 'r';
+	if fn == nil then return io.stdin;
+	elseif fn == '-' then return (mode == 'r' and io.stdin) or io.stdout;
+	elseif fn:sub(-3) == '.gz' then return (mode == 'r' and io.popen('gzip -dc ' .. fn, 'r')) or io.popen('gzip > ' .. fn, 'w');
+	elseif fn:sub(-4) == '.bz2' then return (mode == 'r' and io.popen('bzip2 -dc ' .. fn, 'r')) or io.popen('bgzip2 > ' .. fn, 'w');
+	else return io.open(fn, mode) end
+end
+
+-- Description: find the k-th smallest element in an array (Ref. http://ndevilla.free.fr/median/)
+function table.ksmall(arr, k)
+	local low, high = 1, #arr;
+	while true do
+		if high <= low then return arr[k] end
+		if high == low + 1 then
+			if arr[high] < arr[low] then arr[high], arr[low] = arr[low], arr[high] end;
+			return arr[k];
+		end
+		local mid = math.floor((high + low) / 2);
+		if arr[high] < arr[mid] then arr[mid], arr[high] = arr[high], arr[mid] end
+		if arr[high] < arr[low] then arr[low], arr[high] = arr[high], arr[low] end
+		if arr[low]  < arr[mid] then arr[low], arr[mid]  = arr[mid],  arr[low] end
+		arr[mid], arr[low+1] = arr[low+1], arr[mid];
+		local ll, hh = low + 1, high;
+		while true do
+			repeat ll = ll + 1 until arr[ll] >= arr[low]
+			repeat hh = hh - 1 until arr[low] >= arr[hh]
+			if hh < ll then break end
+			arr[ll], arr[hh] = arr[hh], arr[ll];
+		end
+		arr[low], arr[hh] = arr[hh], arr[low];
+		if hh <= k then low = ll end
+		if hh >= k then high = hh - 1 end
+	end
+end
+
+-- Description: shuffle/permutate an array
+function table.shuffle(a)
+	for i = #a, 1, -1 do
+		local j = math.random(i)
+		a[j], a[i] = a[i], a[j]
+	end
+end
+
+--
+-- Mathematics
+--
+
+-- Description: log gamma function
+-- Required by: math.lbinom()
+-- Reference: AS245, 2nd algorithm, http://lib.stat.cmu.edu/apstat/245
+function math.lgamma(z)
+	local x;
+	x = 0.1659470187408462e-06     / (z+7);
+	x = x + 0.9934937113930748e-05 / (z+6);
+	x = x - 0.1385710331296526     / (z+5);
+	x = x + 12.50734324009056      / (z+4);
+	x = x - 176.6150291498386      / (z+3);
+	x = x + 771.3234287757674      / (z+2);
+	x = x - 1259.139216722289      / (z+1);
+	x = x + 676.5203681218835      / z;
+	x = x + 0.9999999999995183;
+	return math.log(x) - 5.58106146679532777 - z + (z-0.5) * math.log(z+6.5);
+end
+
+-- Description: regularized incomplete gamma function
+-- Dependent on: math.lgamma()
+--[[
+  Formulas are taken from Wiki, with additional input from Numerical
+  Recipes in C (for modified Lentz's algorithm) and AS245
+  (http://lib.stat.cmu.edu/apstat/245).
+ 
+  A good online calculator is available at:
+ 
+    http://www.danielsoper.com/statcalc/calc23.aspx
+ 
+  It calculates upper incomplete gamma function, which equals
+  math.igamma(s,z,true)*math.exp(math.lgamma(s))
+]]--
+function math.igamma(s, z, complement)
+
+	local function _kf_gammap(s, z)
+		local sum, x = 1, 1;
+		for k = 1, 100 do
+			x = x * z / (s + k);
+			sum = sum + x;
+			if x / sum < 1e-14 then break end
+		end
+		return math.exp(s * math.log(z) - z - math.lgamma(s + 1.) + math.log(sum));
+	end
+
+	local function _kf_gammaq(s, z)
+		local C, D, f, TINY;
+		f = 1. + z - s; C = f; D = 0.; TINY = 1e-290;
+		-- Modified Lentz's algorithm for computing continued fraction. See Numerical Recipes in C, 2nd edition, section 5.2
+		for j = 1, 100 do
+			local d;
+			local a, b = j * (s - j), j*2 + 1 + z - s;
+			D = b + a * D;
+			if D < TINY then D = TINY end
+			C = b + a / C;
+			if C < TINY then C = TINY end
+			D = 1. / D;
+			d = C * D;
+			f = f * d;
+			if math.abs(d - 1) < 1e-14 then break end
+		end
+		return math.exp(s * math.log(z) - z - math.lgamma(s) - math.log(f));
+	end
+
+	if complement then
+		return ((z <= 1 or z < s) and 1 - _kf_gammap(s, z)) or _kf_gammaq(s, z);
+	else 
+		return ((z <= 1 or z < s) and _kf_gammap(s, z)) or (1 - _kf_gammaq(s, z));
+	end
+end
+
+math.M_SQRT2   = 1.41421356237309504880  -- sqrt(2)
+math.M_SQRT1_2 = 0.70710678118654752440  -- 1/sqrt(2)
+
+-- Description: complement error function erfc(x): \Phi(x) = 0.5 * erfc(-x/M_SQRT2)
+function math.erfc(x)
+	local z = math.abs(x) * math.M_SQRT2
+	if z > 37 then return (x > 0 and 0) or 2 end
+	local expntl = math.exp(-0.5 * z * z)
+	local p
+	if z < 10. / math.M_SQRT2 then -- for small z
+	    p = expntl * ((((((.03526249659989109 * z + .7003830644436881) * z + 6.37396220353165) * z + 33.912866078383)
+				* z + 112.0792914978709) * z + 221.2135961699311) * z + 220.2068679123761)
+			/ (((((((.08838834764831844 * z + 1.755667163182642) * z + 16.06417757920695) * z + 86.78073220294608)
+				* z + 296.5642487796737) * z + 637.3336333788311) * z + 793.8265125199484) * z + 440.4137358247522);
+	else p = expntl / 2.506628274631001 / (z + 1. / (z + 2. / (z + 3. / (z + 4. / (z + .65))))) end
+	return (x > 0 and 2 * p) or 2 * (1 - p)
+end
+
+-- Description: log binomial coefficient
+-- Dependent on: math.lgamma()
+-- Required by: math.fisher_exact()
+function math.lbinom(n, m)
+	if m == nil then
+		local a = {};
+		a[0], a[n] = 0, 0;
+		local t = math.lgamma(n+1);
+		for m = 1, n-1 do a[m] = t - math.lgamma(m+1) - math.lgamma(n-m+1) end
+		return a;
+	else return math.lgamma(n+1) - math.lgamma(m+1) - math.lgamma(n-m+1) end
+end
+
+-- Description: Berstein polynomials (mainly for Bezier curves)
+-- Dependent on: math.lbinom()
+-- Note: to compute derivative: let beta_new[i]=beta[i+1]-beta[i]
+function math.bernstein_poly(beta)
+	local n = #beta - 1;
+	local lbc = math.lbinom(n); -- log binomial coefficients
+	return function (t)
+		assert(t >= 0 and t <= 1);
+		if t == 0 then return beta[1] end
+		if t == 1 then return beta[n+1] end
+		local sum, logt, logt1 = 0, math.log(t), math.log(1-t);
+		for i = 0, n do sum = sum + beta[i+1] * math.exp(lbc[i] + i * logt + (n-i) * logt1) end
+		return sum;
+	end
+end
+
+-- Description: Fisher's exact test
+-- Dependent on: math.lbinom()
+-- Return: left-, right- and two-tail P-values
+--[[
+  Fisher's exact test for 2x2 congintency tables:
+
+    n11  n12  | n1_
+    n21  n22  | n2_
+   -----------+----
+    n_1  n_2  | n
+
+  Reference: http://www.langsrud.com/fisher.htm
+]]--
+function math.fisher_exact(n11, n12, n21, n22)
+	local aux; -- keep the states of n* for acceleration
+
+	-- Description: hypergeometric function
+	local function hypergeo(n11, n1_, n_1, n)
+		return math.exp(math.lbinom(n1_, n11) + math.lbinom(n-n1_, n_1-n11) - math.lbinom(n, n_1));
+	end
+
+	-- Description: incremental hypergeometric function
+	-- Note: aux = {n11, n1_, n_1, n, p}
+	local function hypergeo_inc(n11, n1_, n_1, n)
+		if n1_ ~= 0 or n_1 ~= 0 or n ~= 0 then
+			aux = {n11, n1_, n_1, n, 1};
+		else -- then only n11 is changed
+			local mod;
+			_, mod = math.modf(n11 / 11);
+			if mod ~= 0 and n11 + aux[4] - aux[2] - aux[3] ~= 0 then
+				if n11 == aux[1] + 1 then -- increase by 1
+					aux[5] = aux[5] * (aux[2] - aux[1]) / n11 * (aux[3] - aux[1]) / (n11 + aux[4] - aux[2] - aux[3]);
+					aux[1] = n11;
+					return aux[5];
+				end
+				if n11 == aux[1] - 1 then -- descrease by 1
+					aux[5] = aux[5] * aux[1] / (aux[2] - n11) * (aux[1] + aux[4] - aux[2] - aux[3]) / (aux[3] - n11);
+					aux[1] = n11;
+					return aux[5];
+				end
+			end
+			aux[1] = n11;
+		end
+		aux[5] = hypergeo(aux[1], aux[2], aux[3], aux[4]);
+		return aux[5];
+	end
+	
+	-- Description: computing the P-value by Fisher's exact test
+	local max, min, left, right, n1_, n_1, n, two, p, q, i, j;
+	n1_, n_1, n = n11 + n12, n11 + n21, n11 + n12 + n21 + n22;
+	max = (n_1 < n1_ and n_1) or n1_; -- max n11, for the right tail
+	min = n1_ + n_1 - n;
+	if min < 0 then min = 0 end -- min n11, for the left tail
+	two, left, right = 1, 1, 1;
+	if min == max then return 1 end -- no need to do test
+	q = hypergeo_inc(n11, n1_, n_1, n); -- the probability of the current table
+	-- left tail
+	i, left, p = min + 1, 0, hypergeo_inc(min, 0, 0, 0);
+	while p < 0.99999999 * q do
+		left, p, i = left + p, hypergeo_inc(i, 0, 0, 0), i + 1;
+	end
+	i = i - 1;
+	if p < 1.00000001 * q then left = left + p;
+	else i = i - 1 end
+	-- right tail
+	j, right, p = max - 1, 0, hypergeo_inc(max, 0, 0, 0);
+	while p < 0.99999999 * q do
+		right, p, j = right + p, hypergeo_inc(j, 0, 0, 0), j - 1;
+	end
+	j = j + 1;
+	if p < 1.00000001 * q then right = right + p;
+	else j = j + 1 end
+	-- two-tail
+	two = left + right;
+	if two > 1 then two = 1 end
+	-- adjust left and right
+	if math.abs(i - n11) < math.abs(j - n11) then right = 1 - left + q;
+	else left = 1 - right + q end
+	return left, right, two;
+end
+
+-- Description: Delete-m Jackknife
+--[[
+  Given g groups of values with a statistics estimated from m[i] samples in
+  i-th group being t[i], compute the mean and the variance. t0 below is the
+  estimate from all samples. Reference:
+
+     Busing et al. (1999) Delete-m Jackknife for unequal m. Statistics and Computing, 9:3-8.
+]]--
+function math.jackknife(g, m, t, t0)
+	local h, n, sum = {}, 0, 0;
+	for j = 1, g do n = n + m[j] end
+	if t0 == nil then -- When t0 is absent, estimate it in a naive way
+		t0 = 0;
+		for j = 1, g do t0 = t0 + m[j] * t[j] end
+		t0 = t0 / n;
+	end
+	local mean, var = 0, 0;
+	for j = 1, g do
+		h[j] = n / m[j];
+		mean = mean + (1 - m[j] / n) * t[j];
+	end
+	mean = g * t0 - mean; -- Eq. (8)
+	for j = 1, g do
+		local x = h[j] * t0 - (h[j] - 1) * t[j] - mean;
+		var = var + 1 / (h[j] - 1) * x * x;
+	end
+	var = var / g;
+	return mean, var;
+end
+
+-- Description: Pearson correlation coefficient
+-- Input: a is an n*2 table
+function math.pearson(a)
+	-- compute the mean
+	local x1, y1 = 0, 0
+	for _, v in pairs(a) do
+		x1, y1 = x1 + v[1], y1 + v[2]
+	end
+	-- compute the coefficient
+	x1, y1 = x1 / #a, y1 / #a
+	local x2, y2, xy = 0, 0, 0
+	for _, v in pairs(a) do
+		local tx, ty = v[1] - x1, v[2] - y1
+		xy, x2, y2 = xy + tx * ty, x2 + tx * tx, y2 + ty * ty
+	end
+	return xy / math.sqrt(x2) / math.sqrt(y2)
+end
+
+-- Description: Spearman correlation coefficient
+function math.spearman(a)
+	local function aux_func(t) -- auxiliary function
+		return (t == 1 and 0) or (t*t - 1) * t / 12
+	end
+
+	for _, v in pairs(a) do v.r = {} end
+	local T, S = {}, {}
+	-- compute the rank
+	for k = 1, 2 do
+		table.sort(a, function(u,v) return u[k]<v[k] end)
+		local same = 1
+		T[k] = 0
+		for i = 2, #a + 1 do
+			if i <= #a and a[i-1][k] == a[i][k] then same = same + 1
+			else
+				local rank = (i-1) * 2 - same + 1
+				for j = i - same, i - 1 do a[j].r[k] = rank end
+				if same > 1 then T[k], same = T[k] + aux_func(same), 1 end
+			end
+		end
+		S[k] = aux_func(#a) - T[k]
+	end
+	-- compute the coefficient
+	local sum = 0
+	for _, v in pairs(a) do -- TODO: use nested loops to reduce loss of precision
+		local t = (v.r[1] - v.r[2]) / 2
+		sum = sum + t * t
+	end
+	return (S[1] + S[2] - sum) / 2 / math.sqrt(S[1] * S[2])
+end
+
+-- Description: Hooke-Jeeves derivative-free optimization
+function math.fmin(func, x, data, r, eps, max_calls)
+	local n, n_calls = #x, 0;
+	r = r or 0.5;
+	eps = eps or 1e-7;
+	max_calls = max_calls or 50000
+
+	function fmin_aux(x1, data, fx1, dx) -- auxiliary function
+		local ftmp;
+		for k = 1, n do
+			x1[k] = x1[k] + dx[k];
+			local ftmp = func(x1, data); n_calls = n_calls + 1;
+			if ftmp < fx1 then fx1 = ftmp;
+			else -- search the opposite direction
+				dx[k] = -dx[k];
+				x1[k] = x1[k] + dx[k] + dx[k];
+				ftmp = func(x1, data); n_calls = n_calls + 1;
+				if ftmp < fx1 then fx1 = ftmp
+				else x1[k] = x1[k] - dx[k] end -- back to the original x[k]
+			end
+		end
+		return fx1; -- here: fx1=f(n,x1)
+	end
+
+	local dx, x1 = {}, {};
+	for k = 1, n do -- initial directions, based on MGJ
+		dx[k] = math.abs(x[k]) * r;
+		if dx[k] == 0 then dx[k] = r end;
+	end
+	local radius = r;
+	local fx1, fx;
+	fx = func(x, data); fx1 = fx; n_calls = n_calls + 1;
+	while true do
+		for i = 1, n do x1[i] = x[i] end; -- x1 = x
+		fx1 = fmin_aux(x1, data, fx, dx);
+		while fx1 < fx do
+			for k = 1, n do
+				local t = x[k];
+				dx[k] = (x1[k] > x[k] and math.abs(dx[k])) or -math.abs(dx[k]);
+				x[k] = x1[k];
+				x1[k] = x1[k] + x1[k] - t;
+			end
+			fx = fx1;
+			if n_calls >= max_calls then break end
+			fx1 = func(x1, data); n_calls = n_calls + 1;
+			fx1 = fmin_aux(x1, data, fx1, dx);
+			if fx1 >= fx then break end
+			local kk = n;
+			for k = 1, n do
+				if math.abs(x1[k] - x[k]) > .5 * math.abs(dx[k]) then
+					kk = k;
+					break;
+				end
+			end
+			if kk == n then break end
+		end
+		if radius >= eps then
+			if n_calls >= max_calls then break end
+			radius = radius * r;
+			for k = 1, n do dx[k] = dx[k] * r end
+		else break end
+	end
+	return fx1, n_calls;
+end
+
+--
+-- Matrix
+--
+
+matrix = {}
+
+-- Description: matrix transpose
+-- Required by: matrix.mul()
+function matrix.T(a)
+	local m, n, x = #a, #a[1], {};
+	for i = 1, n do
+		x[i] = {};
+		for j = 1, m do x[i][j] = a[j][i] end
+	end
+	return x;
+end
+
+-- Description: matrix add
+function matrix.add(a, b)
+	assert(#a == #b and #a[1] == #b[1]);
+	local m, n, x = #a, #a[1], {};
+	for i = 1, m do
+		x[i] = {};
+		local ai, bi, xi = a[i], b[i], x[i];
+		for j = 1, n do xi[j] = ai[j] + bi[j] end
+	end
+	return x;
+end
+
+-- Description: matrix mul
+-- Dependent on: matrix.T()
+-- Note: much slower without transpose
+function matrix.mul(a, b)
+	assert(#a[1] == #b);
+	local m, n, p, x = #a, #a[1], #b[1], {};
+	local c = matrix.T(b); -- transpose for efficiency
+	for i = 1, m do
+		x[i] = {}
+		local xi = x[i];
+		for j = 1, p do
+			local sum, ai, cj = 0, a[i], c[j];
+			for k = 1, n do sum = sum + ai[k] * cj[k] end
+			xi[j] = sum;
+		end
+	end
+	return x;
+end
+
+-- Description: matrix print
+function matrix.tostring(a)
+	local z = {};
+	for i = 1, #a do
+		z[i] = table.concat(a[i], "\t");
+	end
+	return table.concat(z, "\n");
+end
+
+-- Description: chi^2 test for contingency tables
+-- Dependent on: math.igamma()
+function matrix.chi2(a)
+	if #a == 2 and #a[1] == 2 then -- 2x2 table
+		local x, z
+		x = (a[1][1] + a[1][2]) * (a[2][1] + a[2][2]) * (a[1][1] + a[2][1]) * (a[1][2] + a[2][2])
+		if x == 0 then return 0, 1, false end
+		z = a[1][1] * a[2][2] - a[1][2] * a[2][1]
+		z = (a[1][1] + a[1][2] + a[2][1] + a[2][2]) * z * z / x
+		return z, math.igamma(.5, .5 * z, true), true
+	else -- generic table
+		local rs, cs, n, m, N, z = {}, {}, #a, #a[1], 0, 0
+		for i = 1, n do rs[i] = 0 end
+		for j = 1, m do cs[j] = 0 end
+		for i = 1, n do -- compute column sum and row sum
+			for j = 1, m do cs[j], rs[i] = cs[j] + a[i][j], rs[i] + a[i][j] end
+		end
+		for i = 1, n do N = N + rs[i] end
+		for i = 1, n do -- compute the chi^2 statistics
+			for j = 1, m do
+				local E = rs[i] * cs[j] / N;
+				z = z + (a[i][j] - E) * (a[i][j] - E) / E
+			end
+		end
+		return z, math.igamma(.5 * (n-1) * (m-1), .5 * z, true), true;
+	end
+end
+
+-- Description: Gauss-Jordan elimination (solving equations; computing inverse)
+-- Note: on return, a[n][n] is the inverse; b[n][m] is the solution
+-- Reference: Section 2.1, Numerical Recipes in C, 2nd edition
+function matrix.solve(a, b)
+	assert(#a == #a[1]);
+	local n, m = #a, (b and #b[1]) or 0;
+	local xc, xr, ipiv = {}, {}, {};
+	local ic, ir;
+
+	for j = 1, n do ipiv[j] = 0 end
+	for i = 1, n do
+		local big = 0;
+		for j = 1, n do
+			local aj = a[j];
+			if ipiv[j] ~= 1 then
+				for k = 1, n do
+					if ipiv[k] == 0 then
+						if math.abs(aj[k]) >= big then
+							big = math.abs(aj[k]);
+							ir, ic = j, k;
+						end
+					elseif ipiv[k] > 1 then return -2 end -- singular matrix
+				end
+			end
+		end
+		ipiv[ic] = ipiv[ic] + 1;
+		if ir ~= ic then
+			for l = 1, n do a[ir][l], a[ic][l] = a[ic][l], a[ir][l] end
+			if b then
+				for l = 1, m do b[ir][l], b[ic][l] = b[ic][l], b[ir][l] end
+			end
+		end
+		xr[i], xc[i] = ir, ic;
+		if a[ic][ic] == 0 then return -3 end -- singular matrix
+		local pivinv = 1 / a[ic][ic];
+		a[ic][ic] = 1;
+		for l = 1, n do a[ic][l] = a[ic][l] * pivinv end
+		if b then
+			for l = 1, n do b[ic][l] = b[ic][l] * pivinv end
+		end
+		for ll = 1, n do
+			if ll ~= ic then
+				local tmp = a[ll][ic];
+				a[ll][ic] = 0;
+				local all, aic = a[ll], a[ic];
+				for l = 1, n do all[l] = all[l] - aic[l] * tmp end
+				if b then
+					local bll, bic = b[ll], b[ic];
+					for l = 1, m do bll[l] = bll[l] - bic[l] * tmp end
+				end
+			end
+		end
+	end
+	for l = n, 1, -1 do
+		if xr[l] ~= xc[l] then
+			for k = 1, n do a[k][xr[l]], a[k][xc[l]] = a[k][xc[l]], a[k][xr[l]] end
+		end
+	end
+	return 0;
+end
diff --git a/ext/klib/test/Makefile b/ext/klib/test/Makefile
new file mode 100644
index 0000000..c27d721
--- /dev/null
+++ b/ext/klib/test/Makefile
@@ -0,0 +1,72 @@
+CC=gcc
+CXX=g++
+CFLAGS=-g -Wall -O2 -I..
+CXXFLAGS=$(CFLAGS)
+PROGS=kbtree_test khash_keith khash_keith2 khash_test klist_test kseq_test kseq_bench \
+		kseq_bench2 ksort_test ksort_test-stl kvec_test kmin_test kstring_bench kstring_bench2 kstring_test \
+		kavl_test kavl-lite_test kthread_test2
+
+all:$(PROGS)
+
+clean:
+		rm -fr $(PROGS) *.dSYM a.out *.o
+
+kavl_test:kavl_test.c ../kavl.h
+		$(CC) $(CFLAGS) -o $@ kavl_test.c
+
+kavl-lite_test:kavl-lite_test.c ../kavl-lite.h
+		$(CC) $(CFLAGS) -o $@ kavl-lite_test.c
+
+kbtree_test:kbtree_test.c ../kbtree.h
+		$(CC) $(CFLAGS) -o $@ kbtree_test.c
+
+khash_keith:khash_keith.c ../khash.h
+		$(CC) $(CFLAGS) -o $@ khash_keith.c
+
+khash_keith2:khash_keith2.c ../khash.h
+		$(CC) $(CFLAGS) -o $@ khash_keith2.c
+
+khash_test:khash_test.c ../khash.h
+		$(CC) $(CFLAGS) -o $@ khash_test.c
+
+klist_test:klist_test.c ../klist.h
+		$(CC) $(CFLAGS) -o $@ klist_test.c
+
+kseq_test:kseq_test.c ../kseq.h
+		$(CC) $(CFLAGS) -o $@ kseq_test.c -lz
+
+kseq_bench:kseq_bench.c ../kseq.h
+		$(CC) $(CFLAGS) -o $@ kseq_bench.c -lz
+
+kseq_bench2:kseq_bench2.c ../kseq.h
+		$(CC) $(CFLAGS) -o $@ kseq_bench2.c -lz
+
+ksort_test:ksort_test.c ../ksort.h
+		$(CC) $(CFLAGS) -o $@ ksort_test.c
+
+ksort_test-stl:ksort_test.cc ../ksort.h
+		$(CXX) $(CXXFLAGS) -o $@ ksort_test.cc
+
+kvec_test:kvec_test.cc ../kvec.h
+		$(CXX) $(CXXFLAGS) -o $@ kvec_test.cc
+
+kmin_test:kmin_test.c ../kmath.h ../kmath.c
+		$(CC) $(CFLAGS) -o $@ kmin_test.c ../kmath.c
+
+kstring_bench:kstring_bench.c ../kstring.h ../kstring.c
+		$(CC) $(CFLAGS) -o $@ kstring_bench.c ../kstring.c
+
+kstring_bench2:kstring_bench2.c ../kstring.h ../kstring.c
+		$(CC) $(CFLAGS) -o $@ kstring_bench2.c ../kstring.c
+
+kstring_test:kstring_test.c ../kstring.h ../kstring.c
+		$(CC) $(CFLAGS) -o $@ kstring_test.c ../kstring.c
+
+kthread_test:kthread_test.c ../kthread.c
+		$(CC) $(CFLAGS) -fopenmp -o $@ kthread_test.c ../kthread.c
+
+kthread_test2:kthread_test2.c ../kthread.c
+		$(CC) $(CFLAGS) -o $@ kthread_test2.c ../kthread.c
+
+ketopt_test:ketopt_test.c ../ketopt.h
+		$(CC) $(CFLAGS) -o $@ ketopt_test.c
diff --git a/ext/klib/test/kavl-lite_test.c b/ext/klib/test/kavl-lite_test.c
new file mode 100644
index 0000000..375d5d3
--- /dev/null
+++ b/ext/klib/test/kavl-lite_test.c
@@ -0,0 +1,60 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <string.h>
+#include "kavl-lite.h"
+
+#define CALLOC(type, num) ((type*)calloc(num, sizeof(type)))
+
+struct my_node {
+	int key;
+	KAVLL_HEAD(struct my_node) head;
+};
+
+#define my_cmp(p, q) (((p)->key > (q)->key) - ((p)->key < (q)->key))
+KAVLL_INIT(my, struct my_node, head, my_cmp)
+
+void shuffle(int n, char a[])
+{
+	int i, j;
+	for (i = n; i > 1; --i) {
+		char tmp;
+		j = (int)(drand48() * i);
+		tmp = a[j]; a[j] = a[i-1]; a[i-1] = tmp;
+	}
+}
+
+int main(void)
+{
+	char buf[256];
+	int i, n;
+	struct my_node *root = 0;
+	struct my_node *p, *q, t;
+	my_itr_t itr;
+
+	for (i = 33, n = 0; i <= 126; ++i)
+		if (i != '(' && i != ')' && i != '.' && i != ';')
+			buf[n++] = i;
+	shuffle(n, buf);
+	for (i = 0; i < n; ++i) {
+		p = CALLOC(struct my_node, 1);
+		p->key = buf[i];
+		q = my_insert(&root, p);
+		if (p != q) free(p);
+	}
+	shuffle(n, buf);
+	for (i = 0; i < n/2; ++i) {
+		t.key = buf[i];
+		q = my_erase(&root, &t);
+		if (q) free(q);
+	}
+
+	my_itr_first(root, &itr);
+	do {
+		const struct my_node *r = kavll_at(&itr);
+		putchar(r->key);
+		free((void*)r);
+	} while (my_itr_next(&itr));
+	putchar('\n');
+	return 0;
+}
diff --git a/ext/klib/test/kavl_test.c b/ext/klib/test/kavl_test.c
new file mode 100644
index 0000000..15ce395
--- /dev/null
+++ b/ext/klib/test/kavl_test.c
@@ -0,0 +1,104 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <string.h>
+#include "kavl.h"
+
+#define CALLOC(type, num) ((type*)calloc(num, sizeof(type)))
+
+struct my_node {
+	int key;
+	KAVL_HEAD(struct my_node) head;
+};
+
+#define my_cmp(p, q) (((p)->key > (q)->key) - ((p)->key < (q)->key))
+KAVL_INIT(my, struct my_node, head, my_cmp)
+
+int check(struct my_node *p, int *hh)
+{
+	int c = 1, h[2] = {0, 0};
+	*hh = 0;
+	if (p) {
+		if (p->head.p[0]) c += check(p->head.p[0], &h[0]);
+		if (p->head.p[1]) c += check(p->head.p[1], &h[1]);
+		*hh = (h[0] > h[1]? h[0] : h[1]) + 1;
+		if (h[1] - h[0] != (int)p->head.balance)
+			fprintf(stderr, "%d - %d != %d at %c\n", h[1], h[0], p->head.balance, p->key);
+		if (c != (int)p->head.size)
+			fprintf(stderr, "%d != %d at %c\n", p->head.size, c, p->key);
+		return c;
+	} else return 0;
+}
+/*
+int print_tree(const struct my_node *p)
+{
+	int c = 1;
+	if (p == 0) return 0;
+	if (p->head.p[0] || p->head.p[1]) {
+		putchar('(');
+		if (p->head.p[0]) c += print_tree(p->head.p[0]);
+		else putchar('.');
+		putchar(',');
+		if (p->head.p[1]) c += print_tree(p->head.p[1]);
+		else putchar('.');
+		putchar(')');
+	}
+	putchar(p->key);
+	return c;
+}
+
+void check_and_print(struct my_node *root)
+{
+	int h;
+	check(root, &h);
+	print_tree(root);
+	putchar('\n');
+}
+*/
+void shuffle(int n, char a[])
+{
+	int i, j;
+	for (i = n; i > 1; --i) {
+		char tmp;
+		j = (int)(drand48() * i);
+		tmp = a[j]; a[j] = a[i-1]; a[i-1] = tmp;
+	}
+}
+
+int main(void)
+{
+	char buf[256];
+	int i, n, h;
+	struct my_node *root = 0;
+	struct my_node *p, *q, t;
+	kavl_itr_t(my) itr;
+	unsigned cnt;
+
+	for (i = 33, n = 0; i <= 126; ++i)
+		if (i != '(' && i != ')' && i != '.' && i != ';')
+			buf[n++] = i;
+	shuffle(n, buf);
+	for (i = 0; i < n; ++i) {
+		p = CALLOC(struct my_node, 1);
+		p->key = buf[i];
+		q = kavl_insert(my, &root, p, &cnt);
+		if (p != q) free(p);
+		check(root, &h);
+	}
+	shuffle(n, buf);
+	for (i = 0; i < n/2; ++i) {
+		t.key = buf[i];
+		q = kavl_erase(my, &root, &t, 0);
+		if (q) free(q);
+		check(root, &h);
+	}
+
+	kavl_itr_first(my, root, &itr);
+	do {
+		const struct my_node *r = kavl_at(&itr);
+		putchar(r->key);
+		free((void*)r);
+	} while (kavl_itr_next(my, &itr));
+	putchar('\n');
+	return 0;
+}
diff --git a/ext/klib/test/kbit_test.c b/ext/klib/test/kbit_test.c
new file mode 100644
index 0000000..3ae3bd3
--- /dev/null
+++ b/ext/klib/test/kbit_test.c
@@ -0,0 +1,137 @@
+#include <stdlib.h>
+#include <stdio.h>
+#include <time.h>
+#include <emmintrin.h>
+#include "kbit.h"
+
+// from bowtie-0.9.8.1
+inline static int bt1_pop64(uint64_t x) // the kbi_popcount64() equivalence; similar to popcount_2() in wiki
+{
+   x -= ((x >> 1) & 0x5555555555555555llu);
+   x = (x & 0x3333333333333333llu) + ((x >> 2) & 0x3333333333333333llu);
+   x = (x + (x >> 4)) & 0x0F0F0F0F0F0F0F0Fllu;
+   x = x + (x >> 8);
+   x = x + (x >> 16);
+   x = x + (x >> 32);
+   return x & 0x3F;
+}
+
+inline static int bt1_countInU64(uint64_t dw, int c) // the kbi_DNAcount64() equivalence
+{
+	uint64_t dwA  = dw &  0xAAAAAAAAAAAAAAAAllu;
+	uint64_t dwNA = dw & ~0xAAAAAAAAAAAAAAAAllu;
+	uint64_t tmp;
+	switch (c) {
+	case 0: tmp = (dwA >> 1) | dwNA; break;
+	case 1: tmp = ~(dwA >> 1) & dwNA; break;
+	case 2: tmp = (dwA >> 1) & ~dwNA; break;
+	default: tmp = (dwA >> 1) & dwNA;
+	}
+	tmp = bt1_pop64(tmp);
+	if (c == 0) tmp = 32 - tmp;
+	return (int)tmp;
+}
+
+// from bigmagic
+static uint32_t sse2_bit_count32(const __m128i* block, const __m128i* block_end)
+{
+    const unsigned mu1 = 0x55555555;
+    const unsigned mu2 = 0x33333333;
+    const unsigned mu3 = 0x0F0F0F0F;
+    const unsigned mu4 = 0x0000003F;
+
+	uint32_t tcnt[4];
+
+    // Loading masks
+    __m128i m1 = _mm_set_epi32 (mu1, mu1, mu1, mu1);
+    __m128i m2 = _mm_set_epi32 (mu2, mu2, mu2, mu2);
+    __m128i m3 = _mm_set_epi32 (mu3, mu3, mu3, mu3);
+    __m128i m4 = _mm_set_epi32 (mu4, mu4, mu4, mu4);
+    __m128i mcnt;
+    mcnt = _mm_xor_si128(m1, m1); // cnt = 0
+
+    __m128i tmp1, tmp2;
+    do
+    {        
+        __m128i b = _mm_load_si128(block);
+        ++block;
+
+        // b = (b & 0x55555555) + (b >> 1 & 0x55555555);
+        tmp1 = _mm_srli_epi32(b, 1);                    // tmp1 = (b >> 1 & 0x55555555)
+        tmp1 = _mm_and_si128(tmp1, m1); 
+        tmp2 = _mm_and_si128(b, m1);                    // tmp2 = (b & 0x55555555)
+        b    = _mm_add_epi32(tmp1, tmp2);               //  b = tmp1 + tmp2
+
+        // b = (b & 0x33333333) + (b >> 2 & 0x33333333);
+        tmp1 = _mm_srli_epi32(b, 2);                    // (b >> 2 & 0x33333333)
+        tmp1 = _mm_and_si128(tmp1, m2); 
+        tmp2 = _mm_and_si128(b, m2);                    // (b & 0x33333333)
+        b    = _mm_add_epi32(tmp1, tmp2);               // b = tmp1 + tmp2
+
+        // b = (b + (b >> 4)) & 0x0F0F0F0F;
+        tmp1 = _mm_srli_epi32(b, 4);                    // tmp1 = b >> 4
+        b = _mm_add_epi32(b, tmp1);                     // b = b + (b >> 4)
+        b = _mm_and_si128(b, m3);                       //           & 0x0F0F0F0F
+
+        // b = b + (b >> 8);
+        tmp1 = _mm_srli_epi32 (b, 8);                   // tmp1 = b >> 8
+        b = _mm_add_epi32(b, tmp1);                     // b = b + (b >> 8)
+
+        // b = (b + (b >> 16)) & 0x0000003F;
+        tmp1 = _mm_srli_epi32 (b, 16);                  // b >> 16
+        b = _mm_add_epi32(b, tmp1);                     // b + (b >> 16)
+        b = _mm_and_si128(b, m4);                       // (b >> 16) & 0x0000003F;
+
+        mcnt = _mm_add_epi32(mcnt, b);                  // mcnt += b
+
+    } while (block < block_end);
+
+    _mm_store_si128((__m128i*)tcnt, mcnt);
+
+    return tcnt[0] + tcnt[1] + tcnt[2] + tcnt[3];
+}
+
+int main(void)
+{
+	int i, N = 100000000;
+	uint64_t *x, cnt;
+	clock_t t;
+	int c = 1;
+
+	x = (uint64_t*)calloc(N, 8);
+	srand48(11);
+	for (i = 0; i < N; ++i)
+		x[i] = (uint64_t)lrand48() << 32 ^ lrand48();
+
+	fprintf(stderr, "\n===> Calculate # of 1 in an integer (popcount) <===\n");
+
+	t = clock(); cnt = 0;
+	for (i = 0; i < N; ++i) cnt += kbi_popcount64(x[i]);
+	fprintf(stderr, "%20s\t%20ld\t%10.6f\n", "kbit", (long)cnt, (double)(clock() - t) / CLOCKS_PER_SEC);
+
+	t = clock(); cnt = 0;
+	for (i = 0; i < N; ++i) cnt += bt1_pop64(x[i]);
+	fprintf(stderr, "%20s\t%20ld\t%10.6f\n", "wiki-popcount_2", (long)cnt, (double)(clock() - t) / CLOCKS_PER_SEC);
+
+	t = clock(); cnt = 0;
+	for (i = 0; i < N; ++i) cnt += __builtin_popcountl(x[i]);
+	fprintf(stderr, "%20s\t%20ld\t%10.6f\n", "__builtin_popcountl", (long)cnt, (double)(clock() - t) / CLOCKS_PER_SEC);
+
+	t = clock(); cnt = 0;
+	cnt += sse2_bit_count32((__m128i*)x, (__m128i*)(x+N));
+	fprintf(stderr, "%20s\t%20ld\t%10.6f\n", "SSE2-32bit", (long)cnt, (double)(clock() - t) / CLOCKS_PER_SEC);
+
+	fprintf(stderr, "\n===> Count '%c' in 2-bit encoded integers <===\n", "ACGT"[c]);
+
+	t = clock(); cnt = 0;
+	for (i = 0; i < N; ++i) cnt += kbi_DNAcount64(x[i], c);
+	fprintf(stderr, "%20s\t%20ld\t%10.6f\n", "kbit", (long)cnt, (double)(clock() - t) / CLOCKS_PER_SEC);
+
+	t = clock(); cnt = 0;
+	for (i = 0; i < N; ++i) cnt += bt1_countInU64(x[i], c);
+	fprintf(stderr, "%20s\t%20ld\t%10.6f\n", "bowtie1", (long)cnt, (double)(clock() - t) / CLOCKS_PER_SEC);
+
+	fprintf(stderr, "\n");
+	free(x);
+	return 0;
+}
diff --git a/ext/klib/test/kbtree_test.c b/ext/klib/test/kbtree_test.c
new file mode 100644
index 0000000..8e10687
--- /dev/null
+++ b/ext/klib/test/kbtree_test.c
@@ -0,0 +1,94 @@
+#include <stdio.h>
+#include <assert.h>
+#include <time.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <string.h>
+
+typedef const char *str_t;
+
+#include "kbtree.h"
+KBTREE_INIT(int, uint32_t, kb_generic_cmp)
+KBTREE_INIT(str, str_t, kb_str_cmp)
+
+static int data_size = 5000000;
+static unsigned *int_data;
+static char **str_data;
+
+void ht_init_data()
+{
+	int i;
+	char buf[256];
+	printf("--- generating data... ");
+	srand48(11);
+	int_data = (unsigned*)calloc(data_size, sizeof(unsigned));
+	str_data = (char**)calloc(data_size, sizeof(char*));
+	for (i = 0; i < data_size; ++i) {
+		int_data[i] = (unsigned)(data_size * drand48() / 4) * 271828183u;
+		sprintf(buf, "%x", int_data[i]);
+		str_data[i] = strdup(buf);
+	}
+	printf("done!\n");
+}
+void ht_destroy_data()
+{
+	int i;
+	for (i = 0; i < data_size; ++i) free(str_data[i]);
+	free(str_data); free(int_data);
+}
+
+void ht_khash_int()
+{
+	int i;
+	unsigned *data = int_data;
+	uint32_t *l, *u;
+	kbtree_t(int) *h;
+
+	h = kb_init(int, KB_DEFAULT_SIZE);
+	for (i = 0; i < data_size; ++i) {
+		if (kb_get(int, h, data[i]) == 0) kb_put(int, h, data[i]);
+		else kb_del(int, h, data[i]);
+	}
+	printf("[ht_khash_int] size: %d\n", kb_size(h));
+	if (1) {
+		int cnt = 0;
+		uint32_t x, y;
+		kb_interval(int, h, 2174625464u, &l, &u);
+		printf("interval for 2174625464: (%u, %u)\n", l? *l : 0, u? *u : 0);
+#define traverse_f(p) { if (cnt == 0) y = *p; ++cnt; }
+		__kb_traverse(uint32_t, h, traverse_f);
+		__kb_get_first(uint32_t, h, x);
+		printf("# of elements from traversal: %d\n", cnt);
+		printf("first element: %d == %d\n", x, y);
+	}
+	__kb_destroy(h);
+}
+void ht_khash_str()
+{
+	int i;
+	char **data = str_data;
+	kbtree_t(str) *h;
+
+	h = kb_init(str, KB_DEFAULT_SIZE);
+	for (i = 0; i < data_size; ++i) {
+		if (kb_get(str, h, data[i]) == 0) kb_put(str, h, data[i]);
+		else kb_del(str, h, data[i]);
+	}
+	printf("[ht_khash_int] size: %d\n", kb_size(h));
+	__kb_destroy(h);
+}
+void ht_timing(void (*f)(void))
+{
+	clock_t t = clock();
+	(*f)();
+	printf("[ht_timing] %.3lf sec\n", (double)(clock() - t) / CLOCKS_PER_SEC);
+}
+int main(int argc, char *argv[])
+{
+	if (argc > 1) data_size = atoi(argv[1]);
+	ht_init_data();
+	ht_timing(ht_khash_int);
+	ht_timing(ht_khash_str);
+	ht_destroy_data();
+	return 0;
+}
diff --git a/ext/klib/test/ketopt_test.c b/ext/klib/test/ketopt_test.c
new file mode 100644
index 0000000..4009e24
--- /dev/null
+++ b/ext/klib/test/ketopt_test.c
@@ -0,0 +1,89 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <getopt.h>
+#include "ketopt.h"
+
+static void test_opt(int c, int opt, const char *arg)
+{
+	if (c == 'x') fprintf(stderr, "-x\n");
+	else if (c == 'y') fprintf(stderr, "-y %s\n", arg);
+	else if (c == 301) fprintf(stderr, "--foo\n");
+	else if (c == 302) fprintf(stderr, "--bar %s\n", arg? arg : "(null)");
+	else if (c == 303) fprintf(stderr, "--opt %s\n", arg? arg : "(null)");
+	else if (c == '?') fprintf(stderr, "unknown option -%c\n", opt? opt : ':');
+	else if (c == ':') fprintf(stderr, "missing option argument: -%c\n", opt? opt : ':');
+}
+
+static void print_cmd(int argc, char *argv[], int ind)
+{
+	int i;
+	fprintf(stderr, "CMD: %s", argv[0]);
+	if (ind > 1) {
+		fputs(" [", stderr);
+		for (i = 1; i < ind; ++i) {
+			if (i != 1) fputc(' ', stderr);
+			fputs(argv[i], stderr);
+		}
+		fputc(']', stderr);
+	}
+	for (i = ind; i < argc; ++i)
+		fprintf(stderr, " %s", argv[i]);
+	fputc('\n', stderr);
+}
+
+static void test_ketopt(int argc, char *argv[])
+{
+	static ko_longopt_t longopts[] = {
+		{ "foo", ko_no_argument,       301 },
+		{ "bar", ko_required_argument, 302 },
+		{ "opt", ko_optional_argument, 303 },
+		{ NULL, 0, 0 }
+	};
+	ketopt_t opt = KETOPT_INIT;
+	int c;
+	fprintf(stderr, "===> ketopt() <===\n");
+	while ((c = ketopt(&opt, argc, argv, 1, "xy:", longopts)) >= 0)
+		test_opt(c, opt.opt, opt.arg);
+	print_cmd(argc, argv, opt.ind);
+}
+
+static void test_getopt(int argc, char *argv[])
+{
+	static struct option long_options[] = {
+		{ "foo", no_argument,       0, 301 },
+		{ "bar", required_argument, 0, 302 },
+		{ "opt", optional_argument, 0, 303 },
+		{0, 0, 0, 0}
+	};
+	int c, option_index;
+	fprintf(stderr, "===> getopt() <===\n");
+	while ((c = getopt_long(argc, argv, ":xy:", long_options, &option_index)) >= 0)
+		test_opt(c, optopt, optarg);
+	print_cmd(argc, argv, optind);
+}
+
+int main(int argc, char *argv[])
+{
+	int i;
+	char **argv2;
+	if (argc == 1) {
+		fprintf(stderr, "Usage: ketopt_test [options] <argument> [...]\n");
+		fprintf(stderr, "Options:\n");
+		fprintf(stderr, "  -x           no argument\n");
+		fprintf(stderr, "  -y STR       required argument\n");
+		fprintf(stderr, "  --foo        no argument\n");
+		fprintf(stderr, "  --bar=STR    required argument\n");
+		fprintf(stderr, "  --opt[=STR]  optional argument\n");
+		fprintf(stderr, "\nExamples:\n");
+		fprintf(stderr, "  ketopt_test -xy1 -x arg1 -y -x -- arg2 -x\n");
+		fprintf(stderr, "  ketopt_test --foo --bar=1 --bar 2 --opt arg1 --opt=3\n");
+		fprintf(stderr, "  ketopt_test arg1 -y\n");
+		return 1;
+	}
+	argv2 = (char**)malloc(sizeof(char*) * argc);
+	for (i = 0; i < argc; ++i) argv2[i] = argv[i];
+	test_ketopt(argc, argv);
+	test_getopt(argc, argv2);
+	free(argv2);
+	return 0;
+}
diff --git a/ext/klib/test/kgraph_test.c b/ext/klib/test/kgraph_test.c
new file mode 100644
index 0000000..3da1cd7
--- /dev/null
+++ b/ext/klib/test/kgraph_test.c
@@ -0,0 +1,26 @@
+#include <stdio.h>
+#include "kgraph.h"
+
+KHASH_INIT2(e32, extern, uint32_t, int, 1, kh_int_hash_func, kh_int_hash_equal)
+
+typedef struct {
+	int i;
+	khash_t(e32) *_arc;
+} vertex_t;
+
+KGRAPH_INIT(g, extern, vertex_t, int, e32)
+KGRAPH_PRINT(g, extern)
+
+int main()
+{
+	int *pb, *pe;
+	kgraph_t(g) *g;
+	g = kg_init_g();
+	kg_put_a_g(g, 10, 20, 0, &pb, &pe);
+	kg_put_a_g(g, 20, 30, 0, &pb, &pe);
+	kg_put_a_g(g, 30, 10, 1, &pb, &pe);
+	kg_del_v_g(g, 20);
+	kg_print_g(g);
+	kg_destroy_g(g);
+	return 0;
+}
diff --git a/ext/klib/test/khash_keith.c b/ext/klib/test/khash_keith.c
new file mode 100644
index 0000000..ddd755a
--- /dev/null
+++ b/ext/klib/test/khash_keith.c
@@ -0,0 +1,95 @@
+/*
+ * This is an optimized version of the following C++ program:
+ *
+ *   http://keithlea.com/javabench/src/cpp/hash.cpp
+ *
+ * Keith in his benchmark (http://keithlea.com/javabench/data) showed that the
+ * Java implementation is twice as fast as the C++ version. In fact, this is
+ * only because the C++ implementation is substandard. Most importantly, Keith
+ * is using "sprintf()" to convert an integer to a string, which is known to be
+ * extremely inefficient.
+ */
+#include <stdio.h>
+#include "khash.h"
+KHASH_MAP_INIT_STR(str, int)
+
+inline void int2str(int c, int base, char *ret)
+{
+	const char *tab = "0123456789abcdef";
+	if (c == 0) ret[0] = '0', ret[1] = 0;
+	else {
+		int l, x, y;
+		char buf[16];
+		for (l = 0, x = c < 0? -c : c; x > 0; x /= base) buf[l++] = tab[x%base];
+		if (c < 0) buf[l++] = '-';
+		for (x = l - 1, y = 0; x >= 0; --x) ret[y++] = buf[x];
+		ret[y] = 0;
+	}
+}
+
+#ifndef _USE_STRDUP
+#define BLOCK_SIZE 0x100000
+int main(int argc, char *argv[])
+{
+	char **mem = 0;
+	int i, l, n = 1000000, ret, block_end = 0, curr = 0, c = 0;
+	khash_t(str) *h;
+	h = kh_init(str);
+	if (argc > 1) n = atoi(argv[1]);
+	mem = malloc(sizeof(void*));
+	mem[0] = malloc(BLOCK_SIZE); // memory buffer to avoid memory fragmentation
+	curr = block_end = 0;
+	for (i = 1; i <= n; ++i) {
+		char buf[16];
+		int2str(i, 16, buf);
+		khint_t k = kh_put(str, h, buf, &ret);
+		l = strlen(buf) + 1;
+		if (block_end + l > BLOCK_SIZE) {
+			++curr; block_end = 0;
+			mem = realloc(mem, (curr + 1) * sizeof(void*));
+			mem[curr] = malloc(BLOCK_SIZE);
+		}
+		memcpy(mem[curr] + block_end, buf, l);
+		kh_key(h, k) = mem[curr] + block_end;
+		block_end += l;
+		kh_val(h, k) = i;
+	}
+	for (i = 1; i <= n; ++i) {
+		char buf[16];
+		int2str(i, 10, buf);
+		khint_t k = kh_get(str, h, buf);
+		if (k != kh_end(h)) ++c;
+	}
+	printf("%d\n", c);
+	for (ret = 0; ret <= curr; ++ret) free(mem[ret]);
+	free(mem);
+	kh_destroy(str, h);
+	return 0;
+}
+#else // _USE_STRDUP
+int main(int argc, char *argv[])
+{
+	int i, l, n = 1000000, ret, c = 0;
+	khash_t(str) *h;
+	khint_t k;
+	h = kh_init(str);
+	if (argc > 1) n = atoi(argv[1]);
+	for (i = 1; i <= n; ++i) {
+		char buf[16];
+		int2str(i, 16, buf);
+		k = kh_put(str, h, strdup(buf), &ret);
+		kh_val(h, k) = i;
+	}
+	for (i = 1; i <= n; ++i) {
+		char buf[16];
+		int2str(i, 10, buf);
+		k = kh_get(str, h, buf);
+		if (k != kh_end(h)) ++c;
+	}
+	for (k = kh_begin(h); k != kh_end(h); ++k) // explicitly freeing memory takes 10-20% CPU time.
+		if (kh_exist(h, k)) free((char*)kh_key(h, k));
+	printf("%d\n", c);
+	kh_destroy(str, h);
+	return 0;
+}
+#endif
diff --git a/ext/klib/test/khash_keith2.c b/ext/klib/test/khash_keith2.c
new file mode 100644
index 0000000..b9df9b7
--- /dev/null
+++ b/ext/klib/test/khash_keith2.c
@@ -0,0 +1,67 @@
+/*
+ * This is an optimized version of the following C++ program:
+ *
+ *   http://keithlea.com/javabench/src/cpp/hash.cpp
+ *
+ * Keith in his benchmark (http://keithlea.com/javabench/data) showed that the
+ * Java implementation is twice as fast as the C++ version. In fact, this is
+ * only because the C++ implementation is substandard. Most importantly, Keith
+ * is using "sprintf()" to convert an integer to a string, which is known to be
+ * extremely inefficient.
+ */
+#include <stdio.h>
+#include "khash.h"
+KHASH_MAP_INIT_STR(str, int)
+
+inline void int2str(int c, int base, char *ret)
+{
+	const char *tab = "0123456789abcdef";
+	if (c == 0) ret[0] = '0', ret[1] = 0;
+	else {
+		int l, x, y;
+		char buf[16];
+		for (l = 0, x = c < 0? -c : c; x > 0; x /= base) buf[l++] = tab[x%base];
+		if (c < 0) buf[l++] = '-';
+		for (x = l - 1, y = 0; x >= 0; --x) ret[y++] = buf[x];
+		ret[y] = 0;
+	}
+}
+
+int main(int argc, char *argv[])
+{
+	int i, l, n = 1000, ret;
+	khash_t(str) *h, *h2;
+	khint_t k;
+	h = kh_init(str);
+	h2 = kh_init(str);
+	if (argc > 1) n = atoi(argv[1]);
+	for (i = 0; i < 10000; ++i) {
+		char buf[32];
+		strcpy(buf, "foo_");
+		int2str(i, 10, buf+4);
+		k = kh_put(str, h, strdup(buf), &ret);
+		kh_val(h, k) = i;
+	}
+	for (i = 0; i < n; ++i) {
+		for (k = kh_begin(h); k != kh_end(h); ++k) {
+			if (kh_exist(h, k)) {
+				khint_t k2 = kh_put(str, h2, kh_key(h, k), &ret);
+				if (ret) { // absent
+					kh_key(h2, k2) = strdup(kh_key(h, k));
+					kh_val(h2, k2) = kh_val(h, k);
+				} else kh_val(h2, k2) += kh_val(h, k);
+			}
+		}
+	}
+	k = kh_get(str, h, "foo_1"); printf("%d", kh_val(h, k));
+	k = kh_get(str, h, "foo_9999"); printf(" %d", kh_val(h, k));
+	k = kh_get(str, h2, "foo_1"); printf(" %d", kh_val(h2, k));
+	k = kh_get(str, h2, "foo_9999"); printf(" %d\n", kh_val(h2, k));
+	for (k = kh_begin(h); k != kh_end(h); ++k)
+		if (kh_exist(h, k)) free((char*)kh_key(h, k));
+	for (k = kh_begin(h2); k != kh_end(h2); ++k)
+		if (kh_exist(h2, k)) free((char*)kh_key(h2, k));
+	kh_destroy(str, h);
+	kh_destroy(str, h2);
+	return 0;
+}
diff --git a/ext/klib/test/khash_test.c b/ext/klib/test/khash_test.c
new file mode 100644
index 0000000..8d6687f
--- /dev/null
+++ b/ext/klib/test/khash_test.c
@@ -0,0 +1,141 @@
+#include <stdio.h>
+#include <assert.h>
+#include <time.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "khash.h"
+KHASH_SET_INIT_STR(str)
+KHASH_MAP_INIT_INT(int, unsigned char)
+
+typedef struct {
+	unsigned key;
+	unsigned char val;
+} int_unpack_t;
+
+typedef struct {
+	unsigned key;
+	unsigned char val;
+} __attribute__ ((__packed__)) int_packed_t;
+
+#define hash_eq(a, b) ((a).key == (b).key)
+#define hash_func(a) ((a).key)
+
+KHASH_INIT(iun, int_unpack_t, char, 0, hash_func, hash_eq)
+KHASH_INIT(ipk, int_packed_t, char, 0, hash_func, hash_eq)
+
+static int data_size = 5000000;
+static unsigned *int_data;
+static char **str_data;
+
+void ht_init_data()
+{
+	int i;
+	char buf[256];
+	khint32_t x = 11;
+	printf("--- generating data... ");
+	int_data = (unsigned*)calloc(data_size, sizeof(unsigned));
+	str_data = (char**)calloc(data_size, sizeof(char*));
+	for (i = 0; i < data_size; ++i) {
+		int_data[i] = (unsigned)(data_size * ((double)x / UINT_MAX) / 4) * 271828183u;
+		sprintf(buf, "%x", int_data[i]);
+		str_data[i] = strdup(buf);
+		x = 1664525L * x + 1013904223L;
+	}
+	printf("done!\n");
+}
+
+void ht_destroy_data()
+{
+	int i;
+	for (i = 0; i < data_size; ++i) free(str_data[i]);
+	free(str_data); free(int_data);
+}
+
+void ht_khash_int()
+{
+	int i, ret;
+	unsigned *data = int_data;
+	khash_t(int) *h;
+	unsigned k;
+
+	h = kh_init(int);
+	for (i = 0; i < data_size; ++i) {
+		k = kh_put(int, h, data[i], &ret);
+		kh_val(h, k) = i&0xff;
+		if (!ret) kh_del(int, h, k);
+	}
+	printf("[ht_khash_int] size: %u\n", kh_size(h));
+	kh_destroy(int, h);
+}
+
+void ht_khash_str()
+{
+	int i, ret;
+	char **data = str_data;
+	khash_t(str) *h;
+	unsigned k;
+
+	h = kh_init(str);
+	for (i = 0; i < data_size; ++i) {
+		k = kh_put(str, h, data[i], &ret);
+		if (!ret) kh_del(str, h, k);
+	}
+	printf("[ht_khash_int] size: %u\n", kh_size(h));
+	kh_destroy(str, h);
+}
+
+void ht_khash_unpack()
+{
+	int i, ret;
+	unsigned *data = int_data;
+	khash_t(iun) *h;
+	unsigned k;
+
+	h = kh_init(iun);
+	for (i = 0; i < data_size; ++i) {
+		int_unpack_t x;
+		x.key = data[i]; x.val = i&0xff;
+		k = kh_put(iun, h, x, &ret);
+		if (!ret) kh_del(iun, h, k);
+	}
+	printf("[ht_khash_unpack] size: %u (sizeof=%ld)\n", kh_size(h), sizeof(int_unpack_t));
+	kh_destroy(iun, h);
+}
+
+void ht_khash_packed()
+{
+	int i, ret;
+	unsigned *data = int_data;
+	khash_t(ipk) *h;
+	unsigned k;
+
+	h = kh_init(ipk);
+	for (i = 0; i < data_size; ++i) {
+		int_packed_t x;
+		x.key = data[i]; x.val = i&0xff;
+		k = kh_put(ipk, h, x, &ret);
+		if (!ret) kh_del(ipk, h, k);
+	}
+	printf("[ht_khash_packed] size: %u (sizeof=%ld)\n", kh_size(h), sizeof(int_packed_t));
+	kh_destroy(ipk, h);
+}
+
+void ht_timing(void (*f)(void))
+{
+	clock_t t = clock();
+	(*f)();
+	printf("[ht_timing] %.3lf sec\n", (double)(clock() - t) / CLOCKS_PER_SEC);
+}
+
+int main(int argc, char *argv[])
+{
+	if (argc > 1) data_size = atoi(argv[1]);
+	ht_init_data();
+	ht_timing(ht_khash_int);
+	ht_timing(ht_khash_str);
+	ht_timing(ht_khash_unpack);
+	ht_timing(ht_khash_packed);
+	ht_destroy_data();
+	return 0;
+}
diff --git a/ext/klib/test/klist_test.c b/ext/klib/test/klist_test.c
new file mode 100644
index 0000000..cd13813
--- /dev/null
+++ b/ext/klib/test/klist_test.c
@@ -0,0 +1,19 @@
+#include <stdio.h>
+#include "klist.h"
+
+#define __int_free(x)
+KLIST_INIT(32, int, __int_free)
+
+int main()
+{
+	klist_t(32) *kl;
+	kliter_t(32) *p;
+	kl = kl_init(32);
+	*kl_pushp(32, kl) = 1;
+	*kl_pushp(32, kl) = 10;
+	kl_shift(32, kl, 0);
+	for (p = kl_begin(kl); p != kl_end(kl); p = kl_next(p))
+		printf("%d\n", kl_val(p));
+	kl_destroy(32, kl);
+	return 0;
+}
diff --git a/ext/klib/test/kmin_test.c b/ext/klib/test/kmin_test.c
new file mode 100644
index 0000000..33ccd1c
--- /dev/null
+++ b/ext/klib/test/kmin_test.c
@@ -0,0 +1,48 @@
+#include <stdio.h>
+#include <math.h>
+#include "kmath.h"
+
+static int n_evals;
+
+double f_Chebyquad(int n, double *x, void *data)
+{
+    int i, j;
+    double y[20][20], f;
+    int np, iw;
+    double sum;
+    for (j = 0; j != n; ++j) {
+		y[0][j] = 1.;
+		y[1][j] = 2. * x[j] - 1.;
+    }
+    for (i = 1; i != n; ++i)
+		for (j = 0; j != n; ++j)
+			y[i+1][j] = 2. * y[1][j] * y[i][j] - y[i-1][j];
+    f = 0.;
+    np = n + 1;
+    iw = 1;
+    for (i = 0; i != np; ++i) {
+		sum = 0.;
+		for (j = 0; j != n; ++j) sum += y[i][j];
+		sum /= n;
+		if (iw > 0) sum += 1. / ((i - 1) * (i + 1));
+		iw = -iw;
+		f += sum * sum;
+    }
+	++n_evals;
+    return f;
+}
+
+int main()
+{
+	double x[20], y;
+	int n, i;
+	printf("\nMinimizer: Hooke-Jeeves\n");
+	for (n = 2; n <= 8; n += 2) {
+		for (i = 0; i != n; ++i) x[i] = (double)(i + 1) / n;
+		n_evals = 0;
+		y = kmin_hj(f_Chebyquad, n, x, 0, KMIN_RADIUS, KMIN_EPS, KMIN_MAXCALL);
+		printf("n=%d,min=%.8lg,n_evals=%d\n", n, y, n_evals);
+	}
+	printf("\n");
+	return 0;
+}
diff --git a/ext/klib/test/krmq_test.c b/ext/klib/test/krmq_test.c
new file mode 100644
index 0000000..69aff4c
--- /dev/null
+++ b/ext/klib/test/krmq_test.c
@@ -0,0 +1,151 @@
+#include <stdio.h>
+#include <assert.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <string.h>
+#include <limits.h>
+#include "krmq.h"
+
+#define CALLOC(type, num) ((type*)calloc(num, sizeof(type)))
+
+struct my_node {
+	int key;
+	int val;
+	KRMQ_HEAD(struct my_node) head;
+};
+
+#define my_cmp(p, q) (((p)->key > (q)->key) - ((p)->key < (q)->key))
+#define my_lt2(p, q) ((p)->val < (q)->val)
+KRMQ_INIT(my, struct my_node, head, my_cmp, my_lt2)
+
+int check(struct my_node *p, int *hh, int *min)
+{
+	*hh = 0, *min = INT_MAX;
+	if (p) {
+		int c = 1, h[2] = {0, 0}, m[2] = {INT_MAX, INT_MAX};
+		*min = p->val;
+		if (p->head.p[0]) c += check(p->head.p[0], &h[0], &m[0]);
+		if (p->head.p[1]) c += check(p->head.p[1], &h[1], &m[1]);
+		*min = *min < m[0]? *min : m[0];
+		*min = *min < m[1]? *min : m[1];
+		*hh = (h[0] > h[1]? h[0] : h[1]) + 1;
+		if (*min != p->head.s->val)
+			fprintf(stderr, "min %d != %d at %c\n", *min, p->head.s->val, p->key);
+		if (h[1] - h[0] != (int)p->head.balance)
+			fprintf(stderr, "%d - %d != %d at %c\n", h[1], h[0], p->head.balance, p->key);
+		if (c != (int)p->head.size)
+			fprintf(stderr, "%d != %d at %c\n", p->head.size, c, p->key);
+		return c;
+	} else return 0;
+}
+
+int check_rmq(const struct my_node *root, int lo, int hi)
+{
+	struct my_node s, t, *p, *q;
+	krmq_itr_t(my) itr;
+	int min = INT_MAX;
+	s.key = lo, t.key = hi;
+	p = krmq_rmq(my, root, &s, &t);
+	krmq_interval(my, root, &s, 0, &q);
+	if (p == 0) return -1;
+	krmq_itr_find(my, root, q, &itr);
+	do {
+		const struct my_node *r = krmq_at(&itr);
+		if (r->key > hi) break;
+		//fprintf(stderr, "%c\t%d\n", r->key, r->val);
+		if (r->val < min) min = r->val;
+	} while (krmq_itr_next(my, &itr));
+	assert((min == INT_MAX && p == 0) || (min < INT_MAX && p));
+	if (min != p->val) fprintf(stderr, "rmq_min %d != %d\n", p->val, min);
+	return min;
+}
+
+int print_tree(const struct my_node *p)
+{
+	int c = 1;
+	if (p == 0) return 0;
+	if (p->head.p[0] || p->head.p[1]) {
+		putchar('(');
+		if (p->head.p[0]) c += print_tree(p->head.p[0]);
+		else putchar('.');
+		putchar(',');
+		if (p->head.p[1]) c += print_tree(p->head.p[1]);
+		else putchar('.');
+		putchar(')');
+	}
+	printf("%c:%d/%d", p->key, p->val, p->head.s->val);
+	return c;
+}
+
+void check_and_print(struct my_node *root)
+{
+	int h, min;
+	check(root, &h, &min);
+	print_tree(root);
+	putchar('\n');
+}
+
+void shuffle(int n, char a[])
+{
+	int i, j;
+	for (i = n; i > 1; --i) {
+		char tmp;
+		j = (int)(drand48() * i);
+		tmp = a[j]; a[j] = a[i-1]; a[i-1] = tmp;
+	}
+}
+
+int main(void)
+{
+	char buf[256];
+	int i, n, h, min;
+	struct my_node *root = 0;
+	struct my_node *p, *q, t;
+	krmq_itr_t(my) itr;
+	unsigned cnt;
+
+	srand48(123);
+	for (i = 33, n = 0; i <= 126; ++i)
+		if (i != '(' && i != ')' && i != '.' && i != ';')
+			buf[n++] = i;
+	shuffle(n, buf);
+	for (i = 0; i < n; ++i) {
+		p = CALLOC(struct my_node, 1);
+		p->key = buf[i];
+		p->val = i;
+		q = krmq_insert(my, &root, p, &cnt);
+		if (p != q) free(p);
+		check(root, &h, &min);
+	}
+
+	shuffle(n, buf);
+	for (i = 0; i < n/2; ++i) {
+		t.key = buf[i];
+		//fprintf(stderr, "i=%d, key=%c, n/2=%d\n", i, t.key, n/2);
+		q = krmq_erase(my, &root, &t, 0);
+		if (q) free(q);
+		check(root, &h, &min);
+	}
+	check_and_print(root);
+
+	check_rmq(root, '0', '9');
+	check_rmq(root, '!', '~');
+	check_rmq(root, 'A', 'Z');
+	check_rmq(root, 'F', 'G');
+	check_rmq(root, 'a', 'z');
+	for (i = 0; i < n; ++i) {
+		int lo, hi;
+		lo = (int)(drand48() * n);
+		hi = (int)(drand48() * n);
+		check_rmq(root, lo, hi);
+	}
+
+	krmq_itr_first(my, root, &itr);
+	do {
+		const struct my_node *r = krmq_at(&itr);
+		putchar(r->key);
+	} while (krmq_itr_next(my, &itr));
+	putchar('\n');
+	krmq_free(struct my_node, head, root, free);
+	return 0;
+}
diff --git a/ext/klib/test/kseq_bench.c b/ext/klib/test/kseq_bench.c
new file mode 100644
index 0000000..eeda13f
--- /dev/null
+++ b/ext/klib/test/kseq_bench.c
@@ -0,0 +1,69 @@
+#include <zlib.h>
+#include <stdio.h>
+#include <time.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include "kseq.h"
+
+#define BUF_SIZE 4096
+KSTREAM_INIT(gzFile, gzread, BUF_SIZE)
+
+int main(int argc, char *argv[])
+{
+	gzFile fp;
+	clock_t t;
+	if (argc == 1) {
+		fprintf(stderr, "Usage: kseq_bench <in.gz>\n");
+		return 1;
+	}
+	{
+		uint8_t *buf = malloc(BUF_SIZE);
+		fp = gzopen(argv[1], "r");
+		t = clock();
+		while (gzread(fp, buf, BUF_SIZE) > 0);
+		fprintf(stderr, "[gzread] %.2f sec\n", (float)(clock() - t) / CLOCKS_PER_SEC);
+		gzclose(fp);
+		free(buf);
+	}
+	{
+		kstream_t *ks;
+		fp = gzopen(argv[1], "r");
+		ks = ks_init(fp);
+		t = clock();
+		while (ks_getc(ks) >= 0);
+		fprintf(stderr, "[ks_getc] %.2f sec\n", (float)(clock() - t) / CLOCKS_PER_SEC);
+		ks_destroy(ks);
+		gzclose(fp);
+	}
+	{
+		kstream_t *ks;
+		kstring_t *s;
+		int dret;
+		s = calloc(1, sizeof(kstring_t));
+		fp = gzopen(argv[1], "r");
+		ks = ks_init(fp);
+		t = clock();
+		while (ks_getuntil(ks, '\n', s, &dret) >= 0);
+		fprintf(stderr, "[ks_getuntil] %.2f sec\n", (float)(clock() - t) / CLOCKS_PER_SEC);
+		ks_destroy(ks);
+		gzclose(fp);
+		free(s->s); free(s);
+	}
+	if (argc == 2) {
+		fp = gzopen(argv[1], "r");
+		t = clock();
+		while (gzgetc(fp) >= 0);
+		fprintf(stderr, "[gzgetc] %.2f sec\n", (float)(clock() - t) / CLOCKS_PER_SEC);
+		gzclose(fp);
+	}
+	if (argc == 2) {
+		char *buf = malloc(BUF_SIZE);
+		fp = gzopen(argv[1], "r");
+		t = clock();
+		while (gzgets(fp, buf, BUF_SIZE) > 0);
+		fprintf(stderr, "[gzgets] %.2f sec\n", (float)(clock() - t) / CLOCKS_PER_SEC);
+		gzclose(fp);
+		free(buf);
+	}
+	return 0;
+}
diff --git a/ext/klib/test/kseq_bench2.c b/ext/klib/test/kseq_bench2.c
new file mode 100644
index 0000000..3267777
--- /dev/null
+++ b/ext/klib/test/kseq_bench2.c
@@ -0,0 +1,44 @@
+#include <stdio.h>
+#include <time.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include "kseq.h"
+KSTREAM_INIT(int, read, 4096)
+
+#define BUF_SIZE 65536
+
+int main(int argc, char *argv[])
+{
+	clock_t t;
+	if (argc == 1) {
+		fprintf(stderr, "Usage: %s <in.txt>\n", argv[0]);
+		return 1;
+	}
+	{
+		FILE *fp;
+		char *s;
+		t = clock();
+		s = malloc(BUF_SIZE);
+		fp = fopen(argv[1], "r");
+		while (fgets(s, BUF_SIZE, fp));
+		fclose(fp);
+		fprintf(stderr, "[fgets] %.2f sec\n", (float)(clock() - t) / CLOCKS_PER_SEC);
+	}
+	{
+		int fd, dret;
+		kstream_t *ks;
+		kstring_t s;
+		t = clock();
+		s.l = s.m = 0; s.s = 0;
+		fd = open(argv[1], O_RDONLY);
+		ks = ks_init(fd);
+		while (ks_getuntil(ks, '\n', &s, &dret) >= 0);
+		free(s.s);
+		ks_destroy(ks);
+		close(fd);
+		fprintf(stderr, "[kstream] %.2f sec\n", (float)(clock() - t) / CLOCKS_PER_SEC);
+	}
+	return 0;
+}
diff --git a/ext/klib/test/kseq_test.c b/ext/klib/test/kseq_test.c
new file mode 100644
index 0000000..0304dea
--- /dev/null
+++ b/ext/klib/test/kseq_test.c
@@ -0,0 +1,27 @@
+#include <zlib.h>
+#include <stdio.h>
+#include "kseq.h"
+KSEQ_INIT(gzFile, gzread)
+
+int main(int argc, char *argv[])
+{
+	gzFile fp;
+	kseq_t *seq;
+	int l;
+	if (argc == 1) {
+		fprintf(stderr, "Usage: %s <in.fasta>\n", argv[0]);
+		return 1;
+	}
+	fp = gzopen(argv[1], "r");
+	seq = kseq_init(fp);
+	while ((l = kseq_read(seq)) >= 0) {
+		printf("name: %s\n", seq->name.s);
+		if (seq->comment.l) printf("comment: %s\n", seq->comment.s);
+		printf("seq: %s\n", seq->seq.s);
+		if (seq->qual.l) printf("qual: %s\n", seq->qual.s);
+	}
+	printf("return value: %d\n", l);
+	kseq_destroy(seq);
+	gzclose(fp);
+	return 0;
+}
diff --git a/ext/klib/test/kseq_test.dat b/ext/klib/test/kseq_test.dat
new file mode 100644
index 0000000..b774ae2
--- /dev/null
+++ b/ext/klib/test/kseq_test.dat
@@ -0,0 +1,12 @@
+>1
+acgtacgtacgtagc
+>2 test
+acgatcgatc
+@3 test2
+cgctagcatagc
+cgatatgactta
++
+78wo82usd980
+d88fau
+
+238ud8
diff --git a/ext/klib/test/ksort_test.c b/ext/klib/test/ksort_test.c
new file mode 100644
index 0000000..92c7d3d
--- /dev/null
+++ b/ext/klib/test/ksort_test.c
@@ -0,0 +1,104 @@
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <time.h>
+#include "ksort.h"
+
+KSORT_INIT_GENERIC(int)
+
+int main(int argc, char *argv[])
+{
+	int i, N = 10000000;
+	int *array, x;
+	clock_t t1, t2;
+	if (argc > 1) N = atoi(argv[1]);
+	array = (int*)malloc(sizeof(int) * N);
+
+	srand48(11);
+	for (i = 0; i < N; ++i) array[i] = (int)lrand48();
+	t1 = clock();
+	x = ks_ksmall(int, N, array, 10500);
+	t2 = clock();
+	fprintf(stderr, "ksmall [%d]: %.3lf\n", x, (double)(t2-t1)/CLOCKS_PER_SEC);
+
+	srand48(11);
+	for (i = 0; i < N; ++i) array[i] = (int)lrand48();
+	t1 = clock();
+	ks_introsort(int, N, array);
+	t2 = clock();
+	fprintf(stderr, "introsort [%d]: %.3lf\n", array[10500], (double)(t2-t1)/CLOCKS_PER_SEC);
+	for (i = 0; i < N-1; ++i) {
+		if (array[i] > array[i+1]) {
+			fprintf(stderr, "Bug in introsort!\n");
+			exit(1);
+		}
+	}
+
+#ifndef _ALIGNED_ONLY
+	{ // test unaligned ksmall
+		srand48(11);
+		unsigned char *a;
+		int *b;
+		a = malloc(N * sizeof(int) + 1);
+		b = (int*)(a + 1);
+		for (i = 0; i < N; ++i) b[i] = (int)lrand48();
+		t1 = clock();
+		ks_introsort(int, N, b);
+		t2 = clock();
+		fprintf(stderr, "introsort [%d]: %.3lf (unaligned: 0x%lx) \n", b[10500], (double)(t2-t1)/CLOCKS_PER_SEC, (size_t)b);
+	}
+#endif
+
+	t1 = clock();
+	ks_introsort(int, N, array);
+	t2 = clock();
+	fprintf(stderr, "introsort (sorted): %.3lf\n", (double)(t2-t1)/CLOCKS_PER_SEC);
+
+	srand48(11);
+	for (i = 0; i < N; ++i) array[i] = (int)lrand48();
+	t1 = clock();
+	ks_combsort(int, N, array);
+	t2 = clock();
+	fprintf(stderr, "combsort: %.3lf\n", (double)(t2-t1)/CLOCKS_PER_SEC);
+	for (i = 0; i < N-1; ++i) {
+		if (array[i] > array[i+1]) {
+			fprintf(stderr, "Bug in combsort!\n");
+			exit(1);
+		}
+	}
+
+	srand48(11);
+	for (i = 0; i < N; ++i) array[i] = (int)lrand48();
+	t1 = clock();
+	ks_mergesort(int, N, array, 0);
+	t2 = clock();
+	fprintf(stderr, "mergesort: %.3lf\n", (double)(t2-t1)/CLOCKS_PER_SEC);
+	for (i = 0; i < N-1; ++i) {
+		if (array[i] > array[i+1]) {
+			fprintf(stderr, "Bug in mergesort!\n");
+			exit(1);
+		}
+	}
+
+	t1 = clock();
+	ks_mergesort(int, N, array, 0);
+	t2 = clock();
+	fprintf(stderr, "mergesort (sorted): %.3lf\n", (double)(t2-t1)/CLOCKS_PER_SEC);
+
+	srand48(11);
+	for (i = 0; i < N; ++i) array[i] = (int)lrand48();
+	t1 = clock();
+	ks_heapmake(int, N, array);
+	ks_heapsort(int, N, array);
+	t2 = clock();
+	fprintf(stderr, "heapsort: %.3lf\n", (double)(t2-t1)/CLOCKS_PER_SEC);
+	for (i = 0; i < N-1; ++i) {
+		if (array[i] > array[i+1]) {
+			fprintf(stderr, "Bug in heapsort!\n");
+			exit(1);
+		}
+	}
+
+	free(array);
+	return 0;
+}
diff --git a/ext/klib/test/ksort_test.cc b/ext/klib/test/ksort_test.cc
new file mode 100644
index 0000000..8950d80
--- /dev/null
+++ b/ext/klib/test/ksort_test.cc
@@ -0,0 +1,997 @@
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <time.h>
+#include <algorithm>
+
+#include "ksort.h"
+KSORT_INIT_GENERIC(int)
+
+using namespace std;
+
+/**********************************
+ * BEGIN OF PAUL'S IMPLEMENTATION *
+ **********************************/
+
+/* Attractive Chaos: I have added inline where necessary. */
+
+/*
+Copyright (c) 2004 Paul Hsieh
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+    Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+
+    Redistributions in binary form must reproduce the above copyright notice,
+    this list of conditions and the following disclaimer in the documentation
+    and/or other materials provided with the distribution.
+
+    Neither the name of sorttest nor the names of its contributors may be
+    used to endorse or promote products derived from this software without
+    specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+*/
+
+/*
+
+Recommended flags:
+------------------
+
+Intel C/C++:
+icl /O2 /G6 /Qaxi /Qxi /Qip sorttest.c
+
+WATCOM C/C++:
+wcl386 /otexan /6r sorttest.c
+
+GCC:
+gcc -O3 -mcpu=athlon-xp -march=athlon-xp sorttest.c
+
+MSVC:
+cl /O2 /Ot /Og /G6 sorttest.c
+
+*/
+
+static inline void sort2 (int * numbers) {
+int tmp;
+
+    if (numbers[0] <= numbers[1]) return;
+    tmp = numbers[0];
+    numbers[0] = numbers[1];
+    numbers[1] = tmp;
+}
+
+static inline void sort3 (int * numbers) {
+int tmp;
+
+    if (numbers[0] <= numbers[1]) {
+        if (numbers[1] <= numbers[2]) return;
+        if (numbers[2] <= numbers[0]) {
+            tmp = numbers[0];
+            numbers[0] = numbers[2];
+            numbers[2] = numbers[1];
+            numbers[1] = tmp;
+            return;
+        }
+        tmp = numbers[1];
+    } else {
+        tmp = numbers[0];
+        if (numbers[0] <= numbers[2]) {
+            numbers[0] = numbers[1];
+            numbers[1] = tmp;
+            return;
+        }
+        if (numbers[2] <= numbers[1]) {
+            numbers[0] = numbers[2];
+            numbers[2] = tmp;
+            return;
+        }
+        numbers[0] = numbers[1];
+    }
+    numbers[1] = numbers[2];
+    numbers[2] = tmp;
+}
+
+static inline void sort4 (int * num) {
+int tmp;
+  if (num[0] < num[1]) {
+    if (num[1] < num[2]) {
+      if (num[1] < num[3]) {
+        if (num[2] >= num[3]) {
+          tmp = num[2];
+          num[2] = num[3];
+          num[3] = tmp;
+        }
+      } else {
+        tmp = num[1];
+        if (num[0] < num[3]) {
+          num[1] = num[3];
+        } else {
+          num[1] = num[0];
+          num[0] = num[3];
+        }
+        num[3] = num[2];
+        num[2] = tmp;
+      }
+    } else {
+      if (num[0] < num[2]) {
+        if (num[2] < num[3]) {
+          if (num[1] < num[3]) {
+            tmp = num[1];
+          } else {
+            tmp = num[3];
+            num[3] = num[1];
+          }
+          num[1] = num[2];
+          num[2] = tmp;
+        } else {
+          if (num[0] < num[3]) {
+            tmp = num[3];
+          } else {
+            tmp = num[0];
+            num[0] = num[3];
+          }
+          num[3] = num[1];
+          num[1] = tmp;
+        }
+      } else {
+        if (num[0] < num[3]) {
+          tmp = num[0];
+          num[0] = num[2];
+          if (num[1] < num[3]) {
+            num[2] = num[1];
+          } else {
+            num[2] = num[3];
+            num[3] = num[1];
+          }
+          num[1] = tmp;
+        } else {
+          if (num[2] < num[3]) {
+            tmp = num[0];
+            num[0] = num[2];
+            num[2] = tmp;
+            tmp = num[1];
+            num[1] = num[3];
+          } else {
+            tmp = num[1];
+            num[1] = num[2];
+            num[2] = num[0];
+            num[0] = num[3];
+          }
+          num[3] = tmp;
+        }
+      }
+    }
+  } else {
+    tmp = num[0];
+    if (tmp < num[2]) {
+      if (tmp < num[3]) {
+        num[0] = num[1];
+        num[1] = tmp;
+        if (num[2] >= num[3]) {
+          tmp = num[2];
+          num[2] = num[3];
+          num[3] = tmp;
+        }
+      } else {
+        if (num[1] < num[3]) {
+          num[0] = num[1];
+          num[1] = num[3];
+        } else {
+          num[0] = num[3];
+        }
+        num[3] = num[2];
+        num[2] = tmp;
+      }
+    } else {
+      if (num[1] < num[2]) {
+        if (num[2] < num[3]) {
+          num[0] = num[1];
+          num[1] = num[2];
+          if (tmp < num[3]) {
+            num[2] = tmp;
+          } else {
+            num[2] = num[3];
+            num[3] = tmp;
+          }
+        } else {
+          if (num[1] < num[3]) {
+            num[0] = num[1];
+            num[1] = num[3];
+          } else {
+            num[0] = num[3];
+          }
+          num[3] = tmp;
+        }
+      } else {
+        if (num[1] < num[3]) {
+          num[0] = num[2];
+          if (tmp < num[3]) {
+            num[2] = tmp;
+          } else {
+            num[2] = num[3];
+            num[3] = tmp;
+          }
+        } else {
+          if (num[2] < num[3]) {
+            num[0] = num[2];
+            num[2] = num[1];
+            num[1] = num[3];
+            num[3] = tmp;
+          } else {
+            num[0] = num[3];
+            num[3] = tmp;
+            tmp = num[1];
+            num[1] = num[2];
+            num[2] = tmp;
+          }
+        }
+      }
+    }
+  }
+}
+
+static inline void sortAlt2 (int * numbers, int * altNumbers) {
+    if (numbers[0] <= numbers[1]) {
+        altNumbers[0] = numbers[0];
+        altNumbers[1] = numbers[1];
+    } else {
+        altNumbers[0] = numbers[1];
+        altNumbers[1] = numbers[0];
+    }
+}
+
+static inline void sortAlt3 (int * numbers, int * altNumbers) {
+    if (numbers[0] <= numbers[1]) {
+        if (numbers[1] <= numbers[2]) {
+            altNumbers[0] = numbers[0];
+            altNumbers[1] = numbers[1];
+            altNumbers[2] = numbers[2];
+        } else if (numbers[2] <= numbers[0]) {
+            altNumbers[0] = numbers[2];
+            altNumbers[1] = numbers[0];
+            altNumbers[2] = numbers[1];
+        } else {
+            altNumbers[0] = numbers[0];
+            altNumbers[1] = numbers[2];
+            altNumbers[2] = numbers[1];
+        }
+    } else {
+        if (numbers[0] <= numbers[2]) {
+            altNumbers[0] = numbers[1];
+            altNumbers[1] = numbers[0];
+            altNumbers[2] = numbers[2];
+        } else if (numbers[2] <= numbers[1]) {
+            altNumbers[0] = numbers[2];
+            altNumbers[1] = numbers[1];
+            altNumbers[2] = numbers[0];
+        } else {
+            altNumbers[0] = numbers[1];
+            altNumbers[1] = numbers[2];
+            altNumbers[2] = numbers[0];
+        }
+    }
+}
+
+/*
+ *  Insert Sort
+ */
+
+inline void insertSort (int numbers[], int qty) {
+int i, j, idx, q4;
+int tmp;
+
+    if (qty <= 4) {
+        if (qty == 4) sort4 (numbers);
+        else if (qty == 3) sort3 (numbers);
+        else if (qty == 2) sort2 (numbers);
+        return;
+    }
+
+    q4 = qty - 4;
+
+    for (i=0; i < q4; i++) {
+        idx = i;
+        for (j=i+1; j < qty; j++) {
+            if (numbers[j] < numbers[idx]) idx = j;
+        }
+        if (idx != i) {
+            tmp = numbers[idx];
+            numbers[idx] = numbers[i];
+            numbers[i] = tmp;
+        }
+    }
+
+    sort4 (numbers + q4);
+}
+
+/*
+ *  Heap Sort
+ */
+
+/* Assure the heap property for entries from top to last */
+static void siftDown (int numbers[], int top, int last) {
+int tmp = numbers[top];
+int maxIdx = top;
+
+    while (last >= (maxIdx += maxIdx)) {
+
+        /* This is where the comparison occurrs and where a sufficiently
+           good compiler can use a computed conditional result rather
+           than using control logic. */
+        if (maxIdx != last && numbers[maxIdx] < numbers[maxIdx + 1]) maxIdx++;
+
+        if (tmp >= numbers[maxIdx]) break;
+        numbers[top] = numbers[maxIdx];
+        top = maxIdx;
+    }
+    numbers[top] = tmp;
+}
+
+/* Peel off the top siftDown operation since its parameters are trivial to
+   fill in directly (and this saves us some moves.) */
+static void siftDown0 (int numbers[], int last) {
+int tmp;
+
+    if (numbers[0] < numbers[1]) {
+        tmp = numbers[1];
+        numbers[1] = numbers[0];
+        siftDown (numbers, 1, last);
+    } else {
+        tmp = numbers[0];
+    }
+    numbers[0] = numbers[last];
+    numbers[last] = tmp;
+}
+
+void heapSort (int numbers[], int qty) {
+int i;
+
+    if (qty <= 4) {
+        if (qty == 4) sort4 (numbers);
+        else if (qty == 3) sort3 (numbers);
+        else if (qty == 2) sort2 (numbers);
+        return;
+    }
+
+    i = qty / 2;
+    /* Enforce the heap property for each position in the tree */
+    for (  qty--; i >  0; i--) siftDown  (numbers, i, qty);
+    for (i = qty; i > 0; i--) siftDown0 (numbers, i);
+}
+
+/*
+ *  Quick Sort
+ */
+
+static int medianOf3 (int * numbers, int i, int j) {
+int tmp;
+
+    if (numbers[0] <= numbers[i]) {
+        if (numbers[j] <= numbers[0]) return numbers[0]; /* j 0 i */
+        if (numbers[i] <= numbers[j]) j = i;             /* 0 i j */
+                                                         /* 0 j i */
+    } else {
+        if (numbers[0] <= numbers[j]) return numbers[0]; /* i 0 j */
+        if (numbers[j] <= numbers[i]) j = i;             /* j i 0 */
+                                                         /* i j 0 */
+    }
+    tmp = numbers[j];
+    numbers[j] = numbers[0];
+    numbers[0] = tmp;
+    return tmp;
+}
+
+static void quickSortRecurse (int * numbers, int left, int right) {
+int pivot, lTmp, rTmp;
+
+    qsrStart:;
+
+#if defined(__GNUC__)
+    if (right <= left + 8) {
+        insertSort (numbers + left, right - left + 1);
+        return;
+    }
+#else
+    if (right <= left + 3) {
+        if (right == left + 1) {
+            sort2 (numbers + left);
+        } else if (right == left + 2) {
+            sort3 (numbers + left);
+        } else if (right == left + 3) {
+            sort4 (numbers + left);
+        }
+        return;
+    }
+#endif
+
+    lTmp = left;
+    rTmp = right;
+
+    pivot = medianOf3 (numbers + left, (right-left) >> 1, right-1-left);
+
+    goto QStart;
+    while (1) {
+        do {
+            right--;
+            if (left >= right) goto QEnd;
+            QStart:;
+        } while (numbers[right] > pivot);
+        numbers[left] = numbers[right];
+        do { 
+            left++;
+            if (left >= right) {
+                left = right;
+                goto QEnd;
+            }
+        } while (numbers[ left] < pivot);
+        numbers[right] = numbers[left];
+    }
+    QEnd:;
+    numbers[left] = pivot;
+
+    /* Only recurse the smaller partition */
+
+    if (left-1 - lTmp <= rTmp - left - 1) {
+        if (lTmp < left) quickSortRecurse (numbers,   lTmp, left-1);
+
+        /* Set up for larger partition */
+        left++;
+        right = rTmp;
+    } else {
+        if (rTmp > left) quickSortRecurse (numbers, left+1,   rTmp);
+
+        /* Set up for larger partition */
+        right = left - 1;
+        left = lTmp;
+    }
+
+    /* Rerun with larger partition (recursion not required.) */
+    goto qsrStart;
+}
+
+void quickSort (int numbers[], int qty) {
+    if (qty < 2) return;
+    quickSortRecurse (numbers, 0, qty - 1);
+}
+
+/*
+ *  Merge Sort
+ */
+
+static void mergesortInPlace (int * numbers, int * altNumbers, int qty);
+
+/* Perform mergesort, but store results in altNumbers */
+
+static void mergesortExchange (int * numbers, int * altNumbers, int qty) {
+int half, i0, i1, i;
+
+    if (qty == 2) {
+        sortAlt2 (numbers, altNumbers);
+        return;
+    }
+    if (qty == 3) {
+        sortAlt3 (numbers, altNumbers);
+        return;
+    }
+
+    half = (qty + 1)/2;
+
+    mergesortInPlace (numbers, altNumbers, half);
+    mergesortInPlace (numbers + half, altNumbers, qty - half);
+
+    i0 = 0; i1 = half;
+
+    for (i=0; i < qty; i++) {
+        if (i1 >= qty || (i0 < half && numbers[i0] < numbers[i1])) {
+            altNumbers[i] = numbers[i0];
+            i0++;
+        } else {
+            altNumbers[i] = numbers[i1];
+            i1++;
+        }
+    }
+}
+
+/* Perform mergesort and store results in numbers */
+
+static void mergesortInPlace (int * numbers, int * altNumbers, int qty) {
+int half, i0, i1, i;
+
+#if 0
+    if (qty == 2) {
+        sort2 (numbers);
+        return;
+    }
+    if (qty == 3) {
+        sort3 (numbers);
+        return;
+    }
+    if (qty == 4) {
+        sort4 (numbers);
+        return;
+    }
+#else
+    if (qty <= 12) {
+        insertSort (numbers, qty);
+        return;
+    }
+#endif
+
+    half = (qty + 1)/2;
+
+    mergesortExchange (numbers, altNumbers, half);
+    mergesortExchange (numbers + half, altNumbers + half, qty - half);
+
+    i0 = 0; i1 = half;
+
+    for (i=0; i < qty; i++) {
+        if (i1 >= qty || (i0 < half && altNumbers[i0] < altNumbers[i1])) {
+            numbers[i] = altNumbers[i0];
+            i0++;
+        } else {
+            numbers[i] = altNumbers[i1];
+            i1++;
+        }
+    }
+}
+
+#include <stdlib.h>
+
+void mergeSort (int numbers[], int qty) {
+int * tmpArray;
+
+    if (qty <= 12) {
+        insertSort (numbers, qty);
+        return;
+    }
+
+    tmpArray = (int *) malloc (qty * sizeof (int));
+    mergesortInPlace (numbers, tmpArray, qty);
+    free (tmpArray);
+}
+
+/********************************
+ * END OF PAUL'S IMPLEMENTATION *
+ ********************************/
+
+/*************************************************
+ *** Implementation 1: faster on sorted arrays ***
+ *************************************************/
+
+#define rstype_t unsigned
+#define rskey(x) (x)
+
+#define RS_MIN_SIZE 64
+
+typedef struct {
+	rstype_t *b, *e;
+} rsbucket_t;
+
+void rs_sort(rstype_t *beg, rstype_t *end, int n_bits, int s)
+{
+	rstype_t *i;
+	int size = 1<<n_bits, m = size - 1;
+	rsbucket_t *k, b[size], *be = b + size;
+
+	for (k = b; k != be; ++k) k->b = k->e = beg;
+	for (i = beg; i != end; ++i) ++b[rskey(*i)>>s&m].e;
+	for (k = b + 1; k != be; ++k)
+		k->e += (k-1)->e - beg, k->b = (k-1)->e;
+	for (k = b; k != be;) {
+		if (k->b != k->e) {
+			rsbucket_t *l;
+			if ((l = b + (rskey(*k->b)>>s&m)) != k) {
+				rstype_t tmp = *k->b, swap;
+				do {
+					swap = tmp; tmp = *l->b; *l->b++ = swap;
+					l = b + (rskey(tmp)>>s&m);
+				} while (l != k);
+				*k->b++ = tmp;
+			} else ++k->b;
+		} else ++k;
+	}
+	for (b->b = beg, k = b + 1; k != be; ++k) k->b = (k-1)->e;
+	if (s) {
+		s = s > n_bits? s - n_bits : 0;
+		for (k = b; k != be; ++k)
+			if (k->e - k->b > RS_MIN_SIZE) rs_sort(k->b, k->e, n_bits, s);
+			else if (k->e - k->b > 1)
+				for (i = k->b + 1; i < k->e; ++i)
+					if (rskey(*i) < rskey(*(i - 1))) {
+						rstype_t *j, tmp = *i;
+						for (j = i; j > k->b && rskey(tmp) < rskey(*(j-1)); --j)
+							*j = *(j - 1);
+						*j = tmp;
+					}
+	}
+}
+
+/*************************************************
+ *** Implementation 2: faster on random arrays ***
+ *************************************************/
+
+static inline void rs_insertsort(rstype_t *s, rstype_t *t)
+{
+	rstype_t *i;
+	for (i = s + 1; i < t; ++i) {
+		if (rskey(*i) < rskey(*(i - 1))) {
+			rstype_t *j, tmp = *i;
+			for (j = i; j > s && rskey(tmp) < rskey(*(j-1)); --j)
+				*j = *(j - 1);
+			*j = tmp;
+		}
+	}
+}
+/*
+void rs_sort2(rstype_t *beg, rstype_t *end, int n_bits, int s)
+{
+	int j, size = 1<<n_bits, m = size - 1;
+	unsigned long c[size];
+	rstype_t *i, *b[size], *e[size];
+
+	for (j = 0; j < size; ++j) c[j] = 0;
+	for (i = beg; i != end; ++i) ++c[rskey(*i)>>s&m];
+	b[0] = e[0] = beg;
+	for (j = 1; j != size; ++j) b[j] = e[j] = b[j - 1] + c[j - 1];
+	for (i = beg, j = 0; i != end;) {
+		rstype_t tmp = *i, swap;
+		int x;
+		for (;;) {
+			x = rskey(tmp)>>s&m;
+			if (e[x] == i) break;
+			swap = tmp; tmp = *e[x]; *e[x]++ = swap;
+		}
+		*i++ = tmp;
+		++e[x];
+		while (j != size && i >= b[j]) ++j;
+		while (j != size && e[j-1] == b[j]) ++j;
+		if (i < e[j-1]) i = e[j-1];
+	}
+	if (s) {
+		s = s > n_bits? s - n_bits : 0;
+		for (j = 0; j < size; ++j) {
+			if (c[j] >= RS_MIN_SIZE) rs_sort2(b[j], e[j], n_bits, s);
+			else if (c[j] >= 2) rs_insertsort(b[j], e[j]);
+		}
+	}
+}
+*/
+void radix_sort(unsigned *array, int offset, int end, int shift) {
+    int x, y, value, temp;
+    int last[256] = { 0 }, pointer[256];
+
+    for (x=offset; x<end; ++x) {
+        ++last[(array[x] >> shift) & 0xFF];
+    }
+
+    last[0] += offset;
+    pointer[0] = offset;
+    for (x=1; x<256; ++x) {
+        pointer[x] = last[x-1];
+        last[x] += last[x-1];
+    }
+
+    for (x=0; x<256; ++x) {
+        while (pointer[x] != last[x]) {
+            value = array[pointer[x]];
+            y = (value >> shift) & 0xFF;
+            while (x != y) {
+                temp = array[pointer[y]];
+                array[pointer[y]++] = value;
+                value = temp;
+                y = (value >> shift) & 0xFF;
+            }
+            array[pointer[x]++] = value;
+        }
+    }
+
+    if (shift > 0) {
+        shift -= 8;
+        for (x=0; x<256; ++x) {
+            temp = x > 0 ? pointer[x] - pointer[x-1] : pointer[0] - offset;
+            if (temp > 64) {
+                radix_sort(array, pointer[x] - temp, pointer[x], shift);
+            } else if (temp > 1) rs_insertsort(array + pointer[x] - temp, array + pointer[x]);
+        }
+    }
+}
+/*************************
+ *** END OF RADIX SORT ***
+ *************************/
+
+template< class _Type, unsigned long PowerOfTwoRadix, unsigned long Log2ofPowerOfTwoRadix, long Threshold >
+inline void _RadixSort_Unsigned_PowerOf2Radix_1( _Type* a, long last, _Type bitMask, unsigned long shiftRightAmount )
+{
+	const unsigned long numberOfBins = PowerOfTwoRadix;
+	unsigned long count[ numberOfBins ];
+	for( unsigned long i = 0; i < numberOfBins; i++ )
+		count[ i ] = 0;
+	for ( long _current = 0; _current <= last; _current++ ) // Scan the array and count the number of times each value appears
+	{
+		unsigned long digit = (unsigned long)(( a[ _current ] & bitMask ) >> shiftRightAmount ); // extract the digit we are sorting based on
+		count[ digit ]++;
+	}
+	long startOfBin[ numberOfBins ], endOfBin[ numberOfBins ], nextBin;
+	startOfBin[ 0 ] = endOfBin[ 0 ] = nextBin = 0;
+	for( unsigned long i = 1; i < numberOfBins; i++ )
+		startOfBin[ i ] = endOfBin[ i ] = startOfBin[ i - 1 ] + count[ i - 1 ];
+	for ( long _current = 0; _current <= last; )
+	{
+		unsigned long digit;
+		_Type tmp = a[ _current ];  // get the compiler to recognize that a register can be used for the loop instead of a[_current] memory location
+		while ( true ) {
+			digit = (unsigned long)(( tmp & bitMask ) >> shiftRightAmount );   // extract the digit we are sorting based on
+			if ( endOfBin[ digit ] == _current )
+				break;
+			_Type tmp2;
+			//_swap( tmp, a[ endOfBin[ digit ] ] );
+			tmp2 = a[endOfBin[digit]]; a[endOfBin[digit]] = tmp; tmp = tmp2;
+			endOfBin[ digit ]++;
+		}
+		a[ _current ] = tmp;
+		endOfBin[ digit ]++;   // leave the element at its location and grow the bin
+		_current++;  // advance the current pointer to the next element
+		while( _current >= startOfBin[ nextBin ] && nextBin < numberOfBins )
+			nextBin++;
+		while( endOfBin[ nextBin - 1 ] == startOfBin[ nextBin ] && nextBin < numberOfBins )
+			nextBin++;
+		if ( _current < endOfBin[ nextBin - 1 ] )
+			_current = endOfBin[ nextBin - 1 ];
+	}
+	bitMask >>= Log2ofPowerOfTwoRadix;
+	if ( bitMask != 0 )   // end recursion when all the bits have been processes
+	{
+		if ( shiftRightAmount >= Log2ofPowerOfTwoRadix ) shiftRightAmount -= Log2ofPowerOfTwoRadix;
+		else shiftRightAmount  = 0;
+		for( unsigned long i = 0; i < numberOfBins; i++ )
+		{
+			long numberOfElements = endOfBin[ i ] - startOfBin[ i ];
+			if ( numberOfElements >= Threshold )  // endOfBin actually points to one beyond the bin
+				_RadixSort_Unsigned_PowerOf2Radix_1< _Type, PowerOfTwoRadix, Log2ofPowerOfTwoRadix, Threshold >( &a[ startOfBin[ i ]], numberOfElements - 1, bitMask, shiftRightAmount );
+			else if ( numberOfElements >= 2 )
+				rs_insertsort(&a[ startOfBin[ i ]], &a[ endOfBin[ i ]]);
+		}
+	}
+}
+inline void RadixSortInPlace_HybridUnsigned_Radix256( unsigned* a, unsigned long a_size )
+{
+	if ( a_size < 2 ) return;
+	unsigned long bitMask = 0xFF000000; // bitMask controls how many bits we process at a time
+	unsigned long shiftRightAmount = 24;
+	if ( a_size >= 32 )
+		_RadixSort_Unsigned_PowerOf2Radix_1<unsigned, 256, 8, 32>(a, a_size - 1, bitMask, shiftRightAmount );
+	else
+		rs_insertsort(a, a + a_size);
+}
+
+struct intcmp_t {
+	inline int operator() (int a, int b) const {
+		return a < b? -1 : a > b? 1 : 0;
+	}
+};
+
+int compare_int(int a, int b)
+{
+	return a < b? -1 : a > b? 1 : 0;
+}
+int compare(const void *a, const void *b)
+{
+	return *((int*)a) - *((int*)b);
+}
+
+int main(int argc, char *argv[])
+{
+	int i, N = 50000000;
+	int *array, *temp;
+	clock_t t1, t2;
+	if (argc == 1) fprintf(stderr, "Usage: %s [%d]\n", argv[0], N);
+	if (argc > 1) N = atoi(argv[1]);
+	temp = (int*)malloc(sizeof(int) * N);
+	array = (int*)malloc(sizeof(int) * N);
+
+	srand48(11);
+	for (i = 0; i < N; ++i) array[i] = (int)lrand48();
+	t1 = clock();
+	rs_sort((unsigned*)array, (unsigned*)array + N, 8, 24);
+	t2 = clock();
+	fprintf(stderr, "radix sort: %.3lf\n", (double)(t2-t1)/CLOCKS_PER_SEC);
+	for (i = 0; i < N-1; ++i) {
+		if (array[i] > array[i+1]) {
+			fprintf(stderr, "Bug in radix sort!\n");
+			exit(1);
+		}
+	}
+	t1 = clock();
+	rs_sort((unsigned*)array, (unsigned*)array + N, 8, 24);
+	t2 = clock();
+	fprintf(stderr, "radix sort (sorted): %.3lf\n", (double)(t2-t1)/CLOCKS_PER_SEC);
+
+	srand48(11);
+	for (i = 0; i < N; ++i) array[i] = (int)lrand48();
+	t1 = clock();
+	RadixSortInPlace_HybridUnsigned_Radix256((unsigned*)array, N);
+//	radix_sort((unsigned*)array, 0, N, 24);
+	t2 = clock();
+	fprintf(stderr, "vd's radix sort: %.3lf\n", (double)(t2-t1)/CLOCKS_PER_SEC);
+	for (i = 0; i < N-1; ++i) {
+		if (array[i] > array[i+1]) {
+			fprintf(stderr, "Bug in radix sort!\n");
+			exit(1);
+		}
+	}
+	t1 = clock();
+	RadixSortInPlace_HybridUnsigned_Radix256((unsigned*)array, N);
+//	radix_sort((unsigned*)array, 0, N, 24);
+	t2 = clock();
+	fprintf(stderr, "vd's radix sort (sorted): %.3lf\n", (double)(t2-t1)/CLOCKS_PER_SEC);
+
+	srand48(11);
+	for (i = 0; i < N; ++i) array[i] = (int)lrand48();
+	t1 = clock();
+	sort(array, array+N);
+	t2 = clock();
+	fprintf(stderr, "STL introsort: %.3lf\n", (double)(t2-t1)/CLOCKS_PER_SEC);
+	t1 = clock();
+	sort(array, array+N);
+	t2 = clock();
+	fprintf(stderr, "STL introsort (sorted): %.3lf\n", (double)(t2-t1)/CLOCKS_PER_SEC);
+
+	srand48(11);
+	for (i = 0; i < N; ++i) array[i] = (int)lrand48();
+	t1 = clock();
+	stable_sort(array, array+N);
+	t2 = clock();
+	fprintf(stderr, "STL stablesort: %.3lf\n", (double)(t2-t1)/CLOCKS_PER_SEC);
+	t1 = clock();
+	stable_sort(array, array+N);
+	t2 = clock();
+	fprintf(stderr, "STL stablesort (sorted): %.3lf\n", (double)(t2-t1)/CLOCKS_PER_SEC);
+
+	srand48(11);
+	for (i = 0; i < N; ++i) array[i] = (int)lrand48();
+	t1 = clock();
+	make_heap(array, array+N);
+	sort_heap(array, array+N);
+	t2 = clock();
+	fprintf(stderr, "STL heapsort: %.3lf\n", (double)(t2-t1)/CLOCKS_PER_SEC);
+	for (i = 0; i < N-1; ++i) {
+		if (array[i] > array[i+1]) {
+			fprintf(stderr, "Bug in heap_sort!\n");
+			exit(1);
+		}
+	}
+	t1 = clock();
+	make_heap(array, array+N);
+	sort_heap(array, array+N);
+	t2 = clock();
+	fprintf(stderr, "STL heapsort (sorted): %.3lf\n", (double)(t2-t1)/CLOCKS_PER_SEC);
+
+	srand48(11);
+	for (i = 0; i < N; ++i) array[i] = (int)lrand48();
+	t1 = clock();
+	ks_combsort(int, N, array);
+	t2 = clock();
+	fprintf(stderr, "combsort: %.3lf\n", (double)(t2-t1)/CLOCKS_PER_SEC);
+	for (i = 0; i < N-1; ++i) {
+		if (array[i] > array[i+1]) {
+			fprintf(stderr, "Bug in combsort!\n");
+			exit(1);
+		}
+	}
+
+	srand48(11);
+	for (i = 0; i < N; ++i) array[i] = (int)lrand48();
+	t1 = clock();
+	qsort(array, N, sizeof(int), compare);
+	t2 = clock();
+	fprintf(stderr, "libc qsort: %.3lf\n", (double)(t2-t1)/CLOCKS_PER_SEC);
+
+	srand48(11);
+	for (i = 0; i < N; ++i) array[i] = (int)lrand48();
+	t1 = clock();
+	ks_introsort(int, N, array);
+	t2 = clock();
+	fprintf(stderr, "my introsort: %.3lf\n", (double)(t2-t1)/CLOCKS_PER_SEC);
+	for (i = 0; i < N-1; ++i) {
+		if (array[i] > array[i+1]) {
+			fprintf(stderr, "Bug in intro_sort!\n");
+			exit(1);
+		}
+	}
+	t1 = clock();
+	ks_introsort(int, N, array);
+	t2 = clock();
+	fprintf(stderr, "introsort (sorted): %.3lf\n", (double)(t2-t1)/CLOCKS_PER_SEC);
+
+	srand48(11);
+	for (i = 0; i < N; ++i) array[i] = (int)lrand48();
+	t1 = clock();
+	ks_mergesort(int, N, array, 0);
+	t2 = clock();
+	fprintf(stderr, "iterative mergesort: %.3lf\n", (double)(t2-t1)/CLOCKS_PER_SEC);
+	for (i = 0; i < N-1; ++i) {
+		if (array[i] > array[i+1]) {
+			fprintf(stderr, "Bug in merge_sort!\n");
+			exit(1);
+		}
+	}
+	t1 = clock();
+	ks_mergesort(int, N, array, 0);
+	t2 = clock();
+	fprintf(stderr, "iterative mergesort (sorted): %.3lf\n", (double)(t2-t1)/CLOCKS_PER_SEC);
+
+	srand48(11);
+	for (i = 0; i < N; ++i) array[i] = (int)lrand48();
+	t1 = clock();
+	ks_heapmake(int, N, array);
+	ks_heapsort(int, N, array);
+	t2 = clock();
+	fprintf(stderr, "my heapsort: %.3lf\n", (double)(t2-t1)/CLOCKS_PER_SEC);
+	for (i = 0; i < N-1; ++i) {
+		if (array[i] > array[i+1]) {
+			fprintf(stderr, "Bug in heap_sort!\n");
+			exit(1);
+		}
+	}
+	t1 = clock();
+	ks_heapmake(int, N, array);
+	ks_heapsort(int, N, array);
+	t2 = clock();
+	fprintf(stderr, "heapsort (sorted): %.3lf\n", (double)(t2-t1)/CLOCKS_PER_SEC);
+
+	srand48(11);
+	for (i = 0; i < N; ++i) array[i] = (int)lrand48();
+	t1 = clock();
+	heapSort(array, N);
+	t2 = clock();
+	fprintf(stderr, "Paul's heapsort: %.3lf\n", (double)(t2-t1)/CLOCKS_PER_SEC);
+	for (i = 0; i < N-1; ++i) {
+		if (array[i] > array[i+1]) {
+			fprintf(stderr, "Bug in intro_sort!\n");
+			exit(1);
+		}
+	}
+
+	srand48(11);
+	for (i = 0; i < N; ++i) array[i] = (int)lrand48();
+	t1 = clock();
+	quickSort(array, N);
+	t2 = clock();
+	fprintf(stderr, "Paul's quicksort: %.3lf\n", (double)(t2-t1)/CLOCKS_PER_SEC);
+	for (i = 0; i < N-1; ++i) {
+		if (array[i] > array[i+1]) {
+			fprintf(stderr, "Bug in intro_sort!\n");
+			exit(1);
+		}
+	}
+
+	srand48(11);
+	for (i = 0; i < N; ++i) array[i] = (int)lrand48();
+	t1 = clock();
+	mergeSort(array, N);
+	t2 = clock();
+	fprintf(stderr, "Paul's mergesort: %.3lf\n", (double)(t2-t1)/CLOCKS_PER_SEC);
+	for (i = 0; i < N-1; ++i) {
+		if (array[i] > array[i+1]) {
+			fprintf(stderr, "Bug in intro_sort!\n");
+			exit(1);
+		}
+	}
+
+	free(array); free(temp);
+	return 0;
+}
diff --git a/ext/klib/test/kstring_bench.c b/ext/klib/test/kstring_bench.c
new file mode 100644
index 0000000..82598e8
--- /dev/null
+++ b/ext/klib/test/kstring_bench.c
@@ -0,0 +1,51 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+#include "kstring.h"
+
+#define N 10000000
+
+int main()
+{
+	int i;
+	clock_t t;
+	kstring_t s, s2;
+	srand48(11);
+	s.l = s.m = 0; s.s = 0;
+	t = clock();
+	for (i = 0; i < N; ++i) {
+		int x = lrand48();
+		s.l = 0;
+		kputw(x, &s);
+	}
+	fprintf(stderr, "kputw: %lf\n", (double)(clock() - t) / CLOCKS_PER_SEC);
+	srand48(11);
+	t = clock();
+	for (i = 0; i < N; ++i) {
+		int x = lrand48();
+		s.l = 0;
+		ksprintf(&s, "%d", x);
+	}
+	fprintf(stderr, "ksprintf: %lf\n", (double)(clock() - t) / CLOCKS_PER_SEC);
+
+	srand48(11);
+	s2.l = s2.m = 0; s2.s = 0;
+	t = clock();
+	for (i = 0; i < N; ++i) {
+		int x = lrand48();
+		s2.l = s.l = 0;
+		kputw(x, &s2);
+		kputs(s2.s, &s);
+	}
+	fprintf(stderr, "kputw+kputs: %lf\n", (double)(clock() - t) / CLOCKS_PER_SEC);
+	srand48(11);
+	t = clock();
+	for (i = 0; i < N; ++i) {
+		int x = lrand48();
+		s2.l = s.l = 0;
+		kputw(x, &s2);
+		ksprintf(&s, "%s", s2.s);
+	}
+	fprintf(stderr, "kputw+ksprintf: %lf\n", (double)(clock() - t) / CLOCKS_PER_SEC);
+	return 0;
+}
diff --git a/ext/klib/test/kstring_bench2.c b/ext/klib/test/kstring_bench2.c
new file mode 100644
index 0000000..b7707a8
--- /dev/null
+++ b/ext/klib/test/kstring_bench2.c
@@ -0,0 +1,131 @@
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+#include "kstring.h"
+
+#ifdef __APPLE__
+#define HAVE_STRNSTR
+#endif
+
+#ifdef __linux__
+#define HAVE_MEMMEM
+#endif
+
+static int str_len = 1024*1024*128;
+static int pat_len = 30;
+static int alphabet = 2;
+static int repeat = 50;
+
+char *gen_data(int len, int a)
+{
+	char *data;
+	int i;
+	long x;
+	srand48(11);
+	data = malloc(len);
+	for (i = 0; i < len; ++i)
+		data[i] = (int)(a * drand48()) + '!';
+	data[str_len - 1] = 0;
+	return data;
+}
+// http://srcvault.scali.eu.org/cgi-bin/Syntax/c/BoyerMoore.c
+char *BoyerMoore( unsigned char *data, unsigned int dataLength, unsigned char *string, unsigned int strLength )
+{
+	unsigned int skipTable[256], i;
+	unsigned char *search;
+	register unsigned char lastChar;
+
+	if (strLength == 0)
+		return NULL;
+
+	for (i = 0; i < 256; i++)
+		skipTable[i] = strLength;
+	search = string;
+	i = --strLength;
+	do {
+		skipTable[*search++] = i;
+	} while (i--);
+	lastChar = *--search;
+	search = data + strLength;
+	dataLength -= strLength+(strLength-1);
+	while ((int)dataLength > 0 ) {
+		unsigned int skip;
+		skip = skipTable[*search];
+		search += skip;
+		dataLength -= skip;
+		skip = skipTable[*search];
+		search += skip;
+		dataLength -= skip;
+		skip = skipTable[*search];
+		if (*search != lastChar) {
+			search += skip;
+			dataLength -= skip;
+			continue;
+		}
+		i = strLength;
+		do {
+			if (i-- == 0) return search;
+		} while (*--search == string[i]);
+		search += (strLength - i + 1);
+		dataLength--;
+	}
+	return NULL;
+}
+
+int main()
+{
+	char *data;
+	int i;
+	clock_t t;
+	t = clock();
+	data = gen_data(str_len, alphabet);
+	fprintf(stderr, "Generate data in %.3f sec\n", (float)(clock() - t) / CLOCKS_PER_SEC);
+	{
+		t = clock(); srand48(1331);
+		for (i = 0; i < repeat; ++i) {
+			int y = lrand48() % (str_len - pat_len);
+			char *ret;
+			ret = kmemmem(data, str_len, data + y, pat_len, 0);
+//			printf("%d, %d\n", (int)(ret - data), y);
+		}
+		fprintf(stderr, "Search patterns in %.3f sec\n", (float)(clock() - t) / CLOCKS_PER_SEC);
+	}
+	if (1) {
+		t = clock(); srand48(1331);
+		for (i = 0; i < repeat; ++i) {
+			int y = lrand48() % (str_len - pat_len);
+			char *ret;
+			ret = BoyerMoore(data, str_len, data + y, pat_len);
+//			printf("%d, %d\n", (int)(ret - data), y);
+		}
+		fprintf(stderr, "Search patterns in %.3f sec\n", (float)(clock() - t) / CLOCKS_PER_SEC);
+	}
+#ifdef HAVE_STRNSTR
+	if (1) {
+		char *tmp;
+		t = clock(); srand48(1331);
+		tmp = calloc(pat_len+1, 1);
+		for (i = 0; i < repeat; ++i) {
+			int y = lrand48() % (str_len - pat_len);
+			char *ret;
+			memcpy(tmp, data + y, pat_len);
+			ret = strnstr(data, tmp, str_len);
+		}
+		fprintf(stderr, "Search patterns in %.3f sec\n", (float)(clock() - t) / CLOCKS_PER_SEC);		
+	}
+#endif
+#ifdef HAVE_MEMMEM
+	if (1) {
+		t = clock(); srand48(1331);
+		for (i = 0; i < repeat; ++i) {
+			int y = lrand48() % (str_len - pat_len);
+			char *ret;
+			ret = memmem(data, str_len, data + y, pat_len);
+//			printf("%d, %d\n", (int)(ret - data), y);
+		}
+		fprintf(stderr, "Search patterns in %.3f sec\n", (float)(clock() - t) / CLOCKS_PER_SEC);
+	}
+#endif
+	return 0;
+}
diff --git a/ext/klib/test/kstring_test.c b/ext/klib/test/kstring_test.c
new file mode 100644
index 0000000..9f9b6e6
--- /dev/null
+++ b/ext/klib/test/kstring_test.c
@@ -0,0 +1,132 @@
+#include <limits.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "kstring.h"
+
+int nfail = 0;
+
+void check(const char *what, const kstring_t *ks, const char *correct)
+{
+	if (ks->l != strlen(correct) || strcmp(ks->s, correct) != 0) {
+		fprintf(stderr, "%s produced \"%.*s\" (\"%s\" is correct)\tFAIL\n", what, (int)(ks->l), ks->s, correct);
+		nfail++;
+	}
+}
+
+void test_kputw(kstring_t *ks, int n)
+{
+	char buf[16];
+
+	ks->l = 0;
+	kputw(n, ks);
+
+	sprintf(buf, "%d", n);
+	check("kputw()", ks, buf);
+}
+
+void test_kputl(kstring_t *ks, long n)
+{
+	char buf[24];
+
+	ks->l = 0;
+	kputl(n, ks);
+
+	sprintf(buf, "%ld", n);
+	check("kputl()", ks, buf);
+}
+
+static char *mem_gets(char *buf, int buflen, void *vtextp)
+{
+	const char **textp = (const char **) vtextp;
+
+	const char *nl = strchr(*textp, '\n');
+	size_t n = nl? nl - *textp + 1 : strlen(*textp);
+
+	if (n == 0) return NULL;
+
+	if (n > buflen-1) n = buflen-1;
+	memcpy(buf, *textp, n);
+	buf[n] = '\0';
+	*textp += n;
+	return buf;
+}
+
+void test_kgetline(kstring_t *ks, const char *text, ...)
+{
+	const char *exp;
+	va_list arg;
+
+	va_start(arg, text);
+	while ((exp = va_arg(arg, const char *)) != NULL) {
+		ks->l = 0;
+		if (kgetline(ks, mem_gets, &text) != 0) kputs("EOF", ks);
+		check("kgetline()", ks, exp);
+	}
+	va_end(arg);
+
+	ks->l = 0;
+	if (kgetline(ks, mem_gets, &text) == 0) check("kgetline()", ks, "EOF");
+}
+
+int main(int argc, char **argv)
+{
+	kstring_t ks;
+
+	ks.l = ks.m = 0;
+	ks.s = NULL;
+
+	test_kputw(&ks, 0);
+	test_kputw(&ks, 1);
+	test_kputw(&ks, 37);
+	test_kputw(&ks, 12345);
+	test_kputw(&ks, -12345);
+	test_kputw(&ks, INT_MAX);
+	test_kputw(&ks, -INT_MAX);
+	test_kputw(&ks, INT_MIN);
+
+	test_kputl(&ks, 0);
+	test_kputl(&ks, 1);
+	test_kputl(&ks, 37);
+	test_kputl(&ks, 12345);
+	test_kputl(&ks, -12345);
+	test_kputl(&ks, INT_MAX);
+	test_kputl(&ks, -INT_MAX);
+	test_kputl(&ks, INT_MIN);
+	test_kputl(&ks, LONG_MAX);
+	test_kputl(&ks, -LONG_MAX);
+	test_kputl(&ks, LONG_MIN);
+
+	test_kgetline(&ks, "", NULL);
+	test_kgetline(&ks, "apple", "apple", NULL);
+	test_kgetline(&ks, "banana\n", "banana", NULL);
+	test_kgetline(&ks, "carrot\r\n", "carrot", NULL);
+	test_kgetline(&ks, "\n", "", NULL);
+	test_kgetline(&ks, "\n\n", "", "", NULL);
+	test_kgetline(&ks, "foo\nbar", "foo", "bar", NULL);
+	test_kgetline(&ks, "foo\nbar\n", "foo", "bar", NULL);
+	test_kgetline(&ks,
+		"abcdefghijklmnopqrstuvwxyz0123456789\nABCDEFGHIJKLMNOPQRSTUVWXYZ\n",
+		"abcdefghijklmnopqrstuvwxyz0123456789",
+		"ABCDEFGHIJKLMNOPQRSTUVWXYZ", NULL);
+
+	if (argc > 1) {
+		FILE *f = fopen(argv[1], "r");
+		if (f) {
+			for (ks.l = 0; kgetline(&ks, (kgets_func *)fgets, f) == 0; ks.l = 0)
+				puts(ks.s);
+			fclose(f);
+		}
+	}
+
+	free(ks.s);
+
+	if (nfail > 0) {
+		fprintf(stderr, "Total failures: %d\n", nfail);
+		return EXIT_FAILURE;
+	}
+
+	return EXIT_SUCCESS;
+}
diff --git a/ext/klib/test/kthread_test.c b/ext/klib/test/kthread_test.c
new file mode 100644
index 0000000..1b67ed4
--- /dev/null
+++ b/ext/klib/test/kthread_test.c
@@ -0,0 +1,69 @@
+#include <stdio.h>
+#include <assert.h>
+#include <stdlib.h>
+#include <pthread.h>
+#if HAVE_CILK
+#include <cilk/cilk.h>
+#include <cilk/cilk_api.h>
+#endif
+
+typedef struct {
+	int max_iter, w, h;
+	double xmin, xmax, ymin, ymax;
+	int *k;
+} global_t;
+
+static void compute(void *_g, int i, int tid)
+{
+	global_t *g = (global_t*)_g;
+	double x, x0 = g->xmin + (g->xmax - g->xmin) * (i%g->w) / g->w;
+	double y, y0 = g->ymin + (g->ymax - g->ymin) * (i/g->w) / g->h;
+	int k;
+
+	assert(g->k[i] < 0);
+	x = x0, y = y0;
+	for (k = 0; k < g->max_iter; ++k) {
+		double z = x * y;
+		x *= x; y *= y;
+		if (x + y >= 4) break;
+		x = x - y + x0;
+		y = z + z + y0; 
+	}
+	g->k[i] = k;
+}
+
+void kt_for(int n_threads, int n_items, void (*func)(void*,int,int), void *data);
+
+int main(int argc, char *argv[])
+{
+	int i, tmp, tot, type = 0, n_threads = 2;
+	global_t global = { 10240*100, 800, 600, -2., -1.2, -1.2, 1.2, 0 };
+//	global_t global = { 10240*1, 8, 6, -2., -1.2, -1.2, 1.2, 0 };
+
+	if (argc > 1) {
+		type = argv[1][0] == 'o'? 2 : argv[1][0] == 'c'? 3 : argv[1][0] == 'n'? 1 : 0;
+		if (argv[1][0] >= '0' && argv[1][0] <= '9')
+			n_threads = atoi(argv[1]);
+	} else {
+		fprintf(stderr, "Usage: ./a.out [openmp | cilk | #threads]\n");
+	}
+	tot = global.w * global.h;
+	global.k = calloc(tot, sizeof(int));
+	for (i = 0; i < tot; ++i) global.k[i] = -1;
+	if (type == 0) {
+		kt_for(n_threads, tot, compute, &global);
+	} else if (type == 2) {
+		#pragma omp parallel for
+		for (i = 0; i < tot; ++i)
+			compute(&global, i, 0);
+	} else if (type == 3) {
+		#if HAVE_CILK
+		cilk_for (i = 0; i < tot; ++i)
+			compute(&global, i, 0);
+		#endif
+	}
+	for (i = tmp = 0; i < tot; ++i) tmp += (global.k[i] < 0);
+	free(global.k);
+	assert(tmp == 0);
+	return 0;
+}
diff --git a/ext/klib/test/kthread_test2.c b/ext/klib/test/kthread_test2.c
new file mode 100644
index 0000000..68663a5
--- /dev/null
+++ b/ext/klib/test/kthread_test2.c
@@ -0,0 +1,80 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+
+void kt_for(int n_threads, void (*func)(void*,long,int), void *data, long n);
+void kt_pipeline(int n_threads, void *(*func)(void*, int, void*), void *shared_data, int n_steps);
+
+typedef struct {
+	FILE *fp;
+	int max_lines, buf_size, n_threads;
+	char *buf;
+} pipeline_t;
+
+typedef struct {
+	int n_lines;
+	char **lines;
+} step_t;
+
+static void worker_for(void *_data, long i, int tid) // kt_for() callback
+{
+	step_t *step = (step_t*)_data;
+	char *s = step->lines[i];
+	int t, l, j;
+	l = strlen(s) - 1;
+	assert(s[l] == '\n'); // not supporting long lines
+	for (j = 0; j < l>>1; ++j)
+		t = s[j], s[j] = s[l - 1 - j], s[l - 1 - j] = t;
+}
+
+static void *worker_pipeline(void *shared, int step, void *in) // kt_pipeline() callback
+{
+	pipeline_t *p = (pipeline_t*)shared;
+	if (step == 0) { // step 0: read lines into the buffer
+		step_t *s;
+		s = calloc(1, sizeof(step_t));
+		s->lines = calloc(p->max_lines, sizeof(char*));
+		while (fgets(p->buf, p->buf_size, p->fp) != 0) {
+			s->lines[s->n_lines] = strdup(p->buf);
+			if (++s->n_lines >= p->max_lines)
+				break;
+		}
+		if (s->n_lines) return s;
+	} else if (step == 1) { // step 1: reverse lines
+		kt_for(p->n_threads, worker_for, in, ((step_t*)in)->n_lines);
+		return in;
+	} else if (step == 2) { // step 3: write the buffer to output
+		step_t *s = (step_t*)in;
+		while (s->n_lines > 0) {
+			fputs(s->lines[--s->n_lines], stdout);
+			free(s->lines[s->n_lines]);
+		}
+		free(s->lines); free(s);
+	}
+	return 0;
+}
+
+int main(int argc, char *argv[])
+{
+	pipeline_t pl;
+	int pl_threads;
+	if (argc == 1) {
+		fprintf(stderr, "Usage: reverse <in.txt> [pipeline_threads [for_threads]]\n");
+		return 1;
+	}
+	pl.fp = strcmp(argv[1], "-")? fopen(argv[1], "r") : stdin;
+	if (pl.fp == 0) {
+		fprintf(stderr, "ERROR: failed to open the input file.\n");
+		return 1;
+	}
+	pl_threads = argc > 2? atoi(argv[2]) : 3;
+	pl.max_lines = 4096;
+	pl.buf_size = 0x10000;
+	pl.n_threads = argc > 3? atoi(argv[3]) : 1;
+	pl.buf = calloc(pl.buf_size, 1);
+	kt_pipeline(pl_threads, worker_pipeline, &pl, 3);
+	free(pl.buf);
+	if (pl.fp != stdin) fclose(pl.fp);
+	return 0;
+}
diff --git a/ext/klib/test/kvec_test.cc b/ext/klib/test/kvec_test.cc
new file mode 100644
index 0000000..1015574
--- /dev/null
+++ b/ext/klib/test/kvec_test.cc
@@ -0,0 +1,69 @@
+#include <vector>
+#include <time.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include "kvec.h"
+
+int main()
+{
+	int M = 10, N = 20000000, i, j;
+	clock_t t;
+	t = clock();
+	for (i = 0; i < M; ++i) {
+		int *array = (int*)malloc(N * sizeof(int));
+		for (j = 0; j < N; ++j) array[j] = j;
+		free(array);
+	}
+	printf("C array, preallocated: %.3f sec\n",
+		   (float)(clock() - t) / CLOCKS_PER_SEC);
+	t = clock();
+	for (i = 0; i < M; ++i) {
+		int *array = 0, max = 0;
+		for (j = 0; j < N; ++j) {
+			if (j == max) {
+				max = !max? 1 : max << 1;
+				array = (int*)realloc(array, sizeof(int)*max);
+			}
+			array[j] = j;
+		}
+		free(array);
+	}
+	printf("C array, dynamic: %.3f sec\n",
+		   (float)(clock() - t) / CLOCKS_PER_SEC);
+	t = clock();
+	for (i = 0; i < M; ++i) {
+		kvec_t(int) array;
+		kv_init(array);
+		kv_resize(int, array, N);
+		for (j = 0; j < N; ++j) kv_a(int, array, j) = j;
+		kv_destroy(array);
+	}
+	printf("C vector, dynamic(kv_a): %.3f sec\n",
+		   (float)(clock() - t) / CLOCKS_PER_SEC);
+	t = clock();
+	for (i = 0; i < M; ++i) {
+		kvec_t(int) array;
+		kv_init(array);
+		for (j = 0; j < N; ++j)
+			kv_push(int, array, j);
+		kv_destroy(array);
+	}
+	printf("C vector, dynamic(kv_push): %.3f sec\n",
+		   (float)(clock() - t) / CLOCKS_PER_SEC);
+	t = clock();
+	for (i = 0; i < M; ++i) {
+		std::vector<int> array;
+		array.reserve(N);
+		for (j = 0; j < N; ++j) array[j] = j;
+	}
+	printf("C++ vector, preallocated: %.3f sec\n",
+		   (float)(clock() - t) / CLOCKS_PER_SEC);
+	t = clock();
+	for (i = 0; i < M; ++i) {
+		std::vector<int> array;
+		for (j = 0; j < N; ++j) array.push_back(j);
+	}
+	printf("C++ vector, dynamic: %.3f sec\n",
+		   (float)(clock() - t) / CLOCKS_PER_SEC);
+	return 0;
+}
diff --git a/ext/spdlog/.clang-format b/ext/spdlog/.clang-format
new file mode 100644
index 0000000..c8c345f
--- /dev/null
+++ b/ext/spdlog/.clang-format
@@ -0,0 +1,19 @@
+---
+Language:        Cpp
+BasedOnStyle:  Google
+AccessModifierOffset: -4
+Standard:        c++17
+IndentWidth:     4
+TabWidth:        4
+UseTab:          Never
+ColumnLimit:     100
+AlignAfterOpenBracket: Align
+BinPackParameters: false
+AlignEscapedNewlines: Left
+AlwaysBreakTemplateDeclarations: Yes
+PackConstructorInitializers: Never
+BreakConstructorInitializersBeforeComma: false
+IndentPPDirectives: BeforeHash
+SortIncludes:    Never
+...
+
diff --git a/ext/spdlog/.clang-tidy b/ext/spdlog/.clang-tidy
new file mode 100644
index 0000000..bc50ee7
--- /dev/null
+++ b/ext/spdlog/.clang-tidy
@@ -0,0 +1,53 @@
+Checks: 'cppcoreguidelines-*,
+performance-*,
+modernize-*,
+google-*,
+misc-*
+cert-*,
+readability-*,
+clang-analyzer-*,
+-performance-unnecessary-value-param,
+-modernize-use-trailing-return-type,
+-google-runtime-references,
+-misc-non-private-member-variables-in-classes,
+-readability-braces-around-statements,
+-google-readability-braces-around-statements,
+-cppcoreguidelines-avoid-magic-numbers,
+-readability-magic-numbers,
+-readability-magic-numbers,
+-cppcoreguidelines-pro-type-vararg,
+-cppcoreguidelines-pro-bounds-pointer-arithmetic,
+-cppcoreguidelines-avoid-c-arrays,
+-modernize-avoid-c-arrays,
+-cppcoreguidelines-pro-bounds-array-to-pointer-decay,
+-readability-named-parameter,
+-cert-env33-c
+'
+
+
+WarningsAsErrors: ''
+HeaderFilterRegex: '*spdlog/[^f].*'
+FormatStyle:     none
+
+CheckOptions:    
+  - key:             google-readability-braces-around-statements.ShortStatementLines
+    value:           '1'
+  - key:             google-readability-function-size.StatementThreshold
+    value:           '800'
+  - key:             google-readability-namespace-comments.ShortNamespaceLines
+    value:           '10'
+  - key:             google-readability-namespace-comments.SpacesBeforeComments
+    value:           '2'
+  - key:             modernize-loop-convert.MaxCopySize
+    value:           '16'
+  - key:             modernize-loop-convert.MinConfidence
+    value:           reasonable
+  - key:             modernize-loop-convert.NamingStyle
+    value:           CamelCase
+  - key:             modernize-pass-by-value.IncludeStyle
+    value:           llvm
+  - key:             modernize-replace-auto-ptr.IncludeStyle
+    value:           llvm
+  - key:             modernize-use-nullptr.NullMacros
+    value:           'NULL'
+
diff --git a/ext/spdlog/.git-blame-ignore-revs b/ext/spdlog/.git-blame-ignore-revs
new file mode 100644
index 0000000..47b8fed
--- /dev/null
+++ b/ext/spdlog/.git-blame-ignore-revs
@@ -0,0 +1,6 @@
+# clang-format
+1a0bfc7a89f2d58e22605a4dc7e18a9a555b65aa
+95c226e9c92928e20ccdac0d060e7241859e282b
+9d52261185b5f2c454c381d626ec5c84d7b195f4
+4b2a8219d5d1b40062d030441adde7d1fb0d4f84
+0a53eafe18d983c7c8ba4cadd02d0cc7f7308f28
diff --git a/ext/spdlog/.gitattributes b/ext/spdlog/.gitattributes
new file mode 100644
index 0000000..fe505b2
--- /dev/null
+++ b/ext/spdlog/.gitattributes
@@ -0,0 +1 @@
+* text=false
diff --git a/ext/spdlog/.github/workflows/linux.yml b/ext/spdlog/.github/workflows/linux.yml
new file mode 100644
index 0000000..4e97d07
--- /dev/null
+++ b/ext/spdlog/.github/workflows/linux.yml
@@ -0,0 +1,87 @@
+name: linux
+
+on: [push, pull_request]
+
+permissions:
+  contents: read
+
+jobs:
+  # -----------------------------------------------------------------------
+  # Linux build matrix
+  # -----------------------------------------------------------------------
+  build:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - { compiler: gcc, version: 7, build_type: Release, cppstd: 11 }
+          - { compiler: gcc, version: 9, build_type: Release, cppstd: 17 }
+          - { compiler: gcc, version: 11, build_type: Debug, cppstd: 20 }
+          - { compiler: gcc, version: 12, build_type: Release, cppstd: 20 }
+          - { compiler: gcc, version: 12, build_type: Debug, cppstd: 20, asan: ON }
+          - { compiler: clang, version: 12, build_type: Debug, cppstd: 17 }
+          - { compiler: clang, version: 15, build_type: Release, cppstd: 20, tsan: ON }
+    container:
+      image: ${{ matrix.config.compiler == 'clang' && 'teeks99/clang-ubuntu' || matrix.config.compiler }}:${{ matrix.config.version }}
+    name: "${{ matrix.config.compiler}} ${{ matrix.config.version }} (C++${{ matrix.config.cppstd }} ${{ matrix.config.build_type }} ${{ matrix.config.asan == 'ON' && 'ASAN' || '' }}${{ matrix.config.tsan == 'ON' && 'TSAN' || '' }})"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup
+        run: |
+          apt-get update
+          apt-get install -y curl git pkg-config libsystemd-dev
+          CMAKE_VERSION="3.24.2"
+          curl -sSL https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.sh -o install-cmake.sh
+          chmod +x install-cmake.sh
+          ./install-cmake.sh --prefix=/usr/local --skip-license
+      - name: Setup Compiler
+        if: matrix.config.compiler == 'clang'
+        run: |
+          scripts/ci_setup_clang.sh "${{ matrix.config.version }}"
+          echo "CXXFLAGS=-stdlib=libc++" >> $GITHUB_ENV          
+          echo "CC=clang-${{ matrix.config.version }}" >> $GITHUB_ENV
+          echo "CXX=clang++-${{ matrix.config.version }}" >> $GITHUB_ENV
+      - name: Build
+        run: |
+          mkdir -p build && cd build
+          cmake .. \
+            -DCMAKE_BUILD_TYPE=${{ matrix.config.build_type }} \
+            -DCMAKE_CXX_STANDARD=${{ matrix.config.cppstd }} \
+            -DSPDLOG_BUILD_EXAMPLE=${{ matrix.config.examples || 'ON' }} \
+            -DSPDLOG_BUILD_EXAMPLE_HO=${{ matrix.config.examples || 'ON' }} \
+            -DSPDLOG_BUILD_WARNINGS=ON \
+            -DSPDLOG_BUILD_BENCH=OFF \
+            -DSPDLOG_BUILD_TESTS=ON \
+            -DSPDLOG_BUILD_TESTS_HO=OFF \
+            -DSPDLOG_SANITIZE_ADDRESS=${{ matrix.config.asan || 'OFF' }} \
+            -DSPDLOG_SANITIZE_THREAD=${{ matrix.config.tsan || 'OFF' }}
+          make -j 4
+          ctest -j 4 --output-on-failure
+
+  # -----------------------------------------------------------------------
+  # OS X build matrix
+  # -----------------------------------------------------------------------
+  build_osx:
+    runs-on: macOS-latest
+    name: "OS X Clang (C++11, Release)"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build
+        run: |
+          mkdir -p build && cd build
+          cmake .. \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DCMAKE_CXX_STANDARD=11 \
+            -DSPDLOG_BUILD_EXAMPLE=ON \
+            -DSPDLOG_BUILD_EXAMPLE_HO=ON \
+            -DSPDLOG_BUILD_WARNINGS=ON \
+            -DSPDLOG_BUILD_BENCH=OFF \
+            -DSPDLOG_BUILD_TESTS=ON \
+            -DSPDLOG_BUILD_TESTS_HO=OFF \
+            -DSPDLOG_SANITIZE_ADDRESS=OFF
+          make -j 4
+          ctest -j 4 --output-on-failure
diff --git a/ext/spdlog/.github/workflows/macos.yml b/ext/spdlog/.github/workflows/macos.yml
new file mode 100644
index 0000000..60e9e74
--- /dev/null
+++ b/ext/spdlog/.github/workflows/macos.yml
@@ -0,0 +1,28 @@
+name: macos
+
+on: [push, pull_request]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: macOS-latest
+    name: "macOS Clang (C++11, Release)"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build
+        run: |
+          mkdir -p build && cd build
+          cmake .. \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DCMAKE_CXX_STANDARD=11 \
+            -DSPDLOG_BUILD_EXAMPLE=ON \
+            -DSPDLOG_BUILD_EXAMPLE_HO=ON \
+            -DSPDLOG_BUILD_WARNINGS=ON \
+            -DSPDLOG_BUILD_BENCH=OFF \
+            -DSPDLOG_BUILD_TESTS=ON \
+            -DSPDLOG_BUILD_TESTS_HO=OFF \
+            -DSPDLOG_SANITIZE_ADDRESS=OFF
+          make -j 4
+          ctest -j 4 --output-on-failure
diff --git a/ext/spdlog/.github/workflows/windows.yml b/ext/spdlog/.github/workflows/windows.yml
new file mode 100644
index 0000000..710e409
--- /dev/null
+++ b/ext/spdlog/.github/workflows/windows.yml
@@ -0,0 +1,148 @@
+name: windows
+
+on: [push, pull_request]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: windows-latest
+    strategy:
+      fail-fast: true
+      matrix:
+        config:
+          - GENERATOR: "Visual Studio 17 2022"
+            BUILD_TYPE: Release
+            BUILD_SHARED: 'ON'
+            FATAL_ERRORS: 'ON'
+            WCHAR: 'OFF'
+            WCHAR_FILES: 'OFF'
+            BUILD_EXAMPLE: 'OFF'
+            USE_STD_FORMAT: 'ON'
+            CXX_STANDARD: 20
+          - GENERATOR: "Visual Studio 17 2022"
+            BUILD_TYPE: Release
+            BUILD_SHARED: 'ON'
+            FATAL_ERRORS: 'ON'
+            WCHAR: 'ON'
+            WCHAR_FILES: 'ON'
+            BUILD_EXAMPLE: 'OFF'
+            USE_STD_FORMAT: 'ON'
+            CXX_STANDARD: 20
+          - GENERATOR: "Visual Studio 17 2022"
+            BUILD_TYPE: Release
+            BUILD_SHARED: 'ON'
+            FATAL_ERRORS: 'ON'
+            WCHAR: 'OFF'
+            WCHAR_FILES: 'OFF'
+            BUILD_EXAMPLE: 'ON'
+            USE_STD_FORMAT: 'OFF'
+            CXX_STANDARD: 17
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: CMake ${{ matrix.config.GENERATOR }} CXX=${{matrix.config.CXX_STANDARD}} WCHAR=${{matrix.config.WCHAR_FILES}} STD_FORMAT=${{matrix.config.USE_STD_FORMAT}}
+        shell: pwsh
+        run: |
+          mkdir build
+          cd build
+          cmake -G "${{ matrix.config.GENERATOR }}"  -A x64 `
+            -D CMAKE_BUILD_TYPE=${{ matrix.config.BUILD_TYPE }} `
+            -D BUILD_SHARED_LIBS=${{ matrix.config.BUILD_SHARED }} `
+            -D SPDLOG_WCHAR_SUPPORT=${{ matrix.config.WCHAR }} `
+            -D SPDLOG_WCHAR_FILENAMES=${{ matrix.config.WCHAR_FILES }} `
+            -D SPDLOG_BUILD_EXAMPLE=${{ matrix.config.BUILD_EXAMPLE }} `
+            -D SPDLOG_BUILD_EXAMPLE_HO=${{ matrix.config.BUILD_EXAMPLE }} `
+            -D SPDLOG_BUILD_TESTS=ON `
+            -D SPDLOG_BUILD_TESTS_HO=OFF `
+            -D SPDLOG_BUILD_WARNINGS=${{ matrix.config.FATAL_ERRORS }} `
+            -D SPDLOG_USE_STD_FORMAT=${{ matrix.config.USE_STD_FORMAT }} `
+            -D CMAKE_CXX_STANDARD=${{ matrix.config.CXX_STANDARD }} ..
+
+      - name: Build
+        shell: pwsh
+        run: |
+          cd build
+          cmake --build  . --parallel --config ${{ matrix.config.BUILD_TYPE }}
+
+      - name: Run Tests
+        shell: pwsh
+        env:
+          PATH: ${{ env.PATH }};${{ github.workspace }}\build\_deps\catch2-build\src\${{ matrix.config.BUILD_TYPE }};${{ github.workspace }}\build\${{ matrix.config.BUILD_TYPE }}
+        run: |
+          build\tests\${{ matrix.config.BUILD_TYPE }}\spdlog-utests.exe
+
+  # -----------------------------------------------------------------------
+  # MSVC 2019 build matrix
+  # -----------------------------------------------------------------------
+  build_2019:
+    runs-on: windows-2019
+    strategy:
+      fail-fast: true
+      matrix:
+        config:
+          - GENERATOR: "Visual Studio 16 2019"
+            BUILD_TYPE: Release
+            BUILD_SHARED: 'ON'
+            FATAL_ERRORS: 'ON'
+            WCHAR: 'OFF'
+            WCHAR_FILES: 'OFF'
+            BUILD_EXAMPLE: 'ON'
+            USE_STD_FORMAT: 'OFF'
+            CXX_STANDARD: 17
+          - GENERATOR: "Visual Studio 16 2019"
+            BUILD_TYPE: Release
+            BUILD_SHARED: 'ON'
+            FATAL_ERRORS: 'ON'
+            WCHAR: 'OFF'
+            WCHAR_FILES: 'OFF'
+            BUILD_EXAMPLE: 'ON'
+            USE_STD_FORMAT: 'OFF'
+            CXX_STANDARD: 14
+          - GENERATOR: "Visual Studio 16 2019"
+            BUILD_TYPE: Release
+            BUILD_SHARED: 'ON'
+            FATAL_ERRORS: 'ON'
+            WCHAR: 'OFF'
+            WCHAR_FILES: 'OFF'
+            BUILD_EXAMPLE: 'ON'
+            USE_STD_FORMAT: 'OFF'
+            CXX_STANDARD: 11
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: CMake ${{ matrix.config.GENERATOR }} CXX=${{matrix.config.CXX_STANDARD}} WCHAR=${{matrix.config.WCHAR_FILES}} STD_FORMAT=${{matrix.config.USE_STD_FORMAT}}
+        shell: pwsh
+        run: |
+          mkdir build
+          cd build
+          cmake -G "${{ matrix.config.GENERATOR }}"  -A x64 `
+            -D CMAKE_BUILD_TYPE=${{ matrix.config.BUILD_TYPE }} `
+            -D BUILD_SHARED_LIBS=${{ matrix.config.BUILD_SHARED }} `
+            -D SPDLOG_WCHAR_SUPPORT=${{ matrix.config.WCHAR }} `
+            -D SPDLOG_WCHAR_FILENAMES=${{ matrix.config.WCHAR_FILES }} `
+            -D SPDLOG_BUILD_EXAMPLE=${{ matrix.config.BUILD_EXAMPLE }} `
+            -D SPDLOG_BUILD_EXAMPLE_HO=${{ matrix.config.BUILD_EXAMPLE }} `
+            -D SPDLOG_BUILD_TESTS=ON `
+            -D SPDLOG_BUILD_TESTS_HO=OFF `
+            -D SPDLOG_BUILD_WARNINGS=${{ matrix.config.FATAL_ERRORS }} `
+            -D SPDLOG_USE_STD_FORMAT=${{ matrix.config.USE_STD_FORMAT }} `
+            -D CMAKE_CXX_STANDARD=${{ matrix.config.CXX_STANDARD }} ..
+
+      - name: Build
+        shell: pwsh
+        run: |
+          cd build
+          cmake --build  . --parallel --config ${{ matrix.config.BUILD_TYPE }}
+
+      - name: Run Tests
+        shell: pwsh
+        env:
+          PATH: ${{ env.PATH }};${{ github.workspace }}\build\_deps\catch2-build\src\${{ matrix.config.BUILD_TYPE }};${{ github.workspace }}\build\${{ matrix.config.BUILD_TYPE }}
+        run: |
+          build\tests\${{ matrix.config.BUILD_TYPE }}\spdlog-utests.exe
diff --git a/ext/spdlog/.gitignore b/ext/spdlog/.gitignore
new file mode 100644
index 0000000..25f272c
--- /dev/null
+++ b/ext/spdlog/.gitignore
@@ -0,0 +1,98 @@
+# Auto generated files
+[Dd]ebug/
+[Rr]elease/
+build/*
+*.slo
+*.lo
+*.o
+*.obj
+*.suo
+*.tlog
+*.ilk
+*.log
+*.pdb
+*.idb
+*.iobj
+*.ipdb
+*.opensdf
+*.sdf
+
+# Compiled Dynamic libraries
+*.so
+*.dylib
+*.dll
+
+# Compiled Static libraries
+*.lai
+*.la
+*.a
+*.lib
+
+# Executables
+*.exe
+*.out
+*.app
+
+# Codelite
+.codelite
+
+# KDevelop
+*.kdev4
+
+# .orig files
+*.orig
+
+# example  files
+example/*
+!example/example.cpp
+!example/bench.cpp
+!example/utils.h
+!example/Makefile*
+!example/example.sln
+!example/example.vcxproj
+!example/CMakeLists.txt
+!example/meson.build
+!example/multisink.cpp
+!example/jni
+
+# generated files
+generated
+version.rc
+
+# Cmake
+CMakeCache.txt
+CMakeFiles
+CMakeScripts
+Makefile
+cmake_install.cmake
+install_manifest.txt
+/tests/tests.VC.VC.opendb
+/tests/tests.VC.db
+/tests/tests
+/tests/logs/*
+spdlogConfig.cmake
+spdlogConfigVersion.cmake
+compile_commands.json
+
+# idea
+.idea/
+.cache/
+.vscode/
+cmake-build-*/
+*.db
+*.ipch
+*.filters
+*.db-wal
+*.opendb
+*.db-shm
+*.vcxproj
+*.tcl
+*.user
+*.sln
+
+# macos
+*.DS_store
+*.xcodeproj/
+/.vs
+/out/build
+/CMakeSettings.json
diff --git a/ext/spdlog/CMakeLists.txt b/ext/spdlog/CMakeLists.txt
new file mode 100644
index 0000000..e56c569
--- /dev/null
+++ b/ext/spdlog/CMakeLists.txt
@@ -0,0 +1,400 @@
+# Copyright(c) 2019 spdlog authors Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+cmake_minimum_required(VERSION 3.10...3.21)
+
+# ---------------------------------------------------------------------------------------
+# Start spdlog project
+# ---------------------------------------------------------------------------------------
+include(cmake/utils.cmake)
+include(cmake/ide.cmake)
+
+spdlog_extract_version()
+
+project(spdlog VERSION ${SPDLOG_VERSION} LANGUAGES CXX)
+message(STATUS "Build spdlog: ${SPDLOG_VERSION}")
+
+include(GNUInstallDirs)
+
+# ---------------------------------------------------------------------------------------
+# Set default build to release
+# ---------------------------------------------------------------------------------------
+if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
+    set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Choose Release or Debug" FORCE)
+endif()
+
+# ---------------------------------------------------------------------------------------
+# Compiler config
+# ---------------------------------------------------------------------------------------
+if(SPDLOG_USE_STD_FORMAT)
+    set(CMAKE_CXX_STANDARD 20)
+    set(CMAKE_CXX_STANDARD_REQUIRED ON)
+elseif(NOT CMAKE_CXX_STANDARD)
+    set(CMAKE_CXX_STANDARD 11)
+    set(CMAKE_CXX_STANDARD_REQUIRED ON)
+endif()
+
+
+set(CMAKE_CXX_EXTENSIONS OFF)
+
+if(CMAKE_SYSTEM_NAME MATCHES "CYGWIN" OR CMAKE_SYSTEM_NAME MATCHES "MSYS" OR CMAKE_SYSTEM_NAME MATCHES "MINGW")
+    set(CMAKE_CXX_EXTENSIONS ON)
+endif()
+
+# ---------------------------------------------------------------------------------------
+# Set SPDLOG_MASTER_PROJECT to ON if we are building spdlog
+# ---------------------------------------------------------------------------------------
+# Check if spdlog is being used directly or via add_subdirectory, but allow overriding
+if(NOT DEFINED SPDLOG_MASTER_PROJECT)
+    if(CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR)
+        set(SPDLOG_MASTER_PROJECT ON)
+    else()
+        set(SPDLOG_MASTER_PROJECT OFF)
+    endif()
+endif()
+
+option(SPDLOG_BUILD_ALL "Build all artifacts" OFF)
+
+# build shared option
+option(SPDLOG_BUILD_SHARED "Build shared library" OFF)
+
+# precompiled headers option
+option(SPDLOG_ENABLE_PCH "Build static or shared library using precompiled header to speed up compilation time" OFF)
+
+# build position independent code
+option(SPDLOG_BUILD_PIC "Build position independent code (-fPIC)" OFF)
+
+# example options
+option(SPDLOG_BUILD_EXAMPLE "Build example" ${SPDLOG_MASTER_PROJECT})
+option(SPDLOG_BUILD_EXAMPLE_HO "Build header only example" OFF)
+
+# testing options
+option(SPDLOG_BUILD_TESTS "Build tests" OFF)
+option(SPDLOG_BUILD_TESTS_HO "Build tests using the header only version" OFF)
+
+# bench options
+option(SPDLOG_BUILD_BENCH "Build benchmarks (Requires https://github.com/google/benchmark.git to be installed)" OFF)
+
+# sanitizer options
+option(SPDLOG_SANITIZE_ADDRESS "Enable address sanitizer in tests" OFF)
+option(SPDLOG_SANITIZE_THREAD "Enable thread sanitizer in tests" OFF)
+if(SPDLOG_SANITIZE_ADDRESS AND SPDLOG_SANITIZE_THREAD)
+    message(FATAL_ERROR "SPDLOG_SANITIZE_ADDRESS and SPDLOG_SANITIZE_THREAD are mutually exclusive")
+endif()
+
+# warning options
+option(SPDLOG_BUILD_WARNINGS "Enable compiler warnings" OFF)
+
+# install options
+option(SPDLOG_SYSTEM_INCLUDES "Include as system headers (skip for clang-tidy)." OFF)
+option(SPDLOG_INSTALL "Generate the install target" ${SPDLOG_MASTER_PROJECT})
+option(SPDLOG_USE_STD_FORMAT "Use std::format instead of fmt library." OFF)
+option(SPDLOG_FMT_EXTERNAL "Use external fmt library instead of bundled" OFF)
+option(SPDLOG_FMT_EXTERNAL_HO "Use external fmt header-only library instead of bundled" OFF)
+option(SPDLOG_NO_EXCEPTIONS "Compile with -fno-exceptions. Call abort() on any spdlog exceptions" OFF)
+
+if(SPDLOG_FMT_EXTERNAL AND SPDLOG_FMT_EXTERNAL_HO)
+    message(FATAL_ERROR "SPDLOG_FMT_EXTERNAL and SPDLOG_FMT_EXTERNAL_HO are mutually exclusive")
+endif()
+
+if(SPDLOG_USE_STD_FORMAT AND SPDLOG_FMT_EXTERNAL_HO)
+    message(FATAL_ERROR "SPDLOG_USE_STD_FORMAT and SPDLOG_FMT_EXTERNAL_HO are mutually exclusive")
+endif()
+
+if(SPDLOG_USE_STD_FORMAT AND SPDLOG_FMT_EXTERNAL)
+    message(FATAL_ERROR "SPDLOG_USE_STD_FORMAT and SPDLOG_FMT_EXTERNAL are mutually exclusive")
+endif()
+
+# misc tweakme options
+if(WIN32)
+    option(SPDLOG_WCHAR_SUPPORT "Support wchar api" OFF)
+    option(SPDLOG_WCHAR_FILENAMES "Support wchar filenames" OFF)
+    option(SPDLOG_WCHAR_CONSOLE "Support wchar output to console" OFF)
+else()
+    set(SPDLOG_WCHAR_SUPPORT OFF CACHE BOOL "non supported option" FORCE)
+    set(SPDLOG_WCHAR_FILENAMES OFF CACHE BOOL "non supported option" FORCE)
+    set(SPDLOG_WCHAR_CONSOLE OFF CACHE BOOL "non supported option" FORCE)
+endif()
+
+if(MSVC)
+    option(SPDLOG_MSVC_UTF8 "Enable/disable msvc /utf-8 flag required by fmt lib" ON)
+endif()
+
+if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux")
+    option(SPDLOG_CLOCK_COARSE "Use CLOCK_REALTIME_COARSE instead of the regular clock," OFF)
+else()
+    set(SPDLOG_CLOCK_COARSE OFF CACHE BOOL "non supported option" FORCE)
+endif()
+
+option(SPDLOG_PREVENT_CHILD_FD "Prevent from child processes to inherit log file descriptors" OFF)
+option(SPDLOG_NO_THREAD_ID "prevent spdlog from querying the thread id on each log call if thread id is not needed" OFF)
+option(SPDLOG_NO_TLS "prevent spdlog from using thread local storage" OFF)
+option(
+    SPDLOG_NO_ATOMIC_LEVELS
+    "prevent spdlog from using of std::atomic log levels (use only if your code never modifies log levels concurrently"
+    OFF)
+option(SPDLOG_DISABLE_DEFAULT_LOGGER "Disable default logger creation" OFF)
+
+# clang-tidy
+option(SPDLOG_TIDY "run clang-tidy" OFF)
+
+if(SPDLOG_TIDY)
+    set(CMAKE_CXX_CLANG_TIDY "clang-tidy")
+    set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+    message(STATUS "Enabled clang-tidy")
+endif()
+
+if(SPDLOG_BUILD_PIC)
+    set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+endif()
+
+find_package(Threads REQUIRED)
+message(STATUS "Build type: " ${CMAKE_BUILD_TYPE})
+# ---------------------------------------------------------------------------------------
+# Static/Shared library
+# ---------------------------------------------------------------------------------------
+set(SPDLOG_SRCS src/spdlog.cpp src/stdout_sinks.cpp src/color_sinks.cpp src/file_sinks.cpp src/async.cpp src/cfg.cpp)
+
+if(NOT SPDLOG_USE_STD_FORMAT AND NOT SPDLOG_FMT_EXTERNAL AND NOT SPDLOG_FMT_EXTERNAL_HO)
+    list(APPEND SPDLOG_SRCS src/bundled_fmtlib_format.cpp)
+endif()
+
+if(SPDLOG_BUILD_SHARED OR BUILD_SHARED_LIBS)
+    if(WIN32)
+        configure_file(${CMAKE_CURRENT_SOURCE_DIR}/cmake/version.rc.in ${CMAKE_CURRENT_BINARY_DIR}/version.rc @ONLY)
+        list(APPEND SPDLOG_SRCS ${CMAKE_CURRENT_BINARY_DIR}/version.rc)
+    endif()
+    add_library(spdlog SHARED ${SPDLOG_SRCS} ${SPDLOG_ALL_HEADERS})
+    target_compile_definitions(spdlog PUBLIC SPDLOG_SHARED_LIB)
+    if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
+        target_compile_options(spdlog PUBLIC $<$<AND:$<CXX_COMPILER_ID:MSVC>,$<NOT:$<COMPILE_LANGUAGE:CUDA>>>:/wd4251
+                                             /wd4275>)
+    endif()
+    if(NOT SPDLOG_USE_STD_FORMAT AND NOT SPDLOG_FMT_EXTERNAL AND NOT SPDLOG_FMT_EXTERNAL_HO)
+        target_compile_definitions(spdlog PRIVATE FMT_LIB_EXPORT PUBLIC FMT_SHARED)
+    endif()
+else()
+    add_library(spdlog STATIC ${SPDLOG_SRCS} ${SPDLOG_ALL_HEADERS})
+endif()
+
+add_library(spdlog::spdlog ALIAS spdlog)
+
+set(SPDLOG_INCLUDES_LEVEL "")
+if(SPDLOG_SYSTEM_INCLUDES)
+    set(SPDLOG_INCLUDES_LEVEL "SYSTEM")
+endif()
+
+target_compile_definitions(spdlog PUBLIC SPDLOG_COMPILED_LIB)
+target_include_directories(spdlog ${SPDLOG_INCLUDES_LEVEL} PUBLIC "$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/include>"
+                                                                  "$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>")
+target_link_libraries(spdlog PUBLIC Threads::Threads)
+spdlog_enable_warnings(spdlog)
+
+set_target_properties(spdlog PROPERTIES VERSION ${SPDLOG_VERSION} SOVERSION
+                                                                  ${SPDLOG_VERSION_MAJOR}.${SPDLOG_VERSION_MINOR})
+set_target_properties(spdlog PROPERTIES DEBUG_POSTFIX d)
+
+if(COMMAND target_precompile_headers AND SPDLOG_ENABLE_PCH)
+    configure_file(${CMAKE_CURRENT_SOURCE_DIR}/cmake/pch.h.in ${PROJECT_BINARY_DIR}/spdlog_pch.h @ONLY)
+    target_precompile_headers(spdlog PRIVATE ${PROJECT_BINARY_DIR}/spdlog_pch.h)
+endif()
+
+# sanitizer support
+if(SPDLOG_SANITIZE_ADDRESS)
+    spdlog_enable_addr_sanitizer(spdlog)
+elseif (SPDLOG_SANITIZE_THREAD)
+    spdlog_enable_thread_sanitizer(spdlog)
+endif ()
+
+# ---------------------------------------------------------------------------------------
+# Header only version
+# ---------------------------------------------------------------------------------------
+add_library(spdlog_header_only INTERFACE)
+add_library(spdlog::spdlog_header_only ALIAS spdlog_header_only)
+
+target_include_directories(
+    spdlog_header_only ${SPDLOG_INCLUDES_LEVEL} INTERFACE "$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/include>"
+                                                          "$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>")
+target_link_libraries(spdlog_header_only INTERFACE Threads::Threads)
+
+# ---------------------------------------------------------------------------------------
+# Use fmt package if using external fmt
+# ---------------------------------------------------------------------------------------
+if(SPDLOG_FMT_EXTERNAL OR SPDLOG_FMT_EXTERNAL_HO)
+    if(NOT TARGET fmt::fmt)
+        find_package(fmt CONFIG REQUIRED)
+    endif()
+    target_compile_definitions(spdlog PUBLIC SPDLOG_FMT_EXTERNAL)
+    target_compile_definitions(spdlog_header_only INTERFACE SPDLOG_FMT_EXTERNAL)
+
+    # use external fmt-header-only
+    if(SPDLOG_FMT_EXTERNAL_HO)
+        target_link_libraries(spdlog PUBLIC fmt::fmt-header-only)
+        target_link_libraries(spdlog_header_only INTERFACE fmt::fmt-header-only)
+    else() # use external compile fmt
+        target_link_libraries(spdlog PUBLIC fmt::fmt)
+        target_link_libraries(spdlog_header_only INTERFACE fmt::fmt)
+    endif()
+
+    set(PKG_CONFIG_REQUIRES fmt) # add dependency to pkg-config
+endif()
+
+# ---------------------------------------------------------------------------------------
+# Check if fwrite_unlocked/_fwrite_nolock is available
+# ---------------------------------------------------------------------------------------
+include(CheckSymbolExists)
+if(WIN32)
+    check_symbol_exists(_fwrite_nolock "stdio.h" HAVE_FWRITE_UNLOCKED)
+else ()
+    check_symbol_exists(fwrite_unlocked "stdio.h" HAVE_FWRITE_UNLOCKED)
+endif()
+if(HAVE_FWRITE_UNLOCKED)
+    target_compile_definitions(spdlog PRIVATE SPDLOG_FWRITE_UNLOCKED)
+    target_compile_definitions(spdlog_header_only INTERFACE SPDLOG_FWRITE_UNLOCKED)
+endif()
+
+# ---------------------------------------------------------------------------------------
+# Add required libraries for Android CMake build
+# ---------------------------------------------------------------------------------------
+if(ANDROID)
+    target_link_libraries(spdlog PUBLIC log)
+    target_link_libraries(spdlog_header_only INTERFACE log)
+endif()
+
+# ---------------------------------------------------------------------------------------
+# Misc definitions according to tweak options
+# ---------------------------------------------------------------------------------------
+set(SPDLOG_WCHAR_TO_UTF8_SUPPORT ${SPDLOG_WCHAR_SUPPORT})
+set(SPDLOG_UTF8_TO_WCHAR_CONSOLE ${SPDLOG_WCHAR_CONSOLE})
+foreach(
+    SPDLOG_OPTION
+    SPDLOG_WCHAR_TO_UTF8_SUPPORT
+    SPDLOG_UTF8_TO_WCHAR_CONSOLE
+    SPDLOG_WCHAR_FILENAMES
+    SPDLOG_NO_EXCEPTIONS
+    SPDLOG_CLOCK_COARSE
+    SPDLOG_PREVENT_CHILD_FD
+    SPDLOG_NO_THREAD_ID
+    SPDLOG_NO_TLS
+    SPDLOG_NO_ATOMIC_LEVELS
+    SPDLOG_DISABLE_DEFAULT_LOGGER
+    SPDLOG_USE_STD_FORMAT)
+    if(${SPDLOG_OPTION})
+        target_compile_definitions(spdlog PUBLIC ${SPDLOG_OPTION})
+        target_compile_definitions(spdlog_header_only INTERFACE ${SPDLOG_OPTION})
+    endif()
+endforeach()
+
+if(MSVC)
+    target_compile_options(spdlog PRIVATE "/Zc:__cplusplus")
+    target_compile_options(spdlog_header_only INTERFACE "/Zc:__cplusplus")
+    if(SPDLOG_MSVC_UTF8)
+        # fmtlib requires the /utf-8 flag when building with msvc.
+	    # see https://github.com/fmtlib/fmt/pull/4159 on the purpose of the additional
+        # "$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:MSVC>>"
+	    target_compile_options(spdlog PUBLIC $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:MSVC>>:/utf-8>)
+	    target_compile_options(spdlog_header_only INTERFACE $<$<AND:$<COMPILE_LANGUAGE:CXX>,$<CXX_COMPILER_ID:MSVC>>:/utf-8>)
+     endif()
+endif()
+
+# ---------------------------------------------------------------------------------------
+# If exceptions are disabled, disable them in the bundled fmt as well
+# ---------------------------------------------------------------------------------------
+if(SPDLOG_NO_EXCEPTIONS)
+    if(NOT SPDLOG_FMT_EXTERNAL AND NOT SPDLOG_FMT_EXTERNAL_HO)
+        target_compile_definitions(spdlog PUBLIC FMT_EXCEPTIONS=0)
+    endif()
+    if(NOT MSVC)
+        target_compile_options(spdlog PRIVATE -fno-exceptions)
+    else()
+        target_compile_options(spdlog PRIVATE /EHs-c-)
+    endif()
+endif()
+# ---------------------------------------------------------------------------------------
+# Build binaries
+# ---------------------------------------------------------------------------------------
+if(SPDLOG_BUILD_EXAMPLE OR SPDLOG_BUILD_EXAMPLE_HO OR SPDLOG_BUILD_ALL)
+    message(STATUS "Generating example(s)")
+    add_subdirectory(example)
+    spdlog_enable_warnings(example)
+    if(SPDLOG_BUILD_EXAMPLE_HO)
+        spdlog_enable_warnings(example_header_only)
+    endif()
+endif()
+
+if(SPDLOG_BUILD_TESTS OR SPDLOG_BUILD_TESTS_HO OR SPDLOG_BUILD_ALL)
+    message(STATUS "Generating tests")
+    enable_testing()
+    add_subdirectory(tests)
+endif()
+
+if(SPDLOG_BUILD_BENCH OR SPDLOG_BUILD_ALL)
+    message(STATUS "Generating benchmarks")
+    add_subdirectory(bench)
+endif()
+
+# ---------------------------------------------------------------------------------------
+# Install
+# ---------------------------------------------------------------------------------------
+if(SPDLOG_INSTALL)
+    message(STATUS "Generating install")
+    set(project_config_in "${CMAKE_CURRENT_LIST_DIR}/cmake/spdlogConfig.cmake.in")
+    set(project_config_out "${CMAKE_CURRENT_BINARY_DIR}/spdlogConfig.cmake")
+    set(config_targets_file "spdlogConfigTargets.cmake")
+    set(version_config_file "${CMAKE_CURRENT_BINARY_DIR}/spdlogConfigVersion.cmake")
+    set(export_dest_dir "${CMAKE_INSTALL_LIBDIR}/cmake/spdlog")
+    set(pkgconfig_install_dir "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
+    set(pkg_config "${CMAKE_BINARY_DIR}/${PROJECT_NAME}.pc")
+
+    # ---------------------------------------------------------------------------------------
+    # Include files
+    # ---------------------------------------------------------------------------------------
+    install(DIRECTORY include/ DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" PATTERN "fmt/bundled" EXCLUDE)
+    install(
+        TARGETS spdlog spdlog_header_only
+        EXPORT spdlog
+        LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+        ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+        RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
+
+    if(NOT SPDLOG_USE_STD_FORMAT AND NOT SPDLOG_FMT_EXTERNAL AND NOT SPDLOG_FMT_EXTERNAL_HO)
+        install(DIRECTORY include/${PROJECT_NAME}/fmt/bundled/
+                DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME}/fmt/bundled/")
+    endif()
+
+    # ---------------------------------------------------------------------------------------
+    # Install pkg-config file
+    # ---------------------------------------------------------------------------------------
+    if(IS_ABSOLUTE "${CMAKE_INSTALL_INCLUDEDIR}")
+        set(PKG_CONFIG_INCLUDEDIR "${CMAKE_INSTALL_INCLUDEDIR}")
+    else()
+        set(PKG_CONFIG_INCLUDEDIR "\${prefix}/${CMAKE_INSTALL_INCLUDEDIR}")
+    endif()
+    if(IS_ABSOLUTE "${CMAKE_INSTALL_LIBDIR}")
+        set(PKG_CONFIG_LIBDIR "${CMAKE_INSTALL_LIBDIR}")
+    else()
+        set(PKG_CONFIG_LIBDIR "\${exec_prefix}/${CMAKE_INSTALL_LIBDIR}")
+    endif()
+    get_target_property(PKG_CONFIG_DEFINES spdlog INTERFACE_COMPILE_DEFINITIONS)
+    string(REPLACE ";" " -D" PKG_CONFIG_DEFINES "${PKG_CONFIG_DEFINES}")
+    string(CONCAT PKG_CONFIG_DEFINES "-D" "${PKG_CONFIG_DEFINES}")
+    configure_file("cmake/${PROJECT_NAME}.pc.in" "${pkg_config}" @ONLY)
+    install(FILES "${pkg_config}" DESTINATION "${pkgconfig_install_dir}")
+
+    # ---------------------------------------------------------------------------------------
+    # Install CMake config files
+    # ---------------------------------------------------------------------------------------
+    export(TARGETS spdlog spdlog_header_only NAMESPACE spdlog::
+           FILE "${CMAKE_CURRENT_BINARY_DIR}/${config_targets_file}")
+    install(EXPORT spdlog DESTINATION ${export_dest_dir} NAMESPACE spdlog:: FILE ${config_targets_file})
+
+    include(CMakePackageConfigHelpers)
+    configure_package_config_file("${project_config_in}" "${project_config_out}" INSTALL_DESTINATION ${export_dest_dir})
+
+    write_basic_package_version_file("${version_config_file}" COMPATIBILITY SameMajorVersion)
+    install(FILES "${project_config_out}" "${version_config_file}" DESTINATION "${export_dest_dir}")
+
+    # ---------------------------------------------------------------------------------------
+    # Support creation of installable packages
+    # ---------------------------------------------------------------------------------------
+    include(cmake/spdlogCPack.cmake)
+endif()
diff --git a/ext/spdlog/INSTALL b/ext/spdlog/INSTALL
new file mode 100644
index 0000000..4b6fb06
--- /dev/null
+++ b/ext/spdlog/INSTALL
@@ -0,0 +1,27 @@
+Header Only Version
+==================================================================
+Just copy the files to your build tree and use a C++11 compiler.  
+Or use CMake:
+``` 
+  add_executable(example_header_only example.cpp)
+  target_link_libraries(example_header_only spdlog::spdlog_header_only)
+```
+
+Compiled Library Version
+==================================================================
+CMake:
+```  
+  add_executable(example example.cpp)
+  target_link_libraries(example spdlog::spdlog)
+```
+
+Or copy files src/*.cpp to your build tree and pass the -DSPDLOG_COMPILED_LIB to the compiler.
+
+Important Information for Compilation:
+==================================================================
+* If you encounter compilation errors with gcc 4.8.x, please note that gcc 4.8.x does not fully support C++11. In such cases, consider upgrading your compiler or using a different version that fully supports C++11 standards
+
+Tested on:  
+gcc 4.8.1 and above
+clang 3.5
+Visual Studio 2013
\ No newline at end of file
diff --git a/ext/spdlog/LICENSE b/ext/spdlog/LICENSE
new file mode 100644
index 0000000..3e696a7
--- /dev/null
+++ b/ext/spdlog/LICENSE
@@ -0,0 +1,26 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Gabi Melman.                                       
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+-- NOTE: Third party dependency used by this software --
+This software depends on the fmt lib (MIT License),
+and users must comply to its license: https://raw.githubusercontent.com/fmtlib/fmt/master/LICENSE
+
diff --git a/ext/spdlog/README.md b/ext/spdlog/README.md
new file mode 100644
index 0000000..1aab857
--- /dev/null
+++ b/ext/spdlog/README.md
@@ -0,0 +1,524 @@
+# spdlog
+
+ 
+[![ci](https://github.com/gabime/spdlog/actions/workflows/linux.yml/badge.svg)](https://github.com/gabime/spdlog/actions/workflows/linux.yml)&nbsp;
+[![ci](https://github.com/gabime/spdlog/actions/workflows/windows.yml/badge.svg)](https://github.com/gabime/spdlog/actions/workflows/windows.yml)&nbsp;
+[![ci](https://github.com/gabime/spdlog/actions/workflows/macos.yml/badge.svg)](https://github.com/gabime/spdlog/actions/workflows/macos.yml)&nbsp;
+[![Build status](https://ci.appveyor.com/api/projects/status/d2jnxclg20vd0o50?svg=true&branch=v1.x)](https://ci.appveyor.com/project/gabime/spdlog) [![Release](https://img.shields.io/github/release/gabime/spdlog.svg)](https://github.com/gabime/spdlog/releases/latest)
+
+Fast C++ logging library
+
+
+## Install
+#### Header-only version
+Copy the include [folder](https://github.com/gabime/spdlog/tree/v1.x/include/spdlog) to your build tree and use a C++11 compiler.
+
+#### Compiled version (recommended - much faster compile times)
+```console
+$ git clone https://github.com/gabime/spdlog.git
+$ cd spdlog && mkdir build && cd build
+$ cmake .. && cmake --build .
+```
+see example [CMakeLists.txt](https://github.com/gabime/spdlog/blob/v1.x/example/CMakeLists.txt) on how to use.
+
+## Platforms
+* Linux, FreeBSD, OpenBSD, Solaris, AIX
+* Windows (msvc 2013+, cygwin)
+* macOS (clang 3.5+)
+* Android
+
+## Package managers:
+* Debian: `sudo apt install libspdlog-dev`
+* Homebrew: `brew install spdlog`
+* MacPorts: `sudo port install spdlog`
+* FreeBSD:  `pkg install spdlog`
+* Fedora: `dnf install spdlog`
+* Gentoo: `emerge dev-libs/spdlog`
+* Arch Linux: `pacman -S spdlog`
+* openSUSE: `sudo zypper in spdlog-devel`
+* vcpkg: `vcpkg install spdlog`
+* conan: `conan install --requires=spdlog/[*]`
+* conda: `conda install -c conda-forge spdlog`
+* build2: ```depends: spdlog ^1.8.2```
+
+
+## Features
+* Very fast (see [benchmarks](#benchmarks) below).
+* Headers only or compiled
+* Feature-rich formatting, using the excellent [fmt](https://github.com/fmtlib/fmt) library.
+* Asynchronous mode (optional)
+* [Custom](https://github.com/gabime/spdlog/wiki/3.-Custom-formatting) formatting.
+* Multi/Single threaded loggers.
+* Various log targets:
+  * Rotating log files.
+  * Daily log files.
+  * Console logging (colors supported).
+  * syslog.
+  * Windows event log.
+  * Windows debugger (```OutputDebugString(..)```).
+  * Log to Qt widgets ([example](#log-to-qt-with-nice-colors)).
+  * Easily [extendable](https://github.com/gabime/spdlog/wiki/4.-Sinks#implementing-your-own-sink) with custom log targets.
+* Log filtering - log levels can be modified at runtime as well as compile time.
+* Support for loading log levels from argv or environment var.
+* [Backtrace](#backtrace-support) support - store debug messages in a ring buffer and display them later on demand.
+
+## Usage samples
+
+#### Basic usage
+```c++
+#include "spdlog/spdlog.h"
+
+int main() 
+{
+    spdlog::info("Welcome to spdlog!");
+    spdlog::error("Some error message with arg: {}", 1);
+    
+    spdlog::warn("Easy padding in numbers like {:08d}", 12);
+    spdlog::critical("Support for int: {0:d};  hex: {0:x};  oct: {0:o}; bin: {0:b}", 42);
+    spdlog::info("Support for floats {:03.2f}", 1.23456);
+    spdlog::info("Positional args are {1} {0}..", "too", "supported");
+    spdlog::info("{:<30}", "left aligned");
+    
+    spdlog::set_level(spdlog::level::debug); // Set global log level to debug
+    spdlog::debug("This message should be displayed..");    
+    
+    // change log pattern
+    spdlog::set_pattern("[%H:%M:%S %z] [%n] [%^---%L---%$] [thread %t] %v");
+    
+    // Compile time log levels
+    // Note that this does not change the current log level, it will only
+    // remove (depending on SPDLOG_ACTIVE_LEVEL) the call on the release code.
+    SPDLOG_TRACE("Some trace message with param {}", 42);
+    SPDLOG_DEBUG("Some debug message");
+}
+
+```
+---
+#### Create stdout/stderr logger object
+```c++
+#include "spdlog/spdlog.h"
+#include "spdlog/sinks/stdout_color_sinks.h"
+void stdout_example()
+{
+    // create a color multi-threaded logger
+    auto console = spdlog::stdout_color_mt("console");    
+    auto err_logger = spdlog::stderr_color_mt("stderr");    
+    spdlog::get("console")->info("loggers can be retrieved from a global registry using the spdlog::get(logger_name)");
+}
+```
+
+---
+#### Basic file logger
+```c++
+#include "spdlog/sinks/basic_file_sink.h"
+void basic_logfile_example()
+{
+    try 
+    {
+        auto logger = spdlog::basic_logger_mt("basic_logger", "logs/basic-log.txt");
+    }
+    catch (const spdlog::spdlog_ex &ex)
+    {
+        std::cout << "Log init failed: " << ex.what() << std::endl;
+    }
+}
+```
+---
+#### Rotating files
+```c++
+#include "spdlog/sinks/rotating_file_sink.h"
+void rotating_example()
+{
+    // Create a file rotating logger with 5 MB size max and 3 rotated files
+    auto max_size = 1048576 * 5;
+    auto max_files = 3;
+    auto logger = spdlog::rotating_logger_mt("some_logger_name", "logs/rotating.txt", max_size, max_files);
+}
+```
+
+---
+#### Daily files
+```c++
+
+#include "spdlog/sinks/daily_file_sink.h"
+void daily_example()
+{
+    // Create a daily logger - a new file is created every day at 2:30 am
+    auto logger = spdlog::daily_logger_mt("daily_logger", "logs/daily.txt", 2, 30);
+}
+
+```
+
+---
+#### Backtrace support
+```c++
+// Debug messages can be stored in a ring buffer instead of being logged immediately.
+// This is useful to display debug logs only when needed (e.g. when an error happens).
+// When needed, call dump_backtrace() to dump them to your log.
+
+spdlog::enable_backtrace(32); // Store the latest 32 messages in a buffer. 
+// or my_logger->enable_backtrace(32)..
+for(int i = 0; i < 100; i++)
+{
+  spdlog::debug("Backtrace message {}", i); // not logged yet..
+}
+// e.g. if some error happened:
+spdlog::dump_backtrace(); // log them now! show the last 32 messages
+// or my_logger->dump_backtrace(32)..
+```
+
+---
+#### Periodic flush
+```c++
+// periodically flush all *registered* loggers every 3 seconds:
+// warning: only use if all your loggers are thread-safe ("_mt" loggers)
+spdlog::flush_every(std::chrono::seconds(3));
+
+```
+
+---
+#### Stopwatch
+```c++
+// Stopwatch support for spdlog
+#include "spdlog/stopwatch.h"
+void stopwatch_example()
+{
+    spdlog::stopwatch sw;    
+    spdlog::debug("Elapsed {}", sw);
+    spdlog::debug("Elapsed {:.3}", sw);       
+}
+
+```
+
+---
+#### Log binary data in hex
+```c++
+// many types of std::container<char> types can be used.
+// ranges are supported too.
+// format flags:
+// {:X} - print in uppercase.
+// {:s} - don't separate each byte with space.
+// {:p} - don't print the position on each line start.
+// {:n} - don't split the output into lines.
+// {:a} - show ASCII if :n is not set.
+
+#include "spdlog/fmt/bin_to_hex.h"
+
+void binary_example()
+{
+    auto console = spdlog::get("console");
+    std::array<char, 80> buf;
+    console->info("Binary example: {}", spdlog::to_hex(buf));
+    console->info("Another binary example:{:n}", spdlog::to_hex(std::begin(buf), std::begin(buf) + 10));
+    // more examples:
+    // logger->info("uppercase: {:X}", spdlog::to_hex(buf));
+    // logger->info("uppercase, no delimiters: {:Xs}", spdlog::to_hex(buf));
+    // logger->info("uppercase, no delimiters, no position info: {:Xsp}", spdlog::to_hex(buf));
+}
+
+```
+
+---
+#### Logger with multi sinks - each with a different format and log level
+```c++
+
+// create a logger with 2 targets, with different log levels and formats.
+// The console will show only warnings or errors, while the file will log all.
+void multi_sink_example()
+{
+    auto console_sink = std::make_shared<spdlog::sinks::stdout_color_sink_mt>();
+    console_sink->set_level(spdlog::level::warn);
+    console_sink->set_pattern("[multi_sink_example] [%^%l%$] %v");
+
+    auto file_sink = std::make_shared<spdlog::sinks::basic_file_sink_mt>("logs/multisink.txt", true);
+    file_sink->set_level(spdlog::level::trace);
+
+    spdlog::logger logger("multi_sink", {console_sink, file_sink});
+    logger.set_level(spdlog::level::debug);
+    logger.warn("this should appear in both console and file");
+    logger.info("this message should not appear in the console, only in the file");
+}
+```
+
+---
+#### User-defined callbacks about log events
+```c++
+
+// create a logger with a lambda function callback, the callback will be called
+// each time something is logged to the logger
+void callback_example()
+{
+    auto callback_sink = std::make_shared<spdlog::sinks::callback_sink_mt>([](const spdlog::details::log_msg &msg) {
+         // for example you can be notified by sending an email to yourself
+    });
+    callback_sink->set_level(spdlog::level::err);
+
+    auto console_sink = std::make_shared<spdlog::sinks::stdout_color_sink_mt>();
+    spdlog::logger logger("custom_callback_logger", {console_sink, callback_sink});
+
+    logger.info("some info log");
+    logger.error("critical issue"); // will notify you
+}
+```
+
+---
+#### Asynchronous logging
+```c++
+#include "spdlog/async.h"
+#include "spdlog/sinks/basic_file_sink.h"
+void async_example()
+{
+    // default thread pool settings can be modified *before* creating the async logger:
+    // spdlog::init_thread_pool(8192, 1); // queue with 8k items and 1 backing thread.
+    auto async_file = spdlog::basic_logger_mt<spdlog::async_factory>("async_file_logger", "logs/async_log.txt");
+    // alternatively:
+    // auto async_file = spdlog::create_async<spdlog::sinks::basic_file_sink_mt>("async_file_logger", "logs/async_log.txt");   
+}
+
+```
+
+---
+#### Asynchronous logger with multi sinks
+```c++
+#include "spdlog/async.h"
+#include "spdlog/sinks/stdout_color_sinks.h"
+#include "spdlog/sinks/rotating_file_sink.h"
+
+void multi_sink_example2()
+{
+    spdlog::init_thread_pool(8192, 1);
+    auto stdout_sink = std::make_shared<spdlog::sinks::stdout_color_sink_mt >();
+    auto rotating_sink = std::make_shared<spdlog::sinks::rotating_file_sink_mt>("mylog.txt", 1024*1024*10, 3);
+    std::vector<spdlog::sink_ptr> sinks {stdout_sink, rotating_sink};
+    auto logger = std::make_shared<spdlog::async_logger>("loggername", sinks.begin(), sinks.end(), spdlog::thread_pool(), spdlog::async_overflow_policy::block);
+    spdlog::register_logger(logger);
+}
+```
+ 
+---
+#### User-defined types
+```c++
+template<>
+struct fmt::formatter<my_type> : fmt::formatter<std::string>
+{
+    auto format(my_type my, format_context &ctx) const -> decltype(ctx.out())
+    {
+        return fmt::format_to(ctx.out(), "[my_type i={}]", my.i);
+    }
+};
+
+void user_defined_example()
+{
+    spdlog::info("user defined type: {}", my_type(14));
+}
+
+```
+
+---
+#### User-defined flags in the log pattern
+```c++ 
+// Log patterns can contain custom flags.
+// the following example will add new flag '%*' - which will be bound to a <my_formatter_flag> instance.
+#include "spdlog/pattern_formatter.h"
+class my_formatter_flag : public spdlog::custom_flag_formatter
+{
+public:
+    void format(const spdlog::details::log_msg &, const std::tm &, spdlog::memory_buf_t &dest) override
+    {
+        std::string some_txt = "custom-flag";
+        dest.append(some_txt.data(), some_txt.data() + some_txt.size());
+    }
+
+    std::unique_ptr<custom_flag_formatter> clone() const override
+    {
+        return spdlog::details::make_unique<my_formatter_flag>();
+    }
+};
+
+void custom_flags_example()
+{    
+    auto formatter = std::make_unique<spdlog::pattern_formatter>();
+    formatter->add_flag<my_formatter_flag>('*').set_pattern("[%n] [%*] [%^%l%$] %v");
+    spdlog::set_formatter(std::move(formatter));
+}
+
+```
+
+---
+#### Custom error handler
+```c++
+void err_handler_example()
+{
+    // can be set globally or per logger(logger->set_error_handler(..))
+    spdlog::set_error_handler([](const std::string &msg) { spdlog::get("console")->error("*** LOGGER ERROR ***: {}", msg); });
+    spdlog::get("console")->info("some invalid message to trigger an error {}{}{}{}", 3);
+}
+
+```
+
+---
+#### syslog
+```c++
+#include "spdlog/sinks/syslog_sink.h"
+void syslog_example()
+{
+    std::string ident = "spdlog-example";
+    auto syslog_logger = spdlog::syslog_logger_mt("syslog", ident, LOG_PID);
+    syslog_logger->warn("This is warning that will end up in syslog.");
+}
+```
+---
+#### Android example
+```c++
+#include "spdlog/sinks/android_sink.h"
+void android_example()
+{
+    std::string tag = "spdlog-android";
+    auto android_logger = spdlog::android_logger_mt("android", tag);
+    android_logger->critical("Use \"adb shell logcat\" to view this message.");
+}
+```
+
+---
+#### Load log levels from the env variable or argv
+
+```c++
+#include "spdlog/cfg/env.h"
+int main (int argc, char *argv[])
+{
+    spdlog::cfg::load_env_levels();
+    // or from the command line:
+    // ./example SPDLOG_LEVEL=info,mylogger=trace
+    // #include "spdlog/cfg/argv.h" // for loading levels from argv
+    // spdlog::cfg::load_argv_levels(argc, argv);
+}
+```
+So then you can:
+
+```console
+$ export SPDLOG_LEVEL=info,mylogger=trace
+$ ./example
+```
+
+
+---
+#### Log file open/close event handlers
+```c++
+// You can get callbacks from spdlog before/after a log file has been opened or closed. 
+// This is useful for cleanup procedures or for adding something to the start/end of the log file.
+void file_events_example()
+{
+    // pass the spdlog::file_event_handlers to file sinks for open/close log file notifications
+    spdlog::file_event_handlers handlers;
+    handlers.before_open = [](spdlog::filename_t filename) { spdlog::info("Before opening {}", filename); };
+    handlers.after_open = [](spdlog::filename_t filename, std::FILE *fstream) { fputs("After opening\n", fstream); };
+    handlers.before_close = [](spdlog::filename_t filename, std::FILE *fstream) { fputs("Before closing\n", fstream); };
+    handlers.after_close = [](spdlog::filename_t filename) { spdlog::info("After closing {}", filename); };
+    auto my_logger = spdlog::basic_logger_st("some_logger", "logs/events-sample.txt", true, handlers);        
+}
+```
+
+---
+#### Replace the Default Logger
+```c++
+void replace_default_logger_example()
+{
+    auto new_logger = spdlog::basic_logger_mt("new_default_logger", "logs/new-default-log.txt", true);
+    spdlog::set_default_logger(new_logger);
+    spdlog::info("new logger log message");
+}
+```
+
+---
+#### Log to Qt with nice colors
+```c++
+#include "spdlog/spdlog.h"
+#include "spdlog/sinks/qt_sinks.h"
+MainWindow::MainWindow(QWidget *parent) : QMainWindow(parent)
+{
+    setMinimumSize(640, 480);
+    auto log_widget = new QTextEdit(this);
+    setCentralWidget(log_widget);
+    int max_lines = 500; // keep the text widget to max 500 lines. remove old lines if needed.
+    auto logger = spdlog::qt_color_logger_mt("qt_logger", log_widget, max_lines);
+    logger->info("Some info message");
+}
+```
+---
+
+#### Mapped Diagnostic Context
+```c++
+// Mapped Diagnostic Context (MDC) is a map that stores key-value pairs (string values) in thread local storage.
+// Each thread maintains its own MDC, which loggers use to append diagnostic information to log outputs.
+// Note: it is not supported in asynchronous mode due to its reliance on thread-local storage.
+#include "spdlog/mdc.h"
+void mdc_example()
+{
+    spdlog::mdc::put("key1", "value1");
+    spdlog::mdc::put("key2", "value2");
+    // if not using the default format, use the %& formatter to print mdc data
+    // spdlog::set_pattern("[%H:%M:%S %z] [%^%L%$] [%&] %v");
+}
+```
+---
+## Benchmarks
+
+Below are some [benchmarks](https://github.com/gabime/spdlog/blob/v1.x/bench/bench.cpp) done in Ubuntu 64 bit, Intel i7-4770 CPU @ 3.40GHz
+
+#### Synchronous mode
+```
+[info] **************************************************************
+[info] Single thread, 1,000,000 iterations
+[info] **************************************************************
+[info] basic_st         Elapsed: 0.17 secs        5,777,626/sec
+[info] rotating_st      Elapsed: 0.18 secs        5,475,894/sec
+[info] daily_st         Elapsed: 0.20 secs        5,062,659/sec
+[info] empty_logger     Elapsed: 0.07 secs       14,127,300/sec
+[info] **************************************************************
+[info] C-string (400 bytes). Single thread, 1,000,000 iterations
+[info] **************************************************************
+[info] basic_st         Elapsed: 0.41 secs        2,412,483/sec
+[info] rotating_st      Elapsed: 0.72 secs        1,389,196/sec
+[info] daily_st         Elapsed: 0.42 secs        2,393,298/sec
+[info] null_st          Elapsed: 0.04 secs       27,446,957/sec
+[info] **************************************************************
+[info] 10 threads, competing over the same logger object, 1,000,000 iterations
+[info] **************************************************************
+[info] basic_mt         Elapsed: 0.60 secs        1,659,613/sec
+[info] rotating_mt      Elapsed: 0.62 secs        1,612,493/sec
+[info] daily_mt         Elapsed: 0.61 secs        1,638,305/sec
+[info] null_mt          Elapsed: 0.16 secs        6,272,758/sec
+```
+#### Asynchronous mode
+```
+[info] -------------------------------------------------
+[info] Messages     : 1,000,000
+[info] Threads      : 10
+[info] Queue        : 8,192 slots
+[info] Queue memory : 8,192 x 272 = 2,176 KB 
+[info] -------------------------------------------------
+[info] 
+[info] *********************************
+[info] Queue Overflow Policy: block
+[info] *********************************
+[info] Elapsed: 1.70784 secs     585,535/sec
+[info] Elapsed: 1.69805 secs     588,910/sec
+[info] Elapsed: 1.7026 secs      587,337/sec
+[info] 
+[info] *********************************
+[info] Queue Overflow Policy: overrun
+[info] *********************************
+[info] Elapsed: 0.372816 secs    2,682,285/sec
+[info] Elapsed: 0.379758 secs    2,633,255/sec
+[info] Elapsed: 0.373532 secs    2,677,147/sec
+
+```
+
+## Documentation
+Documentation can be found in the [wiki](https://github.com/gabime/spdlog/wiki/1.-QuickStart) pages.
+
+---
+
+Thanks to [JetBrains](https://www.jetbrains.com/?from=spdlog) for donating product licenses to help develop **spdlog** <a href="https://www.jetbrains.com/?from=spdlog"><img src="logos/jetbrains-variant-4.svg" width="94" align="center" /></a>
+
+
diff --git a/ext/spdlog/appveyor.yml b/ext/spdlog/appveyor.yml
new file mode 100644
index 0000000..f2757f3
--- /dev/null
+++ b/ext/spdlog/appveyor.yml
@@ -0,0 +1,89 @@
+version: 1.0.{build}
+image: Visual Studio 2017
+environment:
+  matrix:
+    - GENERATOR: '"Visual Studio 15 2017 Win64"'
+      BUILD_TYPE: Debug
+      BUILD_SHARED: 'OFF'
+      FATAL_ERRORS: 'OFF'
+      WCHAR: 'ON'
+      WCHAR_FILES: 'OFF'
+      BUILD_EXAMPLE: 'ON'
+      USE_STD_FORMAT: 'OFF'
+      CXX_STANDARD: 11
+    - GENERATOR: '"Visual Studio 15 2017 Win64"'
+      BUILD_TYPE: Release
+      BUILD_SHARED: 'OFF'
+      FATAL_ERRORS: 'OFF'
+      WCHAR: 'OFF'
+      WCHAR_FILES: 'OFF'
+      BUILD_EXAMPLE: 'ON'
+      USE_STD_FORMAT: 'OFF'
+      CXX_STANDARD: 11
+    - GENERATOR: '"Visual Studio 15 2017 Win64"'
+      BUILD_TYPE: Release
+      BUILD_SHARED: 'ON'
+      FATAL_ERRORS: 'OFF'
+      WCHAR: 'OFF'
+      WCHAR_FILES: 'OFF'
+      BUILD_EXAMPLE: 'ON'
+      USE_STD_FORMAT: 'OFF'
+      CXX_STANDARD: 11
+    - GENERATOR: '"Visual Studio 15 2017 Win64"'
+      BUILD_TYPE: Release
+      BUILD_SHARED: 'ON'
+      FATAL_ERRORS: 'OFF'
+      WCHAR: 'ON'
+      WCHAR_FILES: 'ON'
+      BUILD_EXAMPLE: 'OFF'
+      USE_STD_FORMAT: 'OFF'
+      CXX_STANDARD: 11
+    - GENERATOR: '"Visual Studio 16 2019" -A x64'
+      BUILD_TYPE: Release
+      BUILD_SHARED: 'ON'
+      FATAL_ERRORS: 'ON'
+      WCHAR: 'OFF'
+      WCHAR_FILES: 'OFF'
+      BUILD_EXAMPLE: 'OFF'
+      USE_STD_FORMAT: 'OFF'
+      CXX_STANDARD: 17
+      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
+    - GENERATOR: '"Visual Studio 17 2022" -A x64'
+      BUILD_TYPE: Release
+      BUILD_SHARED: 'ON'
+      FATAL_ERRORS: 'ON'
+      WCHAR: 'OFF'
+      WCHAR_FILES: 'OFF'
+      BUILD_EXAMPLE: 'OFF'
+      USE_STD_FORMAT: 'ON'
+      CXX_STANDARD: 20
+      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022
+    - GENERATOR: '"Visual Studio 17 2022" -A x64'
+      BUILD_TYPE: Release
+      BUILD_SHARED: 'ON'
+      FATAL_ERRORS: 'ON'
+      WCHAR: 'ON'
+      WCHAR_FILES: 'ON'
+      BUILD_EXAMPLE: 'OFF'
+      USE_STD_FORMAT: 'ON'
+      CXX_STANDARD: 20
+      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022
+build_script:
+  - cmd: >-
+      set
+
+      mkdir build
+
+      cd build
+
+      set PATH=%PATH%;C:\Program Files\Git\usr\bin
+
+      cmake -G %GENERATOR% -D CMAKE_BUILD_TYPE=%BUILD_TYPE% -D BUILD_SHARED_LIBS=%BUILD_SHARED% -D SPDLOG_WCHAR_SUPPORT=%WCHAR% -D SPDLOG_WCHAR_FILENAMES=%WCHAR_FILES% -D SPDLOG_BUILD_EXAMPLE=%BUILD_EXAMPLE% -D SPDLOG_BUILD_EXAMPLE_HO=%BUILD_EXAMPLE% -D SPDLOG_BUILD_TESTS=ON -D SPDLOG_BUILD_TESTS_HO=OFF -D SPDLOG_BUILD_WARNINGS=%FATAL_ERRORS% -D SPDLOG_USE_STD_FORMAT=%USE_STD_FORMAT% -D CMAKE_CXX_STANDARD=%CXX_STANDARD% ..
+
+      cmake --build . --config %BUILD_TYPE%
+
+before_test:
+  - set PATH=%PATH%;C:\projects\spdlog\build\_deps\catch2-build\src\%BUILD_TYPE%;C:\projects\spdlog\build\%BUILD_TYPE%
+  
+test_script:
+  - C:\projects\spdlog\build\tests\%BUILD_TYPE%\spdlog-utests.exe
diff --git a/ext/spdlog/bench/CMakeLists.txt b/ext/spdlog/bench/CMakeLists.txt
new file mode 100644
index 0000000..3806b24
--- /dev/null
+++ b/ext/spdlog/bench/CMakeLists.txt
@@ -0,0 +1,37 @@
+# Copyright(c) 2019 spdlog authors Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+cmake_minimum_required(VERSION 3.11)
+project(spdlog_bench CXX)
+
+if(NOT TARGET spdlog)
+    # Stand-alone build
+    find_package(spdlog CONFIG REQUIRED)
+endif()
+
+find_package(Threads REQUIRED)
+find_package(benchmark CONFIG)
+if(NOT benchmark_FOUND)
+    message(STATUS "Using CMake Version ${CMAKE_VERSION}")
+    # User can fetch googlebenchmark
+    message(STATUS "Downloading GoogleBenchmark")
+    include(FetchContent)
+
+    # disable tests
+    set(BENCHMARK_ENABLE_TESTING OFF CACHE INTERNAL "")
+    # Do not build and run googlebenchmark tests
+    FetchContent_Declare(googlebenchmark GIT_REPOSITORY https://github.com/google/benchmark.git GIT_TAG v1.6.0)
+    FetchContent_MakeAvailable(googlebenchmark)
+endif()
+
+add_executable(bench bench.cpp)
+spdlog_enable_warnings(bench)
+target_link_libraries(bench PRIVATE spdlog::spdlog)
+
+add_executable(async_bench async_bench.cpp)
+target_link_libraries(async_bench PRIVATE spdlog::spdlog)
+
+add_executable(latency latency.cpp)
+target_link_libraries(latency PRIVATE benchmark::benchmark spdlog::spdlog)
+
+add_executable(formatter-bench formatter-bench.cpp)
+target_link_libraries(formatter-bench PRIVATE benchmark::benchmark spdlog::spdlog)
diff --git a/ext/spdlog/bench/async_bench.cpp b/ext/spdlog/bench/async_bench.cpp
new file mode 100644
index 0000000..dcd24b2
--- /dev/null
+++ b/ext/spdlog/bench/async_bench.cpp
@@ -0,0 +1,168 @@
+//
+// Copyright(c) 2015 Gabi Melman.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+//
+
+//
+// bench.cpp : spdlog benchmarks
+//
+#include "spdlog/spdlog.h"
+#include "spdlog/async.h"
+#include "spdlog/sinks/basic_file_sink.h"
+
+#if defined(SPDLOG_USE_STD_FORMAT)
+    #include <format>
+#elif defined(SPDLOG_FMT_EXTERNAL)
+    #include <fmt/format.h>
+#else
+    #include "spdlog/fmt/bundled/format.h"
+#endif
+
+#include "utils.h"
+#include <atomic>
+#include <iostream>
+#include <memory>
+#include <string>
+#include <thread>
+
+using namespace std;
+using namespace std::chrono;
+using namespace spdlog;
+using namespace spdlog::sinks;
+using namespace utils;
+
+void bench_mt(int howmany, std::shared_ptr<spdlog::logger> log, int thread_count);
+
+#ifdef _MSC_VER
+    #pragma warning(push)
+    #pragma warning(disable : 4996)  // disable fopen warning under msvc
+#endif                               // _MSC_VER
+
+int count_lines(const char *filename) {
+    int counter = 0;
+    auto *infile = fopen(filename, "r");
+    int ch;
+    while (EOF != (ch = getc(infile))) {
+        if ('\n' == ch) counter++;
+    }
+    fclose(infile);
+
+    return counter;
+}
+
+void verify_file(const char *filename, int expected_count) {
+    spdlog::info("Verifying {} to contain {} line..", filename, expected_count);
+    auto count = count_lines(filename);
+    if (count != expected_count) {
+        spdlog::error("Test failed. {} has {} lines instead of {}", filename, count,
+                      expected_count);
+        exit(1);
+    }
+    spdlog::info("Line count OK ({})\n", count);
+}
+
+#ifdef _MSC_VER
+    #pragma warning(pop)
+#endif
+
+int main(int argc, char *argv[]) {
+    int howmany = 1000000;
+    int queue_size = std::min(howmany + 2, 8192);
+    int threads = 10;
+    int iters = 3;
+
+    try {
+        spdlog::set_pattern("[%^%l%$] %v");
+        if (argc == 1) {
+            spdlog::info("Usage: {} <message_count> <threads> <q_size> <iterations>", argv[0]);
+            return 0;
+        }
+
+        if (argc > 1) howmany = atoi(argv[1]);
+        if (argc > 2) threads = atoi(argv[2]);
+        if (argc > 3) {
+            queue_size = atoi(argv[3]);
+            if (queue_size > 500000) {
+                spdlog::error("Max queue size allowed: 500,000");
+                exit(1);
+            }
+        }
+
+        if (argc > 4) iters = atoi(argv[4]);
+
+        auto slot_size = sizeof(spdlog::details::async_msg);
+        spdlog::info("-------------------------------------------------");
+        spdlog::info("Messages     : {:L}", howmany);
+        spdlog::info("Threads      : {:L}", threads);
+        spdlog::info("Queue        : {:L} slots", queue_size);
+        spdlog::info("Queue memory : {:L} x {:L} = {:L} KB ", queue_size, slot_size,
+                     (queue_size * slot_size) / 1024);
+        spdlog::info("Total iters  : {:L}", iters);
+        spdlog::info("-------------------------------------------------");
+
+        const char *filename = "logs/basic_async.log";
+        spdlog::info("");
+        spdlog::info("*********************************");
+        spdlog::info("Queue Overflow Policy: block");
+        spdlog::info("*********************************");
+        for (int i = 0; i < iters; i++) {
+            auto tp = std::make_shared<details::thread_pool>(queue_size, 1);
+            auto file_sink = std::make_shared<spdlog::sinks::basic_file_sink_mt>(filename, true);
+            auto logger = std::make_shared<async_logger>(
+                "async_logger", std::move(file_sink), std::move(tp), async_overflow_policy::block);
+            bench_mt(howmany, std::move(logger), threads);
+            // verify_file(filename, howmany);
+        }
+
+        spdlog::info("");
+        spdlog::info("*********************************");
+        spdlog::info("Queue Overflow Policy: overrun");
+        spdlog::info("*********************************");
+        // do same test but discard oldest if queue is full instead of blocking
+        filename = "logs/basic_async-overrun.log";
+        for (int i = 0; i < iters; i++) {
+            auto tp = std::make_shared<details::thread_pool>(queue_size, 1);
+            auto file_sink = std::make_shared<spdlog::sinks::basic_file_sink_mt>(filename, true);
+            auto logger =
+                std::make_shared<async_logger>("async_logger", std::move(file_sink), std::move(tp),
+                                               async_overflow_policy::overrun_oldest);
+            bench_mt(howmany, std::move(logger), threads);
+        }
+        spdlog::shutdown();
+    } catch (std::exception &ex) {
+        std::cerr << "Error: " << ex.what() << std::endl;
+        perror("Last error");
+        return 1;
+    }
+    return 0;
+}
+
+void thread_fun(std::shared_ptr<spdlog::logger> logger, int howmany) {
+    for (int i = 0; i < howmany; i++) {
+        logger->info("Hello logger: msg number {}", i);
+    }
+}
+
+void bench_mt(int howmany, std::shared_ptr<spdlog::logger> logger, int thread_count) {
+    using std::chrono::high_resolution_clock;
+    vector<std::thread> threads;
+    auto start = high_resolution_clock::now();
+
+    int msgs_per_thread = howmany / thread_count;
+    int msgs_per_thread_mod = howmany % thread_count;
+    for (int t = 0; t < thread_count; ++t) {
+        if (t == 0 && msgs_per_thread_mod)
+            threads.push_back(
+                std::thread(thread_fun, logger, msgs_per_thread + msgs_per_thread_mod));
+        else
+            threads.push_back(std::thread(thread_fun, logger, msgs_per_thread));
+    }
+
+    for (auto &t : threads) {
+        t.join();
+    }
+
+    auto delta = high_resolution_clock::now() - start;
+    auto delta_d = duration_cast<duration<double>>(delta).count();
+    spdlog::info("Elapsed: {} secs\t {:L}/sec", delta_d, int(howmany / delta_d));
+}
diff --git a/ext/spdlog/bench/bench.cpp b/ext/spdlog/bench/bench.cpp
new file mode 100644
index 0000000..ceb0edf
--- /dev/null
+++ b/ext/spdlog/bench/bench.cpp
@@ -0,0 +1,246 @@
+//
+// Copyright(c) 2015 Gabi Melman.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+//
+
+//
+// bench.cpp : spdlog benchmarks
+//
+#include "spdlog/spdlog.h"
+#include "spdlog/sinks/basic_file_sink.h"
+#include "spdlog/sinks/daily_file_sink.h"
+#include "spdlog/sinks/null_sink.h"
+#include "spdlog/sinks/rotating_file_sink.h"
+
+#if defined(SPDLOG_USE_STD_FORMAT)
+    #include <format>
+#elif defined(SPDLOG_FMT_EXTERNAL)
+    #include <fmt/format.h>
+#else
+    #include "spdlog/fmt/bundled/format.h"
+#endif
+
+#include "utils.h"
+#include <atomic>
+#include <cstdlib>  // EXIT_FAILURE
+#include <memory>
+#include <string>
+#include <thread>
+
+void bench(int howmany, std::shared_ptr<spdlog::logger> log);
+void bench_mt(int howmany, std::shared_ptr<spdlog::logger> log, size_t thread_count);
+
+// void bench_default_api(int howmany, std::shared_ptr<spdlog::logger> log);
+// void bench_c_string(int howmany, std::shared_ptr<spdlog::logger> log);
+
+static const size_t file_size = 30 * 1024 * 1024;
+static const size_t rotating_files = 5;
+static const int max_threads = 1000;
+
+void bench_threaded_logging(size_t threads, int iters) {
+    spdlog::info("**************************************************************");
+    spdlog::info(spdlog::fmt_lib::format(
+        std::locale("en_US.UTF-8"), "Multi threaded: {:L} threads, {:L} messages", threads, iters));
+    spdlog::info("**************************************************************");
+
+    auto basic_mt = spdlog::basic_logger_mt("basic_mt", "logs/basic_mt.log", true);
+    bench_mt(iters, std::move(basic_mt), threads);
+    auto basic_mt_tracing =
+        spdlog::basic_logger_mt("basic_mt/backtrace-on", "logs/basic_mt.log", true);
+    basic_mt_tracing->enable_backtrace(32);
+    bench_mt(iters, std::move(basic_mt_tracing), threads);
+
+    spdlog::info("");
+    auto rotating_mt = spdlog::rotating_logger_mt("rotating_mt", "logs/rotating_mt.log", file_size,
+                                                  rotating_files);
+    bench_mt(iters, std::move(rotating_mt), threads);
+    auto rotating_mt_tracing = spdlog::rotating_logger_mt(
+        "rotating_mt/backtrace-on", "logs/rotating_mt.log", file_size, rotating_files);
+    rotating_mt_tracing->enable_backtrace(32);
+    bench_mt(iters, std::move(rotating_mt_tracing), threads);
+
+    spdlog::info("");
+    auto daily_mt = spdlog::daily_logger_mt("daily_mt", "logs/daily_mt.log");
+    bench_mt(iters, std::move(daily_mt), threads);
+    auto daily_mt_tracing = spdlog::daily_logger_mt("daily_mt/backtrace-on", "logs/daily_mt.log");
+    daily_mt_tracing->enable_backtrace(32);
+    bench_mt(iters, std::move(daily_mt_tracing), threads);
+
+    spdlog::info("");
+    auto empty_logger = std::make_shared<spdlog::logger>("level-off");
+    empty_logger->set_level(spdlog::level::off);
+    bench(iters, empty_logger);
+    auto empty_logger_tracing = std::make_shared<spdlog::logger>("level-off/backtrace-on");
+    empty_logger_tracing->set_level(spdlog::level::off);
+    empty_logger_tracing->enable_backtrace(32);
+    bench(iters, empty_logger_tracing);
+}
+
+void bench_single_threaded(int iters) {
+    spdlog::info("**************************************************************");
+    spdlog::info(
+        spdlog::fmt_lib::format(std::locale("en_US.UTF-8"), "Single threaded: {} messages", iters));
+    spdlog::info("**************************************************************");
+
+    auto basic_st = spdlog::basic_logger_st("basic_st", "logs/basic_st.log", true);
+    bench(iters, std::move(basic_st));
+
+    auto basic_st_tracing =
+        spdlog::basic_logger_st("basic_st/backtrace-on", "logs/basic_st.log", true);
+    bench(iters, std::move(basic_st_tracing));
+
+    spdlog::info("");
+    auto rotating_st = spdlog::rotating_logger_st("rotating_st", "logs/rotating_st.log", file_size,
+                                                  rotating_files);
+    bench(iters, std::move(rotating_st));
+    auto rotating_st_tracing = spdlog::rotating_logger_st(
+        "rotating_st/backtrace-on", "logs/rotating_st.log", file_size, rotating_files);
+    rotating_st_tracing->enable_backtrace(32);
+    bench(iters, std::move(rotating_st_tracing));
+
+    spdlog::info("");
+    auto daily_st = spdlog::daily_logger_st("daily_st", "logs/daily_st.log");
+    bench(iters, std::move(daily_st));
+    auto daily_st_tracing = spdlog::daily_logger_st("daily_st/backtrace-on", "logs/daily_st.log");
+    daily_st_tracing->enable_backtrace(32);
+    bench(iters, std::move(daily_st_tracing));
+
+    spdlog::info("");
+    auto empty_logger = std::make_shared<spdlog::logger>("level-off");
+    empty_logger->set_level(spdlog::level::off);
+    bench(iters, empty_logger);
+
+    auto empty_logger_tracing = std::make_shared<spdlog::logger>("level-off/backtrace-on");
+    empty_logger_tracing->set_level(spdlog::level::off);
+    empty_logger_tracing->enable_backtrace(32);
+    bench(iters, empty_logger_tracing);
+}
+
+int main(int argc, char *argv[]) {
+    spdlog::set_automatic_registration(false);
+    spdlog::default_logger()->set_pattern("[%^%l%$] %v");
+    int iters = 250000;
+    size_t threads = 4;
+    try {
+        if (argc > 1) {
+            iters = std::stoi(argv[1]);
+        }
+        if (argc > 2) {
+            threads = std::stoul(argv[2]);
+        }
+
+        if (threads > max_threads) {
+            throw std::runtime_error(
+                spdlog::fmt_lib::format("Number of threads exceeds maximum({})", max_threads));
+        }
+
+        bench_single_threaded(iters);
+        bench_threaded_logging(1, iters);
+        bench_threaded_logging(threads, iters);
+    } catch (std::exception &ex) {
+        spdlog::error(ex.what());
+        return EXIT_FAILURE;
+    }
+    return EXIT_SUCCESS;
+}
+
+void bench(int howmany, std::shared_ptr<spdlog::logger> log) {
+    using std::chrono::duration;
+    using std::chrono::duration_cast;
+    using std::chrono::high_resolution_clock;
+
+    auto start = high_resolution_clock::now();
+    for (auto i = 0; i < howmany; ++i) {
+        log->info("Hello logger: msg number {}", i);
+    }
+
+    auto delta = high_resolution_clock::now() - start;
+    auto delta_d = duration_cast<duration<double>>(delta).count();
+
+    spdlog::info(spdlog::fmt_lib::format(std::locale("en_US.UTF-8"),
+                                         "{:<30} Elapsed: {:0.2f} secs {:>16L}/sec", log->name(),
+                                         delta_d, size_t(howmany / delta_d)));
+    spdlog::drop(log->name());
+}
+
+void bench_mt(int howmany, std::shared_ptr<spdlog::logger> log, size_t thread_count) {
+    using std::chrono::duration;
+    using std::chrono::duration_cast;
+    using std::chrono::high_resolution_clock;
+
+    std::vector<std::thread> threads;
+    threads.reserve(thread_count);
+    auto start = high_resolution_clock::now();
+    for (size_t t = 0; t < thread_count; ++t) {
+        threads.emplace_back([&]() {
+            for (int j = 0; j < howmany / static_cast<int>(thread_count); j++) {
+                log->info("Hello logger: msg number {}", j);
+            }
+        });
+    }
+
+    for (auto &t : threads) {
+        t.join();
+    }
+
+    auto delta = high_resolution_clock::now() - start;
+    auto delta_d = duration_cast<duration<double>>(delta).count();
+    spdlog::info(spdlog::fmt_lib::format(std::locale("en_US.UTF-8"),
+                                         "{:<30} Elapsed: {:0.2f} secs {:>16L}/sec", log->name(),
+                                         delta_d, size_t(howmany / delta_d)));
+    spdlog::drop(log->name());
+}
+
+/*
+void bench_default_api(int howmany, std::shared_ptr<spdlog::logger> log)
+{
+    using std::chrono::high_resolution_clock;
+    using std::chrono::duration;
+    using std::chrono::duration_cast;
+
+    auto orig_default = spdlog::default_logger();
+    spdlog::set_default_logger(log);
+    auto start = high_resolution_clock::now();
+    for (auto i = 0; i < howmany; ++i)
+    {
+        spdlog::info("Hello logger: msg number {}", i);
+    }
+
+    auto delta = high_resolution_clock::now() - start;
+    auto delta_d = duration_cast<duration<double>>(delta).count();
+    spdlog::drop(log->name());
+    spdlog::set_default_logger(std::move(orig_default));
+    spdlog::info("{:<30} Elapsed: {:0.2f} secs {:>16}/sec", log->name(), delta_d, int(howmany /
+delta_d));
+}
+
+void bench_c_string(int howmany, std::shared_ptr<spdlog::logger> log)
+{
+    using std::chrono::high_resolution_clock;
+    using std::chrono::duration;
+    using std::chrono::duration_cast;
+
+    const char *msg = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Vestibulum pharetra
+metus cursus " "lacus placerat congue. Nulla egestas, mauris a tincidunt tempus, enim lectus
+volutpat mi, eu consequat sem " "libero nec massa. In dapibus ipsum a diam rhoncus gravida. Etiam
+non dapibus eros. Donec fringilla dui sed " "augue pretium, nec scelerisque est maximus. Nullam
+convallis, sem nec blandit maximus, nisi turpis ornare " "nisl, sit amet volutpat neque massa eu
+odio. Maecenas malesuada quam ex, posuere congue nibh turpis duis.";
+
+    auto orig_default = spdlog::default_logger();
+    spdlog::set_default_logger(log);
+    auto start = high_resolution_clock::now();
+    for (auto i = 0; i < howmany; ++i)
+    {
+        spdlog::log(spdlog::level::info, msg);
+    }
+
+    auto delta = high_resolution_clock::now() - start;
+    auto delta_d = duration_cast<duration<double>>(delta).count();
+    spdlog::drop(log->name());
+    spdlog::set_default_logger(std::move(orig_default));
+    spdlog::info("{:<30} Elapsed: {:0.2f} secs {:>16}/sec", log->name(), delta_d, int(howmany /
+delta_d));
+}
+
+*/
diff --git a/ext/spdlog/bench/formatter-bench.cpp b/ext/spdlog/bench/formatter-bench.cpp
new file mode 100644
index 0000000..375a154
--- /dev/null
+++ b/ext/spdlog/bench/formatter-bench.cpp
@@ -0,0 +1,71 @@
+//
+// Copyright(c) 2018 Gabi Melman.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+//
+
+#include "benchmark/benchmark.h"
+
+#include "spdlog/spdlog.h"
+#include "spdlog/pattern_formatter.h"
+
+void bench_formatter(benchmark::State &state, std::string pattern) {
+    auto formatter = spdlog::details::make_unique<spdlog::pattern_formatter>(pattern);
+    spdlog::memory_buf_t dest;
+    std::string logger_name = "logger-name";
+    const char *text =
+        "Hello. This is some message with length of 80                                   ";
+
+    spdlog::source_loc source_loc{"a/b/c/d/myfile.cpp", 123, "some_func()"};
+    spdlog::details::log_msg msg(source_loc, logger_name, spdlog::level::info, text);
+
+    for (auto _ : state) {
+        dest.clear();
+        formatter->format(msg, dest);
+        benchmark::DoNotOptimize(dest);
+    }
+}
+
+void bench_formatters() {
+    // basic patterns(single flag)
+    std::string all_flags = "+vtPnlLaAbBcCYDmdHIMSefFprRTXzEisg@luioO%";
+    std::vector<std::string> basic_patterns;
+    for (auto &flag : all_flags) {
+        auto pattern = std::string("%") + flag;
+        benchmark::RegisterBenchmark(pattern.c_str(), &bench_formatter, pattern);
+
+        //        pattern = std::string("%16") + flag;
+        //        benchmark::RegisterBenchmark(pattern.c_str(), &bench_formatter, pattern);
+        //
+        //        // bench center padding
+        //        pattern = std::string("%=16") + flag;
+        //        benchmark::RegisterBenchmark(pattern.c_str(), &bench_formatter, pattern);
+    }
+
+    // complex patterns
+    std::vector<std::string> patterns = {
+        "[%D %X] [%l] [%n] %v",
+        "[%Y-%m-%d %H:%M:%S.%e] [%l] [%n] %v",
+        "[%Y-%m-%d %H:%M:%S.%e] [%l] [%n] [%t] %v",
+    };
+    for (auto &pattern : patterns) {
+        benchmark::RegisterBenchmark(pattern.c_str(), &bench_formatter, pattern)
+            ->Iterations(2500000);
+    }
+}
+
+int main(int argc, char *argv[]) {
+    spdlog::set_pattern("[%^%l%$] %v");
+    if (argc != 2) {
+        spdlog::error("Usage: {} <pattern> (or \"all\" to bench all)", argv[0]);
+        exit(1);
+    }
+
+    std::string pattern = argv[1];
+    if (pattern == "all") {
+        bench_formatters();
+    } else {
+        benchmark::RegisterBenchmark(pattern.c_str(), &bench_formatter, pattern);
+    }
+    benchmark::Initialize(&argc, argv);
+    benchmark::RunSpecifiedBenchmarks();
+}
diff --git a/ext/spdlog/bench/latency.cpp b/ext/spdlog/bench/latency.cpp
new file mode 100644
index 0000000..1999317
--- /dev/null
+++ b/ext/spdlog/bench/latency.cpp
@@ -0,0 +1,220 @@
+//
+// Copyright(c) 2018 Gabi Melman.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+//
+
+//
+// latency.cpp : spdlog latency benchmarks
+//
+
+#include "benchmark/benchmark.h"
+
+#include "spdlog/spdlog.h"
+#include "spdlog/async.h"
+#include "spdlog/sinks/basic_file_sink.h"
+#include "spdlog/sinks/daily_file_sink.h"
+#include "spdlog/sinks/null_sink.h"
+#include "spdlog/sinks/rotating_file_sink.h"
+
+void bench_c_string(benchmark::State &state, std::shared_ptr<spdlog::logger> logger) {
+    const char *msg =
+        "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Vestibulum pharetra metus cursus "
+        "lacus placerat congue. Nulla egestas, mauris a tincidunt tempus, enim lectus volutpat mi, "
+        "eu consequat sem "
+        "libero nec massa. In dapibus ipsum a diam rhoncus gravida. Etiam non dapibus eros. Donec "
+        "fringilla dui sed "
+        "augue pretium, nec scelerisque est maximus. Nullam convallis, sem nec blandit maximus, "
+        "nisi turpis ornare "
+        "nisl, sit amet volutpat neque massa eu odio. Maecenas malesuada quam ex, posuere congue "
+        "nibh turpis duis.";
+
+    for (auto _ : state) {
+        logger->info(msg);
+    }
+}
+
+void bench_logger(benchmark::State &state, std::shared_ptr<spdlog::logger> logger) {
+    int i = 0;
+    for (auto _ : state) {
+        logger->info("Hello logger: msg number {}...............", ++i);
+    }
+}
+void bench_global_logger(benchmark::State &state, std::shared_ptr<spdlog::logger> logger) {
+    spdlog::set_default_logger(std::move(logger));
+    int i = 0;
+    for (auto _ : state) {
+        spdlog::info("Hello logger: msg number {}...............", ++i);
+    }
+}
+
+void bench_disabled_macro(benchmark::State &state, std::shared_ptr<spdlog::logger> logger) {
+    int i = 0;
+    benchmark::DoNotOptimize(i);       // prevent unused warnings
+    benchmark::DoNotOptimize(logger);  // prevent unused warnings
+    for (auto _ : state) {
+        SPDLOG_LOGGER_DEBUG(logger, "Hello logger: msg number {}...............", i++);
+    }
+}
+
+void bench_disabled_macro_global_logger(benchmark::State &state,
+                                        std::shared_ptr<spdlog::logger> logger) {
+    spdlog::set_default_logger(std::move(logger));
+    int i = 0;
+    benchmark::DoNotOptimize(i);       // prevent unused warnings
+    benchmark::DoNotOptimize(logger);  // prevent unused warnings
+    for (auto _ : state) {
+        SPDLOG_DEBUG("Hello logger: msg number {}...............", i++);
+    }
+}
+
+#ifdef __linux__
+void bench_dev_null() {
+    auto dev_null_st = spdlog::basic_logger_st("/dev/null_st", "/dev/null");
+    benchmark::RegisterBenchmark("/dev/null_st", bench_logger, std::move(dev_null_st))
+        ->UseRealTime();
+    spdlog::drop("/dev/null_st");
+
+    auto dev_null_mt = spdlog::basic_logger_mt("/dev/null_mt", "/dev/null");
+    benchmark::RegisterBenchmark("/dev/null_mt", bench_logger, std::move(dev_null_mt))
+        ->UseRealTime();
+    spdlog::drop("/dev/null_mt");
+}
+#endif  // __linux__
+
+int main(int argc, char *argv[]) {
+    using spdlog::sinks::null_sink_mt;
+    using spdlog::sinks::null_sink_st;
+
+    size_t file_size = 30 * 1024 * 1024;
+    size_t rotating_files = 5;
+    int n_threads = benchmark::CPUInfo::Get().num_cpus;
+
+    auto full_bench = argc > 1 && std::string(argv[1]) == "full";
+
+    // disabled loggers
+    auto disabled_logger =
+        std::make_shared<spdlog::logger>("bench", std::make_shared<null_sink_mt>());
+    disabled_logger->set_level(spdlog::level::off);
+    benchmark::RegisterBenchmark("disabled-at-compile-time", bench_disabled_macro, disabled_logger);
+    benchmark::RegisterBenchmark("disabled-at-compile-time (global logger)",
+                                 bench_disabled_macro_global_logger, disabled_logger);
+    benchmark::RegisterBenchmark("disabled-at-runtime", bench_logger, disabled_logger);
+    benchmark::RegisterBenchmark("disabled-at-runtime (global logger)", bench_global_logger,
+                                 disabled_logger);
+    // with backtrace of 64
+    auto tracing_disabled_logger =
+        std::make_shared<spdlog::logger>("bench", std::make_shared<null_sink_mt>());
+    tracing_disabled_logger->enable_backtrace(64);
+    benchmark::RegisterBenchmark("disabled-at-runtime/backtrace", bench_logger,
+                                 tracing_disabled_logger);
+
+    auto null_logger_st =
+        std::make_shared<spdlog::logger>("bench", std::make_shared<null_sink_st>());
+    benchmark::RegisterBenchmark("null_sink_st (500_bytes c_str)", bench_c_string,
+                                 std::move(null_logger_st));
+    benchmark::RegisterBenchmark("null_sink_st", bench_logger, null_logger_st);
+    benchmark::RegisterBenchmark("null_sink_st (global logger)", bench_global_logger,
+                                 null_logger_st);
+    // with backtrace of 64
+    auto tracing_null_logger_st =
+        std::make_shared<spdlog::logger>("bench", std::make_shared<null_sink_st>());
+    tracing_null_logger_st->enable_backtrace(64);
+    benchmark::RegisterBenchmark("null_sink_st/backtrace", bench_logger, tracing_null_logger_st);
+
+#ifdef __linux__
+    bench_dev_null();
+#endif  // __linux__
+
+    if (full_bench) {
+        // basic_st
+        auto basic_st = spdlog::basic_logger_st("basic_st", "latency_logs/basic_st.log", true);
+        benchmark::RegisterBenchmark("basic_st", bench_logger, std::move(basic_st))->UseRealTime();
+        spdlog::drop("basic_st");
+        // with backtrace of 64
+        auto tracing_basic_st =
+            spdlog::basic_logger_st("tracing_basic_st", "latency_logs/tracing_basic_st.log", true);
+        tracing_basic_st->enable_backtrace(64);
+        benchmark::RegisterBenchmark("basic_st/backtrace", bench_logger,
+                                     std::move(tracing_basic_st))
+            ->UseRealTime();
+        spdlog::drop("tracing_basic_st");
+
+        // rotating st
+        auto rotating_st = spdlog::rotating_logger_st("rotating_st", "latency_logs/rotating_st.log",
+                                                      file_size, rotating_files);
+        benchmark::RegisterBenchmark("rotating_st", bench_logger, std::move(rotating_st))
+            ->UseRealTime();
+        spdlog::drop("rotating_st");
+        // with backtrace of 64
+        auto tracing_rotating_st = spdlog::rotating_logger_st(
+            "tracing_rotating_st", "latency_logs/tracing_rotating_st.log", file_size,
+            rotating_files);
+        benchmark::RegisterBenchmark("rotating_st/backtrace", bench_logger,
+                                     std::move(tracing_rotating_st))
+            ->UseRealTime();
+        spdlog::drop("tracing_rotating_st");
+
+        // daily st
+        auto daily_st = spdlog::daily_logger_mt("daily_st", "latency_logs/daily_st.log");
+        benchmark::RegisterBenchmark("daily_st", bench_logger, std::move(daily_st))->UseRealTime();
+        spdlog::drop("daily_st");
+        auto tracing_daily_st =
+            spdlog::daily_logger_mt("tracing_daily_st", "latency_logs/daily_st.log");
+        benchmark::RegisterBenchmark("daily_st/backtrace", bench_logger,
+                                     std::move(tracing_daily_st))
+            ->UseRealTime();
+        spdlog::drop("tracing_daily_st");
+
+        //
+        // Multi threaded bench, 10 loggers using same logger concurrently
+        //
+        auto null_logger_mt =
+            std::make_shared<spdlog::logger>("bench", std::make_shared<null_sink_mt>());
+        benchmark::RegisterBenchmark("null_sink_mt", bench_logger, null_logger_mt)
+            ->Threads(n_threads)
+            ->UseRealTime();
+
+        // basic_mt
+        auto basic_mt = spdlog::basic_logger_mt("basic_mt", "latency_logs/basic_mt.log", true);
+        benchmark::RegisterBenchmark("basic_mt", bench_logger, std::move(basic_mt))
+            ->Threads(n_threads)
+            ->UseRealTime();
+        spdlog::drop("basic_mt");
+
+        // rotating mt
+        auto rotating_mt = spdlog::rotating_logger_mt("rotating_mt", "latency_logs/rotating_mt.log",
+                                                      file_size, rotating_files);
+        benchmark::RegisterBenchmark("rotating_mt", bench_logger, std::move(rotating_mt))
+            ->Threads(n_threads)
+            ->UseRealTime();
+        spdlog::drop("rotating_mt");
+
+        // daily mt
+        auto daily_mt = spdlog::daily_logger_mt("daily_mt", "latency_logs/daily_mt.log");
+        benchmark::RegisterBenchmark("daily_mt", bench_logger, std::move(daily_mt))
+            ->Threads(n_threads)
+            ->UseRealTime();
+        spdlog::drop("daily_mt");
+    }
+
+    // async
+    auto queue_size = 1024 * 1024 * 3;
+    auto tp = std::make_shared<spdlog::details::thread_pool>(queue_size, 1);
+    auto async_logger = std::make_shared<spdlog::async_logger>(
+        "async_logger", std::make_shared<null_sink_mt>(), std::move(tp),
+        spdlog::async_overflow_policy::overrun_oldest);
+    benchmark::RegisterBenchmark("async_logger", bench_logger, async_logger)
+        ->Threads(n_threads)
+        ->UseRealTime();
+
+    auto async_logger_tracing = std::make_shared<spdlog::async_logger>(
+        "async_logger_tracing", std::make_shared<null_sink_mt>(), std::move(tp),
+        spdlog::async_overflow_policy::overrun_oldest);
+    async_logger_tracing->enable_backtrace(32);
+    benchmark::RegisterBenchmark("async_logger/tracing", bench_logger, async_logger_tracing)
+        ->Threads(n_threads)
+        ->UseRealTime();
+
+    benchmark::Initialize(&argc, argv);
+    benchmark::RunSpecifiedBenchmarks();
+}
diff --git a/ext/spdlog/bench/utils.h b/ext/spdlog/bench/utils.h
new file mode 100644
index 0000000..1828fc0
--- /dev/null
+++ b/ext/spdlog/bench/utils.h
@@ -0,0 +1,32 @@
+//
+// Copyright(c) 2015 Gabi Melman.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+//
+
+#pragma once
+
+#include <iomanip>
+#include <locale>
+#include <sstream>
+
+namespace utils {
+
+template <typename T>
+inline std::string format(const T &value) {
+    static std::locale loc("");
+    std::stringstream ss;
+    ss.imbue(loc);
+    ss << value;
+    return ss.str();
+}
+
+template <>
+inline std::string format(const double &value) {
+    static std::locale loc("");
+    std::stringstream ss;
+    ss.imbue(loc);
+    ss << std::fixed << std::setprecision(1) << value;
+    return ss.str();
+}
+
+}  // namespace utils
diff --git a/ext/spdlog/cmake/ide.cmake b/ext/spdlog/cmake/ide.cmake
new file mode 100644
index 0000000..a0656a5
--- /dev/null
+++ b/ext/spdlog/cmake/ide.cmake
@@ -0,0 +1,18 @@
+# ---------------------------------------------------------------------------------------
+# IDE support for headers
+# ---------------------------------------------------------------------------------------
+set(SPDLOG_HEADERS_DIR "${CMAKE_CURRENT_LIST_DIR}/../include")
+
+file(GLOB SPDLOG_TOP_HEADERS "${SPDLOG_HEADERS_DIR}/spdlog/*.h")
+file(GLOB SPDLOG_DETAILS_HEADERS "${SPDLOG_HEADERS_DIR}/spdlog/details/*.h")
+file(GLOB SPDLOG_SINKS_HEADERS "${SPDLOG_HEADERS_DIR}/spdlog/sinks/*.h")
+file(GLOB SPDLOG_FMT_HEADERS "${SPDLOG_HEADERS_DIR}/spdlog/fmt/*.h")
+file(GLOB SPDLOG_FMT_BUNDELED_HEADERS "${SPDLOG_HEADERS_DIR}/spdlog/fmt/bundled/*.h")
+set(SPDLOG_ALL_HEADERS ${SPDLOG_TOP_HEADERS} ${SPDLOG_DETAILS_HEADERS} ${SPDLOG_SINKS_HEADERS} ${SPDLOG_FMT_HEADERS}
+                       ${SPDLOG_FMT_BUNDELED_HEADERS})
+
+source_group("Header Files\\spdlog" FILES ${SPDLOG_TOP_HEADERS})
+source_group("Header Files\\spdlog\\details" FILES ${SPDLOG_DETAILS_HEADERS})
+source_group("Header Files\\spdlog\\sinks" FILES ${SPDLOG_SINKS_HEADERS})
+source_group("Header Files\\spdlog\\fmt" FILES ${SPDLOG_FMT_HEADERS})
+source_group("Header Files\\spdlog\\fmt\\bundled\\" FILES ${SPDLOG_FMT_BUNDELED_HEADERS})
diff --git a/ext/spdlog/cmake/pch.h.in b/ext/spdlog/cmake/pch.h.in
new file mode 100644
index 0000000..a5f9415
--- /dev/null
+++ b/ext/spdlog/cmake/pch.h.in
@@ -0,0 +1,258 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+// details/pattern_formatter-inl.h
+// fmt/bin_to_hex.h
+// fmt/bundled/format-inl.h
+#include <cctype>
+
+// details/file_helper-inl.h
+// details/os-inl.h
+// fmt/bundled/core.h
+// fmt/bundled/posix.h
+// logger-inl.h
+// sinks/daily_file_sink.h
+// sinks/stdout_sinks.h
+#include <cstdio>
+
+// details/os-inl.h
+// fmt/bundled/posix.h
+#include <cstdlib>
+
+// details/os-inl.h
+// details/pattern_formatter-inl.h
+// fmt/bundled/core.h
+// fmt/bundled/format-inl.h
+#include <cstring>
+
+// details/os-inl.h
+// details/os.h
+// details/pattern_formatter-inl.h
+// details/pattern_formatter.h
+// fmt/bundled/chrono.h
+// sinks/daily_file_sink.h
+// sinks/rotating_file_sink-inl.h
+#include <ctime>
+
+// fmt/bundled/format-inl.h
+#include <climits>
+
+// fmt/bundled/format-inl.h
+#include <cwchar>
+
+// fmt/bundled/format-inl.h
+// fmt/bundled/format.h
+#include <cmath>
+
+// fmt/bundled/format-inl.h
+#include <cstdarg>
+
+// details/file_helper-inl.h
+// fmt/bundled/format.h
+// fmt/bundled/posix.h
+// sinks/rotating_file_sink-inl.h
+#include <cerrno>
+
+// details/circular_q.h
+// details/thread_pool-inl.h
+// fmt/bundled/format-inl.h
+#include <cassert>
+
+// async_logger-inl.h
+// cfg/helpers-inl.h
+// log_levels.h
+// common.h
+// details/file_helper-inl.h
+// details/log_msg.h
+// details/os-inl.h
+// details/pattern_formatter-inl.h
+// details/pattern_formatter.h
+// details/registry-inl.h
+// details/registry.h
+// details/tcp_client-windows.h
+// details/tcp_client.h
+// fmt/bundled/core.h
+// sinks/android_sink.h
+// sinks/ansicolor_sink.h
+// sinks/basic_file_sink.h
+// sinks/daily_file_sink.h
+// sinks/dup_filter_sink.h
+// sinks/msvc_sink.h
+// sinks/ringbuffer_sink.h
+// sinks/rotating_file_sink-inl.h
+// sinks/rotating_file_sink.h
+// sinks/syslog_sink.h
+// sinks/tcp_sink.h
+// sinks/win_eventlog_sink.h
+// sinks/wincolor_sink.h
+// spdlog.h:
+#include <string>
+
+// cfg/helpers-inl.h
+// fmt/bundled/chrono.h
+#include <sstream>
+
+// fmt/bundled/ostream.h
+// sinks/ostream_sink.h
+#include <ostream>
+
+// cfg/log_levels.h
+// details/registry-inl.h
+// details/registry.h
+#include <unordered_map>
+
+// details/circular_q.h
+// details/pattern_formatter-inl.h
+// details/pattern_formatter.h
+// details/thread_pool.h
+// fmt/bundled/compile.h
+// logger.h
+// sinks/dist_sink.h
+// sinks/ringbuffer_sink.h
+// sinks/win_eventlog_sink.h
+#include <vector>
+
+// details/os-inl.h
+// details/pattern_formatter-inl.h
+// sinks/ansicolor_sink.h
+// sinks/syslog_sink.h
+// sinks/systemd_sink.h
+// sinks/wincolor_sink.h
+#include <array>
+
+// details/file_helper-inl.h
+// details/file_helper.h
+// sinks/rotating_file_sink-inl.h
+#include <tuple>
+
+// details/os-inl.h
+// fmt/bundled/format.h
+// fmt/bundled/printf.h
+#include <limits>
+
+// common.h
+// details/backtracer.h
+// details/null_mutex.h
+#include <atomic>
+
+// common.h
+// details/backtracer.h
+// details/null_mutex.h
+#include <locale>
+
+// common.h
+#include <initializer_list>
+
+// common.h
+#include <exception>
+
+// common.h
+// details/fmt_helper.h
+// fmt/bundled/core.h
+// fmt/bundled/ranges.h
+#include <type_traits>
+
+// cfg/helpers-inl.h
+// details/null_mutex.h
+// details/pattern_formatter-inl.h
+#include <utility>
+
+// async.h
+// async_logger-inl.h
+// common.h
+// details/pattern_formatter-inl.h
+// details/pattern_formatter.h
+// details/registry-inl.h
+// details/registry.h
+// details/thread_pool.h
+// fmt/bundled/format.h
+// sinks/ansicolor_sink.h
+// sinks/base_sink-inl.h
+// sinks/dist_sink.h
+// sinks/stdout_sinks-inl.h
+// sinks/wincolor_sink.h
+// spdlog.h
+#include <memory>
+
+// async.h
+// common.h
+// details/backtracer.h
+// details/periodic_worker.h
+// details/registry-inl.h
+// details/registry.h
+// details/thread_pool.h
+// sinks/tcp_sink.h
+// spdlog.h
+#include <functional>
+
+// details/mpmc_blocking_q.h
+// details/periodic_worker.h
+#include <condition_variable>
+
+// details/os-inl.h
+// fmt/bundled/format.h
+// fmt/bundled/printf.h
+// sinks/dist_sink.h
+#include <algorithm>
+
+// common.h
+// details/file_helper-inl.h
+// details/fmt_helper.h
+// details/os-inl.h
+// details/pattern_formatter-inl.h
+// details/pattern_formatter.h
+// details/periodic_worker.h
+// details/registry-inl.h
+// details/registry.h
+// details/thread_pool.h
+// fmt/bundled/chrono.h
+// sinks/android_sink.h
+// sinks/daily_file_sink.h
+// sinks/dup_filter_sink.h
+// sinks/rotating_file_sink-inl.h
+// sinks/rotating_file_sink.h
+// sinks/tcp_sink.h
+// spdlog.h
+#include <chrono>
+
+// details/file_helper-inl.h
+// details/os-inl.h
+// details/pattern_formatter-inl.h
+// details/periodic_worker.h
+// details/thread_pool.h
+// sinks/android_sink.h
+#include <thread>
+
+// async.h
+// details/backtracer.h
+// details/console_globals.h
+// details/mpmc_blocking_q.h
+// details/pattern_formatter-inl.h
+// details/periodic_worker.h
+// details/registry.h
+// sinks/android_sink.h
+// sinks/ansicolor_sink.h
+// sinks/basic_file_sink.h
+// sinks/daily_file_sink.h
+// sinks/dist_sink.h
+// sinks/dup_filter_sink.h
+// sinks/msvc_sink.h
+// sinks/null_sink.h
+// sinks/ostream_sink.h
+// sinks/ringbuffer_sink.h
+// sinks/rotating_file_sink-inl.h
+// sinks/rotating_file_sink.h
+// sinks/tcp_sink.h
+// sinks/win_eventlog_sink.h
+// sinks/wincolor_sink.h
+//
+// color_sinks.cpp
+// file_sinks.cpp
+// spdlog.cpp
+// stdout_sinks.cpp
+#include <mutex>
+
+// spdlog
+#include <spdlog/common.h>
\ No newline at end of file
diff --git a/ext/spdlog/cmake/spdlog.pc.in b/ext/spdlog/cmake/spdlog.pc.in
new file mode 100644
index 0000000..ffab5d6
--- /dev/null
+++ b/ext/spdlog/cmake/spdlog.pc.in
@@ -0,0 +1,13 @@
+prefix=@CMAKE_INSTALL_PREFIX@
+exec_prefix=${prefix}
+includedir=@PKG_CONFIG_INCLUDEDIR@
+libdir=@PKG_CONFIG_LIBDIR@
+
+Name: lib@PROJECT_NAME@
+Description: Fast C++ logging library.
+URL: https://github.com/gabime/@PROJECT_NAME@
+Version: @SPDLOG_VERSION@
+CFlags: -I${includedir} @PKG_CONFIG_DEFINES@
+Libs: -L${libdir} -lspdlog -pthread
+Requires: @PKG_CONFIG_REQUIRES@
+
diff --git a/ext/spdlog/cmake/spdlogCPack.cmake b/ext/spdlog/cmake/spdlogCPack.cmake
new file mode 100644
index 0000000..58bf401
--- /dev/null
+++ b/ext/spdlog/cmake/spdlogCPack.cmake
@@ -0,0 +1,60 @@
+set(CPACK_GENERATOR "TGZ;ZIP" CACHE STRING "Semicolon separated list of generators")
+
+set(CPACK_INCLUDE_TOPLEVEL_DIRECTORY 0)
+set(CPACK_INSTALL_CMAKE_PROJECTS "${CMAKE_BINARY_DIR}" "${PROJECT_NAME}" ALL .)
+
+set(CPACK_PROJECT_URL "https://github.com/gabime/spdlog")
+set(CPACK_PACKAGE_VENDOR "Gabi Melman")
+set(CPACK_PACKAGE_CONTACT "Gabi Melman <gmelman1@gmail.com>")
+set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Fast C++ logging library")
+set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR})
+set(CPACK_PACKAGE_VERSION_MINOR ${PROJECT_VERSION_MINOR})
+set(CPACK_PACKAGE_VERSION_PATCH ${PROJECT_VERSION_PATCH})
+set(CPACK_PACKAGE_VERSION ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH})
+if(PROJECT_VERSION_TWEAK)
+    set(CPACK_PACKAGE_VERSION ${CPACK_PACKAGE_VERSION}.${PROJECT_VERSION_TWEAK})
+endif()
+set(CPACK_PACKAGE_RELOCATABLE ON CACHE BOOL "Build relocatable package")
+
+set(CPACK_RPM_PACKAGE_LICENSE "MIT")
+set(CPACK_RPM_PACKAGE_GROUP "Development/Libraries")
+set(CPACK_DEBIAN_PACKAGE_SECTION "libs")
+set(CPACK_RPM_PACKAGE_URL ${CPACK_PROJECT_URL})
+set(CPACK_DEBIAN_PACKAGE_HOMEPAGE ${CPACK_PROJECT_URL})
+set(CPACK_RPM_PACKAGE_DESCRIPTION "Very fast, header-only/compiled, C++ logging library.")
+set(CPACK_DEBIAN_PACKAGE_DESCRIPTION "Very fast, header-only/compiled, C++ logging library.")
+
+if(CPACK_PACKAGE_NAME)
+    set(CPACK_RPM_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}")
+    set(CPACK_DEBIAN_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}")
+else()
+    set(CPACK_RPM_FILE_NAME "${PROJECT_NAME}-${CPACK_PACKAGE_VERSION}")
+    set(CPACK_DEBIAN_FILE_NAME "${PROJECT_NAME}-${CPACK_PACKAGE_VERSION}")
+    set(CPACK_RPM_PACKAGE_NAME "${PROJECT_NAME}")
+    set(CPACK_DEBIAN_PACKAGE_NAME "${PROJECT_NAME}")
+endif()
+
+if(CPACK_RPM_PACKAGE_RELEASE)
+    set(CPACK_RPM_FILE_NAME "${CPACK_RPM_FILE_NAME}-${CPACK_RPM_PACKAGE_RELEASE}")
+endif()
+if(CPACK_DEBIAN_PACKAGE_RELEASE)
+    set(CPACK_DEBIAN_FILE_NAME "${CPACK_DEBIAN_FILE_NAME}-${CPACK_DEBIAN_PACKAGE_RELEASE}")
+endif()
+
+if(CPACK_RPM_PACKAGE_ARCHITECTURE)
+    set(CPACK_RPM_FILE_NAME "${CPACK_RPM_FILE_NAME}.${CPACK_RPM_PACKAGE_ARCHITECTURE}")
+endif()
+if(CPACK_DEBIAN_PACKAGE_ARCHITECTURE)
+    set(CPACK_DEBIAN_FILE_NAME "${CPACK_DEBIAN_FILE_NAME}.${CPACK_DEBIAN_PACKAGE_ARCHITECTURE}")
+endif()
+set(CPACK_RPM_FILE_NAME "${CPACK_RPM_FILE_NAME}.rpm")
+set(CPACK_DEBIAN_FILE_NAME "${CPACK_DEBIAN_FILE_NAME}.deb")
+
+if(NOT CPACK_PACKAGE_RELOCATABLE)
+    # Depend on pkgconfig rpm to create the system pkgconfig folder
+    set(CPACK_RPM_PACKAGE_REQUIRES pkgconfig)
+    set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION
+        "${CPACK_PACKAGING_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/pkgconfig")
+endif()
+
+include(CPack)
diff --git a/ext/spdlog/cmake/spdlogConfig.cmake.in b/ext/spdlog/cmake/spdlogConfig.cmake.in
new file mode 100644
index 0000000..1b85b9e
--- /dev/null
+++ b/ext/spdlog/cmake/spdlogConfig.cmake.in
@@ -0,0 +1,20 @@
+# Copyright(c) 2019 spdlog authors
+# Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+@PACKAGE_INIT@
+
+find_package(Threads REQUIRED)
+
+set(SPDLOG_FMT_EXTERNAL @SPDLOG_FMT_EXTERNAL@)
+set(SPDLOG_FMT_EXTERNAL_HO @SPDLOG_FMT_EXTERNAL_HO@)
+set(config_targets_file @config_targets_file@)
+
+if(SPDLOG_FMT_EXTERNAL OR SPDLOG_FMT_EXTERNAL_HO)
+    include(CMakeFindDependencyMacro)
+    find_dependency(fmt CONFIG)
+endif()
+
+
+include("${CMAKE_CURRENT_LIST_DIR}/${config_targets_file}")
+
+check_required_components(spdlog)
diff --git a/ext/spdlog/cmake/utils.cmake b/ext/spdlog/cmake/utils.cmake
new file mode 100644
index 0000000..8926657
--- /dev/null
+++ b/ext/spdlog/cmake/utils.cmake
@@ -0,0 +1,73 @@
+# Get spdlog version from include/spdlog/version.h and put it in SPDLOG_VERSION
+function(spdlog_extract_version)
+    file(READ "${CMAKE_CURRENT_LIST_DIR}/include/spdlog/version.h" file_contents)
+    string(REGEX MATCH "SPDLOG_VER_MAJOR ([0-9]+)" _ "${file_contents}")
+    if(NOT CMAKE_MATCH_COUNT EQUAL 1)
+        message(FATAL_ERROR "Could not extract major version number from spdlog/version.h")
+    endif()
+    set(ver_major ${CMAKE_MATCH_1})
+
+    string(REGEX MATCH "SPDLOG_VER_MINOR ([0-9]+)" _ "${file_contents}")
+    if(NOT CMAKE_MATCH_COUNT EQUAL 1)
+        message(FATAL_ERROR "Could not extract minor version number from spdlog/version.h")
+    endif()
+
+    set(ver_minor ${CMAKE_MATCH_1})
+    string(REGEX MATCH "SPDLOG_VER_PATCH ([0-9]+)" _ "${file_contents}")
+    if(NOT CMAKE_MATCH_COUNT EQUAL 1)
+        message(FATAL_ERROR "Could not extract patch version number from spdlog/version.h")
+    endif()
+    set(ver_patch ${CMAKE_MATCH_1})
+
+    set(SPDLOG_VERSION_MAJOR ${ver_major} PARENT_SCOPE)
+    set(SPDLOG_VERSION_MINOR ${ver_minor} PARENT_SCOPE)
+    set(SPDLOG_VERSION_PATCH ${ver_patch} PARENT_SCOPE)
+    set(SPDLOG_VERSION "${ver_major}.${ver_minor}.${ver_patch}" PARENT_SCOPE)
+endfunction()
+
+# Turn on warnings on the given target
+function(spdlog_enable_warnings target_name)
+    if(SPDLOG_BUILD_WARNINGS)
+        if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
+            list(APPEND MSVC_OPTIONS "/W3")
+            if(MSVC_VERSION GREATER 1900) # Allow non fatal security warnings for msvc 2015
+                list(APPEND MSVC_OPTIONS "/WX")
+            endif()
+        endif()
+
+        target_compile_options(
+            ${target_name}
+            PRIVATE $<$<OR:$<CXX_COMPILER_ID:Clang>,$<CXX_COMPILER_ID:AppleClang>,$<CXX_COMPILER_ID:GNU>>:
+                    -Wall
+                    -Wextra
+                    -Wconversion
+                    -pedantic
+                    -Werror
+                    -Wfatal-errors>
+                    $<$<CXX_COMPILER_ID:MSVC>:${MSVC_OPTIONS}>)
+    endif()
+endfunction()
+
+# Enable address sanitizer (gcc/clang only)
+function(spdlog_enable_addr_sanitizer target_name)
+    if(NOT CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
+        message(FATAL_ERROR "Sanitizer supported only for gcc/clang")
+    endif()
+    message(STATUS "Address sanitizer enabled")
+    target_compile_options(${target_name} PRIVATE -fsanitize=address,undefined)
+    target_compile_options(${target_name} PRIVATE -fno-sanitize=signed-integer-overflow)
+    target_compile_options(${target_name} PRIVATE -fno-sanitize-recover=all)
+    target_compile_options(${target_name} PRIVATE -fno-omit-frame-pointer)
+    target_link_libraries(${target_name} PRIVATE -fsanitize=address,undefined)
+endfunction()
+
+# Enable thread sanitizer (gcc/clang only)
+function(spdlog_enable_thread_sanitizer target_name)
+    if(NOT CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
+        message(FATAL_ERROR "Sanitizer supported only for gcc/clang")
+    endif()
+    message(STATUS "Thread sanitizer enabled")
+    target_compile_options(${target_name} PRIVATE -fsanitize=thread)
+    target_compile_options(${target_name} PRIVATE -fno-omit-frame-pointer)
+    target_link_libraries(${target_name} PRIVATE -fsanitize=thread)
+endfunction()
diff --git a/ext/spdlog/cmake/version.rc.in b/ext/spdlog/cmake/version.rc.in
new file mode 100644
index 0000000..a86c138
--- /dev/null
+++ b/ext/spdlog/cmake/version.rc.in
@@ -0,0 +1,42 @@
+#define APSTUDIO_READONLY_SYMBOLS
+#include <windows.h>
+#undef APSTUDIO_READONLY_SYMBOLS
+
+LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US
+
+
+VS_VERSION_INFO VERSIONINFO
+ FILEVERSION @SPDLOG_VERSION_MAJOR@,@SPDLOG_VERSION_MINOR@,@SPDLOG_VERSION_PATCH@,0
+ PRODUCTVERSION @SPDLOG_VERSION_MAJOR@,@SPDLOG_VERSION_MINOR@,@SPDLOG_VERSION_PATCH@,0
+ FILEFLAGSMASK 0x3fL
+#ifdef _DEBUG
+ FILEFLAGS 0x1L
+#else
+ FILEFLAGS 0x0L
+#endif
+ FILEOS 0x40004L
+ FILETYPE 0x2L
+ FILESUBTYPE 0x0L
+BEGIN
+    BLOCK "StringFileInfo"
+    BEGIN
+        BLOCK "040904b0"
+        BEGIN
+            VALUE "FileDescription", "spdlog dll\0"
+            VALUE "FileVersion", "@SPDLOG_VERSION@.0\0"
+            VALUE "InternalName", "spdlog.dll\0"
+            VALUE "LegalCopyright", "Copyright (C) spdlog\0"
+            VALUE "ProductName", "spdlog\0"
+            VALUE "ProductVersion", "@SPDLOG_VERSION@.0\0"
+        END
+    END
+    BLOCK "VarFileInfo"
+    BEGIN
+        VALUE "Translation", 0x409, 1200
+    END
+END
+
+
+
+
+
diff --git a/ext/spdlog/example/CMakeLists.txt b/ext/spdlog/example/CMakeLists.txt
new file mode 100644
index 0000000..da1ed4e
--- /dev/null
+++ b/ext/spdlog/example/CMakeLists.txt
@@ -0,0 +1,23 @@
+# Copyright(c) 2019 spdlog authors Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+cmake_minimum_required(VERSION 3.11)
+project(spdlog_examples CXX)
+
+if(NOT TARGET spdlog)
+    # Stand-alone build
+    find_package(spdlog REQUIRED)
+endif()
+
+# ---------------------------------------------------------------------------------------
+# Example of using pre-compiled library
+# ---------------------------------------------------------------------------------------
+add_executable(example example.cpp)
+target_link_libraries(example PRIVATE spdlog::spdlog $<$<BOOL:${MINGW}>:ws2_32>)
+
+# ---------------------------------------------------------------------------------------
+# Example of using header-only library
+# ---------------------------------------------------------------------------------------
+if(SPDLOG_BUILD_EXAMPLE_HO)
+    add_executable(example_header_only example.cpp)
+    target_link_libraries(example_header_only PRIVATE spdlog::spdlog_header_only)
+endif()
diff --git a/ext/spdlog/example/example.cpp b/ext/spdlog/example/example.cpp
new file mode 100644
index 0000000..26b92a1
--- /dev/null
+++ b/ext/spdlog/example/example.cpp
@@ -0,0 +1,400 @@
+//
+// Copyright(c) 2015 Gabi Melman.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+// spdlog usage example
+
+#include <cstdio>
+#include <chrono>
+
+void load_levels_example();
+void stdout_logger_example();
+void basic_example();
+void rotating_example();
+void daily_example();
+void callback_example();
+void async_example();
+void binary_example();
+void vector_example();
+void stopwatch_example();
+void trace_example();
+void multi_sink_example();
+void user_defined_example();
+void err_handler_example();
+void syslog_example();
+void udp_example();
+void custom_flags_example();
+void file_events_example();
+void replace_default_logger_example();
+void mdc_example();
+
+#include "spdlog/spdlog.h"
+#include "spdlog/cfg/env.h"   // support for loading levels from the environment variable
+#include "spdlog/fmt/ostr.h"  // support for user defined types
+
+int main(int, char *[]) {
+    // Log levels can be loaded from argv/env using "SPDLOG_LEVEL"
+    load_levels_example();
+
+    spdlog::info("Welcome to spdlog version {}.{}.{}  !", SPDLOG_VER_MAJOR, SPDLOG_VER_MINOR,
+                 SPDLOG_VER_PATCH);
+
+    spdlog::warn("Easy padding in numbers like {:08d}", 12);
+    spdlog::critical("Support for int: {0:d};  hex: {0:x};  oct: {0:o}; bin: {0:b}", 42);
+    spdlog::info("Support for floats {:03.2f}", 1.23456);
+    spdlog::info("Positional args are {1} {0}..", "too", "supported");
+    spdlog::info("{:>8} aligned, {:<8} aligned", "right", "left");
+
+    // Runtime log levels
+    spdlog::set_level(spdlog::level::info);  // Set global log level to info
+    spdlog::debug("This message should not be displayed!");
+    spdlog::set_level(spdlog::level::trace);  // Set specific logger's log level
+    spdlog::debug("This message should be displayed..");
+
+    // Customize msg format for all loggers
+    spdlog::set_pattern("[%H:%M:%S %z] [%^%L%$] [thread %t] %v");
+    spdlog::info("This an info message with custom format");
+    spdlog::set_pattern("%+");  // back to default format
+    spdlog::set_level(spdlog::level::info);
+
+    // Backtrace support
+    // Loggers can store in a ring buffer all messages (including debug/trace) for later inspection.
+    // When needed, call dump_backtrace() to see what happened:
+    spdlog::enable_backtrace(10);  // create ring buffer with capacity of 10  messages
+    for (int i = 0; i < 100; i++) {
+        spdlog::debug("Backtrace message {}", i);  // not logged..
+    }
+    // e.g. if some error happened:
+    spdlog::dump_backtrace();  // log them now!
+
+    try {
+        stdout_logger_example();
+        basic_example();
+        rotating_example();
+        daily_example();
+        callback_example();
+        async_example();
+        binary_example();
+        vector_example();
+        multi_sink_example();
+        user_defined_example();
+        err_handler_example();
+        trace_example();
+        stopwatch_example();
+        udp_example();
+        custom_flags_example();
+        file_events_example();
+        replace_default_logger_example();
+        mdc_example();
+
+        // Flush all *registered* loggers using a worker thread every 3 seconds.
+        // note: registered loggers *must* be thread safe for this to work correctly!
+        spdlog::flush_every(std::chrono::seconds(3));
+
+        // Apply some function on all registered loggers
+        spdlog::apply_all([&](std::shared_ptr<spdlog::logger> l) { l->info("End of example."); });
+
+        // Release all spdlog resources, and drop all loggers in the registry.
+        // This is optional (only mandatory if using windows + async log).
+        spdlog::shutdown();
+    }
+
+    // Exceptions will only be thrown upon failed logger or sink construction (not during logging).
+    catch (const spdlog::spdlog_ex &ex) {
+        std::printf("Log initialization failed: %s\n", ex.what());
+        return 1;
+    }
+}
+
+#include "spdlog/sinks/stdout_color_sinks.h"
+// or #include "spdlog/sinks/stdout_sinks.h" if no colors needed.
+void stdout_logger_example() {
+    // Create color multi threaded logger.
+    auto console = spdlog::stdout_color_mt("console");
+    // or for stderr:
+    // auto console = spdlog::stderr_color_mt("error-logger");
+}
+
+#include "spdlog/sinks/basic_file_sink.h"
+void basic_example() {
+    // Create basic file logger (not rotated).
+    auto my_logger = spdlog::basic_logger_mt("file_logger", "logs/basic-log.txt", true);
+}
+
+#include "spdlog/sinks/rotating_file_sink.h"
+void rotating_example() {
+    // Create a file rotating logger with 5mb size max and 3 rotated files.
+    auto rotating_logger =
+        spdlog::rotating_logger_mt("some_logger_name", "logs/rotating.txt", 1048576 * 5, 3);
+}
+
+#include "spdlog/sinks/daily_file_sink.h"
+void daily_example() {
+    // Create a daily logger - a new file is created every day on 2:30am.
+    auto daily_logger = spdlog::daily_logger_mt("daily_logger", "logs/daily.txt", 2, 30);
+}
+
+#include "spdlog/sinks/callback_sink.h"
+void callback_example() {
+    // Create the logger
+    auto logger = spdlog::callback_logger_mt("custom_callback_logger",
+                                             [](const spdlog::details::log_msg & /*msg*/) {
+                                                 // do what you need to do with msg
+                                             });
+}
+
+#include "spdlog/cfg/env.h"
+void load_levels_example() {
+    // Set the log level to "info" and mylogger to "trace":
+    // SPDLOG_LEVEL=info,mylogger=trace && ./example
+    spdlog::cfg::load_env_levels();
+    // or from command line:
+    // ./example SPDLOG_LEVEL=info,mylogger=trace
+    // #include "spdlog/cfg/argv.h" // for loading levels from argv
+    // spdlog::cfg::load_argv_levels(args, argv);
+}
+
+#include "spdlog/async.h"
+void async_example() {
+    // Default thread pool settings can be modified *before* creating the async logger:
+    // spdlog::init_thread_pool(32768, 1); // queue with max 32k items 1 backing thread.
+    auto async_file =
+        spdlog::basic_logger_mt<spdlog::async_factory>("async_file_logger", "logs/async_log.txt");
+    // alternatively:
+    // auto async_file =
+    // spdlog::create_async<spdlog::sinks::basic_file_sink_mt>("async_file_logger",
+    // "logs/async_log.txt");
+
+    for (int i = 1; i < 101; ++i) {
+        async_file->info("Async message #{}", i);
+    }
+}
+
+// Log binary data as hex.
+// Many types of std::container<char> types can be used.
+// Iterator ranges are supported too.
+// Format flags:
+// {:X} - print in uppercase.
+// {:s} - don't separate each byte with space.
+// {:p} - don't print the position on each line start.
+// {:n} - don't split the output to lines.
+
+#if !defined SPDLOG_USE_STD_FORMAT || defined(_MSC_VER)
+    #include "spdlog/fmt/bin_to_hex.h"
+void binary_example() {
+    std::vector<char> buf;
+    for (int i = 0; i < 80; i++) {
+        buf.push_back(static_cast<char>(i & 0xff));
+    }
+    spdlog::info("Binary example: {}", spdlog::to_hex(buf));
+    spdlog::info("Another binary example:{:n}",
+                 spdlog::to_hex(std::begin(buf), std::begin(buf) + 10));
+    // more examples:
+    // logger->info("uppercase: {:X}", spdlog::to_hex(buf));
+    // logger->info("uppercase, no delimiters: {:Xs}", spdlog::to_hex(buf));
+    // logger->info("uppercase, no delimiters, no position info: {:Xsp}", spdlog::to_hex(buf));
+    // logger->info("hexdump style: {:a}", spdlog::to_hex(buf));
+    // logger->info("hexdump style, 20 chars per line {:a}", spdlog::to_hex(buf, 20));
+}
+#else
+void binary_example() {
+    // not supported with std::format yet
+}
+#endif
+
+// Log a vector of numbers
+#ifndef SPDLOG_USE_STD_FORMAT
+    #include "spdlog/fmt/ranges.h"
+void vector_example() {
+    std::vector<int> vec = {1, 2, 3};
+    spdlog::info("Vector example: {}", vec);
+}
+
+#else
+void vector_example() {}
+#endif
+
+// ! DSPDLOG_USE_STD_FORMAT
+
+// Compile time log levels.
+// define SPDLOG_ACTIVE_LEVEL to required level (e.g. SPDLOG_LEVEL_TRACE)
+void trace_example() {
+    // trace from default logger
+    SPDLOG_TRACE("Some trace message.. {} ,{}", 1, 3.23);
+    // debug from default logger
+    SPDLOG_DEBUG("Some debug message.. {} ,{}", 1, 3.23);
+
+    // trace from logger object
+    auto logger = spdlog::get("file_logger");
+    SPDLOG_LOGGER_TRACE(logger, "another trace message");
+}
+
+// stopwatch example
+#include "spdlog/stopwatch.h"
+#include <thread>
+void stopwatch_example() {
+    spdlog::stopwatch sw;
+    std::this_thread::sleep_for(std::chrono::milliseconds(123));
+    spdlog::info("Stopwatch: {} seconds", sw);
+}
+
+#include "spdlog/sinks/udp_sink.h"
+void udp_example() {
+    spdlog::sinks::udp_sink_config cfg("127.0.0.1", 11091);
+    auto my_logger = spdlog::udp_logger_mt("udplog", cfg);
+    my_logger->set_level(spdlog::level::debug);
+    my_logger->info("hello world");
+}
+
+// A logger with multiple sinks (stdout and file) - each with a different format and log level.
+void multi_sink_example() {
+    auto console_sink = std::make_shared<spdlog::sinks::stdout_color_sink_mt>();
+    console_sink->set_level(spdlog::level::warn);
+    console_sink->set_pattern("[multi_sink_example] [%^%l%$] %v");
+
+    auto file_sink =
+        std::make_shared<spdlog::sinks::basic_file_sink_mt>("logs/multisink.txt", true);
+    file_sink->set_level(spdlog::level::trace);
+
+    spdlog::logger logger("multi_sink", {console_sink, file_sink});
+    logger.set_level(spdlog::level::debug);
+    logger.warn("this should appear in both console and file");
+    logger.info("this message should not appear in the console, only in the file");
+}
+
+// User defined types logging
+struct my_type {
+    int i = 0;
+    explicit my_type(int i)
+        : i(i){}
+};
+
+#ifndef SPDLOG_USE_STD_FORMAT  // when using fmtlib
+template <>
+struct fmt::formatter<my_type> : fmt::formatter<std::string> {
+    auto format(my_type my, format_context &ctx) const -> decltype(ctx.out()) {
+        return fmt::format_to(ctx.out(), "[my_type i={}]", my.i);
+    }
+};
+
+#else  // when using std::format
+template <>
+struct std::formatter<my_type> : std::formatter<std::string> {
+    auto format(my_type my, format_context &ctx) const -> decltype(ctx.out()) {
+        return std::format_to(ctx.out(), "[my_type i={}]", my.i);
+    }
+};
+#endif
+
+void user_defined_example() { spdlog::info("user defined type: {}", my_type(14)); }
+
+// Custom error handler. Will be triggered on log failure.
+void err_handler_example() {
+    // can be set globally or per logger(logger->set_error_handler(..))
+    spdlog::set_error_handler([](const std::string &msg) {
+        printf("*** Custom log error handler: %s ***\n", msg.c_str());
+    });
+}
+
+// syslog example (linux/osx/freebsd)
+#ifndef _WIN32
+    #include "spdlog/sinks/syslog_sink.h"
+void syslog_example() {
+    std::string ident = "spdlog-example";
+    auto syslog_logger = spdlog::syslog_logger_mt("syslog", ident, LOG_PID);
+    syslog_logger->warn("This is warning that will end up in syslog.");
+}
+#endif
+
+// Android example.
+#if defined(__ANDROID__)
+    #include "spdlog/sinks/android_sink.h"
+void android_example() {
+    std::string tag = "spdlog-android";
+    auto android_logger = spdlog::android_logger_mt("android", tag);
+    android_logger->critical("Use \"adb shell logcat\" to view this message.");
+}
+#endif
+
+// Log patterns can contain custom flags.
+// this will add custom flag '%*' which will be bound to a <my_formatter_flag> instance
+#include "spdlog/pattern_formatter.h"
+class my_formatter_flag : public spdlog::custom_flag_formatter {
+public:
+    void format(const spdlog::details::log_msg &,
+                const std::tm &,
+                spdlog::memory_buf_t &dest) override {
+        std::string some_txt = "custom-flag";
+        dest.append(some_txt.data(), some_txt.data() + some_txt.size());
+    }
+
+    std::unique_ptr<custom_flag_formatter> clone() const override {
+        return spdlog::details::make_unique<my_formatter_flag>();
+    }
+};
+
+void custom_flags_example() {
+    using spdlog::details::make_unique;  // for pre c++14
+    auto formatter = make_unique<spdlog::pattern_formatter>();
+    formatter->add_flag<my_formatter_flag>('*').set_pattern("[%n] [%*] [%^%l%$] %v");
+    // set the new formatter using spdlog::set_formatter(formatter) or
+    // logger->set_formatter(formatter) spdlog::set_formatter(std::move(formatter));
+}
+
+void file_events_example() {
+    // pass the spdlog::file_event_handlers to file sinks for open/close log file notifications
+    spdlog::file_event_handlers handlers;
+    handlers.before_open = [](spdlog::filename_t filename) {
+        spdlog::info("Before opening {}", filename);
+    };
+    handlers.after_open = [](spdlog::filename_t filename, std::FILE *fstream) {
+        spdlog::info("After opening {}", filename);
+        fputs("After opening\n", fstream);
+    };
+    handlers.before_close = [](spdlog::filename_t filename, std::FILE *fstream) {
+        spdlog::info("Before closing {}", filename);
+        fputs("Before closing\n", fstream);
+    };
+    handlers.after_close = [](spdlog::filename_t filename) {
+        spdlog::info("After closing {}", filename);
+    };
+    auto file_sink = std::make_shared<spdlog::sinks::basic_file_sink_mt>("logs/events-sample.txt",
+                                                                         true, handlers);
+    spdlog::logger my_logger("some_logger", file_sink);
+    my_logger.info("Some log line");
+}
+
+void replace_default_logger_example() {
+    // store the old logger so we don't break other examples.
+    auto old_logger = spdlog::default_logger();
+
+    auto new_logger =
+        spdlog::basic_logger_mt("new_default_logger", "logs/new-default-log.txt", true);
+    spdlog::set_default_logger(new_logger);
+    spdlog::set_level(spdlog::level::info);
+    spdlog::debug("This message should not be displayed!");
+    spdlog::set_level(spdlog::level::trace);
+    spdlog::debug("This message should be displayed..");
+
+    spdlog::set_default_logger(old_logger);
+}
+
+// Mapped Diagnostic Context (MDC) is a map that stores key-value pairs (string values) in thread local storage.
+// Each thread maintains its own MDC, which loggers use to append diagnostic information to log outputs.
+// Note: it is not supported in asynchronous mode due to its reliance on thread-local storage.
+
+#ifndef SPDLOG_NO_TLS
+    #include "spdlog/mdc.h"
+void mdc_example()
+{
+    spdlog::mdc::put("key1", "value1");
+    spdlog::mdc::put("key2", "value2");
+    // if not using the default format, you can use the %& formatter to print mdc data as well
+    spdlog::set_pattern("[%H:%M:%S %z] [%^%L%$] [%&] %v");
+    spdlog::info("Some log message with context");
+}
+#else
+void mdc_example() {
+    // if TLS feature is disabled
+}
+#endif
diff --git a/ext/spdlog/include/spdlog/async.h b/ext/spdlog/include/spdlog/async.h
new file mode 100644
index 0000000..e96abd1
--- /dev/null
+++ b/ext/spdlog/include/spdlog/async.h
@@ -0,0 +1,100 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+//
+// Async logging using global thread pool
+// All loggers created here share same global thread pool.
+// Each log message is pushed to a queue along with a shared pointer to the
+// logger.
+// If a logger deleted while having pending messages in the queue, it's actual
+// destruction will defer
+// until all its messages are processed by the thread pool.
+// This is because each message in the queue holds a shared_ptr to the
+// originating logger.
+
+#include <spdlog/async_logger.h>
+#include <spdlog/details/registry.h>
+#include <spdlog/details/thread_pool.h>
+
+#include <functional>
+#include <memory>
+#include <mutex>
+
+namespace spdlog {
+
+namespace details {
+static const size_t default_async_q_size = 8192;
+}
+
+// async logger factory - creates async loggers backed with thread pool.
+// if a global thread pool doesn't already exist, create it with default queue
+// size of 8192 items and single thread.
+template <async_overflow_policy OverflowPolicy = async_overflow_policy::block>
+struct async_factory_impl {
+    template <typename Sink, typename... SinkArgs>
+    static std::shared_ptr<async_logger> create(std::string logger_name, SinkArgs &&...args) {
+        auto &registry_inst = details::registry::instance();
+
+        // create global thread pool if not already exists..
+
+        auto &mutex = registry_inst.tp_mutex();
+        std::lock_guard<std::recursive_mutex> tp_lock(mutex);
+        auto tp = registry_inst.get_tp();
+        if (tp == nullptr) {
+            tp = std::make_shared<details::thread_pool>(details::default_async_q_size, 1U);
+            registry_inst.set_tp(tp);
+        }
+
+        auto sink = std::make_shared<Sink>(std::forward<SinkArgs>(args)...);
+        auto new_logger = std::make_shared<async_logger>(std::move(logger_name), std::move(sink),
+                                                         std::move(tp), OverflowPolicy);
+        registry_inst.initialize_logger(new_logger);
+        return new_logger;
+    }
+};
+
+using async_factory = async_factory_impl<async_overflow_policy::block>;
+using async_factory_nonblock = async_factory_impl<async_overflow_policy::overrun_oldest>;
+
+template <typename Sink, typename... SinkArgs>
+inline std::shared_ptr<spdlog::logger> create_async(std::string logger_name,
+                                                    SinkArgs &&...sink_args) {
+    return async_factory::create<Sink>(std::move(logger_name),
+                                       std::forward<SinkArgs>(sink_args)...);
+}
+
+template <typename Sink, typename... SinkArgs>
+inline std::shared_ptr<spdlog::logger> create_async_nb(std::string logger_name,
+                                                       SinkArgs &&...sink_args) {
+    return async_factory_nonblock::create<Sink>(std::move(logger_name),
+                                                std::forward<SinkArgs>(sink_args)...);
+}
+
+// set global thread pool.
+inline void init_thread_pool(size_t q_size,
+                             size_t thread_count,
+                             std::function<void()> on_thread_start,
+                             std::function<void()> on_thread_stop) {
+    auto tp = std::make_shared<details::thread_pool>(q_size, thread_count, on_thread_start,
+                                                     on_thread_stop);
+    details::registry::instance().set_tp(std::move(tp));
+}
+
+inline void init_thread_pool(size_t q_size,
+                             size_t thread_count,
+                             std::function<void()> on_thread_start) {
+    init_thread_pool(q_size, thread_count, on_thread_start, [] {});
+}
+
+inline void init_thread_pool(size_t q_size, size_t thread_count) {
+    init_thread_pool(
+        q_size, thread_count, [] {}, [] {});
+}
+
+// get the global thread pool.
+inline std::shared_ptr<spdlog::details::thread_pool> thread_pool() {
+    return details::registry::instance().get_tp();
+}
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/async_logger-inl.h b/ext/spdlog/include/spdlog/async_logger-inl.h
new file mode 100644
index 0000000..1e79479
--- /dev/null
+++ b/ext/spdlog/include/spdlog/async_logger-inl.h
@@ -0,0 +1,84 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#ifndef SPDLOG_HEADER_ONLY
+    #include <spdlog/async_logger.h>
+#endif
+
+#include <spdlog/details/thread_pool.h>
+#include <spdlog/sinks/sink.h>
+
+#include <memory>
+#include <string>
+
+SPDLOG_INLINE spdlog::async_logger::async_logger(std::string logger_name,
+                                                 sinks_init_list sinks_list,
+                                                 std::weak_ptr<details::thread_pool> tp,
+                                                 async_overflow_policy overflow_policy)
+    : async_logger(std::move(logger_name),
+                   sinks_list.begin(),
+                   sinks_list.end(),
+                   std::move(tp),
+                   overflow_policy) {}
+
+SPDLOG_INLINE spdlog::async_logger::async_logger(std::string logger_name,
+                                                 sink_ptr single_sink,
+                                                 std::weak_ptr<details::thread_pool> tp,
+                                                 async_overflow_policy overflow_policy)
+    : async_logger(
+          std::move(logger_name), {std::move(single_sink)}, std::move(tp), overflow_policy) {}
+
+// send the log message to the thread pool
+SPDLOG_INLINE void spdlog::async_logger::sink_it_(const details::log_msg &msg){
+    SPDLOG_TRY{if (auto pool_ptr = thread_pool_.lock()){
+        pool_ptr->post_log(shared_from_this(), msg, overflow_policy_);
+}
+else {
+    throw_spdlog_ex("async log: thread pool doesn't exist anymore");
+}
+}
+SPDLOG_LOGGER_CATCH(msg.source)
+}
+
+// send flush request to the thread pool
+SPDLOG_INLINE void spdlog::async_logger::flush_(){
+    SPDLOG_TRY{if (auto pool_ptr = thread_pool_.lock()){
+        pool_ptr->post_flush(shared_from_this(), overflow_policy_);
+}
+else {
+    throw_spdlog_ex("async flush: thread pool doesn't exist anymore");
+}
+}
+SPDLOG_LOGGER_CATCH(source_loc())
+}
+
+//
+// backend functions - called from the thread pool to do the actual job
+//
+SPDLOG_INLINE void spdlog::async_logger::backend_sink_it_(const details::log_msg &msg) {
+    for (auto &sink : sinks_) {
+        if (sink->should_log(msg.level)) {
+            SPDLOG_TRY { sink->log(msg); }
+            SPDLOG_LOGGER_CATCH(msg.source)
+        }
+    }
+
+    if (should_flush_(msg)) {
+        backend_flush_();
+    }
+}
+
+SPDLOG_INLINE void spdlog::async_logger::backend_flush_() {
+    for (auto &sink : sinks_) {
+        SPDLOG_TRY { sink->flush(); }
+        SPDLOG_LOGGER_CATCH(source_loc())
+    }
+}
+
+SPDLOG_INLINE std::shared_ptr<spdlog::logger> spdlog::async_logger::clone(std::string new_name) {
+    auto cloned = std::make_shared<spdlog::async_logger>(*this);
+    cloned->name_ = std::move(new_name);
+    return cloned;
+}
diff --git a/ext/spdlog/include/spdlog/async_logger.h b/ext/spdlog/include/spdlog/async_logger.h
new file mode 100644
index 0000000..846c4c6
--- /dev/null
+++ b/ext/spdlog/include/spdlog/async_logger.h
@@ -0,0 +1,74 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+// Fast asynchronous logger.
+// Uses pre allocated queue.
+// Creates a single back thread to pop messages from the queue and log them.
+//
+// Upon each log write the logger:
+//    1. Checks if its log level is enough to log the message
+//    2. Push a new copy of the message to a queue (or block the caller until
+//    space is available in the queue)
+// Upon destruction, logs all remaining messages in the queue before
+// destructing..
+
+#include <spdlog/logger.h>
+
+namespace spdlog {
+
+// Async overflow policy - block by default.
+enum class async_overflow_policy {
+    block,           // Block until message can be enqueued
+    overrun_oldest,  // Discard oldest message in the queue if full when trying to
+                     // add new item.
+    discard_new      // Discard new message if the queue is full when trying to add new item.
+};
+
+namespace details {
+class thread_pool;
+}
+
+class SPDLOG_API async_logger final : public std::enable_shared_from_this<async_logger>,
+                                      public logger {
+    friend class details::thread_pool;
+
+public:
+    template <typename It>
+    async_logger(std::string logger_name,
+                 It begin,
+                 It end,
+                 std::weak_ptr<details::thread_pool> tp,
+                 async_overflow_policy overflow_policy = async_overflow_policy::block)
+        : logger(std::move(logger_name), begin, end),
+          thread_pool_(std::move(tp)),
+          overflow_policy_(overflow_policy) {}
+
+    async_logger(std::string logger_name,
+                 sinks_init_list sinks_list,
+                 std::weak_ptr<details::thread_pool> tp,
+                 async_overflow_policy overflow_policy = async_overflow_policy::block);
+
+    async_logger(std::string logger_name,
+                 sink_ptr single_sink,
+                 std::weak_ptr<details::thread_pool> tp,
+                 async_overflow_policy overflow_policy = async_overflow_policy::block);
+
+    std::shared_ptr<logger> clone(std::string new_name) override;
+
+protected:
+    void sink_it_(const details::log_msg &msg) override;
+    void flush_() override;
+    void backend_sink_it_(const details::log_msg &incoming_log_msg);
+    void backend_flush_();
+
+private:
+    std::weak_ptr<details::thread_pool> thread_pool_;
+    async_overflow_policy overflow_policy_;
+};
+}  // namespace spdlog
+
+#ifdef SPDLOG_HEADER_ONLY
+    #include "async_logger-inl.h"
+#endif
diff --git a/ext/spdlog/include/spdlog/cfg/argv.h b/ext/spdlog/include/spdlog/cfg/argv.h
new file mode 100644
index 0000000..7de2f83
--- /dev/null
+++ b/ext/spdlog/include/spdlog/cfg/argv.h
@@ -0,0 +1,40 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+#include <spdlog/cfg/helpers.h>
+#include <spdlog/details/registry.h>
+
+//
+// Init log levels using each argv entry that starts with "SPDLOG_LEVEL="
+//
+// set all loggers to debug level:
+// example.exe "SPDLOG_LEVEL=debug"
+
+// set logger1 to trace level
+// example.exe "SPDLOG_LEVEL=logger1=trace"
+
+// turn off all logging except for logger1 and logger2:
+// example.exe "SPDLOG_LEVEL=off,logger1=debug,logger2=info"
+
+namespace spdlog {
+namespace cfg {
+
+// search for SPDLOG_LEVEL= in the args and use it to init the levels
+inline void load_argv_levels(int argc, const char **argv) {
+    const std::string spdlog_level_prefix = "SPDLOG_LEVEL=";
+    for (int i = 1; i < argc; i++) {
+        std::string arg = argv[i];
+        if (arg.find(spdlog_level_prefix) == 0) {
+            auto levels_string = arg.substr(spdlog_level_prefix.size());
+            helpers::load_levels(levels_string);
+        }
+    }
+}
+
+inline void load_argv_levels(int argc, char **argv) {
+    load_argv_levels(argc, const_cast<const char **>(argv));
+}
+
+}  // namespace cfg
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/cfg/env.h b/ext/spdlog/include/spdlog/cfg/env.h
new file mode 100644
index 0000000..6e55414
--- /dev/null
+++ b/ext/spdlog/include/spdlog/cfg/env.h
@@ -0,0 +1,36 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+#include <spdlog/cfg/helpers.h>
+#include <spdlog/details/os.h>
+#include <spdlog/details/registry.h>
+
+//
+// Init levels and patterns from env variables SPDLOG_LEVEL
+// Inspired from Rust's "env_logger" crate (https://crates.io/crates/env_logger).
+// Note - fallback to "info" level on unrecognized levels
+//
+// Examples:
+//
+// set global level to debug:
+// export SPDLOG_LEVEL=debug
+//
+// turn off all logging except for logger1:
+// export SPDLOG_LEVEL="*=off,logger1=debug"
+//
+
+// turn off all logging except for logger1 and logger2:
+// export SPDLOG_LEVEL="off,logger1=debug,logger2=info"
+
+namespace spdlog {
+namespace cfg {
+inline void load_env_levels() {
+    auto env_val = details::os::getenv("SPDLOG_LEVEL");
+    if (!env_val.empty()) {
+        helpers::load_levels(env_val);
+    }
+}
+
+}  // namespace cfg
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/cfg/helpers-inl.h b/ext/spdlog/include/spdlog/cfg/helpers-inl.h
new file mode 100644
index 0000000..93650a2
--- /dev/null
+++ b/ext/spdlog/include/spdlog/cfg/helpers-inl.h
@@ -0,0 +1,107 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#ifndef SPDLOG_HEADER_ONLY
+    #include <spdlog/cfg/helpers.h>
+#endif
+
+#include <spdlog/details/os.h>
+#include <spdlog/details/registry.h>
+#include <spdlog/spdlog.h>
+
+#include <algorithm>
+#include <sstream>
+#include <string>
+#include <utility>
+
+namespace spdlog {
+namespace cfg {
+namespace helpers {
+
+// inplace convert to lowercase
+inline std::string &to_lower_(std::string &str) {
+    std::transform(str.begin(), str.end(), str.begin(), [](char ch) {
+        return static_cast<char>((ch >= 'A' && ch <= 'Z') ? ch + ('a' - 'A') : ch);
+    });
+    return str;
+}
+
+// inplace trim spaces
+inline std::string &trim_(std::string &str) {
+    const char *spaces = " \n\r\t";
+    str.erase(str.find_last_not_of(spaces) + 1);
+    str.erase(0, str.find_first_not_of(spaces));
+    return str;
+}
+
+// return (name,value) trimmed pair from given "name=value" string.
+// return empty string on missing parts
+// "key=val" => ("key", "val")
+// " key  =  val " => ("key", "val")
+// "key=" => ("key", "")
+// "val" => ("", "val")
+
+inline std::pair<std::string, std::string> extract_kv_(char sep, const std::string &str) {
+    auto n = str.find(sep);
+    std::string k, v;
+    if (n == std::string::npos) {
+        v = str;
+    } else {
+        k = str.substr(0, n);
+        v = str.substr(n + 1);
+    }
+    return std::make_pair(trim_(k), trim_(v));
+}
+
+// return vector of key/value pairs from sequence of "K1=V1,K2=V2,.."
+// "a=AAA,b=BBB,c=CCC,.." => {("a","AAA"),("b","BBB"),("c", "CCC"),...}
+inline std::unordered_map<std::string, std::string> extract_key_vals_(const std::string &str) {
+    std::string token;
+    std::istringstream token_stream(str);
+    std::unordered_map<std::string, std::string> rv{};
+    while (std::getline(token_stream, token, ',')) {
+        if (token.empty()) {
+            continue;
+        }
+        auto kv = extract_kv_('=', token);
+        rv[kv.first] = kv.second;
+    }
+    return rv;
+}
+
+SPDLOG_INLINE void load_levels(const std::string &input) {
+    if (input.empty() || input.size() > 512) {
+        return;
+    }
+
+    auto key_vals = extract_key_vals_(input);
+    std::unordered_map<std::string, level::level_enum> levels;
+    level::level_enum global_level = level::info;
+    bool global_level_found = false;
+
+    for (auto &name_level : key_vals) {
+        auto &logger_name = name_level.first;
+        auto level_name = to_lower_(name_level.second);
+        auto level = level::from_str(level_name);
+        // ignore unrecognized level names
+        if (level == level::off && level_name != "off") {
+            continue;
+        }
+        if (logger_name.empty())  // no logger name indicate global level
+        {
+            global_level_found = true;
+            global_level = level;
+        } else {
+            levels[logger_name] = level;
+        }
+    }
+
+    details::registry::instance().set_levels(std::move(levels),
+                                             global_level_found ? &global_level : nullptr);
+}
+
+}  // namespace helpers
+}  // namespace cfg
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/cfg/helpers.h b/ext/spdlog/include/spdlog/cfg/helpers.h
new file mode 100644
index 0000000..c023818
--- /dev/null
+++ b/ext/spdlog/include/spdlog/cfg/helpers.h
@@ -0,0 +1,29 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#include <spdlog/common.h>
+#include <unordered_map>
+
+namespace spdlog {
+namespace cfg {
+namespace helpers {
+//
+// Init levels from given string
+//
+// Examples:
+//
+// set global level to debug: "debug"
+// turn off all logging except for logger1: "off,logger1=debug"
+// turn off all logging except for logger1 and logger2: "off,logger1=debug,logger2=info"
+//
+SPDLOG_API void load_levels(const std::string &txt);
+}  // namespace helpers
+
+}  // namespace cfg
+}  // namespace spdlog
+
+#ifdef SPDLOG_HEADER_ONLY
+    #include "helpers-inl.h"
+#endif  // SPDLOG_HEADER_ONLY
diff --git a/ext/spdlog/include/spdlog/common-inl.h b/ext/spdlog/include/spdlog/common-inl.h
new file mode 100644
index 0000000..a8a0453
--- /dev/null
+++ b/ext/spdlog/include/spdlog/common-inl.h
@@ -0,0 +1,68 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#ifndef SPDLOG_HEADER_ONLY
+    #include <spdlog/common.h>
+#endif
+
+#include <algorithm>
+#include <iterator>
+
+namespace spdlog {
+namespace level {
+
+#if __cplusplus >= 201703L
+constexpr
+#endif
+    static string_view_t level_string_views[] SPDLOG_LEVEL_NAMES;
+
+static const char *short_level_names[] SPDLOG_SHORT_LEVEL_NAMES;
+
+SPDLOG_INLINE const string_view_t &to_string_view(spdlog::level::level_enum l) SPDLOG_NOEXCEPT {
+    return level_string_views[l];
+}
+
+SPDLOG_INLINE const char *to_short_c_str(spdlog::level::level_enum l) SPDLOG_NOEXCEPT {
+    return short_level_names[l];
+}
+
+SPDLOG_INLINE spdlog::level::level_enum from_str(const std::string &name) SPDLOG_NOEXCEPT {
+    auto it = std::find(std::begin(level_string_views), std::end(level_string_views), name);
+    if (it != std::end(level_string_views))
+        return static_cast<level::level_enum>(std::distance(std::begin(level_string_views), it));
+
+    // check also for "warn" and "err" before giving up..
+    if (name == "warn") {
+        return level::warn;
+    }
+    if (name == "err") {
+        return level::err;
+    }
+    return level::off;
+}
+}  // namespace level
+
+SPDLOG_INLINE spdlog_ex::spdlog_ex(std::string msg)
+    : msg_(std::move(msg)) {}
+
+SPDLOG_INLINE spdlog_ex::spdlog_ex(const std::string &msg, int last_errno) {
+#ifdef SPDLOG_USE_STD_FORMAT
+    msg_ = std::system_error(std::error_code(last_errno, std::generic_category()), msg).what();
+#else
+    memory_buf_t outbuf;
+    fmt::format_system_error(outbuf, last_errno, msg.c_str());
+    msg_ = fmt::to_string(outbuf);
+#endif
+}
+
+SPDLOG_INLINE const char *spdlog_ex::what() const SPDLOG_NOEXCEPT { return msg_.c_str(); }
+
+SPDLOG_INLINE void throw_spdlog_ex(const std::string &msg, int last_errno) {
+    SPDLOG_THROW(spdlog_ex(msg, last_errno));
+}
+
+SPDLOG_INLINE void throw_spdlog_ex(std::string msg) { SPDLOG_THROW(spdlog_ex(std::move(msg))); }
+
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/common.h b/ext/spdlog/include/spdlog/common.h
new file mode 100644
index 0000000..aca483c
--- /dev/null
+++ b/ext/spdlog/include/spdlog/common.h
@@ -0,0 +1,411 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#include <spdlog/details/null_mutex.h>
+#include <spdlog/tweakme.h>
+
+#include <atomic>
+#include <chrono>
+#include <cstdio>
+#include <exception>
+#include <functional>
+#include <initializer_list>
+#include <memory>
+#include <string>
+#include <type_traits>
+
+#ifdef SPDLOG_USE_STD_FORMAT
+    #include <version>
+    #if __cpp_lib_format >= 202207L
+        #include <format>
+    #else
+        #include <string_view>
+    #endif
+#endif
+
+#ifdef SPDLOG_COMPILED_LIB
+    #undef SPDLOG_HEADER_ONLY
+    #if defined(SPDLOG_SHARED_LIB)
+        #if defined(_WIN32)
+            #ifdef spdlog_EXPORTS
+                #define SPDLOG_API __declspec(dllexport)
+            #else  // !spdlog_EXPORTS
+                #define SPDLOG_API __declspec(dllimport)
+            #endif
+        #else  // !defined(_WIN32)
+            #define SPDLOG_API __attribute__((visibility("default")))
+        #endif
+    #else  // !defined(SPDLOG_SHARED_LIB)
+        #define SPDLOG_API
+    #endif
+    #define SPDLOG_INLINE
+#else  // !defined(SPDLOG_COMPILED_LIB)
+    #define SPDLOG_API
+    #define SPDLOG_HEADER_ONLY
+    #define SPDLOG_INLINE inline
+#endif  // #ifdef SPDLOG_COMPILED_LIB
+
+#include <spdlog/fmt/fmt.h>
+
+#if !defined(SPDLOG_USE_STD_FORMAT) && \
+    FMT_VERSION >= 80000  // backward compatibility with fmt versions older than 8
+    #define SPDLOG_FMT_RUNTIME(format_string) fmt::runtime(format_string)
+    #define SPDLOG_FMT_STRING(format_string) FMT_STRING(format_string)
+    #if defined(SPDLOG_WCHAR_FILENAMES) || defined(SPDLOG_WCHAR_TO_UTF8_SUPPORT)
+        #include <spdlog/fmt/xchar.h>
+    #endif
+#else
+    #define SPDLOG_FMT_RUNTIME(format_string) format_string
+    #define SPDLOG_FMT_STRING(format_string) format_string
+#endif
+
+// visual studio up to 2013 does not support noexcept nor constexpr
+#if defined(_MSC_VER) && (_MSC_VER < 1900)
+    #define SPDLOG_NOEXCEPT _NOEXCEPT
+    #define SPDLOG_CONSTEXPR
+#else
+    #define SPDLOG_NOEXCEPT noexcept
+    #define SPDLOG_CONSTEXPR constexpr
+#endif
+
+// If building with std::format, can just use constexpr, otherwise if building with fmt
+// SPDLOG_CONSTEXPR_FUNC needs to be set the same as FMT_CONSTEXPR to avoid situations where
+// a constexpr function in spdlog could end up calling a non-constexpr function in fmt
+// depending on the compiler
+// If fmt determines it can't use constexpr, we should inline the function instead
+#ifdef SPDLOG_USE_STD_FORMAT
+    #define SPDLOG_CONSTEXPR_FUNC constexpr
+#else  // Being built with fmt
+    #if FMT_USE_CONSTEXPR
+        #define SPDLOG_CONSTEXPR_FUNC FMT_CONSTEXPR
+    #else
+        #define SPDLOG_CONSTEXPR_FUNC inline
+    #endif
+#endif
+
+#if defined(__GNUC__) || defined(__clang__)
+    #define SPDLOG_DEPRECATED __attribute__((deprecated))
+#elif defined(_MSC_VER)
+    #define SPDLOG_DEPRECATED __declspec(deprecated)
+#else
+    #define SPDLOG_DEPRECATED
+#endif
+
+// disable thread local on msvc 2013
+#ifndef SPDLOG_NO_TLS
+    #if (defined(_MSC_VER) && (_MSC_VER < 1900)) || defined(__cplusplus_winrt)
+        #define SPDLOG_NO_TLS 1
+    #endif
+#endif
+
+#ifndef SPDLOG_FUNCTION
+    #define SPDLOG_FUNCTION static_cast<const char *>(__FUNCTION__)
+#endif
+
+#ifdef SPDLOG_NO_EXCEPTIONS
+    #define SPDLOG_TRY
+    #define SPDLOG_THROW(ex)                               \
+        do {                                               \
+            printf("spdlog fatal error: %s\n", ex.what()); \
+            std::abort();                                  \
+        } while (0)
+    #define SPDLOG_CATCH_STD
+#else
+    #define SPDLOG_TRY try
+    #define SPDLOG_THROW(ex) throw(ex)
+    #define SPDLOG_CATCH_STD             \
+        catch (const std::exception &) { \
+        }
+#endif
+
+namespace spdlog {
+
+class formatter;
+
+namespace sinks {
+class sink;
+}
+
+#if defined(_WIN32) && defined(SPDLOG_WCHAR_FILENAMES)
+using filename_t = std::wstring;
+    // allow macro expansion to occur in SPDLOG_FILENAME_T
+    #define SPDLOG_FILENAME_T_INNER(s) L##s
+    #define SPDLOG_FILENAME_T(s) SPDLOG_FILENAME_T_INNER(s)
+#else
+using filename_t = std::string;
+    #define SPDLOG_FILENAME_T(s) s
+#endif
+
+using log_clock = std::chrono::system_clock;
+using sink_ptr = std::shared_ptr<sinks::sink>;
+using sinks_init_list = std::initializer_list<sink_ptr>;
+using err_handler = std::function<void(const std::string &err_msg)>;
+#ifdef SPDLOG_USE_STD_FORMAT
+namespace fmt_lib = std;
+
+using string_view_t = std::string_view;
+using memory_buf_t = std::string;
+
+template <typename... Args>
+    #if __cpp_lib_format >= 202207L
+using format_string_t = std::format_string<Args...>;
+    #else
+using format_string_t = std::string_view;
+    #endif
+
+template <class T, class Char = char>
+struct is_convertible_to_basic_format_string
+    : std::integral_constant<bool, std::is_convertible<T, std::basic_string_view<Char>>::value> {};
+
+    #if defined(SPDLOG_WCHAR_FILENAMES) || defined(SPDLOG_WCHAR_TO_UTF8_SUPPORT)
+using wstring_view_t = std::wstring_view;
+using wmemory_buf_t = std::wstring;
+
+template <typename... Args>
+        #if __cpp_lib_format >= 202207L
+using wformat_string_t = std::wformat_string<Args...>;
+        #else
+using wformat_string_t = std::wstring_view;
+        #endif
+    #endif
+    #define SPDLOG_BUF_TO_STRING(x) x
+#else  // use fmt lib instead of std::format
+namespace fmt_lib = fmt;
+
+using string_view_t = fmt::basic_string_view<char>;
+using memory_buf_t = fmt::basic_memory_buffer<char, 250>;
+
+template <typename... Args>
+using format_string_t = fmt::format_string<Args...>;
+
+template <class T>
+using remove_cvref_t = typename std::remove_cv<typename std::remove_reference<T>::type>::type;
+
+template <typename Char>
+    #if FMT_VERSION >= 90101
+using fmt_runtime_string = fmt::runtime_format_string<Char>;
+    #else
+using fmt_runtime_string = fmt::basic_runtime<Char>;
+    #endif
+
+// clang doesn't like SFINAE disabled constructor in std::is_convertible<> so have to repeat the
+// condition from basic_format_string here, in addition, fmt::basic_runtime<Char> is only
+// convertible to basic_format_string<Char> but not basic_string_view<Char>
+template <class T, class Char = char>
+struct is_convertible_to_basic_format_string
+    : std::integral_constant<bool,
+                             std::is_convertible<T, fmt::basic_string_view<Char>>::value ||
+                                 std::is_same<remove_cvref_t<T>, fmt_runtime_string<Char>>::value> {
+};
+
+    #if defined(SPDLOG_WCHAR_FILENAMES) || defined(SPDLOG_WCHAR_TO_UTF8_SUPPORT)
+using wstring_view_t = fmt::basic_string_view<wchar_t>;
+using wmemory_buf_t = fmt::basic_memory_buffer<wchar_t, 250>;
+
+template <typename... Args>
+using wformat_string_t = fmt::wformat_string<Args...>;
+    #endif
+    #define SPDLOG_BUF_TO_STRING(x) fmt::to_string(x)
+#endif
+
+#ifdef SPDLOG_WCHAR_TO_UTF8_SUPPORT
+    #ifndef _WIN32
+        #error SPDLOG_WCHAR_TO_UTF8_SUPPORT only supported on windows
+    #endif  // _WIN32
+#endif      // SPDLOG_WCHAR_TO_UTF8_SUPPORT
+
+template <class T>
+struct is_convertible_to_any_format_string
+    : std::integral_constant<bool,
+                             is_convertible_to_basic_format_string<T, char>::value ||
+                                 is_convertible_to_basic_format_string<T, wchar_t>::value> {};
+
+#if defined(SPDLOG_NO_ATOMIC_LEVELS)
+using level_t = details::null_atomic_int;
+#else
+using level_t = std::atomic<int>;
+#endif
+
+#define SPDLOG_LEVEL_TRACE 0
+#define SPDLOG_LEVEL_DEBUG 1
+#define SPDLOG_LEVEL_INFO 2
+#define SPDLOG_LEVEL_WARN 3
+#define SPDLOG_LEVEL_ERROR 4
+#define SPDLOG_LEVEL_CRITICAL 5
+#define SPDLOG_LEVEL_OFF 6
+
+#if !defined(SPDLOG_ACTIVE_LEVEL)
+    #define SPDLOG_ACTIVE_LEVEL SPDLOG_LEVEL_INFO
+#endif
+
+// Log level enum
+namespace level {
+enum level_enum : int {
+    trace = SPDLOG_LEVEL_TRACE,
+    debug = SPDLOG_LEVEL_DEBUG,
+    info = SPDLOG_LEVEL_INFO,
+    warn = SPDLOG_LEVEL_WARN,
+    err = SPDLOG_LEVEL_ERROR,
+    critical = SPDLOG_LEVEL_CRITICAL,
+    off = SPDLOG_LEVEL_OFF,
+    n_levels
+};
+
+#define SPDLOG_LEVEL_NAME_TRACE spdlog::string_view_t("trace", 5)
+#define SPDLOG_LEVEL_NAME_DEBUG spdlog::string_view_t("debug", 5)
+#define SPDLOG_LEVEL_NAME_INFO spdlog::string_view_t("info", 4)
+#define SPDLOG_LEVEL_NAME_WARNING spdlog::string_view_t("warning", 7)
+#define SPDLOG_LEVEL_NAME_ERROR spdlog::string_view_t("error", 5)
+#define SPDLOG_LEVEL_NAME_CRITICAL spdlog::string_view_t("critical", 8)
+#define SPDLOG_LEVEL_NAME_OFF spdlog::string_view_t("off", 3)
+
+#if !defined(SPDLOG_LEVEL_NAMES)
+    #define SPDLOG_LEVEL_NAMES                                                                  \
+        {                                                                                       \
+            SPDLOG_LEVEL_NAME_TRACE, SPDLOG_LEVEL_NAME_DEBUG, SPDLOG_LEVEL_NAME_INFO,           \
+                SPDLOG_LEVEL_NAME_WARNING, SPDLOG_LEVEL_NAME_ERROR, SPDLOG_LEVEL_NAME_CRITICAL, \
+                SPDLOG_LEVEL_NAME_OFF                                                           \
+        }
+#endif
+
+#if !defined(SPDLOG_SHORT_LEVEL_NAMES)
+
+    #define SPDLOG_SHORT_LEVEL_NAMES \
+        { "T", "D", "I", "W", "E", "C", "O" }
+#endif
+
+SPDLOG_API const string_view_t &to_string_view(spdlog::level::level_enum l) SPDLOG_NOEXCEPT;
+SPDLOG_API const char *to_short_c_str(spdlog::level::level_enum l) SPDLOG_NOEXCEPT;
+SPDLOG_API spdlog::level::level_enum from_str(const std::string &name) SPDLOG_NOEXCEPT;
+
+}  // namespace level
+
+//
+// Color mode used by sinks with color support.
+//
+enum class color_mode { always, automatic, never };
+
+//
+// Pattern time - specific time getting to use for pattern_formatter.
+// local time by default
+//
+enum class pattern_time_type {
+    local,  // log localtime
+    utc     // log utc
+};
+
+//
+// Log exception
+//
+class SPDLOG_API spdlog_ex : public std::exception {
+public:
+    explicit spdlog_ex(std::string msg);
+    spdlog_ex(const std::string &msg, int last_errno);
+    const char *what() const SPDLOG_NOEXCEPT override;
+
+private:
+    std::string msg_;
+};
+
+[[noreturn]] SPDLOG_API void throw_spdlog_ex(const std::string &msg, int last_errno);
+[[noreturn]] SPDLOG_API void throw_spdlog_ex(std::string msg);
+
+struct source_loc {
+    SPDLOG_CONSTEXPR source_loc() = default;
+    SPDLOG_CONSTEXPR source_loc(const char *filename_in, int line_in, const char *funcname_in)
+        : filename{filename_in},
+          line{line_in},
+          funcname{funcname_in} {}
+
+    SPDLOG_CONSTEXPR bool empty() const SPDLOG_NOEXCEPT { return line <= 0; }
+    const char *filename{nullptr};
+    int line{0};
+    const char *funcname{nullptr};
+};
+
+struct file_event_handlers {
+    file_event_handlers()
+        : before_open(nullptr),
+          after_open(nullptr),
+          before_close(nullptr),
+          after_close(nullptr) {}
+
+    std::function<void(const filename_t &filename)> before_open;
+    std::function<void(const filename_t &filename, std::FILE *file_stream)> after_open;
+    std::function<void(const filename_t &filename, std::FILE *file_stream)> before_close;
+    std::function<void(const filename_t &filename)> after_close;
+};
+
+namespace details {
+
+// to_string_view
+
+SPDLOG_CONSTEXPR_FUNC spdlog::string_view_t to_string_view(const memory_buf_t &buf)
+    SPDLOG_NOEXCEPT {
+    return spdlog::string_view_t{buf.data(), buf.size()};
+}
+
+SPDLOG_CONSTEXPR_FUNC spdlog::string_view_t to_string_view(spdlog::string_view_t str)
+    SPDLOG_NOEXCEPT {
+    return str;
+}
+
+#if defined(SPDLOG_WCHAR_FILENAMES) || defined(SPDLOG_WCHAR_TO_UTF8_SUPPORT)
+SPDLOG_CONSTEXPR_FUNC spdlog::wstring_view_t to_string_view(const wmemory_buf_t &buf)
+    SPDLOG_NOEXCEPT {
+    return spdlog::wstring_view_t{buf.data(), buf.size()};
+}
+
+SPDLOG_CONSTEXPR_FUNC spdlog::wstring_view_t to_string_view(spdlog::wstring_view_t str)
+    SPDLOG_NOEXCEPT {
+    return str;
+}
+#endif
+
+#ifndef SPDLOG_USE_STD_FORMAT
+template <typename T, typename... Args>
+inline fmt::basic_string_view<T> to_string_view(fmt::basic_format_string<T, Args...> fmt) {
+    return fmt;
+}
+#elif __cpp_lib_format >= 202207L
+template <typename T, typename... Args>
+SPDLOG_CONSTEXPR_FUNC std::basic_string_view<T> to_string_view(
+    std::basic_format_string<T, Args...> fmt) SPDLOG_NOEXCEPT {
+    return fmt.get();
+}
+#endif
+
+// make_unique support for pre c++14
+#if __cplusplus >= 201402L  // C++14 and beyond
+using std::enable_if_t;
+using std::make_unique;
+#else
+template <bool B, class T = void>
+using enable_if_t = typename std::enable_if<B, T>::type;
+
+template <typename T, typename... Args>
+std::unique_ptr<T> make_unique(Args &&...args) {
+    static_assert(!std::is_array<T>::value, "arrays not supported");
+    return std::unique_ptr<T>(new T(std::forward<Args>(args)...));
+}
+#endif
+
+// to avoid useless casts (see https://github.com/nlohmann/json/issues/2893#issuecomment-889152324)
+template <typename T, typename U, enable_if_t<!std::is_same<T, U>::value, int> = 0>
+constexpr T conditional_static_cast(U value) {
+    return static_cast<T>(value);
+}
+
+template <typename T, typename U, enable_if_t<std::is_same<T, U>::value, int> = 0>
+constexpr T conditional_static_cast(U value) {
+    return value;
+}
+
+}  // namespace details
+}  // namespace spdlog
+
+#ifdef SPDLOG_HEADER_ONLY
+    #include "common-inl.h"
+#endif
diff --git a/ext/spdlog/include/spdlog/details/backtracer-inl.h b/ext/spdlog/include/spdlog/details/backtracer-inl.h
new file mode 100644
index 0000000..43d1002
--- /dev/null
+++ b/ext/spdlog/include/spdlog/details/backtracer-inl.h
@@ -0,0 +1,63 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#ifndef SPDLOG_HEADER_ONLY
+    #include <spdlog/details/backtracer.h>
+#endif
+namespace spdlog {
+namespace details {
+SPDLOG_INLINE backtracer::backtracer(const backtracer &other) {
+    std::lock_guard<std::mutex> lock(other.mutex_);
+    enabled_ = other.enabled();
+    messages_ = other.messages_;
+}
+
+SPDLOG_INLINE backtracer::backtracer(backtracer &&other) SPDLOG_NOEXCEPT {
+    std::lock_guard<std::mutex> lock(other.mutex_);
+    enabled_ = other.enabled();
+    messages_ = std::move(other.messages_);
+}
+
+SPDLOG_INLINE backtracer &backtracer::operator=(backtracer other) {
+    std::lock_guard<std::mutex> lock(mutex_);
+    enabled_ = other.enabled();
+    messages_ = std::move(other.messages_);
+    return *this;
+}
+
+SPDLOG_INLINE void backtracer::enable(size_t size) {
+    std::lock_guard<std::mutex> lock{mutex_};
+    enabled_.store(true, std::memory_order_relaxed);
+    messages_ = circular_q<log_msg_buffer>{size};
+}
+
+SPDLOG_INLINE void backtracer::disable() {
+    std::lock_guard<std::mutex> lock{mutex_};
+    enabled_.store(false, std::memory_order_relaxed);
+}
+
+SPDLOG_INLINE bool backtracer::enabled() const { return enabled_.load(std::memory_order_relaxed); }
+
+SPDLOG_INLINE void backtracer::push_back(const log_msg &msg) {
+    std::lock_guard<std::mutex> lock{mutex_};
+    messages_.push_back(log_msg_buffer{msg});
+}
+
+SPDLOG_INLINE bool backtracer::empty() const {
+    std::lock_guard<std::mutex> lock{mutex_};
+    return messages_.empty();
+}
+
+// pop all items in the q and apply the given fun on each of them.
+SPDLOG_INLINE void backtracer::foreach_pop(std::function<void(const details::log_msg &)> fun) {
+    std::lock_guard<std::mutex> lock{mutex_};
+    while (!messages_.empty()) {
+        auto &front_msg = messages_.front();
+        fun(front_msg);
+        messages_.pop_front();
+    }
+}
+}  // namespace details
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/details/backtracer.h b/ext/spdlog/include/spdlog/details/backtracer.h
new file mode 100644
index 0000000..541339c
--- /dev/null
+++ b/ext/spdlog/include/spdlog/details/backtracer.h
@@ -0,0 +1,45 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#include <spdlog/details/circular_q.h>
+#include <spdlog/details/log_msg_buffer.h>
+
+#include <atomic>
+#include <functional>
+#include <mutex>
+
+// Store log messages in circular buffer.
+// Useful for storing debug data in case of error/warning happens.
+
+namespace spdlog {
+namespace details {
+class SPDLOG_API backtracer {
+    mutable std::mutex mutex_;
+    std::atomic<bool> enabled_{false};
+    circular_q<log_msg_buffer> messages_;
+
+public:
+    backtracer() = default;
+    backtracer(const backtracer &other);
+
+    backtracer(backtracer &&other) SPDLOG_NOEXCEPT;
+    backtracer &operator=(backtracer other);
+
+    void enable(size_t size);
+    void disable();
+    bool enabled() const;
+    void push_back(const log_msg &msg);
+    bool empty() const;
+
+    // pop all items in the q and apply the given fun on each of them.
+    void foreach_pop(std::function<void(const details::log_msg &)> fun);
+};
+
+}  // namespace details
+}  // namespace spdlog
+
+#ifdef SPDLOG_HEADER_ONLY
+    #include "backtracer-inl.h"
+#endif
diff --git a/ext/spdlog/include/spdlog/details/circular_q.h b/ext/spdlog/include/spdlog/details/circular_q.h
new file mode 100644
index 0000000..29e9d25
--- /dev/null
+++ b/ext/spdlog/include/spdlog/details/circular_q.h
@@ -0,0 +1,115 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+// circular q view of std::vector.
+#pragma once
+
+#include <cassert>
+#include <vector>
+
+#include "spdlog/common.h"
+
+namespace spdlog {
+namespace details {
+template <typename T>
+class circular_q {
+    size_t max_items_ = 0;
+    typename std::vector<T>::size_type head_ = 0;
+    typename std::vector<T>::size_type tail_ = 0;
+    size_t overrun_counter_ = 0;
+    std::vector<T> v_;
+
+public:
+    using value_type = T;
+
+    // empty ctor - create a disabled queue with no elements allocated at all
+    circular_q() = default;
+
+    explicit circular_q(size_t max_items)
+        : max_items_(max_items + 1)  // one item is reserved as marker for full q
+          ,
+          v_(max_items_) {}
+
+    circular_q(const circular_q &) = default;
+    circular_q &operator=(const circular_q &) = default;
+
+    // move cannot be default,
+    // since we need to reset head_, tail_, etc to zero in the moved object
+    circular_q(circular_q &&other) SPDLOG_NOEXCEPT { copy_moveable(std::move(other)); }
+
+    circular_q &operator=(circular_q &&other) SPDLOG_NOEXCEPT {
+        copy_moveable(std::move(other));
+        return *this;
+    }
+
+    // push back, overrun (oldest) item if no room left
+    void push_back(T &&item) {
+        if (max_items_ > 0) {
+            v_[tail_] = std::move(item);
+            tail_ = (tail_ + 1) % max_items_;
+
+            if (tail_ == head_)  // overrun last item if full
+            {
+                head_ = (head_ + 1) % max_items_;
+                ++overrun_counter_;
+            }
+        }
+    }
+
+    // Return reference to the front item.
+    // If there are no elements in the container, the behavior is undefined.
+    const T &front() const { return v_[head_]; }
+
+    T &front() { return v_[head_]; }
+
+    // Return number of elements actually stored
+    size_t size() const {
+        if (tail_ >= head_) {
+            return tail_ - head_;
+        } else {
+            return max_items_ - (head_ - tail_);
+        }
+    }
+
+    // Return const reference to item by index.
+    // If index is out of range 0…size()-1, the behavior is undefined.
+    const T &at(size_t i) const {
+        assert(i < size());
+        return v_[(head_ + i) % max_items_];
+    }
+
+    // Pop item from front.
+    // If there are no elements in the container, the behavior is undefined.
+    void pop_front() { head_ = (head_ + 1) % max_items_; }
+
+    bool empty() const { return tail_ == head_; }
+
+    bool full() const {
+        // head is ahead of the tail by 1
+        if (max_items_ > 0) {
+            return ((tail_ + 1) % max_items_) == head_;
+        }
+        return false;
+    }
+
+    size_t overrun_counter() const { return overrun_counter_; }
+
+    void reset_overrun_counter() { overrun_counter_ = 0; }
+
+private:
+    // copy from other&& and reset it to disabled state
+    void copy_moveable(circular_q &&other) SPDLOG_NOEXCEPT {
+        max_items_ = other.max_items_;
+        head_ = other.head_;
+        tail_ = other.tail_;
+        overrun_counter_ = other.overrun_counter_;
+        v_ = std::move(other.v_);
+
+        // put &&other in disabled, but valid state
+        other.max_items_ = 0;
+        other.head_ = other.tail_ = 0;
+        other.overrun_counter_ = 0;
+    }
+};
+}  // namespace details
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/details/console_globals.h b/ext/spdlog/include/spdlog/details/console_globals.h
new file mode 100644
index 0000000..9c55210
--- /dev/null
+++ b/ext/spdlog/include/spdlog/details/console_globals.h
@@ -0,0 +1,28 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#include <mutex>
+#include <spdlog/details/null_mutex.h>
+
+namespace spdlog {
+namespace details {
+
+struct console_mutex {
+    using mutex_t = std::mutex;
+    static mutex_t &mutex() {
+        static mutex_t s_mutex;
+        return s_mutex;
+    }
+};
+
+struct console_nullmutex {
+    using mutex_t = null_mutex;
+    static mutex_t &mutex() {
+        static mutex_t s_mutex;
+        return s_mutex;
+    }
+};
+}  // namespace details
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/details/file_helper-inl.h b/ext/spdlog/include/spdlog/details/file_helper-inl.h
new file mode 100644
index 0000000..8742b96
--- /dev/null
+++ b/ext/spdlog/include/spdlog/details/file_helper-inl.h
@@ -0,0 +1,153 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#ifndef SPDLOG_HEADER_ONLY
+    #include <spdlog/details/file_helper.h>
+#endif
+
+#include <spdlog/common.h>
+#include <spdlog/details/os.h>
+
+#include <cerrno>
+#include <chrono>
+#include <cstdio>
+#include <string>
+#include <thread>
+#include <tuple>
+
+namespace spdlog {
+namespace details {
+
+SPDLOG_INLINE file_helper::file_helper(const file_event_handlers &event_handlers)
+    : event_handlers_(event_handlers) {}
+
+SPDLOG_INLINE file_helper::~file_helper() { close(); }
+
+SPDLOG_INLINE void file_helper::open(const filename_t &fname, bool truncate) {
+    close();
+    filename_ = fname;
+
+    auto *mode = SPDLOG_FILENAME_T("ab");
+    auto *trunc_mode = SPDLOG_FILENAME_T("wb");
+
+    if (event_handlers_.before_open) {
+        event_handlers_.before_open(filename_);
+    }
+    for (int tries = 0; tries < open_tries_; ++tries) {
+        // create containing folder if not exists already.
+        os::create_dir(os::dir_name(fname));
+        if (truncate) {
+            // Truncate by opening-and-closing a tmp file in "wb" mode, always
+            // opening the actual log-we-write-to in "ab" mode, since that
+            // interacts more politely with eternal processes that might
+            // rotate/truncate the file underneath us.
+            std::FILE *tmp;
+            if (os::fopen_s(&tmp, fname, trunc_mode)) {
+                continue;
+            }
+            std::fclose(tmp);
+        }
+        if (!os::fopen_s(&fd_, fname, mode)) {
+            if (event_handlers_.after_open) {
+                event_handlers_.after_open(filename_, fd_);
+            }
+            return;
+        }
+
+        details::os::sleep_for_millis(open_interval_);
+    }
+
+    throw_spdlog_ex("Failed opening file " + os::filename_to_str(filename_) + " for writing",
+                    errno);
+}
+
+SPDLOG_INLINE void file_helper::reopen(bool truncate) {
+    if (filename_.empty()) {
+        throw_spdlog_ex("Failed re opening file - was not opened before");
+    }
+    this->open(filename_, truncate);
+}
+
+SPDLOG_INLINE void file_helper::flush() {
+    if (std::fflush(fd_) != 0) {
+        throw_spdlog_ex("Failed flush to file " + os::filename_to_str(filename_), errno);
+    }
+}
+
+SPDLOG_INLINE void file_helper::sync() {
+    if (!os::fsync(fd_)) {
+        throw_spdlog_ex("Failed to fsync file " + os::filename_to_str(filename_), errno);
+    }
+}
+
+SPDLOG_INLINE void file_helper::close() {
+    if (fd_ != nullptr) {
+        if (event_handlers_.before_close) {
+            event_handlers_.before_close(filename_, fd_);
+        }
+
+        std::fclose(fd_);
+        fd_ = nullptr;
+
+        if (event_handlers_.after_close) {
+            event_handlers_.after_close(filename_);
+        }
+    }
+}
+
+SPDLOG_INLINE void file_helper::write(const memory_buf_t &buf) {
+    if (fd_ == nullptr) return;
+    size_t msg_size = buf.size();
+    auto data = buf.data();
+
+    if (!details::os::fwrite_bytes(data, msg_size, fd_)) {
+        throw_spdlog_ex("Failed writing to file " + os::filename_to_str(filename_), errno);
+    }
+}
+
+SPDLOG_INLINE size_t file_helper::size() const {
+    if (fd_ == nullptr) {
+        throw_spdlog_ex("Cannot use size() on closed file " + os::filename_to_str(filename_));
+    }
+    return os::filesize(fd_);
+}
+
+SPDLOG_INLINE const filename_t &file_helper::filename() const { return filename_; }
+
+//
+// return file path and its extension:
+//
+// "mylog.txt" => ("mylog", ".txt")
+// "mylog" => ("mylog", "")
+// "mylog." => ("mylog.", "")
+// "/dir1/dir2/mylog.txt" => ("/dir1/dir2/mylog", ".txt")
+//
+// the starting dot in filenames is ignored (hidden files):
+//
+// ".mylog" => (".mylog". "")
+// "my_folder/.mylog" => ("my_folder/.mylog", "")
+// "my_folder/.mylog.txt" => ("my_folder/.mylog", ".txt")
+SPDLOG_INLINE std::tuple<filename_t, filename_t> file_helper::split_by_extension(
+    const filename_t &fname) {
+    auto ext_index = fname.rfind('.');
+
+    // no valid extension found - return whole path and empty string as
+    // extension
+    if (ext_index == filename_t::npos || ext_index == 0 || ext_index == fname.size() - 1) {
+        return std::make_tuple(fname, filename_t());
+    }
+
+    // treat cases like "/etc/rc.d/somelogfile or "/abc/.hiddenfile"
+    auto folder_index = fname.find_last_of(details::os::folder_seps_filename);
+    if (folder_index != filename_t::npos && folder_index >= ext_index - 1) {
+        return std::make_tuple(fname, filename_t());
+    }
+
+    // finally - return a valid base and extension tuple
+    return std::make_tuple(fname.substr(0, ext_index), fname.substr(ext_index));
+}
+
+}  // namespace details
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/details/file_helper.h b/ext/spdlog/include/spdlog/details/file_helper.h
new file mode 100644
index 0000000..f0e5d18
--- /dev/null
+++ b/ext/spdlog/include/spdlog/details/file_helper.h
@@ -0,0 +1,61 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#include <spdlog/common.h>
+#include <tuple>
+
+namespace spdlog {
+namespace details {
+
+// Helper class for file sinks.
+// When failing to open a file, retry several times(5) with a delay interval(10 ms).
+// Throw spdlog_ex exception on errors.
+
+class SPDLOG_API file_helper {
+public:
+    file_helper() = default;
+    explicit file_helper(const file_event_handlers &event_handlers);
+
+    file_helper(const file_helper &) = delete;
+    file_helper &operator=(const file_helper &) = delete;
+    ~file_helper();
+
+    void open(const filename_t &fname, bool truncate = false);
+    void reopen(bool truncate);
+    void flush();
+    void sync();
+    void close();
+    void write(const memory_buf_t &buf);
+    size_t size() const;
+    const filename_t &filename() const;
+
+    //
+    // return file path and its extension:
+    //
+    // "mylog.txt" => ("mylog", ".txt")
+    // "mylog" => ("mylog", "")
+    // "mylog." => ("mylog.", "")
+    // "/dir1/dir2/mylog.txt" => ("/dir1/dir2/mylog", ".txt")
+    //
+    // the starting dot in filenames is ignored (hidden files):
+    //
+    // ".mylog" => (".mylog". "")
+    // "my_folder/.mylog" => ("my_folder/.mylog", "")
+    // "my_folder/.mylog.txt" => ("my_folder/.mylog", ".txt")
+    static std::tuple<filename_t, filename_t> split_by_extension(const filename_t &fname);
+
+private:
+    const int open_tries_ = 5;
+    const unsigned int open_interval_ = 10;
+    std::FILE *fd_{nullptr};
+    filename_t filename_;
+    file_event_handlers event_handlers_;
+};
+}  // namespace details
+}  // namespace spdlog
+
+#ifdef SPDLOG_HEADER_ONLY
+    #include "file_helper-inl.h"
+#endif
diff --git a/ext/spdlog/include/spdlog/details/fmt_helper.h b/ext/spdlog/include/spdlog/details/fmt_helper.h
new file mode 100644
index 0000000..6130600
--- /dev/null
+++ b/ext/spdlog/include/spdlog/details/fmt_helper.h
@@ -0,0 +1,141 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+#pragma once
+
+#include <chrono>
+#include <iterator>
+#include <spdlog/common.h>
+#include <spdlog/fmt/fmt.h>
+#include <type_traits>
+
+#ifdef SPDLOG_USE_STD_FORMAT
+    #include <charconv>
+    #include <limits>
+#endif
+
+// Some fmt helpers to efficiently format and pad ints and strings
+namespace spdlog {
+namespace details {
+namespace fmt_helper {
+
+inline void append_string_view(spdlog::string_view_t view, memory_buf_t &dest) {
+    auto *buf_ptr = view.data();
+    dest.append(buf_ptr, buf_ptr + view.size());
+}
+
+#ifdef SPDLOG_USE_STD_FORMAT
+template <typename T>
+inline void append_int(T n, memory_buf_t &dest) {
+    // Buffer should be large enough to hold all digits (digits10 + 1) and a sign
+    SPDLOG_CONSTEXPR const auto BUF_SIZE = std::numeric_limits<T>::digits10 + 2;
+    char buf[BUF_SIZE];
+
+    auto [ptr, ec] = std::to_chars(buf, buf + BUF_SIZE, n, 10);
+    if (ec == std::errc()) {
+        dest.append(buf, ptr);
+    } else {
+        throw_spdlog_ex("Failed to format int", static_cast<int>(ec));
+    }
+}
+#else
+template <typename T>
+inline void append_int(T n, memory_buf_t &dest) {
+    fmt::format_int i(n);
+    dest.append(i.data(), i.data() + i.size());
+}
+#endif
+
+template <typename T>
+SPDLOG_CONSTEXPR_FUNC unsigned int count_digits_fallback(T n) {
+    // taken from fmt: https://github.com/fmtlib/fmt/blob/8.0.1/include/fmt/format.h#L899-L912
+    unsigned int count = 1;
+    for (;;) {
+        // Integer division is slow so do it for a group of four digits instead
+        // of for every digit. The idea comes from the talk by Alexandrescu
+        // "Three Optimization Tips for C++". See speed-test for a comparison.
+        if (n < 10) return count;
+        if (n < 100) return count + 1;
+        if (n < 1000) return count + 2;
+        if (n < 10000) return count + 3;
+        n /= 10000u;
+        count += 4;
+    }
+}
+
+template <typename T>
+inline unsigned int count_digits(T n) {
+    using count_type =
+        typename std::conditional<(sizeof(T) > sizeof(uint32_t)), uint64_t, uint32_t>::type;
+#ifdef SPDLOG_USE_STD_FORMAT
+    return count_digits_fallback(static_cast<count_type>(n));
+#else
+    return static_cast<unsigned int>(fmt::
+    // fmt 7.0.0 renamed the internal namespace to detail.
+    // See: https://github.com/fmtlib/fmt/issues/1538
+    #if FMT_VERSION < 70000
+                                         internal
+    #else
+                                         detail
+    #endif
+                                     ::count_digits(static_cast<count_type>(n)));
+#endif
+}
+
+inline void pad2(int n, memory_buf_t &dest) {
+    if (n >= 0 && n < 100)  // 0-99
+    {
+        dest.push_back(static_cast<char>('0' + n / 10));
+        dest.push_back(static_cast<char>('0' + n % 10));
+    } else  // unlikely, but just in case, let fmt deal with it
+    {
+        fmt_lib::format_to(std::back_inserter(dest), SPDLOG_FMT_STRING("{:02}"), n);
+    }
+}
+
+template <typename T>
+inline void pad_uint(T n, unsigned int width, memory_buf_t &dest) {
+    static_assert(std::is_unsigned<T>::value, "pad_uint must get unsigned T");
+    for (auto digits = count_digits(n); digits < width; digits++) {
+        dest.push_back('0');
+    }
+    append_int(n, dest);
+}
+
+template <typename T>
+inline void pad3(T n, memory_buf_t &dest) {
+    static_assert(std::is_unsigned<T>::value, "pad3 must get unsigned T");
+    if (n < 1000) {
+        dest.push_back(static_cast<char>(n / 100 + '0'));
+        n = n % 100;
+        dest.push_back(static_cast<char>((n / 10) + '0'));
+        dest.push_back(static_cast<char>((n % 10) + '0'));
+    } else {
+        append_int(n, dest);
+    }
+}
+
+template <typename T>
+inline void pad6(T n, memory_buf_t &dest) {
+    pad_uint(n, 6, dest);
+}
+
+template <typename T>
+inline void pad9(T n, memory_buf_t &dest) {
+    pad_uint(n, 9, dest);
+}
+
+// return fraction of a second of the given time_point.
+// e.g.
+// fraction<std::milliseconds>(tp) -> will return the millis part of the second
+template <typename ToDuration>
+inline ToDuration time_fraction(log_clock::time_point tp) {
+    using std::chrono::duration_cast;
+    using std::chrono::seconds;
+    auto duration = tp.time_since_epoch();
+    auto secs = duration_cast<seconds>(duration);
+    return duration_cast<ToDuration>(duration) - duration_cast<ToDuration>(secs);
+}
+
+}  // namespace fmt_helper
+}  // namespace details
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/details/log_msg-inl.h b/ext/spdlog/include/spdlog/details/log_msg-inl.h
new file mode 100644
index 0000000..aa3a957
--- /dev/null
+++ b/ext/spdlog/include/spdlog/details/log_msg-inl.h
@@ -0,0 +1,44 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#ifndef SPDLOG_HEADER_ONLY
+    #include <spdlog/details/log_msg.h>
+#endif
+
+#include <spdlog/details/os.h>
+
+namespace spdlog {
+namespace details {
+
+SPDLOG_INLINE log_msg::log_msg(spdlog::log_clock::time_point log_time,
+                               spdlog::source_loc loc,
+                               string_view_t a_logger_name,
+                               spdlog::level::level_enum lvl,
+                               spdlog::string_view_t msg)
+    : logger_name(a_logger_name),
+      level(lvl),
+      time(log_time)
+#ifndef SPDLOG_NO_THREAD_ID
+      ,
+      thread_id(os::thread_id())
+#endif
+      ,
+      source(loc),
+      payload(msg) {
+}
+
+SPDLOG_INLINE log_msg::log_msg(spdlog::source_loc loc,
+                               string_view_t a_logger_name,
+                               spdlog::level::level_enum lvl,
+                               spdlog::string_view_t msg)
+    : log_msg(os::now(), loc, a_logger_name, lvl, msg) {}
+
+SPDLOG_INLINE log_msg::log_msg(string_view_t a_logger_name,
+                               spdlog::level::level_enum lvl,
+                               spdlog::string_view_t msg)
+    : log_msg(os::now(), source_loc{}, a_logger_name, lvl, msg) {}
+
+}  // namespace details
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/details/log_msg.h b/ext/spdlog/include/spdlog/details/log_msg.h
new file mode 100644
index 0000000..87df1e8
--- /dev/null
+++ b/ext/spdlog/include/spdlog/details/log_msg.h
@@ -0,0 +1,40 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#include <spdlog/common.h>
+#include <string>
+
+namespace spdlog {
+namespace details {
+struct SPDLOG_API log_msg {
+    log_msg() = default;
+    log_msg(log_clock::time_point log_time,
+            source_loc loc,
+            string_view_t logger_name,
+            level::level_enum lvl,
+            string_view_t msg);
+    log_msg(source_loc loc, string_view_t logger_name, level::level_enum lvl, string_view_t msg);
+    log_msg(string_view_t logger_name, level::level_enum lvl, string_view_t msg);
+    log_msg(const log_msg &other) = default;
+    log_msg &operator=(const log_msg &other) = default;
+
+    string_view_t logger_name;
+    level::level_enum level{level::off};
+    log_clock::time_point time;
+    size_t thread_id{0};
+
+    // wrapping the formatted text with color (updated by pattern_formatter).
+    mutable size_t color_range_start{0};
+    mutable size_t color_range_end{0};
+
+    source_loc source;
+    string_view_t payload;
+};
+}  // namespace details
+}  // namespace spdlog
+
+#ifdef SPDLOG_HEADER_ONLY
+    #include "log_msg-inl.h"
+#endif
diff --git a/ext/spdlog/include/spdlog/details/log_msg_buffer-inl.h b/ext/spdlog/include/spdlog/details/log_msg_buffer-inl.h
new file mode 100644
index 0000000..2eb2428
--- /dev/null
+++ b/ext/spdlog/include/spdlog/details/log_msg_buffer-inl.h
@@ -0,0 +1,54 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#ifndef SPDLOG_HEADER_ONLY
+    #include <spdlog/details/log_msg_buffer.h>
+#endif
+
+namespace spdlog {
+namespace details {
+
+SPDLOG_INLINE log_msg_buffer::log_msg_buffer(const log_msg &orig_msg)
+    : log_msg{orig_msg} {
+    buffer.append(logger_name.begin(), logger_name.end());
+    buffer.append(payload.begin(), payload.end());
+    update_string_views();
+}
+
+SPDLOG_INLINE log_msg_buffer::log_msg_buffer(const log_msg_buffer &other)
+    : log_msg{other} {
+    buffer.append(logger_name.begin(), logger_name.end());
+    buffer.append(payload.begin(), payload.end());
+    update_string_views();
+}
+
+SPDLOG_INLINE log_msg_buffer::log_msg_buffer(log_msg_buffer &&other) SPDLOG_NOEXCEPT
+    : log_msg{other},
+      buffer{std::move(other.buffer)} {
+    update_string_views();
+}
+
+SPDLOG_INLINE log_msg_buffer &log_msg_buffer::operator=(const log_msg_buffer &other) {
+    log_msg::operator=(other);
+    buffer.clear();
+    buffer.append(other.buffer.data(), other.buffer.data() + other.buffer.size());
+    update_string_views();
+    return *this;
+}
+
+SPDLOG_INLINE log_msg_buffer &log_msg_buffer::operator=(log_msg_buffer &&other) SPDLOG_NOEXCEPT {
+    log_msg::operator=(other);
+    buffer = std::move(other.buffer);
+    update_string_views();
+    return *this;
+}
+
+SPDLOG_INLINE void log_msg_buffer::update_string_views() {
+    logger_name = string_view_t{buffer.data(), logger_name.size()};
+    payload = string_view_t{buffer.data() + logger_name.size(), payload.size()};
+}
+
+}  // namespace details
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/details/log_msg_buffer.h b/ext/spdlog/include/spdlog/details/log_msg_buffer.h
new file mode 100644
index 0000000..1143b3b
--- /dev/null
+++ b/ext/spdlog/include/spdlog/details/log_msg_buffer.h
@@ -0,0 +1,32 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#include <spdlog/details/log_msg.h>
+
+namespace spdlog {
+namespace details {
+
+// Extend log_msg with internal buffer to store its payload.
+// This is needed since log_msg holds string_views that points to stack data.
+
+class SPDLOG_API log_msg_buffer : public log_msg {
+    memory_buf_t buffer;
+    void update_string_views();
+
+public:
+    log_msg_buffer() = default;
+    explicit log_msg_buffer(const log_msg &orig_msg);
+    log_msg_buffer(const log_msg_buffer &other);
+    log_msg_buffer(log_msg_buffer &&other) SPDLOG_NOEXCEPT;
+    log_msg_buffer &operator=(const log_msg_buffer &other);
+    log_msg_buffer &operator=(log_msg_buffer &&other) SPDLOG_NOEXCEPT;
+};
+
+}  // namespace details
+}  // namespace spdlog
+
+#ifdef SPDLOG_HEADER_ONLY
+    #include "log_msg_buffer-inl.h"
+#endif
diff --git a/ext/spdlog/include/spdlog/details/mpmc_blocking_q.h b/ext/spdlog/include/spdlog/details/mpmc_blocking_q.h
new file mode 100644
index 0000000..5848cca
--- /dev/null
+++ b/ext/spdlog/include/spdlog/details/mpmc_blocking_q.h
@@ -0,0 +1,177 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+// multi producer-multi consumer blocking queue.
+// enqueue(..) - will block until room found to put the new message.
+// enqueue_nowait(..) - will return immediately with false if no room left in
+// the queue.
+// dequeue_for(..) - will block until the queue is not empty or timeout have
+// passed.
+
+#include <spdlog/details/circular_q.h>
+
+#include <atomic>
+#include <condition_variable>
+#include <mutex>
+
+namespace spdlog {
+namespace details {
+
+template <typename T>
+class mpmc_blocking_queue {
+public:
+    using item_type = T;
+    explicit mpmc_blocking_queue(size_t max_items)
+        : q_(max_items) {}
+
+#ifndef __MINGW32__
+    // try to enqueue and block if no room left
+    void enqueue(T &&item) {
+        {
+            std::unique_lock<std::mutex> lock(queue_mutex_);
+            pop_cv_.wait(lock, [this] { return !this->q_.full(); });
+            q_.push_back(std::move(item));
+        }
+        push_cv_.notify_one();
+    }
+
+    // enqueue immediately. overrun oldest message in the queue if no room left.
+    void enqueue_nowait(T &&item) {
+        {
+            std::unique_lock<std::mutex> lock(queue_mutex_);
+            q_.push_back(std::move(item));
+        }
+        push_cv_.notify_one();
+    }
+
+    void enqueue_if_have_room(T &&item) {
+        bool pushed = false;
+        {
+            std::unique_lock<std::mutex> lock(queue_mutex_);
+            if (!q_.full()) {
+                q_.push_back(std::move(item));
+                pushed = true;
+            }
+        }
+
+        if (pushed) {
+            push_cv_.notify_one();
+        } else {
+            ++discard_counter_;
+        }
+    }
+
+    // dequeue with a timeout.
+    // Return true, if succeeded dequeue item, false otherwise
+    bool dequeue_for(T &popped_item, std::chrono::milliseconds wait_duration) {
+        {
+            std::unique_lock<std::mutex> lock(queue_mutex_);
+            if (!push_cv_.wait_for(lock, wait_duration, [this] { return !this->q_.empty(); })) {
+                return false;
+            }
+            popped_item = std::move(q_.front());
+            q_.pop_front();
+        }
+        pop_cv_.notify_one();
+        return true;
+    }
+
+    // blocking dequeue without a timeout.
+    void dequeue(T &popped_item) {
+        {
+            std::unique_lock<std::mutex> lock(queue_mutex_);
+            push_cv_.wait(lock, [this] { return !this->q_.empty(); });
+            popped_item = std::move(q_.front());
+            q_.pop_front();
+        }
+        pop_cv_.notify_one();
+    }
+
+#else
+    // apparently mingw deadlocks if the mutex is released before cv.notify_one(),
+    // so release the mutex at the very end each function.
+
+    // try to enqueue and block if no room left
+    void enqueue(T &&item) {
+        std::unique_lock<std::mutex> lock(queue_mutex_);
+        pop_cv_.wait(lock, [this] { return !this->q_.full(); });
+        q_.push_back(std::move(item));
+        push_cv_.notify_one();
+    }
+
+    // enqueue immediately. overrun oldest message in the queue if no room left.
+    void enqueue_nowait(T &&item) {
+        std::unique_lock<std::mutex> lock(queue_mutex_);
+        q_.push_back(std::move(item));
+        push_cv_.notify_one();
+    }
+
+    void enqueue_if_have_room(T &&item) {
+        bool pushed = false;
+        std::unique_lock<std::mutex> lock(queue_mutex_);
+        if (!q_.full()) {
+            q_.push_back(std::move(item));
+            pushed = true;
+        }
+
+        if (pushed) {
+            push_cv_.notify_one();
+        } else {
+            ++discard_counter_;
+        }
+    }
+
+    // dequeue with a timeout.
+    // Return true, if succeeded dequeue item, false otherwise
+    bool dequeue_for(T &popped_item, std::chrono::milliseconds wait_duration) {
+        std::unique_lock<std::mutex> lock(queue_mutex_);
+        if (!push_cv_.wait_for(lock, wait_duration, [this] { return !this->q_.empty(); })) {
+            return false;
+        }
+        popped_item = std::move(q_.front());
+        q_.pop_front();
+        pop_cv_.notify_one();
+        return true;
+    }
+
+    // blocking dequeue without a timeout.
+    void dequeue(T &popped_item) {
+        std::unique_lock<std::mutex> lock(queue_mutex_);
+        push_cv_.wait(lock, [this] { return !this->q_.empty(); });
+        popped_item = std::move(q_.front());
+        q_.pop_front();
+        pop_cv_.notify_one();
+    }
+
+#endif
+
+    size_t overrun_counter() {
+        std::lock_guard<std::mutex> lock(queue_mutex_);
+        return q_.overrun_counter();
+    }
+
+    size_t discard_counter() { return discard_counter_.load(std::memory_order_relaxed); }
+
+    size_t size() {
+        std::lock_guard<std::mutex> lock(queue_mutex_);
+        return q_.size();
+    }
+
+    void reset_overrun_counter() {
+        std::lock_guard<std::mutex> lock(queue_mutex_);
+        q_.reset_overrun_counter();
+    }
+
+    void reset_discard_counter() { discard_counter_.store(0, std::memory_order_relaxed); }
+
+private:
+    std::mutex queue_mutex_;
+    std::condition_variable push_cv_;
+    std::condition_variable pop_cv_;
+    spdlog::details::circular_q<T> q_;
+    std::atomic<size_t> discard_counter_{0};
+};
+}  // namespace details
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/details/null_mutex.h b/ext/spdlog/include/spdlog/details/null_mutex.h
new file mode 100644
index 0000000..e3b3220
--- /dev/null
+++ b/ext/spdlog/include/spdlog/details/null_mutex.h
@@ -0,0 +1,35 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#include <atomic>
+#include <utility>
+// null, no cost dummy "mutex" and dummy "atomic" int
+
+namespace spdlog {
+namespace details {
+struct null_mutex {
+    void lock() const {}
+    void unlock() const {}
+};
+
+struct null_atomic_int {
+    int value;
+    null_atomic_int() = default;
+
+    explicit null_atomic_int(int new_value)
+        : value(new_value) {}
+
+    int load(std::memory_order = std::memory_order_relaxed) const { return value; }
+
+    void store(int new_value, std::memory_order = std::memory_order_relaxed) { value = new_value; }
+
+    int exchange(int new_value, std::memory_order = std::memory_order_relaxed) {
+        std::swap(new_value, value);
+        return new_value;  // return value before the call
+    }
+};
+
+}  // namespace details
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/details/os-inl.h b/ext/spdlog/include/spdlog/details/os-inl.h
new file mode 100644
index 0000000..8ee4230
--- /dev/null
+++ b/ext/spdlog/include/spdlog/details/os-inl.h
@@ -0,0 +1,606 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#ifndef SPDLOG_HEADER_ONLY
+    #include <spdlog/details/os.h>
+#endif
+
+#include <spdlog/common.h>
+
+#include <algorithm>
+#include <array>
+#include <chrono>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <ctime>
+#include <string>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <thread>
+
+#ifdef _WIN32
+    #include <spdlog/details/windows_include.h>
+    #include <fileapi.h>  // for FlushFileBuffers
+    #include <io.h>       // for _get_osfhandle, _isatty, _fileno
+    #include <process.h>  // for _get_pid
+
+    #ifdef __MINGW32__
+        #include <share.h>
+    #endif
+
+    #if defined(SPDLOG_WCHAR_TO_UTF8_SUPPORT) || defined(SPDLOG_WCHAR_FILENAMES)
+        #include <cassert>
+        #include <limits>
+    #endif
+
+    #include <direct.h>  // for _mkdir/_wmkdir
+
+#else  // unix
+
+    #include <fcntl.h>
+    #include <unistd.h>
+
+    #ifdef __linux__
+        #include <sys/syscall.h>  //Use gettid() syscall under linux to get thread id
+
+    #elif defined(_AIX)
+        #include <pthread.h>  // for pthread_getthrds_np
+
+    #elif defined(__DragonFly__) || defined(__FreeBSD__)
+        #include <pthread_np.h>  // for pthread_getthreadid_np
+
+    #elif defined(__NetBSD__)
+        #include <lwp.h>  // for _lwp_self
+
+    #elif defined(__sun)
+        #include <thread.h>  // for thr_self
+    #endif
+
+#endif  // unix
+
+#if defined __APPLE__
+    #include <AvailabilityMacros.h>
+#endif
+
+#ifndef __has_feature           // Clang - feature checking macros.
+    #define __has_feature(x) 0  // Compatibility with non-clang compilers.
+#endif
+
+namespace spdlog {
+namespace details {
+namespace os {
+
+SPDLOG_INLINE spdlog::log_clock::time_point now() SPDLOG_NOEXCEPT {
+#if defined __linux__ && defined SPDLOG_CLOCK_COARSE
+    timespec ts;
+    ::clock_gettime(CLOCK_REALTIME_COARSE, &ts);
+    return std::chrono::time_point<log_clock, typename log_clock::duration>(
+        std::chrono::duration_cast<typename log_clock::duration>(
+            std::chrono::seconds(ts.tv_sec) + std::chrono::nanoseconds(ts.tv_nsec)));
+
+#else
+    return log_clock::now();
+#endif
+}
+SPDLOG_INLINE std::tm localtime(const std::time_t &time_tt) SPDLOG_NOEXCEPT {
+#ifdef _WIN32
+    std::tm tm;
+    ::localtime_s(&tm, &time_tt);
+#else
+    std::tm tm;
+    ::localtime_r(&time_tt, &tm);
+#endif
+    return tm;
+}
+
+SPDLOG_INLINE std::tm localtime() SPDLOG_NOEXCEPT {
+    std::time_t now_t = ::time(nullptr);
+    return localtime(now_t);
+}
+
+SPDLOG_INLINE std::tm gmtime(const std::time_t &time_tt) SPDLOG_NOEXCEPT {
+#ifdef _WIN32
+    std::tm tm;
+    ::gmtime_s(&tm, &time_tt);
+#else
+    std::tm tm;
+    ::gmtime_r(&time_tt, &tm);
+#endif
+    return tm;
+}
+
+SPDLOG_INLINE std::tm gmtime() SPDLOG_NOEXCEPT {
+    std::time_t now_t = ::time(nullptr);
+    return gmtime(now_t);
+}
+
+// fopen_s on non windows for writing
+SPDLOG_INLINE bool fopen_s(FILE **fp, const filename_t &filename, const filename_t &mode) {
+#ifdef _WIN32
+    #ifdef SPDLOG_WCHAR_FILENAMES
+    *fp = ::_wfsopen((filename.c_str()), mode.c_str(), _SH_DENYNO);
+    #else
+    *fp = ::_fsopen((filename.c_str()), mode.c_str(), _SH_DENYNO);
+    #endif
+    #if defined(SPDLOG_PREVENT_CHILD_FD)
+    if (*fp != nullptr) {
+        auto file_handle = reinterpret_cast<HANDLE>(_get_osfhandle(::_fileno(*fp)));
+        if (!::SetHandleInformation(file_handle, HANDLE_FLAG_INHERIT, 0)) {
+            ::fclose(*fp);
+            *fp = nullptr;
+        }
+    }
+    #endif
+#else  // unix
+    #if defined(SPDLOG_PREVENT_CHILD_FD)
+    const int mode_flag = mode == SPDLOG_FILENAME_T("ab") ? O_APPEND : O_TRUNC;
+    const int fd =
+        ::open((filename.c_str()), O_CREAT | O_WRONLY | O_CLOEXEC | mode_flag, mode_t(0644));
+    if (fd == -1) {
+        return true;
+    }
+    *fp = ::fdopen(fd, mode.c_str());
+    if (*fp == nullptr) {
+        ::close(fd);
+    }
+    #else
+    *fp = ::fopen((filename.c_str()), mode.c_str());
+    #endif
+#endif
+
+    return *fp == nullptr;
+}
+
+SPDLOG_INLINE int remove(const filename_t &filename) SPDLOG_NOEXCEPT {
+#if defined(_WIN32) && defined(SPDLOG_WCHAR_FILENAMES)
+    return ::_wremove(filename.c_str());
+#else
+    return std::remove(filename.c_str());
+#endif
+}
+
+SPDLOG_INLINE int remove_if_exists(const filename_t &filename) SPDLOG_NOEXCEPT {
+    return path_exists(filename) ? remove(filename) : 0;
+}
+
+SPDLOG_INLINE int rename(const filename_t &filename1, const filename_t &filename2) SPDLOG_NOEXCEPT {
+#if defined(_WIN32) && defined(SPDLOG_WCHAR_FILENAMES)
+    return ::_wrename(filename1.c_str(), filename2.c_str());
+#else
+    return std::rename(filename1.c_str(), filename2.c_str());
+#endif
+}
+
+// Return true if path exists (file or directory)
+SPDLOG_INLINE bool path_exists(const filename_t &filename) SPDLOG_NOEXCEPT {
+#ifdef _WIN32
+    struct _stat buffer;
+    #ifdef SPDLOG_WCHAR_FILENAMES
+    return (::_wstat(filename.c_str(), &buffer) == 0);
+    #else
+    return (::_stat(filename.c_str(), &buffer) == 0);
+    #endif
+#else  // common linux/unix all have the stat system call
+    struct stat buffer;
+    return (::stat(filename.c_str(), &buffer) == 0);
+#endif
+}
+
+#ifdef _MSC_VER
+    // avoid warning about unreachable statement at the end of filesize()
+    #pragma warning(push)
+    #pragma warning(disable : 4702)
+#endif
+
+// Return file size according to open FILE* object
+SPDLOG_INLINE size_t filesize(FILE *f) {
+    if (f == nullptr) {
+        throw_spdlog_ex("Failed getting file size. fd is null");
+    }
+#if defined(_WIN32) && !defined(__CYGWIN__)
+    int fd = ::_fileno(f);
+    #if defined(_WIN64)  // 64 bits
+    __int64 ret = ::_filelengthi64(fd);
+    if (ret >= 0) {
+        return static_cast<size_t>(ret);
+    }
+
+    #else  // windows 32 bits
+    long ret = ::_filelength(fd);
+    if (ret >= 0) {
+        return static_cast<size_t>(ret);
+    }
+    #endif
+
+#else  // unix
+    // OpenBSD and AIX doesn't compile with :: before the fileno(..)
+    #if defined(__OpenBSD__) || defined(_AIX)
+    int fd = fileno(f);
+    #else
+    int fd = ::fileno(f);
+    #endif
+    // 64 bits(but not in osx, linux/musl or cygwin, where fstat64 is deprecated)
+    #if ((defined(__linux__) && defined(__GLIBC__)) || defined(__sun) || defined(_AIX)) && \
+        (defined(__LP64__) || defined(_LP64))
+    struct stat64 st;
+    if (::fstat64(fd, &st) == 0) {
+        return static_cast<size_t>(st.st_size);
+    }
+    #else  // other unix or linux 32 bits or cygwin
+    struct stat st;
+    if (::fstat(fd, &st) == 0) {
+        return static_cast<size_t>(st.st_size);
+    }
+    #endif
+#endif
+    throw_spdlog_ex("Failed getting file size from fd", errno);
+    return 0;  // will not be reached.
+}
+
+#ifdef _MSC_VER
+    #pragma warning(pop)
+#endif
+
+// Return utc offset in minutes or throw spdlog_ex on failure
+SPDLOG_INLINE int utc_minutes_offset(const std::tm &tm) {
+#ifdef _WIN32
+    #if _WIN32_WINNT < _WIN32_WINNT_WS08
+    TIME_ZONE_INFORMATION tzinfo;
+    auto rv = ::GetTimeZoneInformation(&tzinfo);
+    #else
+    DYNAMIC_TIME_ZONE_INFORMATION tzinfo;
+    auto rv = ::GetDynamicTimeZoneInformation(&tzinfo);
+    #endif
+    if (rv == TIME_ZONE_ID_INVALID) throw_spdlog_ex("Failed getting timezone info. ", errno);
+
+    int offset = -tzinfo.Bias;
+    if (tm.tm_isdst) {
+        offset -= tzinfo.DaylightBias;
+    } else {
+        offset -= tzinfo.StandardBias;
+    }
+    return offset;
+#else
+
+    #if defined(sun) || defined(__sun) || defined(_AIX) || \
+        (defined(__NEWLIB__) && !defined(__TM_GMTOFF)) ||  \
+        (!defined(_BSD_SOURCE) && !defined(_GNU_SOURCE))
+    // 'tm_gmtoff' field is BSD extension and it's missing on SunOS/Solaris
+    struct helper {
+        static long int calculate_gmt_offset(const std::tm &localtm = details::os::localtime(),
+                                             const std::tm &gmtm = details::os::gmtime()) {
+            int local_year = localtm.tm_year + (1900 - 1);
+            int gmt_year = gmtm.tm_year + (1900 - 1);
+
+            long int days = (
+                // difference in day of year
+                localtm.tm_yday -
+                gmtm.tm_yday
+
+                // + intervening leap days
+                + ((local_year >> 2) - (gmt_year >> 2)) - (local_year / 100 - gmt_year / 100) +
+                ((local_year / 100 >> 2) - (gmt_year / 100 >> 2))
+
+                // + difference in years * 365 */
+                + static_cast<long int>(local_year - gmt_year) * 365);
+
+            long int hours = (24 * days) + (localtm.tm_hour - gmtm.tm_hour);
+            long int mins = (60 * hours) + (localtm.tm_min - gmtm.tm_min);
+            long int secs = (60 * mins) + (localtm.tm_sec - gmtm.tm_sec);
+
+            return secs;
+        }
+    };
+
+    auto offset_seconds = helper::calculate_gmt_offset(tm);
+    #else
+    auto offset_seconds = tm.tm_gmtoff;
+    #endif
+
+    return static_cast<int>(offset_seconds / 60);
+#endif
+}
+
+// Return current thread id as size_t
+// It exists because the std::this_thread::get_id() is much slower(especially
+// under VS 2013)
+SPDLOG_INLINE size_t _thread_id() SPDLOG_NOEXCEPT {
+#ifdef _WIN32
+    return static_cast<size_t>(::GetCurrentThreadId());
+#elif defined(__linux__)
+    #if defined(__ANDROID__) && defined(__ANDROID_API__) && (__ANDROID_API__ < 21)
+        #define SYS_gettid __NR_gettid
+    #endif
+    return static_cast<size_t>(::syscall(SYS_gettid));
+#elif defined(_AIX)
+    struct __pthrdsinfo buf;
+    int reg_size = 0;
+    pthread_t pt = pthread_self();
+    int retval = pthread_getthrds_np(&pt, PTHRDSINFO_QUERY_TID, &buf, sizeof(buf), NULL, &reg_size);
+    int tid = (!retval) ? buf.__pi_tid : 0;
+    return static_cast<size_t>(tid);
+#elif defined(__DragonFly__) || defined(__FreeBSD__)
+    return static_cast<size_t>(::pthread_getthreadid_np());
+#elif defined(__NetBSD__)
+    return static_cast<size_t>(::_lwp_self());
+#elif defined(__OpenBSD__)
+    return static_cast<size_t>(::getthrid());
+#elif defined(__sun)
+    return static_cast<size_t>(::thr_self());
+#elif __APPLE__
+    uint64_t tid;
+    // There is no pthread_threadid_np prior to Mac OS X 10.6, and it is not supported on any PPC,
+    // including 10.6.8 Rosetta. __POWERPC__ is Apple-specific define encompassing ppc and ppc64.
+    #ifdef MAC_OS_X_VERSION_MAX_ALLOWED
+    {
+        #if (MAC_OS_X_VERSION_MAX_ALLOWED < 1060) || defined(__POWERPC__)
+        tid = pthread_mach_thread_np(pthread_self());
+        #elif MAC_OS_X_VERSION_MIN_REQUIRED < 1060
+        if (&pthread_threadid_np) {
+            pthread_threadid_np(nullptr, &tid);
+        } else {
+            tid = pthread_mach_thread_np(pthread_self());
+        }
+        #else
+        pthread_threadid_np(nullptr, &tid);
+        #endif
+    }
+    #else
+    pthread_threadid_np(nullptr, &tid);
+    #endif
+    return static_cast<size_t>(tid);
+#else  // Default to standard C++11 (other Unix)
+    return static_cast<size_t>(std::hash<std::thread::id>()(std::this_thread::get_id()));
+#endif
+}
+
+// Return current thread id as size_t (from thread local storage)
+SPDLOG_INLINE size_t thread_id() SPDLOG_NOEXCEPT {
+#if defined(SPDLOG_NO_TLS)
+    return _thread_id();
+#else  // cache thread id in tls
+    static thread_local const size_t tid = _thread_id();
+    return tid;
+#endif
+}
+
+// This is avoid msvc issue in sleep_for that happens if the clock changes.
+// See https://github.com/gabime/spdlog/issues/609
+SPDLOG_INLINE void sleep_for_millis(unsigned int milliseconds) SPDLOG_NOEXCEPT {
+#if defined(_WIN32)
+    ::Sleep(milliseconds);
+#else
+    std::this_thread::sleep_for(std::chrono::milliseconds(milliseconds));
+#endif
+}
+
+// wchar support for windows file names (SPDLOG_WCHAR_FILENAMES must be defined)
+#if defined(_WIN32) && defined(SPDLOG_WCHAR_FILENAMES)
+SPDLOG_INLINE std::string filename_to_str(const filename_t &filename) {
+    memory_buf_t buf;
+    wstr_to_utf8buf(filename, buf);
+    return SPDLOG_BUF_TO_STRING(buf);
+}
+#else
+SPDLOG_INLINE std::string filename_to_str(const filename_t &filename) { return filename; }
+#endif
+
+SPDLOG_INLINE int pid() SPDLOG_NOEXCEPT {
+#ifdef _WIN32
+    return conditional_static_cast<int>(::GetCurrentProcessId());
+#else
+    return conditional_static_cast<int>(::getpid());
+#endif
+}
+
+// Determine if the terminal supports colors
+// Based on: https://github.com/agauniyal/rang/
+SPDLOG_INLINE bool is_color_terminal() SPDLOG_NOEXCEPT {
+#ifdef _WIN32
+    return true;
+#else
+
+    static const bool result = []() {
+        const char *env_colorterm_p = std::getenv("COLORTERM");
+        if (env_colorterm_p != nullptr) {
+            return true;
+        }
+
+        static constexpr std::array<const char *, 16> terms = {
+            {"ansi", "color", "console", "cygwin", "gnome", "konsole", "kterm", "linux", "msys",
+             "putty", "rxvt", "screen", "vt100", "xterm", "alacritty", "vt102"}};
+
+        const char *env_term_p = std::getenv("TERM");
+        if (env_term_p == nullptr) {
+            return false;
+        }
+
+        return std::any_of(terms.begin(), terms.end(), [&](const char *term) {
+            return std::strstr(env_term_p, term) != nullptr;
+        });
+    }();
+
+    return result;
+#endif
+}
+
+// Determine if the terminal attached
+// Source: https://github.com/agauniyal/rang/
+SPDLOG_INLINE bool in_terminal(FILE *file) SPDLOG_NOEXCEPT {
+#ifdef _WIN32
+    return ::_isatty(_fileno(file)) != 0;
+#else
+    return ::isatty(fileno(file)) != 0;
+#endif
+}
+
+#if (defined(SPDLOG_WCHAR_TO_UTF8_SUPPORT) || defined(SPDLOG_WCHAR_FILENAMES)) && defined(_WIN32)
+SPDLOG_INLINE void wstr_to_utf8buf(wstring_view_t wstr, memory_buf_t &target) {
+    if (wstr.size() > static_cast<size_t>((std::numeric_limits<int>::max)()) / 4 - 1) {
+        throw_spdlog_ex("UTF-16 string is too big to be converted to UTF-8");
+    }
+
+    int wstr_size = static_cast<int>(wstr.size());
+    if (wstr_size == 0) {
+        target.resize(0);
+        return;
+    }
+
+    int result_size = static_cast<int>(target.capacity());
+    if ((wstr_size + 1) * 4 > result_size) {
+        result_size =
+            ::WideCharToMultiByte(CP_UTF8, 0, wstr.data(), wstr_size, NULL, 0, NULL, NULL);
+    }
+
+    if (result_size > 0) {
+        target.resize(result_size);
+        result_size = ::WideCharToMultiByte(CP_UTF8, 0, wstr.data(), wstr_size, target.data(),
+                                            result_size, NULL, NULL);
+
+        if (result_size > 0) {
+            target.resize(result_size);
+            return;
+        }
+    }
+
+    throw_spdlog_ex(
+        fmt_lib::format("WideCharToMultiByte failed. Last error: {}", ::GetLastError()));
+}
+
+SPDLOG_INLINE void utf8_to_wstrbuf(string_view_t str, wmemory_buf_t &target) {
+    if (str.size() > static_cast<size_t>((std::numeric_limits<int>::max)()) - 1) {
+        throw_spdlog_ex("UTF-8 string is too big to be converted to UTF-16");
+    }
+
+    int str_size = static_cast<int>(str.size());
+    if (str_size == 0) {
+        target.resize(0);
+        return;
+    }
+
+    // find the size to allocate for the result buffer
+    int result_size =
+        ::MultiByteToWideChar(CP_UTF8, 0, str.data(), str_size, NULL, 0);
+
+    if (result_size > 0) {
+        target.resize(result_size);
+        result_size = ::MultiByteToWideChar(CP_UTF8, 0, str.data(), str_size, target.data(),
+                                            result_size);
+        if (result_size > 0) {
+            assert(result_size == target.size());
+            return;
+        }
+    }
+
+    throw_spdlog_ex(
+        fmt_lib::format("MultiByteToWideChar failed. Last error: {}", ::GetLastError()));
+}
+#endif  // (defined(SPDLOG_WCHAR_TO_UTF8_SUPPORT) || defined(SPDLOG_WCHAR_FILENAMES)) &&
+        // defined(_WIN32)
+
+// return true on success
+static SPDLOG_INLINE bool mkdir_(const filename_t &path) {
+#ifdef _WIN32
+    #ifdef SPDLOG_WCHAR_FILENAMES
+    return ::_wmkdir(path.c_str()) == 0;
+    #else
+    return ::_mkdir(path.c_str()) == 0;
+    #endif
+#else
+    return ::mkdir(path.c_str(), mode_t(0755)) == 0;
+#endif
+}
+
+// create the given directory - and all directories leading to it
+// return true on success or if the directory already exists
+SPDLOG_INLINE bool create_dir(const filename_t &path) {
+    if (path_exists(path)) {
+        return true;
+    }
+
+    if (path.empty()) {
+        return false;
+    }
+
+    size_t search_offset = 0;
+    do {
+        auto token_pos = path.find_first_of(folder_seps_filename, search_offset);
+        // treat the entire path as a folder if no folder separator not found
+        if (token_pos == filename_t::npos) {
+            token_pos = path.size();
+        }
+
+        auto subdir = path.substr(0, token_pos);
+#ifdef _WIN32
+        // if subdir is just a drive letter, add a slash e.g. "c:"=>"c:\",
+        // otherwise path_exists(subdir) returns false (issue #3079)
+        const bool is_drive = subdir.length() == 2 && subdir[1] == ':';
+        if (is_drive) {
+            subdir += '\\';
+            token_pos++;
+        }
+#endif
+
+        if (!subdir.empty() && !path_exists(subdir) && !mkdir_(subdir)) {
+            return false;  // return error if failed creating dir
+        }
+        search_offset = token_pos + 1;
+    } while (search_offset < path.size());
+
+    return true;
+}
+
+// Return directory name from given path or empty string
+// "abc/file" => "abc"
+// "abc/" => "abc"
+// "abc" => ""
+// "abc///" => "abc//"
+SPDLOG_INLINE filename_t dir_name(const filename_t &path) {
+    auto pos = path.find_last_of(folder_seps_filename);
+    return pos != filename_t::npos ? path.substr(0, pos) : filename_t{};
+}
+
+std::string SPDLOG_INLINE getenv(const char *field) {
+#if defined(_MSC_VER)
+    #if defined(__cplusplus_winrt)
+    return std::string{};  // not supported under uwp
+    #else
+    size_t len = 0;
+    char buf[128];
+    bool ok = ::getenv_s(&len, buf, sizeof(buf), field) == 0;
+    return ok ? buf : std::string{};
+    #endif
+#else  // revert to getenv
+    char *buf = ::getenv(field);
+    return buf ? buf : std::string{};
+#endif
+}
+
+// Do fsync by FILE handlerpointer
+// Return true on success
+SPDLOG_INLINE bool fsync(FILE *fp) {
+#ifdef _WIN32
+    return FlushFileBuffers(reinterpret_cast<HANDLE>(_get_osfhandle(_fileno(fp)))) != 0;
+#else
+    return ::fsync(fileno(fp)) == 0;
+#endif
+}
+
+// Do non-locking fwrite if possible by the os or use the regular locking fwrite
+// Return true on success.
+SPDLOG_INLINE bool fwrite_bytes(const void *ptr, const size_t n_bytes, FILE *fp) {
+    #if defined(_WIN32) && defined(SPDLOG_FWRITE_UNLOCKED)
+    return _fwrite_nolock(ptr, 1, n_bytes, fp) == n_bytes;
+    #elif defined(SPDLOG_FWRITE_UNLOCKED)
+    return ::fwrite_unlocked(ptr, 1, n_bytes, fp) == n_bytes;
+    #else
+    return std::fwrite(ptr, 1, n_bytes, fp) == n_bytes;
+    #endif
+}
+
+}  // namespace os
+}  // namespace details
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/details/os.h b/ext/spdlog/include/spdlog/details/os.h
new file mode 100644
index 0000000..5fd12ba
--- /dev/null
+++ b/ext/spdlog/include/spdlog/details/os.h
@@ -0,0 +1,127 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#include <ctime>  // std::time_t
+#include <spdlog/common.h>
+
+namespace spdlog {
+namespace details {
+namespace os {
+
+SPDLOG_API spdlog::log_clock::time_point now() SPDLOG_NOEXCEPT;
+
+SPDLOG_API std::tm localtime(const std::time_t &time_tt) SPDLOG_NOEXCEPT;
+
+SPDLOG_API std::tm localtime() SPDLOG_NOEXCEPT;
+
+SPDLOG_API std::tm gmtime(const std::time_t &time_tt) SPDLOG_NOEXCEPT;
+
+SPDLOG_API std::tm gmtime() SPDLOG_NOEXCEPT;
+
+// eol definition
+#if !defined(SPDLOG_EOL)
+    #ifdef _WIN32
+        #define SPDLOG_EOL "\r\n"
+    #else
+        #define SPDLOG_EOL "\n"
+    #endif
+#endif
+
+SPDLOG_CONSTEXPR static const char *default_eol = SPDLOG_EOL;
+
+// folder separator
+#if !defined(SPDLOG_FOLDER_SEPS)
+    #ifdef _WIN32
+        #define SPDLOG_FOLDER_SEPS "\\/"
+    #else
+        #define SPDLOG_FOLDER_SEPS "/"
+    #endif
+#endif
+
+SPDLOG_CONSTEXPR static const char folder_seps[] = SPDLOG_FOLDER_SEPS;
+SPDLOG_CONSTEXPR static const filename_t::value_type folder_seps_filename[] =
+    SPDLOG_FILENAME_T(SPDLOG_FOLDER_SEPS);
+
+// fopen_s on non windows for writing
+SPDLOG_API bool fopen_s(FILE **fp, const filename_t &filename, const filename_t &mode);
+
+// Remove filename. return 0 on success
+SPDLOG_API int remove(const filename_t &filename) SPDLOG_NOEXCEPT;
+
+// Remove file if exists. return 0 on success
+// Note: Non atomic (might return failure to delete if concurrently deleted by other process/thread)
+SPDLOG_API int remove_if_exists(const filename_t &filename) SPDLOG_NOEXCEPT;
+
+SPDLOG_API int rename(const filename_t &filename1, const filename_t &filename2) SPDLOG_NOEXCEPT;
+
+// Return if file exists.
+SPDLOG_API bool path_exists(const filename_t &filename) SPDLOG_NOEXCEPT;
+
+// Return file size according to open FILE* object
+SPDLOG_API size_t filesize(FILE *f);
+
+// Return utc offset in minutes or throw spdlog_ex on failure
+SPDLOG_API int utc_minutes_offset(const std::tm &tm = details::os::localtime());
+
+// Return current thread id as size_t
+// It exists because the std::this_thread::get_id() is much slower(especially
+// under VS 2013)
+SPDLOG_API size_t _thread_id() SPDLOG_NOEXCEPT;
+
+// Return current thread id as size_t (from thread local storage)
+SPDLOG_API size_t thread_id() SPDLOG_NOEXCEPT;
+
+// This is avoid msvc issue in sleep_for that happens if the clock changes.
+// See https://github.com/gabime/spdlog/issues/609
+SPDLOG_API void sleep_for_millis(unsigned int milliseconds) SPDLOG_NOEXCEPT;
+
+SPDLOG_API std::string filename_to_str(const filename_t &filename);
+
+SPDLOG_API int pid() SPDLOG_NOEXCEPT;
+
+// Determine if the terminal supports colors
+// Source: https://github.com/agauniyal/rang/
+SPDLOG_API bool is_color_terminal() SPDLOG_NOEXCEPT;
+
+// Determine if the terminal attached
+// Source: https://github.com/agauniyal/rang/
+SPDLOG_API bool in_terminal(FILE *file) SPDLOG_NOEXCEPT;
+
+#if (defined(SPDLOG_WCHAR_TO_UTF8_SUPPORT) || defined(SPDLOG_WCHAR_FILENAMES)) && defined(_WIN32)
+SPDLOG_API void wstr_to_utf8buf(wstring_view_t wstr, memory_buf_t &target);
+
+SPDLOG_API void utf8_to_wstrbuf(string_view_t str, wmemory_buf_t &target);
+#endif
+
+// Return directory name from given path or empty string
+// "abc/file" => "abc"
+// "abc/" => "abc"
+// "abc" => ""
+// "abc///" => "abc//"
+SPDLOG_API filename_t dir_name(const filename_t &path);
+
+// Create a dir from the given path.
+// Return true if succeeded or if this dir already exists.
+SPDLOG_API bool create_dir(const filename_t &path);
+
+// non thread safe, cross platform getenv/getenv_s
+// return empty string if field not found
+SPDLOG_API std::string getenv(const char *field);
+
+// Do fsync by FILE objectpointer.
+// Return true on success.
+SPDLOG_API bool fsync(FILE *fp);
+
+// Do non-locking fwrite if possible by the os or use the regular locking fwrite
+// Return true on success.
+SPDLOG_API bool fwrite_bytes(const void *ptr, const size_t n_bytes, FILE *fp);
+
+}  // namespace os
+}  // namespace details
+}  // namespace spdlog
+
+#ifdef SPDLOG_HEADER_ONLY
+    #include "os-inl.h"
+#endif
diff --git a/ext/spdlog/include/spdlog/details/periodic_worker-inl.h b/ext/spdlog/include/spdlog/details/periodic_worker-inl.h
new file mode 100644
index 0000000..18f11fb
--- /dev/null
+++ b/ext/spdlog/include/spdlog/details/periodic_worker-inl.h
@@ -0,0 +1,26 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#ifndef SPDLOG_HEADER_ONLY
+    #include <spdlog/details/periodic_worker.h>
+#endif
+
+namespace spdlog {
+namespace details {
+
+// stop the worker thread and join it
+SPDLOG_INLINE periodic_worker::~periodic_worker() {
+    if (worker_thread_.joinable()) {
+        {
+            std::lock_guard<std::mutex> lock(mutex_);
+            active_ = false;
+        }
+        cv_.notify_one();
+        worker_thread_.join();
+    }
+}
+
+}  // namespace details
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/details/periodic_worker.h b/ext/spdlog/include/spdlog/details/periodic_worker.h
new file mode 100644
index 0000000..d647b66
--- /dev/null
+++ b/ext/spdlog/include/spdlog/details/periodic_worker.h
@@ -0,0 +1,58 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+// periodic worker thread - periodically executes the given callback function.
+//
+// RAII over the owned thread:
+//    creates the thread on construction.
+//    stops and joins the thread on destruction (if the thread is executing a callback, wait for it
+//    to finish first).
+
+#include <chrono>
+#include <condition_variable>
+#include <functional>
+#include <mutex>
+#include <thread>
+namespace spdlog {
+namespace details {
+
+class SPDLOG_API periodic_worker {
+public:
+    template <typename Rep, typename Period>
+    periodic_worker(const std::function<void()> &callback_fun,
+                    std::chrono::duration<Rep, Period> interval) {
+        active_ = (interval > std::chrono::duration<Rep, Period>::zero());
+        if (!active_) {
+            return;
+        }
+
+        worker_thread_ = std::thread([this, callback_fun, interval]() {
+            for (;;) {
+                std::unique_lock<std::mutex> lock(this->mutex_);
+                if (this->cv_.wait_for(lock, interval, [this] { return !this->active_; })) {
+                    return;  // active_ == false, so exit this thread
+                }
+                callback_fun();
+            }
+        });
+    }
+    std::thread &get_thread() { return worker_thread_; }
+    periodic_worker(const periodic_worker &) = delete;
+    periodic_worker &operator=(const periodic_worker &) = delete;
+    // stop the worker thread and join it
+    ~periodic_worker();
+
+private:
+    bool active_;
+    std::thread worker_thread_;
+    std::mutex mutex_;
+    std::condition_variable cv_;
+};
+}  // namespace details
+}  // namespace spdlog
+
+#ifdef SPDLOG_HEADER_ONLY
+    #include "periodic_worker-inl.h"
+#endif
diff --git a/ext/spdlog/include/spdlog/details/registry-inl.h b/ext/spdlog/include/spdlog/details/registry-inl.h
new file mode 100644
index 0000000..f447848
--- /dev/null
+++ b/ext/spdlog/include/spdlog/details/registry-inl.h
@@ -0,0 +1,261 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#ifndef SPDLOG_HEADER_ONLY
+    #include <spdlog/details/registry.h>
+#endif
+
+#include <spdlog/common.h>
+#include <spdlog/details/periodic_worker.h>
+#include <spdlog/logger.h>
+#include <spdlog/pattern_formatter.h>
+
+#ifndef SPDLOG_DISABLE_DEFAULT_LOGGER
+    // support for the default stdout color logger
+    #ifdef _WIN32
+        #include <spdlog/sinks/wincolor_sink.h>
+    #else
+        #include <spdlog/sinks/ansicolor_sink.h>
+    #endif
+#endif  // SPDLOG_DISABLE_DEFAULT_LOGGER
+
+#include <chrono>
+#include <functional>
+#include <memory>
+#include <string>
+#include <unordered_map>
+
+namespace spdlog {
+namespace details {
+
+SPDLOG_INLINE registry::registry()
+    : formatter_(new pattern_formatter()) {
+#ifndef SPDLOG_DISABLE_DEFAULT_LOGGER
+    // create default logger (ansicolor_stdout_sink_mt or wincolor_stdout_sink_mt in windows).
+    #ifdef _WIN32
+    auto color_sink = std::make_shared<sinks::wincolor_stdout_sink_mt>();
+    #else
+    auto color_sink = std::make_shared<sinks::ansicolor_stdout_sink_mt>();
+    #endif
+
+    const char *default_logger_name = "";
+    default_logger_ = std::make_shared<spdlog::logger>(default_logger_name, std::move(color_sink));
+    loggers_[default_logger_name] = default_logger_;
+
+#endif  // SPDLOG_DISABLE_DEFAULT_LOGGER
+}
+
+SPDLOG_INLINE registry::~registry() = default;
+
+SPDLOG_INLINE void registry::register_logger(std::shared_ptr<logger> new_logger) {
+    std::lock_guard<std::mutex> lock(logger_map_mutex_);
+    register_logger_(std::move(new_logger));
+}
+
+SPDLOG_INLINE void registry::initialize_logger(std::shared_ptr<logger> new_logger) {
+    std::lock_guard<std::mutex> lock(logger_map_mutex_);
+    new_logger->set_formatter(formatter_->clone());
+
+    if (err_handler_) {
+        new_logger->set_error_handler(err_handler_);
+    }
+
+    // set new level according to previously configured level or default level
+    auto it = log_levels_.find(new_logger->name());
+    auto new_level = it != log_levels_.end() ? it->second : global_log_level_;
+    new_logger->set_level(new_level);
+
+    new_logger->flush_on(flush_level_);
+
+    if (backtrace_n_messages_ > 0) {
+        new_logger->enable_backtrace(backtrace_n_messages_);
+    }
+
+    if (automatic_registration_) {
+        register_logger_(std::move(new_logger));
+    }
+}
+
+SPDLOG_INLINE std::shared_ptr<logger> registry::get(const std::string &logger_name) {
+    std::lock_guard<std::mutex> lock(logger_map_mutex_);
+    auto found = loggers_.find(logger_name);
+    return found == loggers_.end() ? nullptr : found->second;
+}
+
+SPDLOG_INLINE std::shared_ptr<logger> registry::default_logger() {
+    std::lock_guard<std::mutex> lock(logger_map_mutex_);
+    return default_logger_;
+}
+
+// Return raw ptr to the default logger.
+// To be used directly by the spdlog default api (e.g. spdlog::info)
+// This make the default API faster, but cannot be used concurrently with set_default_logger().
+// e.g do not call set_default_logger() from one thread while calling spdlog::info() from another.
+SPDLOG_INLINE logger *registry::get_default_raw() { return default_logger_.get(); }
+
+// set default logger.
+// default logger is stored in default_logger_ (for faster retrieval) and in the loggers_ map.
+SPDLOG_INLINE void registry::set_default_logger(std::shared_ptr<logger> new_default_logger) {
+    std::lock_guard<std::mutex> lock(logger_map_mutex_);
+    if (new_default_logger != nullptr) {
+        loggers_[new_default_logger->name()] = new_default_logger;
+    }
+    default_logger_ = std::move(new_default_logger);
+}
+
+SPDLOG_INLINE void registry::set_tp(std::shared_ptr<thread_pool> tp) {
+    std::lock_guard<std::recursive_mutex> lock(tp_mutex_);
+    tp_ = std::move(tp);
+}
+
+SPDLOG_INLINE std::shared_ptr<thread_pool> registry::get_tp() {
+    std::lock_guard<std::recursive_mutex> lock(tp_mutex_);
+    return tp_;
+}
+
+// Set global formatter. Each sink in each logger will get a clone of this object
+SPDLOG_INLINE void registry::set_formatter(std::unique_ptr<formatter> formatter) {
+    std::lock_guard<std::mutex> lock(logger_map_mutex_);
+    formatter_ = std::move(formatter);
+    for (auto &l : loggers_) {
+        l.second->set_formatter(formatter_->clone());
+    }
+}
+
+SPDLOG_INLINE void registry::enable_backtrace(size_t n_messages) {
+    std::lock_guard<std::mutex> lock(logger_map_mutex_);
+    backtrace_n_messages_ = n_messages;
+
+    for (auto &l : loggers_) {
+        l.second->enable_backtrace(n_messages);
+    }
+}
+
+SPDLOG_INLINE void registry::disable_backtrace() {
+    std::lock_guard<std::mutex> lock(logger_map_mutex_);
+    backtrace_n_messages_ = 0;
+    for (auto &l : loggers_) {
+        l.second->disable_backtrace();
+    }
+}
+
+SPDLOG_INLINE void registry::set_level(level::level_enum log_level) {
+    std::lock_guard<std::mutex> lock(logger_map_mutex_);
+    for (auto &l : loggers_) {
+        l.second->set_level(log_level);
+    }
+    global_log_level_ = log_level;
+}
+
+SPDLOG_INLINE void registry::flush_on(level::level_enum log_level) {
+    std::lock_guard<std::mutex> lock(logger_map_mutex_);
+    for (auto &l : loggers_) {
+        l.second->flush_on(log_level);
+    }
+    flush_level_ = log_level;
+}
+
+SPDLOG_INLINE void registry::set_error_handler(err_handler handler) {
+    std::lock_guard<std::mutex> lock(logger_map_mutex_);
+    for (auto &l : loggers_) {
+        l.second->set_error_handler(handler);
+    }
+    err_handler_ = std::move(handler);
+}
+
+SPDLOG_INLINE void registry::apply_all(
+    const std::function<void(const std::shared_ptr<logger>)> &fun) {
+    std::lock_guard<std::mutex> lock(logger_map_mutex_);
+    for (auto &l : loggers_) {
+        fun(l.second);
+    }
+}
+
+SPDLOG_INLINE void registry::flush_all() {
+    std::lock_guard<std::mutex> lock(logger_map_mutex_);
+    for (auto &l : loggers_) {
+        l.second->flush();
+    }
+}
+
+SPDLOG_INLINE void registry::drop(const std::string &logger_name) {
+    std::lock_guard<std::mutex> lock(logger_map_mutex_);
+    auto is_default_logger = default_logger_ && default_logger_->name() == logger_name;
+    loggers_.erase(logger_name);
+    if (is_default_logger) {
+        default_logger_.reset();
+    }
+}
+
+SPDLOG_INLINE void registry::drop_all() {
+    std::lock_guard<std::mutex> lock(logger_map_mutex_);
+    loggers_.clear();
+    default_logger_.reset();
+}
+
+// clean all resources and threads started by the registry
+SPDLOG_INLINE void registry::shutdown() {
+    {
+        std::lock_guard<std::mutex> lock(flusher_mutex_);
+        periodic_flusher_.reset();
+    }
+
+    drop_all();
+
+    {
+        std::lock_guard<std::recursive_mutex> lock(tp_mutex_);
+        tp_.reset();
+    }
+}
+
+SPDLOG_INLINE std::recursive_mutex &registry::tp_mutex() { return tp_mutex_; }
+
+SPDLOG_INLINE void registry::set_automatic_registration(bool automatic_registration) {
+    std::lock_guard<std::mutex> lock(logger_map_mutex_);
+    automatic_registration_ = automatic_registration;
+}
+
+SPDLOG_INLINE void registry::set_levels(log_levels levels, level::level_enum *global_level) {
+    std::lock_guard<std::mutex> lock(logger_map_mutex_);
+    log_levels_ = std::move(levels);
+    auto global_level_requested = global_level != nullptr;
+    global_log_level_ = global_level_requested ? *global_level : global_log_level_;
+
+    for (auto &logger : loggers_) {
+        auto logger_entry = log_levels_.find(logger.first);
+        if (logger_entry != log_levels_.end()) {
+            logger.second->set_level(logger_entry->second);
+        } else if (global_level_requested) {
+            logger.second->set_level(*global_level);
+        }
+    }
+}
+
+SPDLOG_INLINE registry &registry::instance() {
+    static registry s_instance;
+    return s_instance;
+}
+
+SPDLOG_INLINE void registry::apply_logger_env_levels(std::shared_ptr<logger> new_logger) {
+    std::lock_guard<std::mutex> lock(logger_map_mutex_);
+    auto it = log_levels_.find(new_logger->name());
+    auto new_level = it != log_levels_.end() ? it->second : global_log_level_;
+    new_logger->set_level(new_level);
+}
+
+SPDLOG_INLINE void registry::throw_if_exists_(const std::string &logger_name) {
+    if (loggers_.find(logger_name) != loggers_.end()) {
+        throw_spdlog_ex("logger with name '" + logger_name + "' already exists");
+    }
+}
+
+SPDLOG_INLINE void registry::register_logger_(std::shared_ptr<logger> new_logger) {
+    auto logger_name = new_logger->name();
+    throw_if_exists_(logger_name);
+    loggers_[logger_name] = std::move(new_logger);
+}
+
+}  // namespace details
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/details/registry.h b/ext/spdlog/include/spdlog/details/registry.h
new file mode 100644
index 0000000..8afcbd6
--- /dev/null
+++ b/ext/spdlog/include/spdlog/details/registry.h
@@ -0,0 +1,129 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+// Loggers registry of unique name->logger pointer
+// An attempt to create a logger with an already existing name will result with spdlog_ex exception.
+// If user requests a non existing logger, nullptr will be returned
+// This class is thread safe
+
+#include <spdlog/common.h>
+#include <spdlog/details/periodic_worker.h>
+
+#include <chrono>
+#include <functional>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <unordered_map>
+
+namespace spdlog {
+class logger;
+
+namespace details {
+class thread_pool;
+
+class SPDLOG_API registry {
+public:
+    using log_levels = std::unordered_map<std::string, level::level_enum>;
+    registry(const registry &) = delete;
+    registry &operator=(const registry &) = delete;
+
+    void register_logger(std::shared_ptr<logger> new_logger);
+    void initialize_logger(std::shared_ptr<logger> new_logger);
+    std::shared_ptr<logger> get(const std::string &logger_name);
+    std::shared_ptr<logger> default_logger();
+
+    // Return raw ptr to the default logger.
+    // To be used directly by the spdlog default api (e.g. spdlog::info)
+    // This make the default API faster, but cannot be used concurrently with set_default_logger().
+    // e.g do not call set_default_logger() from one thread while calling spdlog::info() from
+    // another.
+    logger *get_default_raw();
+
+    // set default logger and add it to the registry if not registered already.
+    // default logger is stored in default_logger_ (for faster retrieval) and in the loggers_ map.
+    // Note: Make sure to unregister it when no longer needed or before calling again with a new
+    // logger.
+    void set_default_logger(std::shared_ptr<logger> new_default_logger);
+
+    void set_tp(std::shared_ptr<thread_pool> tp);
+
+    std::shared_ptr<thread_pool> get_tp();
+
+    // Set global formatter. Each sink in each logger will get a clone of this object
+    void set_formatter(std::unique_ptr<formatter> formatter);
+
+    void enable_backtrace(size_t n_messages);
+
+    void disable_backtrace();
+
+    void set_level(level::level_enum log_level);
+
+    void flush_on(level::level_enum log_level);
+
+    template <typename Rep, typename Period>
+    void flush_every(std::chrono::duration<Rep, Period> interval) {
+        std::lock_guard<std::mutex> lock(flusher_mutex_);
+        auto clbk = [this]() { this->flush_all(); };
+        periodic_flusher_ = details::make_unique<periodic_worker>(clbk, interval);
+    }
+
+    std::unique_ptr<periodic_worker> &get_flusher() {
+        std::lock_guard<std::mutex> lock(flusher_mutex_);
+        return periodic_flusher_;
+    }
+
+    void set_error_handler(err_handler handler);
+
+    void apply_all(const std::function<void(const std::shared_ptr<logger>)> &fun);
+
+    void flush_all();
+
+    void drop(const std::string &logger_name);
+
+    void drop_all();
+
+    // clean all resources and threads started by the registry
+    void shutdown();
+
+    std::recursive_mutex &tp_mutex();
+
+    void set_automatic_registration(bool automatic_registration);
+
+    // set levels for all existing/future loggers. global_level can be null if should not set.
+    void set_levels(log_levels levels, level::level_enum *global_level);
+
+    static registry &instance();
+
+    void apply_logger_env_levels(std::shared_ptr<logger> new_logger);
+
+private:
+    registry();
+    ~registry();
+
+    void throw_if_exists_(const std::string &logger_name);
+    void register_logger_(std::shared_ptr<logger> new_logger);
+    bool set_level_from_cfg_(logger *logger);
+    std::mutex logger_map_mutex_, flusher_mutex_;
+    std::recursive_mutex tp_mutex_;
+    std::unordered_map<std::string, std::shared_ptr<logger>> loggers_;
+    log_levels log_levels_;
+    std::unique_ptr<formatter> formatter_;
+    spdlog::level::level_enum global_log_level_ = level::info;
+    level::level_enum flush_level_ = level::off;
+    err_handler err_handler_;
+    std::shared_ptr<thread_pool> tp_;
+    std::unique_ptr<periodic_worker> periodic_flusher_;
+    std::shared_ptr<logger> default_logger_;
+    bool automatic_registration_ = true;
+    size_t backtrace_n_messages_ = 0;
+};
+
+}  // namespace details
+}  // namespace spdlog
+
+#ifdef SPDLOG_HEADER_ONLY
+    #include "registry-inl.h"
+#endif
diff --git a/ext/spdlog/include/spdlog/details/synchronous_factory.h b/ext/spdlog/include/spdlog/details/synchronous_factory.h
new file mode 100644
index 0000000..4bd5a51
--- /dev/null
+++ b/ext/spdlog/include/spdlog/details/synchronous_factory.h
@@ -0,0 +1,22 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#include "registry.h"
+
+namespace spdlog {
+
+// Default logger factory-  creates synchronous loggers
+class logger;
+
+struct synchronous_factory {
+    template <typename Sink, typename... SinkArgs>
+    static std::shared_ptr<spdlog::logger> create(std::string logger_name, SinkArgs &&...args) {
+        auto sink = std::make_shared<Sink>(std::forward<SinkArgs>(args)...);
+        auto new_logger = std::make_shared<spdlog::logger>(std::move(logger_name), std::move(sink));
+        details::registry::instance().initialize_logger(new_logger);
+        return new_logger;
+    }
+};
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/details/tcp_client-windows.h b/ext/spdlog/include/spdlog/details/tcp_client-windows.h
new file mode 100644
index 0000000..bf8f7b8
--- /dev/null
+++ b/ext/spdlog/include/spdlog/details/tcp_client-windows.h
@@ -0,0 +1,135 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#define WIN32_LEAN_AND_MEAN
+// tcp client helper
+#include <spdlog/common.h>
+#include <spdlog/details/os.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string>
+#include <windows.h>
+#include <winsock2.h>
+#include <ws2tcpip.h>
+
+#pragma comment(lib, "Ws2_32.lib")
+#pragma comment(lib, "Mswsock.lib")
+#pragma comment(lib, "AdvApi32.lib")
+
+namespace spdlog {
+namespace details {
+class tcp_client {
+    SOCKET socket_ = INVALID_SOCKET;
+
+    static void init_winsock_() {
+        WSADATA wsaData;
+        auto rv = WSAStartup(MAKEWORD(2, 2), &wsaData);
+        if (rv != 0) {
+            throw_winsock_error_("WSAStartup failed", ::WSAGetLastError());
+        }
+    }
+
+    static void throw_winsock_error_(const std::string &msg, int last_error) {
+        char buf[512];
+        ::FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL,
+                         last_error, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), buf,
+                         (sizeof(buf) / sizeof(char)), NULL);
+
+        throw_spdlog_ex(fmt_lib::format("tcp_sink - {}: {}", msg, buf));
+    }
+
+public:
+    tcp_client() { init_winsock_(); }
+
+    ~tcp_client() {
+        close();
+        ::WSACleanup();
+    }
+
+    bool is_connected() const { return socket_ != INVALID_SOCKET; }
+
+    void close() {
+        ::closesocket(socket_);
+        socket_ = INVALID_SOCKET;
+    }
+
+    SOCKET fd() const { return socket_; }
+
+    // try to connect or throw on failure
+    void connect(const std::string &host, int port) {
+        if (is_connected()) {
+            close();
+        }
+        struct addrinfo hints {};
+        ZeroMemory(&hints, sizeof(hints));
+
+        hints.ai_family = AF_UNSPEC;      // To work with IPv4, IPv6, and so on
+        hints.ai_socktype = SOCK_STREAM;  // TCP
+        hints.ai_flags = AI_NUMERICSERV;  // port passed as as numeric value
+        hints.ai_protocol = 0;
+
+        auto port_str = std::to_string(port);
+        struct addrinfo *addrinfo_result;
+        auto rv = ::getaddrinfo(host.c_str(), port_str.c_str(), &hints, &addrinfo_result);
+        int last_error = 0;
+        if (rv != 0) {
+            last_error = ::WSAGetLastError();
+            WSACleanup();
+            throw_winsock_error_("getaddrinfo failed", last_error);
+        }
+
+        // Try each address until we successfully connect(2).
+
+        for (auto *rp = addrinfo_result; rp != nullptr; rp = rp->ai_next) {
+            socket_ = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol);
+            if (socket_ == INVALID_SOCKET) {
+                last_error = ::WSAGetLastError();
+                WSACleanup();
+                continue;
+            }
+            if (::connect(socket_, rp->ai_addr, (int)rp->ai_addrlen) == 0) {
+                break;
+            } else {
+                last_error = ::WSAGetLastError();
+                close();
+            }
+        }
+        ::freeaddrinfo(addrinfo_result);
+        if (socket_ == INVALID_SOCKET) {
+            WSACleanup();
+            throw_winsock_error_("connect failed", last_error);
+        }
+
+        // set TCP_NODELAY
+        int enable_flag = 1;
+        ::setsockopt(socket_, IPPROTO_TCP, TCP_NODELAY, reinterpret_cast<char *>(&enable_flag),
+                     sizeof(enable_flag));
+    }
+
+    // Send exactly n_bytes of the given data.
+    // On error close the connection and throw.
+    void send(const char *data, size_t n_bytes) {
+        size_t bytes_sent = 0;
+        while (bytes_sent < n_bytes) {
+            const int send_flags = 0;
+            auto write_result =
+                ::send(socket_, data + bytes_sent, (int)(n_bytes - bytes_sent), send_flags);
+            if (write_result == SOCKET_ERROR) {
+                int last_error = ::WSAGetLastError();
+                close();
+                throw_winsock_error_("send failed", last_error);
+            }
+
+            if (write_result == 0)  // (probably should not happen but in any case..)
+            {
+                break;
+            }
+            bytes_sent += static_cast<size_t>(write_result);
+        }
+    }
+};
+}  // namespace details
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/details/tcp_client.h b/ext/spdlog/include/spdlog/details/tcp_client.h
new file mode 100644
index 0000000..9d3c40d
--- /dev/null
+++ b/ext/spdlog/include/spdlog/details/tcp_client.h
@@ -0,0 +1,127 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#ifdef _WIN32
+    #error include tcp_client-windows.h instead
+#endif
+
+// tcp client helper
+#include <spdlog/common.h>
+#include <spdlog/details/os.h>
+
+#include <arpa/inet.h>
+#include <netdb.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+#include <sys/socket.h>
+#include <unistd.h>
+
+#include <string>
+
+namespace spdlog {
+namespace details {
+class tcp_client {
+    int socket_ = -1;
+
+public:
+    bool is_connected() const { return socket_ != -1; }
+
+    void close() {
+        if (is_connected()) {
+            ::close(socket_);
+            socket_ = -1;
+        }
+    }
+
+    int fd() const { return socket_; }
+
+    ~tcp_client() { close(); }
+
+    // try to connect or throw on failure
+    void connect(const std::string &host, int port) {
+        close();
+        struct addrinfo hints {};
+        memset(&hints, 0, sizeof(struct addrinfo));
+        hints.ai_family = AF_UNSPEC;      // To work with IPv4, IPv6, and so on
+        hints.ai_socktype = SOCK_STREAM;  // TCP
+        hints.ai_flags = AI_NUMERICSERV;  // port passed as as numeric value
+        hints.ai_protocol = 0;
+
+        auto port_str = std::to_string(port);
+        struct addrinfo *addrinfo_result;
+        auto rv = ::getaddrinfo(host.c_str(), port_str.c_str(), &hints, &addrinfo_result);
+        if (rv != 0) {
+            throw_spdlog_ex(fmt_lib::format("::getaddrinfo failed: {}", gai_strerror(rv)));
+        }
+
+        // Try each address until we successfully connect(2).
+        int last_errno = 0;
+        for (auto *rp = addrinfo_result; rp != nullptr; rp = rp->ai_next) {
+#if defined(SOCK_CLOEXEC)
+            const int flags = SOCK_CLOEXEC;
+#else
+            const int flags = 0;
+#endif
+            socket_ = ::socket(rp->ai_family, rp->ai_socktype | flags, rp->ai_protocol);
+            if (socket_ == -1) {
+                last_errno = errno;
+                continue;
+            }
+            rv = ::connect(socket_, rp->ai_addr, rp->ai_addrlen);
+            if (rv == 0) {
+                break;
+            }
+            last_errno = errno;
+            ::close(socket_);
+            socket_ = -1;
+        }
+        ::freeaddrinfo(addrinfo_result);
+        if (socket_ == -1) {
+            throw_spdlog_ex("::connect failed", last_errno);
+        }
+
+        // set TCP_NODELAY
+        int enable_flag = 1;
+        ::setsockopt(socket_, IPPROTO_TCP, TCP_NODELAY, reinterpret_cast<char *>(&enable_flag),
+                     sizeof(enable_flag));
+
+        // prevent sigpipe on systems where MSG_NOSIGNAL is not available
+#if defined(SO_NOSIGPIPE) && !defined(MSG_NOSIGNAL)
+        ::setsockopt(socket_, SOL_SOCKET, SO_NOSIGPIPE, reinterpret_cast<char *>(&enable_flag),
+                     sizeof(enable_flag));
+#endif
+
+#if !defined(SO_NOSIGPIPE) && !defined(MSG_NOSIGNAL)
+    #error "tcp_sink would raise SIGPIPE since neither SO_NOSIGPIPE nor MSG_NOSIGNAL are available"
+#endif
+    }
+
+    // Send exactly n_bytes of the given data.
+    // On error close the connection and throw.
+    void send(const char *data, size_t n_bytes) {
+        size_t bytes_sent = 0;
+        while (bytes_sent < n_bytes) {
+#if defined(MSG_NOSIGNAL)
+            const int send_flags = MSG_NOSIGNAL;
+#else
+            const int send_flags = 0;
+#endif
+            auto write_result =
+                ::send(socket_, data + bytes_sent, n_bytes - bytes_sent, send_flags);
+            if (write_result < 0) {
+                close();
+                throw_spdlog_ex("write(2) failed", errno);
+            }
+
+            if (write_result == 0)  // (probably should not happen but in any case..)
+            {
+                break;
+            }
+            bytes_sent += static_cast<size_t>(write_result);
+        }
+    }
+};
+}  // namespace details
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/details/thread_pool-inl.h b/ext/spdlog/include/spdlog/details/thread_pool-inl.h
new file mode 100644
index 0000000..17e01c0
--- /dev/null
+++ b/ext/spdlog/include/spdlog/details/thread_pool-inl.h
@@ -0,0 +1,127 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#ifndef SPDLOG_HEADER_ONLY
+    #include <spdlog/details/thread_pool.h>
+#endif
+
+#include <cassert>
+#include <spdlog/common.h>
+
+namespace spdlog {
+namespace details {
+
+SPDLOG_INLINE thread_pool::thread_pool(size_t q_max_items,
+                                       size_t threads_n,
+                                       std::function<void()> on_thread_start,
+                                       std::function<void()> on_thread_stop)
+    : q_(q_max_items) {
+    if (threads_n == 0 || threads_n > 1000) {
+        throw_spdlog_ex(
+            "spdlog::thread_pool(): invalid threads_n param (valid "
+            "range is 1-1000)");
+    }
+    for (size_t i = 0; i < threads_n; i++) {
+        threads_.emplace_back([this, on_thread_start, on_thread_stop] {
+            on_thread_start();
+            this->thread_pool::worker_loop_();
+            on_thread_stop();
+        });
+    }
+}
+
+SPDLOG_INLINE thread_pool::thread_pool(size_t q_max_items,
+                                       size_t threads_n,
+                                       std::function<void()> on_thread_start)
+    : thread_pool(q_max_items, threads_n, on_thread_start, [] {}) {}
+
+SPDLOG_INLINE thread_pool::thread_pool(size_t q_max_items, size_t threads_n)
+    : thread_pool(
+          q_max_items, threads_n, [] {}, [] {}) {}
+
+// message all threads to terminate gracefully join them
+SPDLOG_INLINE thread_pool::~thread_pool() {
+    SPDLOG_TRY {
+        for (size_t i = 0; i < threads_.size(); i++) {
+            post_async_msg_(async_msg(async_msg_type::terminate), async_overflow_policy::block);
+        }
+
+        for (auto &t : threads_) {
+            t.join();
+        }
+    }
+    SPDLOG_CATCH_STD
+}
+
+void SPDLOG_INLINE thread_pool::post_log(async_logger_ptr &&worker_ptr,
+                                         const details::log_msg &msg,
+                                         async_overflow_policy overflow_policy) {
+    async_msg async_m(std::move(worker_ptr), async_msg_type::log, msg);
+    post_async_msg_(std::move(async_m), overflow_policy);
+}
+
+void SPDLOG_INLINE thread_pool::post_flush(async_logger_ptr &&worker_ptr,
+                                           async_overflow_policy overflow_policy) {
+    post_async_msg_(async_msg(std::move(worker_ptr), async_msg_type::flush), overflow_policy);
+}
+
+size_t SPDLOG_INLINE thread_pool::overrun_counter() { return q_.overrun_counter(); }
+
+void SPDLOG_INLINE thread_pool::reset_overrun_counter() { q_.reset_overrun_counter(); }
+
+size_t SPDLOG_INLINE thread_pool::discard_counter() { return q_.discard_counter(); }
+
+void SPDLOG_INLINE thread_pool::reset_discard_counter() { q_.reset_discard_counter(); }
+
+size_t SPDLOG_INLINE thread_pool::queue_size() { return q_.size(); }
+
+void SPDLOG_INLINE thread_pool::post_async_msg_(async_msg &&new_msg,
+                                                async_overflow_policy overflow_policy) {
+    if (overflow_policy == async_overflow_policy::block) {
+        q_.enqueue(std::move(new_msg));
+    } else if (overflow_policy == async_overflow_policy::overrun_oldest) {
+        q_.enqueue_nowait(std::move(new_msg));
+    } else {
+        assert(overflow_policy == async_overflow_policy::discard_new);
+        q_.enqueue_if_have_room(std::move(new_msg));
+    }
+}
+
+void SPDLOG_INLINE thread_pool::worker_loop_() {
+    while (process_next_msg_()) {
+    }
+}
+
+// process next message in the queue
+// return true if this thread should still be active (while no terminate msg
+// was received)
+bool SPDLOG_INLINE thread_pool::process_next_msg_() {
+    async_msg incoming_async_msg;
+    q_.dequeue(incoming_async_msg);
+
+    switch (incoming_async_msg.msg_type) {
+        case async_msg_type::log: {
+            incoming_async_msg.worker_ptr->backend_sink_it_(incoming_async_msg);
+            return true;
+        }
+        case async_msg_type::flush: {
+            incoming_async_msg.worker_ptr->backend_flush_();
+            return true;
+        }
+
+        case async_msg_type::terminate: {
+            return false;
+        }
+
+        default: {
+            assert(false);
+        }
+    }
+
+    return true;
+}
+
+}  // namespace details
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/details/thread_pool.h b/ext/spdlog/include/spdlog/details/thread_pool.h
new file mode 100644
index 0000000..f22b078
--- /dev/null
+++ b/ext/spdlog/include/spdlog/details/thread_pool.h
@@ -0,0 +1,117 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#include <spdlog/details/log_msg_buffer.h>
+#include <spdlog/details/mpmc_blocking_q.h>
+#include <spdlog/details/os.h>
+
+#include <chrono>
+#include <functional>
+#include <memory>
+#include <thread>
+#include <vector>
+
+namespace spdlog {
+class async_logger;
+
+namespace details {
+
+using async_logger_ptr = std::shared_ptr<spdlog::async_logger>;
+
+enum class async_msg_type { log, flush, terminate };
+
+// Async msg to move to/from the queue
+// Movable only. should never be copied
+struct async_msg : log_msg_buffer {
+    async_msg_type msg_type{async_msg_type::log};
+    async_logger_ptr worker_ptr;
+
+    async_msg() = default;
+    ~async_msg() = default;
+
+    // should only be moved in or out of the queue..
+    async_msg(const async_msg &) = delete;
+
+// support for vs2013 move
+#if defined(_MSC_VER) && _MSC_VER <= 1800
+    async_msg(async_msg &&other)
+        : log_msg_buffer(std::move(other)),
+          msg_type(other.msg_type),
+          worker_ptr(std::move(other.worker_ptr)) {}
+
+    async_msg &operator=(async_msg &&other) {
+        *static_cast<log_msg_buffer *>(this) = std::move(other);
+        msg_type = other.msg_type;
+        worker_ptr = std::move(other.worker_ptr);
+        return *this;
+    }
+#else  // (_MSC_VER) && _MSC_VER <= 1800
+    async_msg(async_msg &&) = default;
+    async_msg &operator=(async_msg &&) = default;
+#endif
+
+    // construct from log_msg with given type
+    async_msg(async_logger_ptr &&worker, async_msg_type the_type, const details::log_msg &m)
+        : log_msg_buffer{m},
+          msg_type{the_type},
+          worker_ptr{std::move(worker)} {}
+
+    async_msg(async_logger_ptr &&worker, async_msg_type the_type)
+        : log_msg_buffer{},
+          msg_type{the_type},
+          worker_ptr{std::move(worker)} {}
+
+    explicit async_msg(async_msg_type the_type)
+        : async_msg{nullptr, the_type} {}
+};
+
+class SPDLOG_API thread_pool {
+public:
+    using item_type = async_msg;
+    using q_type = details::mpmc_blocking_queue<item_type>;
+
+    thread_pool(size_t q_max_items,
+                size_t threads_n,
+                std::function<void()> on_thread_start,
+                std::function<void()> on_thread_stop);
+    thread_pool(size_t q_max_items, size_t threads_n, std::function<void()> on_thread_start);
+    thread_pool(size_t q_max_items, size_t threads_n);
+
+    // message all threads to terminate gracefully and join them
+    ~thread_pool();
+
+    thread_pool(const thread_pool &) = delete;
+    thread_pool &operator=(thread_pool &&) = delete;
+
+    void post_log(async_logger_ptr &&worker_ptr,
+                  const details::log_msg &msg,
+                  async_overflow_policy overflow_policy);
+    void post_flush(async_logger_ptr &&worker_ptr, async_overflow_policy overflow_policy);
+    size_t overrun_counter();
+    void reset_overrun_counter();
+    size_t discard_counter();
+    void reset_discard_counter();
+    size_t queue_size();
+
+private:
+    q_type q_;
+
+    std::vector<std::thread> threads_;
+
+    void post_async_msg_(async_msg &&new_msg, async_overflow_policy overflow_policy);
+    void worker_loop_();
+
+    // process next message in the queue
+    // return true if this thread should still be active (while no terminate msg
+    // was received)
+    bool process_next_msg_();
+};
+
+}  // namespace details
+}  // namespace spdlog
+
+#ifdef SPDLOG_HEADER_ONLY
+    #include "thread_pool-inl.h"
+#endif
diff --git a/ext/spdlog/include/spdlog/details/udp_client-windows.h b/ext/spdlog/include/spdlog/details/udp_client-windows.h
new file mode 100644
index 0000000..8b7c223
--- /dev/null
+++ b/ext/spdlog/include/spdlog/details/udp_client-windows.h
@@ -0,0 +1,98 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+// Helper RAII over winsock udp client socket.
+// Will throw on construction if socket creation failed.
+
+#include <spdlog/common.h>
+#include <spdlog/details/os.h>
+#include <spdlog/details/windows_include.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string>
+#include <winsock2.h>
+#include <ws2tcpip.h>
+
+#if defined(_MSC_VER)
+    #pragma comment(lib, "Ws2_32.lib")
+    #pragma comment(lib, "Mswsock.lib")
+    #pragma comment(lib, "AdvApi32.lib")
+#endif
+
+namespace spdlog {
+namespace details {
+class udp_client {
+    static constexpr int TX_BUFFER_SIZE = 1024 * 10;
+    SOCKET socket_ = INVALID_SOCKET;
+    sockaddr_in addr_ = {};
+
+    static void init_winsock_() {
+        WSADATA wsaData;
+        auto rv = ::WSAStartup(MAKEWORD(2, 2), &wsaData);
+        if (rv != 0) {
+            throw_winsock_error_("WSAStartup failed", ::WSAGetLastError());
+        }
+    }
+
+    static void throw_winsock_error_(const std::string &msg, int last_error) {
+        char buf[512];
+        ::FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL,
+                         last_error, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), buf,
+                         (sizeof(buf) / sizeof(char)), NULL);
+
+        throw_spdlog_ex(fmt_lib::format("udp_sink - {}: {}", msg, buf));
+    }
+
+    void cleanup_() {
+        if (socket_ != INVALID_SOCKET) {
+            ::closesocket(socket_);
+        }
+        socket_ = INVALID_SOCKET;
+        ::WSACleanup();
+    }
+
+public:
+    udp_client(const std::string &host, uint16_t port) {
+        init_winsock_();
+
+        addr_.sin_family = PF_INET;
+        addr_.sin_port = htons(port);
+        addr_.sin_addr.s_addr = INADDR_ANY;
+        if (InetPtonA(PF_INET, host.c_str(), &addr_.sin_addr.s_addr) != 1) {
+            int last_error = ::WSAGetLastError();
+            ::WSACleanup();
+            throw_winsock_error_("error: Invalid address!", last_error);
+        }
+
+        socket_ = ::socket(PF_INET, SOCK_DGRAM, 0);
+        if (socket_ == INVALID_SOCKET) {
+            int last_error = ::WSAGetLastError();
+            ::WSACleanup();
+            throw_winsock_error_("error: Create Socket failed", last_error);
+        }
+
+        int option_value = TX_BUFFER_SIZE;
+        if (::setsockopt(socket_, SOL_SOCKET, SO_SNDBUF,
+                         reinterpret_cast<const char *>(&option_value), sizeof(option_value)) < 0) {
+            int last_error = ::WSAGetLastError();
+            cleanup_();
+            throw_winsock_error_("error: setsockopt(SO_SNDBUF) Failed!", last_error);
+        }
+    }
+
+    ~udp_client() { cleanup_(); }
+
+    SOCKET fd() const { return socket_; }
+
+    void send(const char *data, size_t n_bytes) {
+        socklen_t tolen = sizeof(struct sockaddr);
+        if (::sendto(socket_, data, static_cast<int>(n_bytes), 0, (struct sockaddr *)&addr_,
+                     tolen) == -1) {
+            throw_spdlog_ex("sendto(2) failed", errno);
+        }
+    }
+};
+}  // namespace details
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/details/udp_client.h b/ext/spdlog/include/spdlog/details/udp_client.h
new file mode 100644
index 0000000..95826f5
--- /dev/null
+++ b/ext/spdlog/include/spdlog/details/udp_client.h
@@ -0,0 +1,81 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+// Helper RAII over unix udp client socket.
+// Will throw on construction if the socket creation failed.
+
+#ifdef _WIN32
+    #error "include udp_client-windows.h instead"
+#endif
+
+#include <arpa/inet.h>
+#include <cstring>
+#include <netdb.h>
+#include <netinet/in.h>
+#include <netinet/udp.h>
+#include <spdlog/common.h>
+#include <spdlog/details/os.h>
+#include <sys/socket.h>
+#include <unistd.h>
+
+#include <string>
+
+namespace spdlog {
+namespace details {
+
+class udp_client {
+    static constexpr int TX_BUFFER_SIZE = 1024 * 10;
+    int socket_ = -1;
+    struct sockaddr_in sockAddr_;
+
+    void cleanup_() {
+        if (socket_ != -1) {
+            ::close(socket_);
+            socket_ = -1;
+        }
+    }
+
+public:
+    udp_client(const std::string &host, uint16_t port) {
+        socket_ = ::socket(PF_INET, SOCK_DGRAM, 0);
+        if (socket_ < 0) {
+            throw_spdlog_ex("error: Create Socket Failed!");
+        }
+
+        int option_value = TX_BUFFER_SIZE;
+        if (::setsockopt(socket_, SOL_SOCKET, SO_SNDBUF,
+                         reinterpret_cast<const char *>(&option_value), sizeof(option_value)) < 0) {
+            cleanup_();
+            throw_spdlog_ex("error: setsockopt(SO_SNDBUF) Failed!");
+        }
+
+        sockAddr_.sin_family = AF_INET;
+        sockAddr_.sin_port = htons(port);
+
+        if (::inet_aton(host.c_str(), &sockAddr_.sin_addr) == 0) {
+            cleanup_();
+            throw_spdlog_ex("error: Invalid address!");
+        }
+
+        ::memset(sockAddr_.sin_zero, 0x00, sizeof(sockAddr_.sin_zero));
+    }
+
+    ~udp_client() { cleanup_(); }
+
+    int fd() const { return socket_; }
+
+    // Send exactly n_bytes of the given data.
+    // On error close the connection and throw.
+    void send(const char *data, size_t n_bytes) {
+        ssize_t toslen = 0;
+        socklen_t tolen = sizeof(struct sockaddr);
+        if ((toslen = ::sendto(socket_, data, n_bytes, 0, (struct sockaddr *)&sockAddr_, tolen)) ==
+            -1) {
+            throw_spdlog_ex("sendto(2) failed", errno);
+        }
+    }
+};
+}  // namespace details
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/details/windows_include.h b/ext/spdlog/include/spdlog/details/windows_include.h
new file mode 100644
index 0000000..bbab59b
--- /dev/null
+++ b/ext/spdlog/include/spdlog/details/windows_include.h
@@ -0,0 +1,11 @@
+#pragma once
+
+#ifndef NOMINMAX
+    #define NOMINMAX  // prevent windows redefining min/max
+#endif
+
+#ifndef WIN32_LEAN_AND_MEAN
+    #define WIN32_LEAN_AND_MEAN
+#endif
+
+#include <windows.h>
diff --git a/ext/spdlog/include/spdlog/fmt/bin_to_hex.h b/ext/spdlog/include/spdlog/fmt/bin_to_hex.h
new file mode 100644
index 0000000..c2998d5
--- /dev/null
+++ b/ext/spdlog/include/spdlog/fmt/bin_to_hex.h
@@ -0,0 +1,224 @@
+//
+// Copyright(c) 2015 Gabi Melman.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+//
+
+#pragma once
+
+#include <cctype>
+#include <spdlog/common.h>
+
+#if defined(__has_include)
+    #if __has_include(<version>)
+        #include <version>
+    #endif
+#endif
+
+#if __cpp_lib_span >= 202002L
+    #include <span>
+#endif
+
+//
+// Support for logging binary data as hex
+// format flags, any combination of the following:
+// {:X} - print in uppercase.
+// {:s} - don't separate each byte with space.
+// {:p} - don't print the position on each line start.
+// {:n} - don't split the output to lines.
+// {:a} - show ASCII if :n is not set
+
+//
+// Examples:
+//
+// std::vector<char> v(200, 0x0b);
+// logger->info("Some buffer {}", spdlog::to_hex(v));
+// char buf[128];
+// logger->info("Some buffer {:X}", spdlog::to_hex(std::begin(buf), std::end(buf)));
+// logger->info("Some buffer {:X}", spdlog::to_hex(std::begin(buf), std::end(buf), 16));
+
+namespace spdlog {
+namespace details {
+
+template <typename It>
+class dump_info {
+public:
+    dump_info(It range_begin, It range_end, size_t size_per_line)
+        : begin_(range_begin),
+          end_(range_end),
+          size_per_line_(size_per_line) {}
+
+    // do not use begin() and end() to avoid collision with fmt/ranges
+    It get_begin() const { return begin_; }
+    It get_end() const { return end_; }
+    size_t size_per_line() const { return size_per_line_; }
+
+private:
+    It begin_, end_;
+    size_t size_per_line_;
+};
+}  // namespace details
+
+// create a dump_info that wraps the given container
+template <typename Container>
+inline details::dump_info<typename Container::const_iterator> to_hex(const Container &container,
+                                                                     size_t size_per_line = 32) {
+    static_assert(sizeof(typename Container::value_type) == 1,
+                  "sizeof(Container::value_type) != 1");
+    using Iter = typename Container::const_iterator;
+    return details::dump_info<Iter>(std::begin(container), std::end(container), size_per_line);
+}
+
+#if __cpp_lib_span >= 202002L
+
+template <typename Value, size_t Extent>
+inline details::dump_info<typename std::span<Value, Extent>::iterator> to_hex(
+    const std::span<Value, Extent> &container, size_t size_per_line = 32) {
+    using Container = std::span<Value, Extent>;
+    static_assert(sizeof(typename Container::value_type) == 1,
+                  "sizeof(Container::value_type) != 1");
+    using Iter = typename Container::iterator;
+    return details::dump_info<Iter>(std::begin(container), std::end(container), size_per_line);
+}
+
+#endif
+
+// create dump_info from ranges
+template <typename It>
+inline details::dump_info<It> to_hex(const It range_begin,
+                                     const It range_end,
+                                     size_t size_per_line = 32) {
+    return details::dump_info<It>(range_begin, range_end, size_per_line);
+}
+
+}  // namespace spdlog
+
+namespace
+#ifdef SPDLOG_USE_STD_FORMAT
+    std
+#else
+    fmt
+#endif
+{
+
+template <typename T>
+struct formatter<spdlog::details::dump_info<T>, char> {
+    const char delimiter = ' ';
+    bool put_newlines = true;
+    bool put_delimiters = true;
+    bool use_uppercase = false;
+    bool put_positions = true;  // position on start of each line
+    bool show_ascii = false;
+
+    // parse the format string flags
+    template <typename ParseContext>
+    SPDLOG_CONSTEXPR_FUNC auto parse(ParseContext &ctx) -> decltype(ctx.begin()) {
+        auto it = ctx.begin();
+        while (it != ctx.end() && *it != '}') {
+            switch (*it) {
+                case 'X':
+                    use_uppercase = true;
+                    break;
+                case 's':
+                    put_delimiters = false;
+                    break;
+                case 'p':
+                    put_positions = false;
+                    break;
+                case 'n':
+                    put_newlines = false;
+                    show_ascii = false;
+                    break;
+                case 'a':
+                    if (put_newlines) {
+                        show_ascii = true;
+                    }
+                    break;
+            }
+
+            ++it;
+        }
+        return it;
+    }
+
+    // format the given bytes range as hex
+    template <typename FormatContext, typename Container>
+    auto format(const spdlog::details::dump_info<Container> &the_range, FormatContext &ctx) const
+        -> decltype(ctx.out()) {
+        SPDLOG_CONSTEXPR const char *hex_upper = "0123456789ABCDEF";
+        SPDLOG_CONSTEXPR const char *hex_lower = "0123456789abcdef";
+        const char *hex_chars = use_uppercase ? hex_upper : hex_lower;
+
+#if !defined(SPDLOG_USE_STD_FORMAT) && FMT_VERSION < 60000
+        auto inserter = ctx.begin();
+#else
+        auto inserter = ctx.out();
+#endif
+
+        int size_per_line = static_cast<int>(the_range.size_per_line());
+        auto start_of_line = the_range.get_begin();
+        for (auto i = the_range.get_begin(); i != the_range.get_end(); i++) {
+            auto ch = static_cast<unsigned char>(*i);
+
+            if (put_newlines &&
+                (i == the_range.get_begin() || i - start_of_line >= size_per_line)) {
+                if (show_ascii && i != the_range.get_begin()) {
+                    *inserter++ = delimiter;
+                    *inserter++ = delimiter;
+                    for (auto j = start_of_line; j < i; j++) {
+                        auto pc = static_cast<unsigned char>(*j);
+                        *inserter++ = std::isprint(pc) ? static_cast<char>(*j) : '.';
+                    }
+                }
+
+                put_newline(inserter, static_cast<size_t>(i - the_range.get_begin()));
+
+                // put first byte without delimiter in front of it
+                *inserter++ = hex_chars[(ch >> 4) & 0x0f];
+                *inserter++ = hex_chars[ch & 0x0f];
+                start_of_line = i;
+                continue;
+            }
+
+            if (put_delimiters && i != the_range.get_begin()) {
+                *inserter++ = delimiter;
+            }
+
+            *inserter++ = hex_chars[(ch >> 4) & 0x0f];
+            *inserter++ = hex_chars[ch & 0x0f];
+        }
+        if (show_ascii)  // add ascii to last line
+        {
+            if (the_range.get_end() - the_range.get_begin() > size_per_line) {
+                auto blank_num = size_per_line - (the_range.get_end() - start_of_line);
+                while (blank_num-- > 0) {
+                    *inserter++ = delimiter;
+                    *inserter++ = delimiter;
+                    if (put_delimiters) {
+                        *inserter++ = delimiter;
+                    }
+                }
+            }
+            *inserter++ = delimiter;
+            *inserter++ = delimiter;
+            for (auto j = start_of_line; j != the_range.get_end(); j++) {
+                auto pc = static_cast<unsigned char>(*j);
+                *inserter++ = std::isprint(pc) ? static_cast<char>(*j) : '.';
+            }
+        }
+        return inserter;
+    }
+
+    // put newline(and position header)
+    template <typename It>
+    void put_newline(It inserter, std::size_t pos) const {
+#ifdef _WIN32
+        *inserter++ = '\r';
+#endif
+        *inserter++ = '\n';
+
+        if (put_positions) {
+            spdlog::fmt_lib::format_to(inserter, SPDLOG_FMT_STRING("{:04X}: "), pos);
+        }
+    }
+};
+}  // namespace std
diff --git a/ext/spdlog/include/spdlog/fmt/bundled/args.h b/ext/spdlog/include/spdlog/fmt/bundled/args.h
new file mode 100644
index 0000000..31a60e8
--- /dev/null
+++ b/ext/spdlog/include/spdlog/fmt/bundled/args.h
@@ -0,0 +1,228 @@
+// Formatting library for C++ - dynamic argument lists
+//
+// Copyright (c) 2012 - present, Victor Zverovich
+// All rights reserved.
+//
+// For the license information refer to format.h.
+
+#ifndef FMT_ARGS_H_
+#define FMT_ARGS_H_
+
+#ifndef FMT_MODULE
+#  include <functional>  // std::reference_wrapper
+#  include <memory>      // std::unique_ptr
+#  include <vector>
+#endif
+
+#include "format.h"  // std_string_view
+
+FMT_BEGIN_NAMESPACE
+
+namespace detail {
+
+template <typename T> struct is_reference_wrapper : std::false_type {};
+template <typename T>
+struct is_reference_wrapper<std::reference_wrapper<T>> : std::true_type {};
+
+template <typename T> auto unwrap(const T& v) -> const T& { return v; }
+template <typename T>
+auto unwrap(const std::reference_wrapper<T>& v) -> const T& {
+  return static_cast<const T&>(v);
+}
+
+// node is defined outside dynamic_arg_list to workaround a C2504 bug in MSVC
+// 2022 (v17.10.0).
+//
+// Workaround for clang's -Wweak-vtables. Unlike for regular classes, for
+// templates it doesn't complain about inability to deduce single translation
+// unit for placing vtable. So node is made a fake template.
+template <typename = void> struct node {
+  virtual ~node() = default;
+  std::unique_ptr<node<>> next;
+};
+
+class dynamic_arg_list {
+  template <typename T> struct typed_node : node<> {
+    T value;
+
+    template <typename Arg>
+    FMT_CONSTEXPR typed_node(const Arg& arg) : value(arg) {}
+
+    template <typename Char>
+    FMT_CONSTEXPR typed_node(const basic_string_view<Char>& arg)
+        : value(arg.data(), arg.size()) {}
+  };
+
+  std::unique_ptr<node<>> head_;
+
+ public:
+  template <typename T, typename Arg> auto push(const Arg& arg) -> const T& {
+    auto new_node = std::unique_ptr<typed_node<T>>(new typed_node<T>(arg));
+    auto& value = new_node->value;
+    new_node->next = std::move(head_);
+    head_ = std::move(new_node);
+    return value;
+  }
+};
+}  // namespace detail
+
+/**
+ * A dynamic list of formatting arguments with storage.
+ *
+ * It can be implicitly converted into `fmt::basic_format_args` for passing
+ * into type-erased formatting functions such as `fmt::vformat`.
+ */
+template <typename Context>
+class dynamic_format_arg_store
+#if FMT_GCC_VERSION && FMT_GCC_VERSION < 409
+    // Workaround a GCC template argument substitution bug.
+    : public basic_format_args<Context>
+#endif
+{
+ private:
+  using char_type = typename Context::char_type;
+
+  template <typename T> struct need_copy {
+    static constexpr detail::type mapped_type =
+        detail::mapped_type_constant<T, Context>::value;
+
+    enum {
+      value = !(detail::is_reference_wrapper<T>::value ||
+                std::is_same<T, basic_string_view<char_type>>::value ||
+                std::is_same<T, detail::std_string_view<char_type>>::value ||
+                (mapped_type != detail::type::cstring_type &&
+                 mapped_type != detail::type::string_type &&
+                 mapped_type != detail::type::custom_type))
+    };
+  };
+
+  template <typename T>
+  using stored_type = conditional_t<
+      std::is_convertible<T, std::basic_string<char_type>>::value &&
+          !detail::is_reference_wrapper<T>::value,
+      std::basic_string<char_type>, T>;
+
+  // Storage of basic_format_arg must be contiguous.
+  std::vector<basic_format_arg<Context>> data_;
+  std::vector<detail::named_arg_info<char_type>> named_info_;
+
+  // Storage of arguments not fitting into basic_format_arg must grow
+  // without relocation because items in data_ refer to it.
+  detail::dynamic_arg_list dynamic_args_;
+
+  friend class basic_format_args<Context>;
+
+  auto get_types() const -> unsigned long long {
+    return detail::is_unpacked_bit | data_.size() |
+           (named_info_.empty()
+                ? 0ULL
+                : static_cast<unsigned long long>(detail::has_named_args_bit));
+  }
+
+  auto data() const -> const basic_format_arg<Context>* {
+    return named_info_.empty() ? data_.data() : data_.data() + 1;
+  }
+
+  template <typename T> void emplace_arg(const T& arg) {
+    data_.emplace_back(detail::make_arg<Context>(arg));
+  }
+
+  template <typename T>
+  void emplace_arg(const detail::named_arg<char_type, T>& arg) {
+    if (named_info_.empty()) {
+      constexpr const detail::named_arg_info<char_type>* zero_ptr{nullptr};
+      data_.insert(data_.begin(), {zero_ptr, 0});
+    }
+    data_.emplace_back(detail::make_arg<Context>(detail::unwrap(arg.value)));
+    auto pop_one = [](std::vector<basic_format_arg<Context>>* data) {
+      data->pop_back();
+    };
+    std::unique_ptr<std::vector<basic_format_arg<Context>>, decltype(pop_one)>
+        guard{&data_, pop_one};
+    named_info_.push_back({arg.name, static_cast<int>(data_.size() - 2u)});
+    data_[0].value_.named_args = {named_info_.data(), named_info_.size()};
+    guard.release();
+  }
+
+ public:
+  constexpr dynamic_format_arg_store() = default;
+
+  /**
+   * Adds an argument into the dynamic store for later passing to a formatting
+   * function.
+   *
+   * Note that custom types and string types (but not string views) are copied
+   * into the store dynamically allocating memory if necessary.
+   *
+   * **Example**:
+   *
+   *     fmt::dynamic_format_arg_store<fmt::format_context> store;
+   *     store.push_back(42);
+   *     store.push_back("abc");
+   *     store.push_back(1.5f);
+   *     std::string result = fmt::vformat("{} and {} and {}", store);
+   */
+  template <typename T> void push_back(const T& arg) {
+    if (detail::const_check(need_copy<T>::value))
+      emplace_arg(dynamic_args_.push<stored_type<T>>(arg));
+    else
+      emplace_arg(detail::unwrap(arg));
+  }
+
+  /**
+   * Adds a reference to the argument into the dynamic store for later passing
+   * to a formatting function.
+   *
+   * **Example**:
+   *
+   *     fmt::dynamic_format_arg_store<fmt::format_context> store;
+   *     char band[] = "Rolling Stones";
+   *     store.push_back(std::cref(band));
+   *     band[9] = 'c'; // Changing str affects the output.
+   *     std::string result = fmt::vformat("{}", store);
+   *     // result == "Rolling Scones"
+   */
+  template <typename T> void push_back(std::reference_wrapper<T> arg) {
+    static_assert(
+        need_copy<T>::value,
+        "objects of built-in types and string views are always copied");
+    emplace_arg(arg.get());
+  }
+
+  /**
+   * Adds named argument into the dynamic store for later passing to a
+   * formatting function. `std::reference_wrapper` is supported to avoid
+   * copying of the argument. The name is always copied into the store.
+   */
+  template <typename T>
+  void push_back(const detail::named_arg<char_type, T>& arg) {
+    const char_type* arg_name =
+        dynamic_args_.push<std::basic_string<char_type>>(arg.name).c_str();
+    if (detail::const_check(need_copy<T>::value)) {
+      emplace_arg(
+          fmt::arg(arg_name, dynamic_args_.push<stored_type<T>>(arg.value)));
+    } else {
+      emplace_arg(fmt::arg(arg_name, arg.value));
+    }
+  }
+
+  /// Erase all elements from the store.
+  void clear() {
+    data_.clear();
+    named_info_.clear();
+    dynamic_args_ = detail::dynamic_arg_list();
+  }
+
+  /// Reserves space to store at least `new_cap` arguments including
+  /// `new_cap_named` named arguments.
+  void reserve(size_t new_cap, size_t new_cap_named) {
+    FMT_ASSERT(new_cap >= new_cap_named,
+               "Set of arguments includes set of named arguments");
+    data_.reserve(new_cap);
+    named_info_.reserve(new_cap_named);
+  }
+};
+
+FMT_END_NAMESPACE
+
+#endif  // FMT_ARGS_H_
diff --git a/ext/spdlog/include/spdlog/fmt/bundled/base.h b/ext/spdlog/include/spdlog/fmt/bundled/base.h
new file mode 100644
index 0000000..6276494
--- /dev/null
+++ b/ext/spdlog/include/spdlog/fmt/bundled/base.h
@@ -0,0 +1,3077 @@
+// Formatting library for C++ - the base API for char/UTF-8
+//
+// Copyright (c) 2012 - present, Victor Zverovich
+// All rights reserved.
+//
+// For the license information refer to format.h.
+
+#ifndef FMT_BASE_H_
+#define FMT_BASE_H_
+
+#if defined(FMT_IMPORT_STD) && !defined(FMT_MODULE)
+#  define FMT_MODULE
+#endif
+
+#ifndef FMT_MODULE
+#  include <limits.h>  // CHAR_BIT
+#  include <stdio.h>   // FILE
+#  include <string.h>  // strlen
+
+// <cstddef> is also included transitively from <type_traits>.
+#  include <cstddef>      // std::byte
+#  include <type_traits>  // std::enable_if
+#endif
+
+// The fmt library version in the form major * 10000 + minor * 100 + patch.
+#define FMT_VERSION 110002
+
+// Detect compiler versions.
+#if defined(__clang__) && !defined(__ibmxl__)
+#  define FMT_CLANG_VERSION (__clang_major__ * 100 + __clang_minor__)
+#else
+#  define FMT_CLANG_VERSION 0
+#endif
+#if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER)
+#  define FMT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+#else
+#  define FMT_GCC_VERSION 0
+#endif
+#if defined(__ICL)
+#  define FMT_ICC_VERSION __ICL
+#elif defined(__INTEL_COMPILER)
+#  define FMT_ICC_VERSION __INTEL_COMPILER
+#else
+#  define FMT_ICC_VERSION 0
+#endif
+#if defined(_MSC_VER)
+#  define FMT_MSC_VERSION _MSC_VER
+#else
+#  define FMT_MSC_VERSION 0
+#endif
+
+// Detect standard library versions.
+#ifdef _GLIBCXX_RELEASE
+#  define FMT_GLIBCXX_RELEASE _GLIBCXX_RELEASE
+#else
+#  define FMT_GLIBCXX_RELEASE 0
+#endif
+#ifdef _LIBCPP_VERSION
+#  define FMT_LIBCPP_VERSION _LIBCPP_VERSION
+#else
+#  define FMT_LIBCPP_VERSION 0
+#endif
+
+#ifdef _MSVC_LANG
+#  define FMT_CPLUSPLUS _MSVC_LANG
+#else
+#  define FMT_CPLUSPLUS __cplusplus
+#endif
+
+// Detect __has_*.
+#ifdef __has_feature
+#  define FMT_HAS_FEATURE(x) __has_feature(x)
+#else
+#  define FMT_HAS_FEATURE(x) 0
+#endif
+#ifdef __has_include
+#  define FMT_HAS_INCLUDE(x) __has_include(x)
+#else
+#  define FMT_HAS_INCLUDE(x) 0
+#endif
+#ifdef __has_cpp_attribute
+#  define FMT_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x)
+#else
+#  define FMT_HAS_CPP_ATTRIBUTE(x) 0
+#endif
+
+#define FMT_HAS_CPP14_ATTRIBUTE(attribute) \
+  (FMT_CPLUSPLUS >= 201402L && FMT_HAS_CPP_ATTRIBUTE(attribute))
+
+#define FMT_HAS_CPP17_ATTRIBUTE(attribute) \
+  (FMT_CPLUSPLUS >= 201703L && FMT_HAS_CPP_ATTRIBUTE(attribute))
+
+// Detect C++14 relaxed constexpr.
+#ifdef FMT_USE_CONSTEXPR
+// Use the provided definition.
+#elif FMT_GCC_VERSION >= 600 && FMT_CPLUSPLUS >= 201402L
+// GCC only allows throw in constexpr since version 6:
+// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=67371.
+#  define FMT_USE_CONSTEXPR 1
+#elif FMT_ICC_VERSION
+#  define FMT_USE_CONSTEXPR 0  // https://github.com/fmtlib/fmt/issues/1628
+#elif FMT_HAS_FEATURE(cxx_relaxed_constexpr) || FMT_MSC_VERSION >= 1912
+#  define FMT_USE_CONSTEXPR 1
+#else
+#  define FMT_USE_CONSTEXPR 0
+#endif
+#if FMT_USE_CONSTEXPR
+#  define FMT_CONSTEXPR constexpr
+#else
+#  define FMT_CONSTEXPR
+#endif
+
+// Detect consteval, C++20 constexpr extensions and std::is_constant_evaluated.
+#if !defined(__cpp_lib_is_constant_evaluated)
+#  define FMT_USE_CONSTEVAL 0
+#elif FMT_CPLUSPLUS < 201709L
+#  define FMT_USE_CONSTEVAL 0
+#elif FMT_GLIBCXX_RELEASE && FMT_GLIBCXX_RELEASE < 10
+#  define FMT_USE_CONSTEVAL 0
+#elif FMT_LIBCPP_VERSION && FMT_LIBCPP_VERSION < 10000
+#  define FMT_USE_CONSTEVAL 0
+#elif defined(__apple_build_version__) && __apple_build_version__ < 14000029L
+#  define FMT_USE_CONSTEVAL 0  // consteval is broken in Apple clang < 14.
+#elif FMT_MSC_VERSION && FMT_MSC_VERSION < 1929
+#  define FMT_USE_CONSTEVAL 0  // consteval is broken in MSVC VS2019 < 16.10.
+#elif defined(__cpp_consteval)
+#  define FMT_USE_CONSTEVAL 1
+#elif FMT_GCC_VERSION >= 1002 || FMT_CLANG_VERSION >= 1101
+#  define FMT_USE_CONSTEVAL 1
+#else
+#  define FMT_USE_CONSTEVAL 0
+#endif
+#if FMT_USE_CONSTEVAL
+#  define FMT_CONSTEVAL consteval
+#  define FMT_CONSTEXPR20 constexpr
+#else
+#  define FMT_CONSTEVAL
+#  define FMT_CONSTEXPR20
+#endif
+
+#if defined(FMT_USE_NONTYPE_TEMPLATE_ARGS)
+// Use the provided definition.
+#elif defined(__NVCOMPILER)
+#  define FMT_USE_NONTYPE_TEMPLATE_ARGS 0
+#elif FMT_GCC_VERSION >= 903 && FMT_CPLUSPLUS >= 201709L
+#  define FMT_USE_NONTYPE_TEMPLATE_ARGS 1
+#elif defined(__cpp_nontype_template_args) && \
+    __cpp_nontype_template_args >= 201911L
+#  define FMT_USE_NONTYPE_TEMPLATE_ARGS 1
+#elif FMT_CLANG_VERSION >= 1200 && FMT_CPLUSPLUS >= 202002L
+#  define FMT_USE_NONTYPE_TEMPLATE_ARGS 1
+#else
+#  define FMT_USE_NONTYPE_TEMPLATE_ARGS 0
+#endif
+
+#ifdef FMT_USE_CONCEPTS
+// Use the provided definition.
+#elif defined(__cpp_concepts)
+#  define FMT_USE_CONCEPTS 1
+#else
+#  define FMT_USE_CONCEPTS 0
+#endif
+
+// Check if exceptions are disabled.
+#ifdef FMT_EXCEPTIONS
+// Use the provided definition.
+#elif defined(__GNUC__) && !defined(__EXCEPTIONS)
+#  define FMT_EXCEPTIONS 0
+#elif FMT_MSC_VERSION && !_HAS_EXCEPTIONS
+#  define FMT_EXCEPTIONS 0
+#else
+#  define FMT_EXCEPTIONS 1
+#endif
+#if FMT_EXCEPTIONS
+#  define FMT_TRY try
+#  define FMT_CATCH(x) catch (x)
+#else
+#  define FMT_TRY if (true)
+#  define FMT_CATCH(x) if (false)
+#endif
+
+#if FMT_HAS_CPP17_ATTRIBUTE(fallthrough)
+#  define FMT_FALLTHROUGH [[fallthrough]]
+#elif defined(__clang__)
+#  define FMT_FALLTHROUGH [[clang::fallthrough]]
+#elif FMT_GCC_VERSION >= 700 && \
+    (!defined(__EDG_VERSION__) || __EDG_VERSION__ >= 520)
+#  define FMT_FALLTHROUGH [[gnu::fallthrough]]
+#else
+#  define FMT_FALLTHROUGH
+#endif
+
+// Disable [[noreturn]] on MSVC/NVCC because of bogus unreachable code warnings.
+#if FMT_HAS_CPP_ATTRIBUTE(noreturn) && !FMT_MSC_VERSION && !defined(__NVCC__)
+#  define FMT_NORETURN [[noreturn]]
+#else
+#  define FMT_NORETURN
+#endif
+
+#ifndef FMT_NODISCARD
+#  if FMT_HAS_CPP17_ATTRIBUTE(nodiscard)
+#    define FMT_NODISCARD [[nodiscard]]
+#  else
+#    define FMT_NODISCARD
+#  endif
+#endif
+
+#ifdef FMT_DEPRECATED
+// Use the provided definition.
+#elif FMT_HAS_CPP14_ATTRIBUTE(deprecated)
+#  define FMT_DEPRECATED [[deprecated]]
+#else
+#  define FMT_DEPRECATED /* deprecated */
+#endif
+
+#ifdef FMT_INLINE
+// Use the provided definition.
+#elif FMT_GCC_VERSION || FMT_CLANG_VERSION
+#  define FMT_ALWAYS_INLINE inline __attribute__((always_inline))
+#else
+#  define FMT_ALWAYS_INLINE inline
+#endif
+// A version of FMT_INLINE to prevent code bloat in debug mode.
+#ifdef NDEBUG
+#  define FMT_INLINE FMT_ALWAYS_INLINE
+#else
+#  define FMT_INLINE inline
+#endif
+
+#if FMT_GCC_VERSION || FMT_CLANG_VERSION
+#  define FMT_VISIBILITY(value) __attribute__((visibility(value)))
+#else
+#  define FMT_VISIBILITY(value)
+#endif
+
+#ifndef FMT_GCC_PRAGMA
+// Workaround a _Pragma bug https://gcc.gnu.org/bugzilla/show_bug.cgi?id=59884
+// and an nvhpc warning: https://github.com/fmtlib/fmt/pull/2582.
+#  if FMT_GCC_VERSION >= 504 && !defined(__NVCOMPILER)
+#    define FMT_GCC_PRAGMA(arg) _Pragma(arg)
+#  else
+#    define FMT_GCC_PRAGMA(arg)
+#  endif
+#endif
+
+// GCC < 5 requires this-> in decltype.
+#if FMT_GCC_VERSION && FMT_GCC_VERSION < 500
+#  define FMT_DECLTYPE_THIS this->
+#else
+#  define FMT_DECLTYPE_THIS
+#endif
+
+#if FMT_MSC_VERSION
+#  define FMT_MSC_WARNING(...) __pragma(warning(__VA_ARGS__))
+#  define FMT_UNCHECKED_ITERATOR(It) \
+    using _Unchecked_type = It  // Mark iterator as checked.
+#else
+#  define FMT_MSC_WARNING(...)
+#  define FMT_UNCHECKED_ITERATOR(It) using unchecked_type = It
+#endif
+
+#ifndef FMT_BEGIN_NAMESPACE
+#  define FMT_BEGIN_NAMESPACE \
+    namespace fmt {           \
+    inline namespace v11 {
+#  define FMT_END_NAMESPACE \
+    }                       \
+    }
+#endif
+
+#ifndef FMT_EXPORT
+#  define FMT_EXPORT
+#  define FMT_BEGIN_EXPORT
+#  define FMT_END_EXPORT
+#endif
+
+#if !defined(FMT_HEADER_ONLY) && defined(_WIN32)
+#  if defined(FMT_LIB_EXPORT)
+#    define FMT_API __declspec(dllexport)
+#  elif defined(FMT_SHARED)
+#    define FMT_API __declspec(dllimport)
+#  endif
+#elif defined(FMT_LIB_EXPORT) || defined(FMT_SHARED)
+#  define FMT_API FMT_VISIBILITY("default")
+#endif
+#ifndef FMT_API
+#  define FMT_API
+#endif
+
+#ifndef FMT_UNICODE
+#  define FMT_UNICODE 1
+#endif
+
+// Check if rtti is available.
+#ifndef FMT_USE_RTTI
+// __RTTI is for EDG compilers. _CPPRTTI is for MSVC.
+#  if defined(__GXX_RTTI) || FMT_HAS_FEATURE(cxx_rtti) || defined(_CPPRTTI) || \
+      defined(__INTEL_RTTI__) || defined(__RTTI)
+#    define FMT_USE_RTTI 1
+#  else
+#    define FMT_USE_RTTI 0
+#  endif
+#endif
+
+#define FMT_FWD(...) static_cast<decltype(__VA_ARGS__)&&>(__VA_ARGS__)
+
+// Enable minimal optimizations for more compact code in debug mode.
+FMT_GCC_PRAGMA("GCC push_options")
+#if !defined(__OPTIMIZE__) && !defined(__CUDACC__)
+FMT_GCC_PRAGMA("GCC optimize(\"Og\")")
+#endif
+
+FMT_BEGIN_NAMESPACE
+
+// Implementations of enable_if_t and other metafunctions for older systems.
+template <bool B, typename T = void>
+using enable_if_t = typename std::enable_if<B, T>::type;
+template <bool B, typename T, typename F>
+using conditional_t = typename std::conditional<B, T, F>::type;
+template <bool B> using bool_constant = std::integral_constant<bool, B>;
+template <typename T>
+using remove_reference_t = typename std::remove_reference<T>::type;
+template <typename T>
+using remove_const_t = typename std::remove_const<T>::type;
+template <typename T>
+using remove_cvref_t = typename std::remove_cv<remove_reference_t<T>>::type;
+template <typename T> struct type_identity {
+  using type = T;
+};
+template <typename T> using type_identity_t = typename type_identity<T>::type;
+template <typename T>
+using make_unsigned_t = typename std::make_unsigned<T>::type;
+template <typename T>
+using underlying_t = typename std::underlying_type<T>::type;
+
+#if FMT_GCC_VERSION && FMT_GCC_VERSION < 500
+// A workaround for gcc 4.8 to make void_t work in a SFINAE context.
+template <typename...> struct void_t_impl {
+  using type = void;
+};
+template <typename... T> using void_t = typename void_t_impl<T...>::type;
+#else
+template <typename...> using void_t = void;
+#endif
+
+struct monostate {
+  constexpr monostate() {}
+};
+
+// An enable_if helper to be used in template parameters which results in much
+// shorter symbols: https://godbolt.org/z/sWw4vP. Extra parentheses are needed
+// to workaround a bug in MSVC 2019 (see #1140 and #1186).
+#ifdef FMT_DOC
+#  define FMT_ENABLE_IF(...)
+#else
+#  define FMT_ENABLE_IF(...) fmt::enable_if_t<(__VA_ARGS__), int> = 0
+#endif
+
+// This is defined in base.h instead of format.h to avoid injecting in std.
+// It is a template to avoid undesirable implicit conversions to std::byte.
+#ifdef __cpp_lib_byte
+template <typename T, FMT_ENABLE_IF(std::is_same<T, std::byte>::value)>
+inline auto format_as(T b) -> unsigned char {
+  return static_cast<unsigned char>(b);
+}
+#endif
+
+namespace detail {
+// Suppresses "unused variable" warnings with the method described in
+// https://herbsutter.com/2009/10/18/mailbag-shutting-up-compiler-warnings/.
+// (void)var does not work on many Intel compilers.
+template <typename... T> FMT_CONSTEXPR void ignore_unused(const T&...) {}
+
+constexpr auto is_constant_evaluated(bool default_value = false) noexcept
+    -> bool {
+// Workaround for incompatibility between libstdc++ consteval-based
+// std::is_constant_evaluated() implementation and clang-14:
+// https://github.com/fmtlib/fmt/issues/3247.
+#if FMT_CPLUSPLUS >= 202002L && FMT_GLIBCXX_RELEASE >= 12 && \
+    (FMT_CLANG_VERSION >= 1400 && FMT_CLANG_VERSION < 1500)
+  ignore_unused(default_value);
+  return __builtin_is_constant_evaluated();
+#elif defined(__cpp_lib_is_constant_evaluated)
+  ignore_unused(default_value);
+  return std::is_constant_evaluated();
+#else
+  return default_value;
+#endif
+}
+
+// Suppresses "conditional expression is constant" warnings.
+template <typename T> constexpr auto const_check(T value) -> T { return value; }
+
+FMT_NORETURN FMT_API void assert_fail(const char* file, int line,
+                                      const char* message);
+
+#if defined(FMT_ASSERT)
+// Use the provided definition.
+#elif defined(NDEBUG)
+// FMT_ASSERT is not empty to avoid -Wempty-body.
+#  define FMT_ASSERT(condition, message) \
+    fmt::detail::ignore_unused((condition), (message))
+#else
+#  define FMT_ASSERT(condition, message)                                    \
+    ((condition) /* void() fails with -Winvalid-constexpr on clang 4.0.1 */ \
+         ? (void)0                                                          \
+         : fmt::detail::assert_fail(__FILE__, __LINE__, (message)))
+#endif
+
+#ifdef FMT_USE_INT128
+// Do nothing.
+#elif defined(__SIZEOF_INT128__) && !defined(__NVCC__) && \
+    !(FMT_CLANG_VERSION && FMT_MSC_VERSION)
+#  define FMT_USE_INT128 1
+using int128_opt = __int128_t;  // An optional native 128-bit integer.
+using uint128_opt = __uint128_t;
+template <typename T> inline auto convert_for_visit(T value) -> T {
+  return value;
+}
+#else
+#  define FMT_USE_INT128 0
+#endif
+#if !FMT_USE_INT128
+enum class int128_opt {};
+enum class uint128_opt {};
+// Reduce template instantiations.
+template <typename T> auto convert_for_visit(T) -> monostate { return {}; }
+#endif
+
+// Casts a nonnegative integer to unsigned.
+template <typename Int>
+FMT_CONSTEXPR auto to_unsigned(Int value) -> make_unsigned_t<Int> {
+  FMT_ASSERT(std::is_unsigned<Int>::value || value >= 0, "negative value");
+  return static_cast<make_unsigned_t<Int>>(value);
+}
+
+// A heuristic to detect std::string and std::[experimental::]string_view.
+// It is mainly used to avoid dependency on <[experimental/]string_view>.
+template <typename T, typename Enable = void>
+struct is_std_string_like : std::false_type {};
+template <typename T>
+struct is_std_string_like<T, void_t<decltype(std::declval<T>().find_first_of(
+                                 typename T::value_type(), 0))>>
+    : std::is_convertible<decltype(std::declval<T>().data()),
+                          const typename T::value_type*> {};
+
+// Returns true iff the literal encoding is UTF-8.
+constexpr auto is_utf8_enabled() -> bool {
+  // Avoid an MSVC sign extension bug: https://github.com/fmtlib/fmt/pull/2297.
+  using uchar = unsigned char;
+  return sizeof("\u00A7") == 3 && uchar("\u00A7"[0]) == 0xC2 &&
+         uchar("\u00A7"[1]) == 0xA7;
+}
+constexpr auto use_utf8() -> bool {
+  return !FMT_MSC_VERSION || is_utf8_enabled();
+}
+
+static_assert(!FMT_UNICODE || use_utf8(),
+              "Unicode support requires compiling with /utf-8");
+
+template <typename Char> FMT_CONSTEXPR auto length(const Char* s) -> size_t {
+  size_t len = 0;
+  while (*s++) ++len;
+  return len;
+}
+
+template <typename Char>
+FMT_CONSTEXPR auto compare(const Char* s1, const Char* s2, std::size_t n)
+    -> int {
+  if (!is_constant_evaluated() && sizeof(Char) == 1) return memcmp(s1, s2, n);
+  for (; n != 0; ++s1, ++s2, --n) {
+    if (*s1 < *s2) return -1;
+    if (*s1 > *s2) return 1;
+  }
+  return 0;
+}
+
+namespace adl {
+using namespace std;
+
+template <typename Container>
+auto invoke_back_inserter()
+    -> decltype(back_inserter(std::declval<Container&>()));
+}  // namespace adl
+
+template <typename It, typename Enable = std::true_type>
+struct is_back_insert_iterator : std::false_type {};
+
+template <typename It>
+struct is_back_insert_iterator<
+    It, bool_constant<std::is_same<
+            decltype(adl::invoke_back_inserter<typename It::container_type>()),
+            It>::value>> : std::true_type {};
+
+// Extracts a reference to the container from *insert_iterator.
+template <typename OutputIt>
+inline auto get_container(OutputIt it) -> typename OutputIt::container_type& {
+  struct accessor : OutputIt {
+    accessor(OutputIt base) : OutputIt(base) {}
+    using OutputIt::container;
+  };
+  return *accessor(it).container;
+}
+}  // namespace detail
+
+// Checks whether T is a container with contiguous storage.
+template <typename T> struct is_contiguous : std::false_type {};
+
+/**
+ * An implementation of `std::basic_string_view` for pre-C++17. It provides a
+ * subset of the API. `fmt::basic_string_view` is used for format strings even
+ * if `std::basic_string_view` is available to prevent issues when a library is
+ * compiled with a different `-std` option than the client code (which is not
+ * recommended).
+ */
+FMT_EXPORT
+template <typename Char> class basic_string_view {
+ private:
+  const Char* data_;
+  size_t size_;
+
+ public:
+  using value_type = Char;
+  using iterator = const Char*;
+
+  constexpr basic_string_view() noexcept : data_(nullptr), size_(0) {}
+
+  /// Constructs a string reference object from a C string and a size.
+  constexpr basic_string_view(const Char* s, size_t count) noexcept
+      : data_(s), size_(count) {}
+
+  constexpr basic_string_view(std::nullptr_t) = delete;
+
+  /// Constructs a string reference object from a C string.
+  FMT_CONSTEXPR20
+  basic_string_view(const Char* s)
+      : data_(s),
+        size_(detail::const_check(std::is_same<Char, char>::value &&
+                                  !detail::is_constant_evaluated(false))
+                  ? strlen(reinterpret_cast<const char*>(s))
+                  : detail::length(s)) {}
+
+  /// Constructs a string reference from a `std::basic_string` or a
+  /// `std::basic_string_view` object.
+  template <typename S,
+            FMT_ENABLE_IF(detail::is_std_string_like<S>::value&& std::is_same<
+                          typename S::value_type, Char>::value)>
+  FMT_CONSTEXPR basic_string_view(const S& s) noexcept
+      : data_(s.data()), size_(s.size()) {}
+
+  /// Returns a pointer to the string data.
+  constexpr auto data() const noexcept -> const Char* { return data_; }
+
+  /// Returns the string size.
+  constexpr auto size() const noexcept -> size_t { return size_; }
+
+  constexpr auto begin() const noexcept -> iterator { return data_; }
+  constexpr auto end() const noexcept -> iterator { return data_ + size_; }
+
+  constexpr auto operator[](size_t pos) const noexcept -> const Char& {
+    return data_[pos];
+  }
+
+  FMT_CONSTEXPR void remove_prefix(size_t n) noexcept {
+    data_ += n;
+    size_ -= n;
+  }
+
+  FMT_CONSTEXPR auto starts_with(basic_string_view<Char> sv) const noexcept
+      -> bool {
+    return size_ >= sv.size_ && detail::compare(data_, sv.data_, sv.size_) == 0;
+  }
+  FMT_CONSTEXPR auto starts_with(Char c) const noexcept -> bool {
+    return size_ >= 1 && *data_ == c;
+  }
+  FMT_CONSTEXPR auto starts_with(const Char* s) const -> bool {
+    return starts_with(basic_string_view<Char>(s));
+  }
+
+  // Lexicographically compare this string reference to other.
+  FMT_CONSTEXPR auto compare(basic_string_view other) const -> int {
+    size_t str_size = size_ < other.size_ ? size_ : other.size_;
+    int result = detail::compare(data_, other.data_, str_size);
+    if (result == 0)
+      result = size_ == other.size_ ? 0 : (size_ < other.size_ ? -1 : 1);
+    return result;
+  }
+
+  FMT_CONSTEXPR friend auto operator==(basic_string_view lhs,
+                                       basic_string_view rhs) -> bool {
+    return lhs.compare(rhs) == 0;
+  }
+  friend auto operator!=(basic_string_view lhs, basic_string_view rhs) -> bool {
+    return lhs.compare(rhs) != 0;
+  }
+  friend auto operator<(basic_string_view lhs, basic_string_view rhs) -> bool {
+    return lhs.compare(rhs) < 0;
+  }
+  friend auto operator<=(basic_string_view lhs, basic_string_view rhs) -> bool {
+    return lhs.compare(rhs) <= 0;
+  }
+  friend auto operator>(basic_string_view lhs, basic_string_view rhs) -> bool {
+    return lhs.compare(rhs) > 0;
+  }
+  friend auto operator>=(basic_string_view lhs, basic_string_view rhs) -> bool {
+    return lhs.compare(rhs) >= 0;
+  }
+};
+
+FMT_EXPORT
+using string_view = basic_string_view<char>;
+
+/// Specifies if `T` is a character type. Can be specialized by users.
+FMT_EXPORT
+template <typename T> struct is_char : std::false_type {};
+template <> struct is_char<char> : std::true_type {};
+
+namespace detail {
+
+// Constructs fmt::basic_string_view<Char> from types implicitly convertible
+// to it, deducing Char. Explicitly convertible types such as the ones returned
+// from FMT_STRING are intentionally excluded.
+template <typename Char, FMT_ENABLE_IF(is_char<Char>::value)>
+constexpr auto to_string_view(const Char* s) -> basic_string_view<Char> {
+  return s;
+}
+template <typename T, FMT_ENABLE_IF(is_std_string_like<T>::value)>
+constexpr auto to_string_view(const T& s)
+    -> basic_string_view<typename T::value_type> {
+  return s;
+}
+template <typename Char>
+constexpr auto to_string_view(basic_string_view<Char> s)
+    -> basic_string_view<Char> {
+  return s;
+}
+
+template <typename T, typename Enable = void>
+struct has_to_string_view : std::false_type {};
+// detail:: is intentional since to_string_view is not an extension point.
+template <typename T>
+struct has_to_string_view<
+    T, void_t<decltype(detail::to_string_view(std::declval<T>()))>>
+    : std::true_type {};
+
+template <typename Char, Char... C> struct string_literal {
+  static constexpr Char value[sizeof...(C)] = {C...};
+  constexpr operator basic_string_view<Char>() const {
+    return {value, sizeof...(C)};
+  }
+};
+#if FMT_CPLUSPLUS < 201703L
+template <typename Char, Char... C>
+constexpr Char string_literal<Char, C...>::value[sizeof...(C)];
+#endif
+
+enum class type {
+  none_type,
+  // Integer types should go first,
+  int_type,
+  uint_type,
+  long_long_type,
+  ulong_long_type,
+  int128_type,
+  uint128_type,
+  bool_type,
+  char_type,
+  last_integer_type = char_type,
+  // followed by floating-point types.
+  float_type,
+  double_type,
+  long_double_type,
+  last_numeric_type = long_double_type,
+  cstring_type,
+  string_type,
+  pointer_type,
+  custom_type
+};
+
+// Maps core type T to the corresponding type enum constant.
+template <typename T, typename Char>
+struct type_constant : std::integral_constant<type, type::custom_type> {};
+
+#define FMT_TYPE_CONSTANT(Type, constant) \
+  template <typename Char>                \
+  struct type_constant<Type, Char>        \
+      : std::integral_constant<type, type::constant> {}
+
+FMT_TYPE_CONSTANT(int, int_type);
+FMT_TYPE_CONSTANT(unsigned, uint_type);
+FMT_TYPE_CONSTANT(long long, long_long_type);
+FMT_TYPE_CONSTANT(unsigned long long, ulong_long_type);
+FMT_TYPE_CONSTANT(int128_opt, int128_type);
+FMT_TYPE_CONSTANT(uint128_opt, uint128_type);
+FMT_TYPE_CONSTANT(bool, bool_type);
+FMT_TYPE_CONSTANT(Char, char_type);
+FMT_TYPE_CONSTANT(float, float_type);
+FMT_TYPE_CONSTANT(double, double_type);
+FMT_TYPE_CONSTANT(long double, long_double_type);
+FMT_TYPE_CONSTANT(const Char*, cstring_type);
+FMT_TYPE_CONSTANT(basic_string_view<Char>, string_type);
+FMT_TYPE_CONSTANT(const void*, pointer_type);
+
+constexpr auto is_integral_type(type t) -> bool {
+  return t > type::none_type && t <= type::last_integer_type;
+}
+constexpr auto is_arithmetic_type(type t) -> bool {
+  return t > type::none_type && t <= type::last_numeric_type;
+}
+
+constexpr auto set(type rhs) -> int { return 1 << static_cast<int>(rhs); }
+constexpr auto in(type t, int set) -> bool {
+  return ((set >> static_cast<int>(t)) & 1) != 0;
+}
+
+// Bitsets of types.
+enum {
+  sint_set =
+      set(type::int_type) | set(type::long_long_type) | set(type::int128_type),
+  uint_set = set(type::uint_type) | set(type::ulong_long_type) |
+             set(type::uint128_type),
+  bool_set = set(type::bool_type),
+  char_set = set(type::char_type),
+  float_set = set(type::float_type) | set(type::double_type) |
+              set(type::long_double_type),
+  string_set = set(type::string_type),
+  cstring_set = set(type::cstring_type),
+  pointer_set = set(type::pointer_type)
+};
+}  // namespace detail
+
+/// Reports a format error at compile time or, via a `format_error` exception,
+/// at runtime.
+// This function is intentionally not constexpr to give a compile-time error.
+FMT_NORETURN FMT_API void report_error(const char* message);
+
+FMT_DEPRECATED FMT_NORETURN inline void throw_format_error(
+    const char* message) {
+  report_error(message);
+}
+
+/// String's character (code unit) type.
+template <typename S,
+          typename V = decltype(detail::to_string_view(std::declval<S>()))>
+using char_t = typename V::value_type;
+
+/**
+ * Parsing context consisting of a format string range being parsed and an
+ * argument counter for automatic indexing.
+ * You can use the `format_parse_context` type alias for `char` instead.
+ */
+FMT_EXPORT
+template <typename Char> class basic_format_parse_context {
+ private:
+  basic_string_view<Char> format_str_;
+  int next_arg_id_;
+
+  FMT_CONSTEXPR void do_check_arg_id(int id);
+
+ public:
+  using char_type = Char;
+  using iterator = const Char*;
+
+  explicit constexpr basic_format_parse_context(
+      basic_string_view<Char> format_str, int next_arg_id = 0)
+      : format_str_(format_str), next_arg_id_(next_arg_id) {}
+
+  /// Returns an iterator to the beginning of the format string range being
+  /// parsed.
+  constexpr auto begin() const noexcept -> iterator {
+    return format_str_.begin();
+  }
+
+  /// Returns an iterator past the end of the format string range being parsed.
+  constexpr auto end() const noexcept -> iterator { return format_str_.end(); }
+
+  /// Advances the begin iterator to `it`.
+  FMT_CONSTEXPR void advance_to(iterator it) {
+    format_str_.remove_prefix(detail::to_unsigned(it - begin()));
+  }
+
+  /// Reports an error if using the manual argument indexing; otherwise returns
+  /// the next argument index and switches to the automatic indexing.
+  FMT_CONSTEXPR auto next_arg_id() -> int {
+    if (next_arg_id_ < 0) {
+      report_error("cannot switch from manual to automatic argument indexing");
+      return 0;
+    }
+    int id = next_arg_id_++;
+    do_check_arg_id(id);
+    return id;
+  }
+
+  /// Reports an error if using the automatic argument indexing; otherwise
+  /// switches to the manual indexing.
+  FMT_CONSTEXPR void check_arg_id(int id) {
+    if (next_arg_id_ > 0) {
+      report_error("cannot switch from automatic to manual argument indexing");
+      return;
+    }
+    next_arg_id_ = -1;
+    do_check_arg_id(id);
+  }
+  FMT_CONSTEXPR void check_arg_id(basic_string_view<Char>) {
+    next_arg_id_ = -1;
+  }
+  FMT_CONSTEXPR void check_dynamic_spec(int arg_id);
+};
+
+FMT_EXPORT
+using format_parse_context = basic_format_parse_context<char>;
+
+namespace detail {
+// A parse context with extra data used only in compile-time checks.
+template <typename Char>
+class compile_parse_context : public basic_format_parse_context<Char> {
+ private:
+  int num_args_;
+  const type* types_;
+  using base = basic_format_parse_context<Char>;
+
+ public:
+  explicit FMT_CONSTEXPR compile_parse_context(
+      basic_string_view<Char> format_str, int num_args, const type* types,
+      int next_arg_id = 0)
+      : base(format_str, next_arg_id), num_args_(num_args), types_(types) {}
+
+  constexpr auto num_args() const -> int { return num_args_; }
+  constexpr auto arg_type(int id) const -> type { return types_[id]; }
+
+  FMT_CONSTEXPR auto next_arg_id() -> int {
+    int id = base::next_arg_id();
+    if (id >= num_args_) report_error("argument not found");
+    return id;
+  }
+
+  FMT_CONSTEXPR void check_arg_id(int id) {
+    base::check_arg_id(id);
+    if (id >= num_args_) report_error("argument not found");
+  }
+  using base::check_arg_id;
+
+  FMT_CONSTEXPR void check_dynamic_spec(int arg_id) {
+    detail::ignore_unused(arg_id);
+    if (arg_id < num_args_ && types_ && !is_integral_type(types_[arg_id]))
+      report_error("width/precision is not integer");
+  }
+};
+
+/// A contiguous memory buffer with an optional growing ability. It is an
+/// internal class and shouldn't be used directly, only via `memory_buffer`.
+template <typename T> class buffer {
+ private:
+  T* ptr_;
+  size_t size_;
+  size_t capacity_;
+
+  using grow_fun = void (*)(buffer& buf, size_t capacity);
+  grow_fun grow_;
+
+ protected:
+  // Don't initialize ptr_ since it is not accessed to save a few cycles.
+  FMT_MSC_WARNING(suppress : 26495)
+  FMT_CONSTEXPR20 buffer(grow_fun grow, size_t sz) noexcept
+      : size_(sz), capacity_(sz), grow_(grow) {}
+
+  constexpr buffer(grow_fun grow, T* p = nullptr, size_t sz = 0,
+                   size_t cap = 0) noexcept
+      : ptr_(p), size_(sz), capacity_(cap), grow_(grow) {}
+
+  FMT_CONSTEXPR20 ~buffer() = default;
+  buffer(buffer&&) = default;
+
+  /// Sets the buffer data and capacity.
+  FMT_CONSTEXPR void set(T* buf_data, size_t buf_capacity) noexcept {
+    ptr_ = buf_data;
+    capacity_ = buf_capacity;
+  }
+
+ public:
+  using value_type = T;
+  using const_reference = const T&;
+
+  buffer(const buffer&) = delete;
+  void operator=(const buffer&) = delete;
+
+  auto begin() noexcept -> T* { return ptr_; }
+  auto end() noexcept -> T* { return ptr_ + size_; }
+
+  auto begin() const noexcept -> const T* { return ptr_; }
+  auto end() const noexcept -> const T* { return ptr_ + size_; }
+
+  /// Returns the size of this buffer.
+  constexpr auto size() const noexcept -> size_t { return size_; }
+
+  /// Returns the capacity of this buffer.
+  constexpr auto capacity() const noexcept -> size_t { return capacity_; }
+
+  /// Returns a pointer to the buffer data (not null-terminated).
+  FMT_CONSTEXPR auto data() noexcept -> T* { return ptr_; }
+  FMT_CONSTEXPR auto data() const noexcept -> const T* { return ptr_; }
+
+  /// Clears this buffer.
+  void clear() { size_ = 0; }
+
+  // Tries resizing the buffer to contain `count` elements. If T is a POD type
+  // the new elements may not be initialized.
+  FMT_CONSTEXPR void try_resize(size_t count) {
+    try_reserve(count);
+    size_ = count <= capacity_ ? count : capacity_;
+  }
+
+  // Tries increasing the buffer capacity to `new_capacity`. It can increase the
+  // capacity by a smaller amount than requested but guarantees there is space
+  // for at least one additional element either by increasing the capacity or by
+  // flushing the buffer if it is full.
+  FMT_CONSTEXPR void try_reserve(size_t new_capacity) {
+    if (new_capacity > capacity_) grow_(*this, new_capacity);
+  }
+
+  FMT_CONSTEXPR void push_back(const T& value) {
+    try_reserve(size_ + 1);
+    ptr_[size_++] = value;
+  }
+
+  /// Appends data to the end of the buffer.
+  template <typename U> void append(const U* begin, const U* end) {
+    while (begin != end) {
+      auto count = to_unsigned(end - begin);
+      try_reserve(size_ + count);
+      auto free_cap = capacity_ - size_;
+      if (free_cap < count) count = free_cap;
+      // A loop is faster than memcpy on small sizes.
+      T* out = ptr_ + size_;
+      for (size_t i = 0; i < count; ++i) out[i] = begin[i];
+      size_ += count;
+      begin += count;
+    }
+  }
+
+  template <typename Idx> FMT_CONSTEXPR auto operator[](Idx index) -> T& {
+    return ptr_[index];
+  }
+  template <typename Idx>
+  FMT_CONSTEXPR auto operator[](Idx index) const -> const T& {
+    return ptr_[index];
+  }
+};
+
+struct buffer_traits {
+  explicit buffer_traits(size_t) {}
+  auto count() const -> size_t { return 0; }
+  auto limit(size_t size) -> size_t { return size; }
+};
+
+class fixed_buffer_traits {
+ private:
+  size_t count_ = 0;
+  size_t limit_;
+
+ public:
+  explicit fixed_buffer_traits(size_t limit) : limit_(limit) {}
+  auto count() const -> size_t { return count_; }
+  auto limit(size_t size) -> size_t {
+    size_t n = limit_ > count_ ? limit_ - count_ : 0;
+    count_ += size;
+    return size < n ? size : n;
+  }
+};
+
+// A buffer that writes to an output iterator when flushed.
+template <typename OutputIt, typename T, typename Traits = buffer_traits>
+class iterator_buffer : public Traits, public buffer<T> {
+ private:
+  OutputIt out_;
+  enum { buffer_size = 256 };
+  T data_[buffer_size];
+
+  static FMT_CONSTEXPR void grow(buffer<T>& buf, size_t) {
+    if (buf.size() == buffer_size) static_cast<iterator_buffer&>(buf).flush();
+  }
+
+  void flush() {
+    auto size = this->size();
+    this->clear();
+    const T* begin = data_;
+    const T* end = begin + this->limit(size);
+    while (begin != end) *out_++ = *begin++;
+  }
+
+ public:
+  explicit iterator_buffer(OutputIt out, size_t n = buffer_size)
+      : Traits(n), buffer<T>(grow, data_, 0, buffer_size), out_(out) {}
+  iterator_buffer(iterator_buffer&& other) noexcept
+      : Traits(other),
+        buffer<T>(grow, data_, 0, buffer_size),
+        out_(other.out_) {}
+  ~iterator_buffer() {
+    // Don't crash if flush fails during unwinding.
+    FMT_TRY { flush(); }
+    FMT_CATCH(...) {}
+  }
+
+  auto out() -> OutputIt {
+    flush();
+    return out_;
+  }
+  auto count() const -> size_t { return Traits::count() + this->size(); }
+};
+
+template <typename T>
+class iterator_buffer<T*, T, fixed_buffer_traits> : public fixed_buffer_traits,
+                                                    public buffer<T> {
+ private:
+  T* out_;
+  enum { buffer_size = 256 };
+  T data_[buffer_size];
+
+  static FMT_CONSTEXPR void grow(buffer<T>& buf, size_t) {
+    if (buf.size() == buf.capacity())
+      static_cast<iterator_buffer&>(buf).flush();
+  }
+
+  void flush() {
+    size_t n = this->limit(this->size());
+    if (this->data() == out_) {
+      out_ += n;
+      this->set(data_, buffer_size);
+    }
+    this->clear();
+  }
+
+ public:
+  explicit iterator_buffer(T* out, size_t n = buffer_size)
+      : fixed_buffer_traits(n), buffer<T>(grow, out, 0, n), out_(out) {}
+  iterator_buffer(iterator_buffer&& other) noexcept
+      : fixed_buffer_traits(other),
+        buffer<T>(static_cast<iterator_buffer&&>(other)),
+        out_(other.out_) {
+    if (this->data() != out_) {
+      this->set(data_, buffer_size);
+      this->clear();
+    }
+  }
+  ~iterator_buffer() { flush(); }
+
+  auto out() -> T* {
+    flush();
+    return out_;
+  }
+  auto count() const -> size_t {
+    return fixed_buffer_traits::count() + this->size();
+  }
+};
+
+template <typename T> class iterator_buffer<T*, T> : public buffer<T> {
+ public:
+  explicit iterator_buffer(T* out, size_t = 0)
+      : buffer<T>([](buffer<T>&, size_t) {}, out, 0, ~size_t()) {}
+
+  auto out() -> T* { return &*this->end(); }
+};
+
+// A buffer that writes to a container with the contiguous storage.
+template <typename OutputIt>
+class iterator_buffer<
+    OutputIt,
+    enable_if_t<detail::is_back_insert_iterator<OutputIt>::value &&
+                    is_contiguous<typename OutputIt::container_type>::value,
+                typename OutputIt::container_type::value_type>>
+    : public buffer<typename OutputIt::container_type::value_type> {
+ private:
+  using container_type = typename OutputIt::container_type;
+  using value_type = typename container_type::value_type;
+  container_type& container_;
+
+  static FMT_CONSTEXPR void grow(buffer<value_type>& buf, size_t capacity) {
+    auto& self = static_cast<iterator_buffer&>(buf);
+    self.container_.resize(capacity);
+    self.set(&self.container_[0], capacity);
+  }
+
+ public:
+  explicit iterator_buffer(container_type& c)
+      : buffer<value_type>(grow, c.size()), container_(c) {}
+  explicit iterator_buffer(OutputIt out, size_t = 0)
+      : iterator_buffer(get_container(out)) {}
+
+  auto out() -> OutputIt { return back_inserter(container_); }
+};
+
+// A buffer that counts the number of code units written discarding the output.
+template <typename T = char> class counting_buffer : public buffer<T> {
+ private:
+  enum { buffer_size = 256 };
+  T data_[buffer_size];
+  size_t count_ = 0;
+
+  static FMT_CONSTEXPR void grow(buffer<T>& buf, size_t) {
+    if (buf.size() != buffer_size) return;
+    static_cast<counting_buffer&>(buf).count_ += buf.size();
+    buf.clear();
+  }
+
+ public:
+  counting_buffer() : buffer<T>(grow, data_, 0, buffer_size) {}
+
+  auto count() -> size_t { return count_ + this->size(); }
+};
+}  // namespace detail
+
+template <typename Char>
+FMT_CONSTEXPR void basic_format_parse_context<Char>::do_check_arg_id(int id) {
+  // Argument id is only checked at compile-time during parsing because
+  // formatting has its own validation.
+  if (detail::is_constant_evaluated() &&
+      (!FMT_GCC_VERSION || FMT_GCC_VERSION >= 1200)) {
+    using context = detail::compile_parse_context<Char>;
+    if (id >= static_cast<context*>(this)->num_args())
+      report_error("argument not found");
+  }
+}
+
+template <typename Char>
+FMT_CONSTEXPR void basic_format_parse_context<Char>::check_dynamic_spec(
+    int arg_id) {
+  if (detail::is_constant_evaluated() &&
+      (!FMT_GCC_VERSION || FMT_GCC_VERSION >= 1200)) {
+    using context = detail::compile_parse_context<Char>;
+    static_cast<context*>(this)->check_dynamic_spec(arg_id);
+  }
+}
+
+FMT_EXPORT template <typename Context> class basic_format_arg;
+FMT_EXPORT template <typename Context> class basic_format_args;
+FMT_EXPORT template <typename Context> class dynamic_format_arg_store;
+
+// A formatter for objects of type T.
+FMT_EXPORT
+template <typename T, typename Char = char, typename Enable = void>
+struct formatter {
+  // A deleted default constructor indicates a disabled formatter.
+  formatter() = delete;
+};
+
+// Specifies if T has an enabled formatter specialization. A type can be
+// formattable even if it doesn't have a formatter e.g. via a conversion.
+template <typename T, typename Context>
+using has_formatter =
+    std::is_constructible<typename Context::template formatter_type<T>>;
+
+// An output iterator that appends to a buffer. It is used instead of
+// back_insert_iterator to reduce symbol sizes and avoid <iterator> dependency.
+template <typename T> class basic_appender {
+ private:
+  detail::buffer<T>* buffer_;
+
+  friend auto get_container(basic_appender app) -> detail::buffer<T>& {
+    return *app.buffer_;
+  }
+
+ public:
+  using iterator_category = int;
+  using value_type = T;
+  using difference_type = ptrdiff_t;
+  using pointer = T*;
+  using reference = T&;
+  using container_type = detail::buffer<T>;
+  FMT_UNCHECKED_ITERATOR(basic_appender);
+
+  FMT_CONSTEXPR basic_appender(detail::buffer<T>& buf) : buffer_(&buf) {}
+
+  auto operator=(T c) -> basic_appender& {
+    buffer_->push_back(c);
+    return *this;
+  }
+  auto operator*() -> basic_appender& { return *this; }
+  auto operator++() -> basic_appender& { return *this; }
+  auto operator++(int) -> basic_appender { return *this; }
+};
+
+using appender = basic_appender<char>;
+
+namespace detail {
+template <typename T>
+struct is_back_insert_iterator<basic_appender<T>> : std::true_type {};
+
+template <typename T, typename Enable = void>
+struct locking : std::true_type {};
+template <typename T>
+struct locking<T, void_t<typename formatter<remove_cvref_t<T>>::nonlocking>>
+    : std::false_type {};
+
+template <typename T = int> FMT_CONSTEXPR inline auto is_locking() -> bool {
+  return locking<T>::value;
+}
+template <typename T1, typename T2, typename... Tail>
+FMT_CONSTEXPR inline auto is_locking() -> bool {
+  return locking<T1>::value || is_locking<T2, Tail...>();
+}
+
+// An optimized version of std::copy with the output value type (T).
+template <typename T, typename InputIt, typename OutputIt,
+          FMT_ENABLE_IF(is_back_insert_iterator<OutputIt>::value)>
+auto copy(InputIt begin, InputIt end, OutputIt out) -> OutputIt {
+  get_container(out).append(begin, end);
+  return out;
+}
+
+template <typename T, typename InputIt, typename OutputIt,
+          FMT_ENABLE_IF(!is_back_insert_iterator<OutputIt>::value)>
+FMT_CONSTEXPR auto copy(InputIt begin, InputIt end, OutputIt out) -> OutputIt {
+  while (begin != end) *out++ = static_cast<T>(*begin++);
+  return out;
+}
+
+template <typename T, typename V, typename OutputIt>
+FMT_CONSTEXPR auto copy(basic_string_view<V> s, OutputIt out) -> OutputIt {
+  return copy<T>(s.begin(), s.end(), out);
+}
+
+template <typename Context, typename T>
+constexpr auto has_const_formatter_impl(T*)
+    -> decltype(typename Context::template formatter_type<T>().format(
+                    std::declval<const T&>(), std::declval<Context&>()),
+                true) {
+  return true;
+}
+template <typename Context>
+constexpr auto has_const_formatter_impl(...) -> bool {
+  return false;
+}
+template <typename T, typename Context>
+constexpr auto has_const_formatter() -> bool {
+  return has_const_formatter_impl<Context>(static_cast<T*>(nullptr));
+}
+
+template <typename It, typename Enable = std::true_type>
+struct is_buffer_appender : std::false_type {};
+template <typename It>
+struct is_buffer_appender<
+    It, bool_constant<
+            is_back_insert_iterator<It>::value &&
+            std::is_base_of<buffer<typename It::container_type::value_type>,
+                            typename It::container_type>::value>>
+    : std::true_type {};
+
+// Maps an output iterator to a buffer.
+template <typename T, typename OutputIt,
+          FMT_ENABLE_IF(!is_buffer_appender<OutputIt>::value)>
+auto get_buffer(OutputIt out) -> iterator_buffer<OutputIt, T> {
+  return iterator_buffer<OutputIt, T>(out);
+}
+template <typename T, typename OutputIt,
+          FMT_ENABLE_IF(is_buffer_appender<OutputIt>::value)>
+auto get_buffer(OutputIt out) -> buffer<T>& {
+  return get_container(out);
+}
+
+template <typename Buf, typename OutputIt>
+auto get_iterator(Buf& buf, OutputIt) -> decltype(buf.out()) {
+  return buf.out();
+}
+template <typename T, typename OutputIt>
+auto get_iterator(buffer<T>&, OutputIt out) -> OutputIt {
+  return out;
+}
+
+struct view {};
+
+template <typename Char, typename T> struct named_arg : view {
+  const Char* name;
+  const T& value;
+  named_arg(const Char* n, const T& v) : name(n), value(v) {}
+};
+
+template <typename Char> struct named_arg_info {
+  const Char* name;
+  int id;
+};
+
+template <typename T> struct is_named_arg : std::false_type {};
+template <typename T> struct is_statically_named_arg : std::false_type {};
+
+template <typename T, typename Char>
+struct is_named_arg<named_arg<Char, T>> : std::true_type {};
+
+template <bool B = false> constexpr auto count() -> size_t { return B ? 1 : 0; }
+template <bool B1, bool B2, bool... Tail> constexpr auto count() -> size_t {
+  return (B1 ? 1 : 0) + count<B2, Tail...>();
+}
+
+template <typename... Args> constexpr auto count_named_args() -> size_t {
+  return count<is_named_arg<Args>::value...>();
+}
+
+template <typename... Args>
+constexpr auto count_statically_named_args() -> size_t {
+  return count<is_statically_named_arg<Args>::value...>();
+}
+
+struct unformattable {};
+struct unformattable_char : unformattable {};
+struct unformattable_pointer : unformattable {};
+
+template <typename Char> struct string_value {
+  const Char* data;
+  size_t size;
+};
+
+template <typename Char> struct named_arg_value {
+  const named_arg_info<Char>* data;
+  size_t size;
+};
+
+template <typename Context> struct custom_value {
+  using parse_context = typename Context::parse_context_type;
+  void* value;
+  void (*format)(void* arg, parse_context& parse_ctx, Context& ctx);
+};
+
+// A formatting argument value.
+template <typename Context> class value {
+ public:
+  using char_type = typename Context::char_type;
+
+  union {
+    monostate no_value;
+    int int_value;
+    unsigned uint_value;
+    long long long_long_value;
+    unsigned long long ulong_long_value;
+    int128_opt int128_value;
+    uint128_opt uint128_value;
+    bool bool_value;
+    char_type char_value;
+    float float_value;
+    double double_value;
+    long double long_double_value;
+    const void* pointer;
+    string_value<char_type> string;
+    custom_value<Context> custom;
+    named_arg_value<char_type> named_args;
+  };
+
+  constexpr FMT_ALWAYS_INLINE value() : no_value() {}
+  constexpr FMT_ALWAYS_INLINE value(int val) : int_value(val) {}
+  constexpr FMT_ALWAYS_INLINE value(unsigned val) : uint_value(val) {}
+  constexpr FMT_ALWAYS_INLINE value(long long val) : long_long_value(val) {}
+  constexpr FMT_ALWAYS_INLINE value(unsigned long long val)
+      : ulong_long_value(val) {}
+  FMT_ALWAYS_INLINE value(int128_opt val) : int128_value(val) {}
+  FMT_ALWAYS_INLINE value(uint128_opt val) : uint128_value(val) {}
+  constexpr FMT_ALWAYS_INLINE value(float val) : float_value(val) {}
+  constexpr FMT_ALWAYS_INLINE value(double val) : double_value(val) {}
+  FMT_ALWAYS_INLINE value(long double val) : long_double_value(val) {}
+  constexpr FMT_ALWAYS_INLINE value(bool val) : bool_value(val) {}
+  constexpr FMT_ALWAYS_INLINE value(char_type val) : char_value(val) {}
+  FMT_CONSTEXPR FMT_ALWAYS_INLINE value(const char_type* val) {
+    string.data = val;
+    if (is_constant_evaluated()) string.size = {};
+  }
+  FMT_CONSTEXPR FMT_ALWAYS_INLINE value(basic_string_view<char_type> val) {
+    string.data = val.data();
+    string.size = val.size();
+  }
+  FMT_ALWAYS_INLINE value(const void* val) : pointer(val) {}
+  FMT_ALWAYS_INLINE value(const named_arg_info<char_type>* args, size_t size)
+      : named_args{args, size} {}
+
+  template <typename T> FMT_CONSTEXPR20 FMT_ALWAYS_INLINE value(T& val) {
+    using value_type = remove_const_t<T>;
+    // T may overload operator& e.g. std::vector<bool>::reference in libc++.
+#if defined(__cpp_if_constexpr)
+    if constexpr (std::is_same<decltype(&val), T*>::value)
+      custom.value = const_cast<value_type*>(&val);
+#endif
+    if (!is_constant_evaluated())
+      custom.value = const_cast<char*>(&reinterpret_cast<const char&>(val));
+    // Get the formatter type through the context to allow different contexts
+    // have different extension points, e.g. `formatter<T>` for `format` and
+    // `printf_formatter<T>` for `printf`.
+    custom.format = format_custom_arg<
+        value_type, typename Context::template formatter_type<value_type>>;
+  }
+  value(unformattable);
+  value(unformattable_char);
+  value(unformattable_pointer);
+
+ private:
+  // Formats an argument of a custom type, such as a user-defined class.
+  template <typename T, typename Formatter>
+  static void format_custom_arg(void* arg,
+                                typename Context::parse_context_type& parse_ctx,
+                                Context& ctx) {
+    auto f = Formatter();
+    parse_ctx.advance_to(f.parse(parse_ctx));
+    using qualified_type =
+        conditional_t<has_const_formatter<T, Context>(), const T, T>;
+    // format must be const for compatibility with std::format and compilation.
+    const auto& cf = f;
+    ctx.advance_to(cf.format(*static_cast<qualified_type*>(arg), ctx));
+  }
+};
+
+// To minimize the number of types we need to deal with, long is translated
+// either to int or to long long depending on its size.
+enum { long_short = sizeof(long) == sizeof(int) };
+using long_type = conditional_t<long_short, int, long long>;
+using ulong_type = conditional_t<long_short, unsigned, unsigned long long>;
+
+template <typename T> struct format_as_result {
+  template <typename U,
+            FMT_ENABLE_IF(std::is_enum<U>::value || std::is_class<U>::value)>
+  static auto map(U*) -> remove_cvref_t<decltype(format_as(std::declval<U>()))>;
+  static auto map(...) -> void;
+
+  using type = decltype(map(static_cast<T*>(nullptr)));
+};
+template <typename T> using format_as_t = typename format_as_result<T>::type;
+
+template <typename T>
+struct has_format_as
+    : bool_constant<!std::is_same<format_as_t<T>, void>::value> {};
+
+#define FMT_MAP_API FMT_CONSTEXPR FMT_ALWAYS_INLINE
+
+// Maps formatting arguments to core types.
+// arg_mapper reports errors by returning unformattable instead of using
+// static_assert because it's used in the is_formattable trait.
+template <typename Context> struct arg_mapper {
+  using char_type = typename Context::char_type;
+
+  FMT_MAP_API auto map(signed char val) -> int { return val; }
+  FMT_MAP_API auto map(unsigned char val) -> unsigned { return val; }
+  FMT_MAP_API auto map(short val) -> int { return val; }
+  FMT_MAP_API auto map(unsigned short val) -> unsigned { return val; }
+  FMT_MAP_API auto map(int val) -> int { return val; }
+  FMT_MAP_API auto map(unsigned val) -> unsigned { return val; }
+  FMT_MAP_API auto map(long val) -> long_type { return val; }
+  FMT_MAP_API auto map(unsigned long val) -> ulong_type { return val; }
+  FMT_MAP_API auto map(long long val) -> long long { return val; }
+  FMT_MAP_API auto map(unsigned long long val) -> unsigned long long {
+    return val;
+  }
+  FMT_MAP_API auto map(int128_opt val) -> int128_opt { return val; }
+  FMT_MAP_API auto map(uint128_opt val) -> uint128_opt { return val; }
+  FMT_MAP_API auto map(bool val) -> bool { return val; }
+
+  template <typename T, FMT_ENABLE_IF(std::is_same<T, char>::value ||
+                                      std::is_same<T, char_type>::value)>
+  FMT_MAP_API auto map(T val) -> char_type {
+    return val;
+  }
+  template <typename T, enable_if_t<(std::is_same<T, wchar_t>::value ||
+#ifdef __cpp_char8_t
+                                     std::is_same<T, char8_t>::value ||
+#endif
+                                     std::is_same<T, char16_t>::value ||
+                                     std::is_same<T, char32_t>::value) &&
+                                        !std::is_same<T, char_type>::value,
+                                    int> = 0>
+  FMT_MAP_API auto map(T) -> unformattable_char {
+    return {};
+  }
+
+  FMT_MAP_API auto map(float val) -> float { return val; }
+  FMT_MAP_API auto map(double val) -> double { return val; }
+  FMT_MAP_API auto map(long double val) -> long double { return val; }
+
+  FMT_MAP_API auto map(char_type* val) -> const char_type* { return val; }
+  FMT_MAP_API auto map(const char_type* val) -> const char_type* { return val; }
+  template <typename T, typename Char = char_t<T>,
+            FMT_ENABLE_IF(std::is_same<Char, char_type>::value &&
+                          !std::is_pointer<T>::value)>
+  FMT_MAP_API auto map(const T& val) -> basic_string_view<Char> {
+    return to_string_view(val);
+  }
+  template <typename T, typename Char = char_t<T>,
+            FMT_ENABLE_IF(!std::is_same<Char, char_type>::value &&
+                          !std::is_pointer<T>::value)>
+  FMT_MAP_API auto map(const T&) -> unformattable_char {
+    return {};
+  }
+
+  FMT_MAP_API auto map(void* val) -> const void* { return val; }
+  FMT_MAP_API auto map(const void* val) -> const void* { return val; }
+  FMT_MAP_API auto map(volatile void* val) -> const void* {
+    return const_cast<const void*>(val);
+  }
+  FMT_MAP_API auto map(const volatile void* val) -> const void* {
+    return const_cast<const void*>(val);
+  }
+  FMT_MAP_API auto map(std::nullptr_t val) -> const void* { return val; }
+
+  // Use SFINAE instead of a const T* parameter to avoid a conflict with the
+  // array overload.
+  template <
+      typename T,
+      FMT_ENABLE_IF(
+          std::is_pointer<T>::value || std::is_member_pointer<T>::value ||
+          std::is_function<typename std::remove_pointer<T>::type>::value ||
+          (std::is_array<T>::value &&
+           !std::is_convertible<T, const char_type*>::value))>
+  FMT_CONSTEXPR auto map(const T&) -> unformattable_pointer {
+    return {};
+  }
+
+  template <typename T, std::size_t N,
+            FMT_ENABLE_IF(!std::is_same<T, wchar_t>::value)>
+  FMT_MAP_API auto map(const T (&values)[N]) -> const T (&)[N] {
+    return values;
+  }
+
+  // Only map owning types because mapping views can be unsafe.
+  template <typename T, typename U = format_as_t<T>,
+            FMT_ENABLE_IF(std::is_arithmetic<U>::value)>
+  FMT_MAP_API auto map(const T& val) -> decltype(FMT_DECLTYPE_THIS map(U())) {
+    return map(format_as(val));
+  }
+
+  template <typename T, typename U = remove_const_t<T>>
+  struct formattable : bool_constant<has_const_formatter<U, Context>() ||
+                                     (has_formatter<U, Context>::value &&
+                                      !std::is_const<T>::value)> {};
+
+  template <typename T, FMT_ENABLE_IF(formattable<T>::value)>
+  FMT_MAP_API auto do_map(T& val) -> T& {
+    return val;
+  }
+  template <typename T, FMT_ENABLE_IF(!formattable<T>::value)>
+  FMT_MAP_API auto do_map(T&) -> unformattable {
+    return {};
+  }
+
+  // is_fundamental is used to allow formatters for extended FP types.
+  template <typename T, typename U = remove_const_t<T>,
+            FMT_ENABLE_IF(
+                (std::is_class<U>::value || std::is_enum<U>::value ||
+                 std::is_union<U>::value || std::is_fundamental<U>::value) &&
+                !has_to_string_view<U>::value && !is_char<U>::value &&
+                !is_named_arg<U>::value && !std::is_integral<U>::value &&
+                !std::is_arithmetic<format_as_t<U>>::value)>
+  FMT_MAP_API auto map(T& val) -> decltype(FMT_DECLTYPE_THIS do_map(val)) {
+    return do_map(val);
+  }
+
+  template <typename T, FMT_ENABLE_IF(is_named_arg<T>::value)>
+  FMT_MAP_API auto map(const T& named_arg)
+      -> decltype(FMT_DECLTYPE_THIS map(named_arg.value)) {
+    return map(named_arg.value);
+  }
+
+  auto map(...) -> unformattable { return {}; }
+};
+
+// A type constant after applying arg_mapper<Context>.
+template <typename T, typename Context>
+using mapped_type_constant =
+    type_constant<decltype(arg_mapper<Context>().map(std::declval<const T&>())),
+                  typename Context::char_type>;
+
+enum { packed_arg_bits = 4 };
+// Maximum number of arguments with packed types.
+enum { max_packed_args = 62 / packed_arg_bits };
+enum : unsigned long long { is_unpacked_bit = 1ULL << 63 };
+enum : unsigned long long { has_named_args_bit = 1ULL << 62 };
+
+template <typename It, typename T, typename Enable = void>
+struct is_output_iterator : std::false_type {};
+
+template <> struct is_output_iterator<appender, char> : std::true_type {};
+
+template <typename It, typename T>
+struct is_output_iterator<
+    It, T, void_t<decltype(*std::declval<It&>()++ = std::declval<T>())>>
+    : std::true_type {};
+
+// A type-erased reference to an std::locale to avoid a heavy <locale> include.
+class locale_ref {
+ private:
+  const void* locale_;  // A type-erased pointer to std::locale.
+
+ public:
+  constexpr locale_ref() : locale_(nullptr) {}
+  template <typename Locale> explicit locale_ref(const Locale& loc);
+
+  explicit operator bool() const noexcept { return locale_ != nullptr; }
+
+  template <typename Locale> auto get() const -> Locale;
+};
+
+template <typename> constexpr auto encode_types() -> unsigned long long {
+  return 0;
+}
+
+template <typename Context, typename Arg, typename... Args>
+constexpr auto encode_types() -> unsigned long long {
+  return static_cast<unsigned>(mapped_type_constant<Arg, Context>::value) |
+         (encode_types<Context, Args...>() << packed_arg_bits);
+}
+
+template <typename Context, typename... T, size_t NUM_ARGS = sizeof...(T)>
+constexpr unsigned long long make_descriptor() {
+  return NUM_ARGS <= max_packed_args ? encode_types<Context, T...>()
+                                     : is_unpacked_bit | NUM_ARGS;
+}
+
+// This type is intentionally undefined, only used for errors.
+template <typename T, typename Char>
+#if FMT_CLANG_VERSION && FMT_CLANG_VERSION <= 1500
+// https://github.com/fmtlib/fmt/issues/3796
+struct type_is_unformattable_for {
+};
+#else
+struct type_is_unformattable_for;
+#endif
+
+template <bool PACKED, typename Context, typename T, FMT_ENABLE_IF(PACKED)>
+FMT_CONSTEXPR auto make_arg(T& val) -> value<Context> {
+  using arg_type = remove_cvref_t<decltype(arg_mapper<Context>().map(val))>;
+
+  // Use enum instead of constexpr because the latter may generate code.
+  enum {
+    formattable_char = !std::is_same<arg_type, unformattable_char>::value
+  };
+  static_assert(formattable_char, "Mixing character types is disallowed.");
+
+  // Formatting of arbitrary pointers is disallowed. If you want to format a
+  // pointer cast it to `void*` or `const void*`. In particular, this forbids
+  // formatting of `[const] volatile char*` printed as bool by iostreams.
+  enum {
+    formattable_pointer = !std::is_same<arg_type, unformattable_pointer>::value
+  };
+  static_assert(formattable_pointer,
+                "Formatting of non-void pointers is disallowed.");
+
+  enum { formattable = !std::is_same<arg_type, unformattable>::value };
+#if defined(__cpp_if_constexpr)
+  if constexpr (!formattable)
+    type_is_unformattable_for<T, typename Context::char_type> _;
+#endif
+  static_assert(
+      formattable,
+      "Cannot format an argument. To make type T formattable provide a "
+      "formatter<T> specialization: https://fmt.dev/latest/api.html#udt");
+  return {arg_mapper<Context>().map(val)};
+}
+
+template <typename Context, typename T>
+FMT_CONSTEXPR auto make_arg(T& val) -> basic_format_arg<Context> {
+  auto arg = basic_format_arg<Context>();
+  arg.type_ = mapped_type_constant<T, Context>::value;
+  arg.value_ = make_arg<true, Context>(val);
+  return arg;
+}
+
+template <bool PACKED, typename Context, typename T, FMT_ENABLE_IF(!PACKED)>
+FMT_CONSTEXPR inline auto make_arg(T& val) -> basic_format_arg<Context> {
+  return make_arg<Context>(val);
+}
+
+template <typename Context, size_t NUM_ARGS>
+using arg_t = conditional_t<NUM_ARGS <= max_packed_args, value<Context>,
+                            basic_format_arg<Context>>;
+
+template <typename Char, typename T, FMT_ENABLE_IF(!is_named_arg<T>::value)>
+void init_named_arg(named_arg_info<Char>*, int& arg_index, int&, const T&) {
+  ++arg_index;
+}
+template <typename Char, typename T, FMT_ENABLE_IF(is_named_arg<T>::value)>
+void init_named_arg(named_arg_info<Char>* named_args, int& arg_index,
+                    int& named_arg_index, const T& arg) {
+  named_args[named_arg_index++] = {arg.name, arg_index++};
+}
+
+// An array of references to arguments. It can be implicitly converted to
+// `fmt::basic_format_args` for passing into type-erased formatting functions
+// such as `fmt::vformat`.
+template <typename Context, size_t NUM_ARGS, size_t NUM_NAMED_ARGS,
+          unsigned long long DESC>
+struct format_arg_store {
+  // args_[0].named_args points to named_args to avoid bloating format_args.
+  // +1 to workaround a bug in gcc 7.5 that causes duplicated-branches warning.
+  static constexpr size_t ARGS_ARR_SIZE = 1 + (NUM_ARGS != 0 ? NUM_ARGS : +1);
+
+  arg_t<Context, NUM_ARGS> args[ARGS_ARR_SIZE];
+  named_arg_info<typename Context::char_type> named_args[NUM_NAMED_ARGS];
+
+  template <typename... T>
+  FMT_MAP_API format_arg_store(T&... values)
+      : args{{named_args, NUM_NAMED_ARGS},
+             make_arg<NUM_ARGS <= max_packed_args, Context>(values)...} {
+    using dummy = int[];
+    int arg_index = 0, named_arg_index = 0;
+    (void)dummy{
+        0,
+        (init_named_arg(named_args, arg_index, named_arg_index, values), 0)...};
+  }
+
+  format_arg_store(format_arg_store&& rhs) {
+    args[0] = {named_args, NUM_NAMED_ARGS};
+    for (size_t i = 1; i < ARGS_ARR_SIZE; ++i) args[i] = rhs.args[i];
+    for (size_t i = 0; i < NUM_NAMED_ARGS; ++i)
+      named_args[i] = rhs.named_args[i];
+  }
+
+  format_arg_store(const format_arg_store& rhs) = delete;
+  format_arg_store& operator=(const format_arg_store& rhs) = delete;
+  format_arg_store& operator=(format_arg_store&& rhs) = delete;
+};
+
+// A specialization of format_arg_store without named arguments.
+// It is a plain struct to reduce binary size in debug mode.
+template <typename Context, size_t NUM_ARGS, unsigned long long DESC>
+struct format_arg_store<Context, NUM_ARGS, 0, DESC> {
+  // +1 to workaround a bug in gcc 7.5 that causes duplicated-branches warning.
+  arg_t<Context, NUM_ARGS> args[NUM_ARGS != 0 ? NUM_ARGS : +1];
+};
+
+}  // namespace detail
+FMT_BEGIN_EXPORT
+
+// A formatting argument. Context is a template parameter for the compiled API
+// where output can be unbuffered.
+template <typename Context> class basic_format_arg {
+ private:
+  detail::value<Context> value_;
+  detail::type type_;
+
+  template <typename ContextType, typename T>
+  friend FMT_CONSTEXPR auto detail::make_arg(T& value)
+      -> basic_format_arg<ContextType>;
+
+  friend class basic_format_args<Context>;
+  friend class dynamic_format_arg_store<Context>;
+
+  using char_type = typename Context::char_type;
+
+  template <typename, size_t, size_t, unsigned long long>
+  friend struct detail::format_arg_store;
+
+  basic_format_arg(const detail::named_arg_info<char_type>* args, size_t size)
+      : value_(args, size) {}
+
+ public:
+  class handle {
+   public:
+    explicit handle(detail::custom_value<Context> custom) : custom_(custom) {}
+
+    void format(typename Context::parse_context_type& parse_ctx,
+                Context& ctx) const {
+      custom_.format(custom_.value, parse_ctx, ctx);
+    }
+
+   private:
+    detail::custom_value<Context> custom_;
+  };
+
+  constexpr basic_format_arg() : type_(detail::type::none_type) {}
+
+  constexpr explicit operator bool() const noexcept {
+    return type_ != detail::type::none_type;
+  }
+
+  auto type() const -> detail::type { return type_; }
+
+  auto is_integral() const -> bool { return detail::is_integral_type(type_); }
+  auto is_arithmetic() const -> bool {
+    return detail::is_arithmetic_type(type_);
+  }
+
+  /**
+   * Visits an argument dispatching to the appropriate visit method based on
+   * the argument type. For example, if the argument type is `double` then
+   * `vis(value)` will be called with the value of type `double`.
+   */
+  template <typename Visitor>
+  FMT_CONSTEXPR FMT_INLINE auto visit(Visitor&& vis) const -> decltype(vis(0)) {
+    switch (type_) {
+    case detail::type::none_type:
+      break;
+    case detail::type::int_type:
+      return vis(value_.int_value);
+    case detail::type::uint_type:
+      return vis(value_.uint_value);
+    case detail::type::long_long_type:
+      return vis(value_.long_long_value);
+    case detail::type::ulong_long_type:
+      return vis(value_.ulong_long_value);
+    case detail::type::int128_type:
+      return vis(detail::convert_for_visit(value_.int128_value));
+    case detail::type::uint128_type:
+      return vis(detail::convert_for_visit(value_.uint128_value));
+    case detail::type::bool_type:
+      return vis(value_.bool_value);
+    case detail::type::char_type:
+      return vis(value_.char_value);
+    case detail::type::float_type:
+      return vis(value_.float_value);
+    case detail::type::double_type:
+      return vis(value_.double_value);
+    case detail::type::long_double_type:
+      return vis(value_.long_double_value);
+    case detail::type::cstring_type:
+      return vis(value_.string.data);
+    case detail::type::string_type:
+      using sv = basic_string_view<typename Context::char_type>;
+      return vis(sv(value_.string.data, value_.string.size));
+    case detail::type::pointer_type:
+      return vis(value_.pointer);
+    case detail::type::custom_type:
+      return vis(typename basic_format_arg<Context>::handle(value_.custom));
+    }
+    return vis(monostate());
+  }
+
+  auto format_custom(const char_type* parse_begin,
+                     typename Context::parse_context_type& parse_ctx,
+                     Context& ctx) -> bool {
+    if (type_ != detail::type::custom_type) return false;
+    parse_ctx.advance_to(parse_begin);
+    value_.custom.format(value_.custom.value, parse_ctx, ctx);
+    return true;
+  }
+};
+
+template <typename Visitor, typename Context>
+FMT_DEPRECATED FMT_CONSTEXPR auto visit_format_arg(
+    Visitor&& vis, const basic_format_arg<Context>& arg) -> decltype(vis(0)) {
+  return arg.visit(static_cast<Visitor&&>(vis));
+}
+
+/**
+ * A view of a collection of formatting arguments. To avoid lifetime issues it
+ * should only be used as a parameter type in type-erased functions such as
+ * `vformat`:
+ *
+ *     void vlog(fmt::string_view fmt, fmt::format_args args);  // OK
+ *     fmt::format_args args = fmt::make_format_args();  // Dangling reference
+ */
+template <typename Context> class basic_format_args {
+ public:
+  using size_type = int;
+  using format_arg = basic_format_arg<Context>;
+
+ private:
+  // A descriptor that contains information about formatting arguments.
+  // If the number of arguments is less or equal to max_packed_args then
+  // argument types are passed in the descriptor. This reduces binary code size
+  // per formatting function call.
+  unsigned long long desc_;
+  union {
+    // If is_packed() returns true then argument values are stored in values_;
+    // otherwise they are stored in args_. This is done to improve cache
+    // locality and reduce compiled code size since storing larger objects
+    // may require more code (at least on x86-64) even if the same amount of
+    // data is actually copied to stack. It saves ~10% on the bloat test.
+    const detail::value<Context>* values_;
+    const format_arg* args_;
+  };
+
+  constexpr auto is_packed() const -> bool {
+    return (desc_ & detail::is_unpacked_bit) == 0;
+  }
+  constexpr auto has_named_args() const -> bool {
+    return (desc_ & detail::has_named_args_bit) != 0;
+  }
+
+  FMT_CONSTEXPR auto type(int index) const -> detail::type {
+    int shift = index * detail::packed_arg_bits;
+    unsigned int mask = (1 << detail::packed_arg_bits) - 1;
+    return static_cast<detail::type>((desc_ >> shift) & mask);
+  }
+
+ public:
+  constexpr basic_format_args() : desc_(0), args_(nullptr) {}
+
+  /// Constructs a `basic_format_args` object from `format_arg_store`.
+  template <size_t NUM_ARGS, size_t NUM_NAMED_ARGS, unsigned long long DESC,
+            FMT_ENABLE_IF(NUM_ARGS <= detail::max_packed_args)>
+  constexpr FMT_ALWAYS_INLINE basic_format_args(
+      const detail::format_arg_store<Context, NUM_ARGS, NUM_NAMED_ARGS, DESC>&
+          store)
+      : desc_(DESC), values_(store.args + (NUM_NAMED_ARGS != 0 ? 1 : 0)) {}
+
+  template <size_t NUM_ARGS, size_t NUM_NAMED_ARGS, unsigned long long DESC,
+            FMT_ENABLE_IF(NUM_ARGS > detail::max_packed_args)>
+  constexpr basic_format_args(
+      const detail::format_arg_store<Context, NUM_ARGS, NUM_NAMED_ARGS, DESC>&
+          store)
+      : desc_(DESC), args_(store.args + (NUM_NAMED_ARGS != 0 ? 1 : 0)) {}
+
+  /// Constructs a `basic_format_args` object from `dynamic_format_arg_store`.
+  constexpr basic_format_args(const dynamic_format_arg_store<Context>& store)
+      : desc_(store.get_types()), args_(store.data()) {}
+
+  /// Constructs a `basic_format_args` object from a dynamic list of arguments.
+  constexpr basic_format_args(const format_arg* args, int count)
+      : desc_(detail::is_unpacked_bit | detail::to_unsigned(count)),
+        args_(args) {}
+
+  /// Returns the argument with the specified id.
+  FMT_CONSTEXPR auto get(int id) const -> format_arg {
+    format_arg arg;
+    if (!is_packed()) {
+      if (id < max_size()) arg = args_[id];
+      return arg;
+    }
+    if (static_cast<unsigned>(id) >= detail::max_packed_args) return arg;
+    arg.type_ = type(id);
+    if (arg.type_ == detail::type::none_type) return arg;
+    arg.value_ = values_[id];
+    return arg;
+  }
+
+  template <typename Char>
+  auto get(basic_string_view<Char> name) const -> format_arg {
+    int id = get_id(name);
+    return id >= 0 ? get(id) : format_arg();
+  }
+
+  template <typename Char>
+  FMT_CONSTEXPR auto get_id(basic_string_view<Char> name) const -> int {
+    if (!has_named_args()) return -1;
+    const auto& named_args =
+        (is_packed() ? values_[-1] : args_[-1].value_).named_args;
+    for (size_t i = 0; i < named_args.size; ++i) {
+      if (named_args.data[i].name == name) return named_args.data[i].id;
+    }
+    return -1;
+  }
+
+  auto max_size() const -> int {
+    unsigned long long max_packed = detail::max_packed_args;
+    return static_cast<int>(is_packed() ? max_packed
+                                        : desc_ & ~detail::is_unpacked_bit);
+  }
+};
+
+// A formatting context.
+class context {
+ private:
+  appender out_;
+  basic_format_args<context> args_;
+  detail::locale_ref loc_;
+
+ public:
+  /// The character type for the output.
+  using char_type = char;
+
+  using iterator = appender;
+  using format_arg = basic_format_arg<context>;
+  using parse_context_type = basic_format_parse_context<char>;
+  template <typename T> using formatter_type = formatter<T, char>;
+
+  /// Constructs a `basic_format_context` object. References to the arguments
+  /// are stored in the object so make sure they have appropriate lifetimes.
+  FMT_CONSTEXPR context(iterator out, basic_format_args<context> ctx_args,
+                        detail::locale_ref loc = {})
+      : out_(out), args_(ctx_args), loc_(loc) {}
+  context(context&&) = default;
+  context(const context&) = delete;
+  void operator=(const context&) = delete;
+
+  FMT_CONSTEXPR auto arg(int id) const -> format_arg { return args_.get(id); }
+  auto arg(string_view name) -> format_arg { return args_.get(name); }
+  FMT_CONSTEXPR auto arg_id(string_view name) -> int {
+    return args_.get_id(name);
+  }
+  auto args() const -> const basic_format_args<context>& { return args_; }
+
+  // Returns an iterator to the beginning of the output range.
+  FMT_CONSTEXPR auto out() -> iterator { return out_; }
+
+  // Advances the begin iterator to `it`.
+  void advance_to(iterator) {}
+
+  FMT_CONSTEXPR auto locale() -> detail::locale_ref { return loc_; }
+};
+
+template <typename OutputIt, typename Char> class generic_context;
+
+// Longer aliases for C++20 compatibility.
+template <typename OutputIt, typename Char>
+using basic_format_context =
+    conditional_t<std::is_same<OutputIt, appender>::value, context,
+                  generic_context<OutputIt, Char>>;
+using format_context = context;
+
+template <typename Char>
+using buffered_context = basic_format_context<basic_appender<Char>, Char>;
+
+template <typename T, typename Char = char>
+using is_formattable = bool_constant<!std::is_base_of<
+    detail::unformattable, decltype(detail::arg_mapper<buffered_context<Char>>()
+                                        .map(std::declval<T&>()))>::value>;
+
+#if FMT_USE_CONCEPTS
+template <typename T, typename Char = char>
+concept formattable = is_formattable<remove_reference_t<T>, Char>::value;
+#endif
+
+/**
+ * Constructs an object that stores references to arguments and can be
+ * implicitly converted to `format_args`. `Context` can be omitted in which case
+ * it defaults to `format_context`. See `arg` for lifetime considerations.
+ */
+// Take arguments by lvalue references to avoid some lifetime issues, e.g.
+//   auto args = make_format_args(std::string());
+template <typename Context = format_context, typename... T,
+          size_t NUM_ARGS = sizeof...(T),
+          size_t NUM_NAMED_ARGS = detail::count_named_args<T...>(),
+          unsigned long long DESC = detail::make_descriptor<Context, T...>(),
+          FMT_ENABLE_IF(NUM_NAMED_ARGS == 0)>
+constexpr FMT_ALWAYS_INLINE auto make_format_args(T&... args)
+    -> detail::format_arg_store<Context, NUM_ARGS, 0, DESC> {
+  return {{detail::make_arg<NUM_ARGS <= detail::max_packed_args, Context>(
+      args)...}};
+}
+
+#ifndef FMT_DOC
+template <typename Context = format_context, typename... T,
+          size_t NUM_NAMED_ARGS = detail::count_named_args<T...>(),
+          unsigned long long DESC =
+              detail::make_descriptor<Context, T...>() |
+              static_cast<unsigned long long>(detail::has_named_args_bit),
+          FMT_ENABLE_IF(NUM_NAMED_ARGS != 0)>
+constexpr auto make_format_args(T&... args)
+    -> detail::format_arg_store<Context, sizeof...(T), NUM_NAMED_ARGS, DESC> {
+  return {args...};
+}
+#endif
+
+/**
+ * Returns a named argument to be used in a formatting function.
+ * It should only be used in a call to a formatting function or
+ * `dynamic_format_arg_store::push_back`.
+ *
+ * **Example**:
+ *
+ *     fmt::print("The answer is {answer}.", fmt::arg("answer", 42));
+ */
+template <typename Char, typename T>
+inline auto arg(const Char* name, const T& arg) -> detail::named_arg<Char, T> {
+  static_assert(!detail::is_named_arg<T>(), "nested named arguments");
+  return {name, arg};
+}
+FMT_END_EXPORT
+
+/// An alias for `basic_format_args<format_context>`.
+// A separate type would result in shorter symbols but break ABI compatibility
+// between clang and gcc on ARM (#1919).
+FMT_EXPORT using format_args = basic_format_args<format_context>;
+
+// We cannot use enum classes as bit fields because of a gcc bug, so we put them
+// in namespaces instead (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61414).
+// Additionally, if an underlying type is specified, older gcc incorrectly warns
+// that the type is too small. Both bugs are fixed in gcc 9.3.
+#if FMT_GCC_VERSION && FMT_GCC_VERSION < 903
+#  define FMT_ENUM_UNDERLYING_TYPE(type)
+#else
+#  define FMT_ENUM_UNDERLYING_TYPE(type) : type
+#endif
+namespace align {
+enum type FMT_ENUM_UNDERLYING_TYPE(unsigned char){none, left, right, center,
+                                                  numeric};
+}
+using align_t = align::type;
+namespace sign {
+enum type FMT_ENUM_UNDERLYING_TYPE(unsigned char){none, minus, plus, space};
+}
+using sign_t = sign::type;
+
+namespace detail {
+
+template <typename Char>
+using unsigned_char = typename conditional_t<std::is_integral<Char>::value,
+                                             std::make_unsigned<Char>,
+                                             type_identity<unsigned>>::type;
+
+// Character (code unit) type is erased to prevent template bloat.
+struct fill_t {
+ private:
+  enum { max_size = 4 };
+  char data_[max_size] = {' '};
+  unsigned char size_ = 1;
+
+ public:
+  template <typename Char>
+  FMT_CONSTEXPR void operator=(basic_string_view<Char> s) {
+    auto size = s.size();
+    size_ = static_cast<unsigned char>(size);
+    if (size == 1) {
+      unsigned uchar = static_cast<unsigned_char<Char>>(s[0]);
+      data_[0] = static_cast<char>(uchar);
+      data_[1] = static_cast<char>(uchar >> 8);
+      return;
+    }
+    FMT_ASSERT(size <= max_size, "invalid fill");
+    for (size_t i = 0; i < size; ++i) data_[i] = static_cast<char>(s[i]);
+  }
+
+  FMT_CONSTEXPR void operator=(char c) {
+    data_[0] = c;
+    size_ = 1;
+  }
+
+  constexpr auto size() const -> size_t { return size_; }
+
+  template <typename Char> constexpr auto get() const -> Char {
+    using uchar = unsigned char;
+    return static_cast<Char>(static_cast<uchar>(data_[0]) |
+                             (static_cast<uchar>(data_[1]) << 8));
+  }
+
+  template <typename Char, FMT_ENABLE_IF(std::is_same<Char, char>::value)>
+  constexpr auto data() const -> const Char* {
+    return data_;
+  }
+  template <typename Char, FMT_ENABLE_IF(!std::is_same<Char, char>::value)>
+  constexpr auto data() const -> const Char* {
+    return nullptr;
+  }
+};
+}  // namespace detail
+
+enum class presentation_type : unsigned char {
+  // Common specifiers:
+  none = 0,
+  debug = 1,   // '?'
+  string = 2,  // 's' (string, bool)
+
+  // Integral, bool and character specifiers:
+  dec = 3,  // 'd'
+  hex,      // 'x' or 'X'
+  oct,      // 'o'
+  bin,      // 'b' or 'B'
+  chr,      // 'c'
+
+  // String and pointer specifiers:
+  pointer = 3,  // 'p'
+
+  // Floating-point specifiers:
+  exp = 1,  // 'e' or 'E' (1 since there is no FP debug presentation)
+  fixed,    // 'f' or 'F'
+  general,  // 'g' or 'G'
+  hexfloat  // 'a' or 'A'
+};
+
+// Format specifiers for built-in and string types.
+struct format_specs {
+  int width;
+  int precision;
+  presentation_type type;
+  align_t align : 4;
+  sign_t sign : 3;
+  bool upper : 1;  // An uppercase version e.g. 'X' for 'x'.
+  bool alt : 1;    // Alternate form ('#').
+  bool localized : 1;
+  detail::fill_t fill;
+
+  constexpr format_specs()
+      : width(0),
+        precision(-1),
+        type(presentation_type::none),
+        align(align::none),
+        sign(sign::none),
+        upper(false),
+        alt(false),
+        localized(false) {}
+};
+
+namespace detail {
+
+enum class arg_id_kind { none, index, name };
+
+// An argument reference.
+template <typename Char> struct arg_ref {
+  FMT_CONSTEXPR arg_ref() : kind(arg_id_kind::none), val() {}
+
+  FMT_CONSTEXPR explicit arg_ref(int index)
+      : kind(arg_id_kind::index), val(index) {}
+  FMT_CONSTEXPR explicit arg_ref(basic_string_view<Char> name)
+      : kind(arg_id_kind::name), val(name) {}
+
+  FMT_CONSTEXPR auto operator=(int idx) -> arg_ref& {
+    kind = arg_id_kind::index;
+    val.index = idx;
+    return *this;
+  }
+
+  arg_id_kind kind;
+  union value {
+    FMT_CONSTEXPR value(int idx = 0) : index(idx) {}
+    FMT_CONSTEXPR value(basic_string_view<Char> n) : name(n) {}
+
+    int index;
+    basic_string_view<Char> name;
+  } val;
+};
+
+// Format specifiers with width and precision resolved at formatting rather
+// than parsing time to allow reusing the same parsed specifiers with
+// different sets of arguments (precompilation of format strings).
+template <typename Char = char> struct dynamic_format_specs : format_specs {
+  arg_ref<Char> width_ref;
+  arg_ref<Char> precision_ref;
+};
+
+// Converts a character to ASCII. Returns '\0' on conversion failure.
+template <typename Char, FMT_ENABLE_IF(std::is_integral<Char>::value)>
+constexpr auto to_ascii(Char c) -> char {
+  return c <= 0xff ? static_cast<char>(c) : '\0';
+}
+
+// Returns the number of code units in a code point or 1 on error.
+template <typename Char>
+FMT_CONSTEXPR auto code_point_length(const Char* begin) -> int {
+  if (const_check(sizeof(Char) != 1)) return 1;
+  auto c = static_cast<unsigned char>(*begin);
+  return static_cast<int>((0x3a55000000000000ull >> (2 * (c >> 3))) & 0x3) + 1;
+}
+
+// Return the result via the out param to workaround gcc bug 77539.
+template <bool IS_CONSTEXPR, typename T, typename Ptr = const T*>
+FMT_CONSTEXPR auto find(Ptr first, Ptr last, T value, Ptr& out) -> bool {
+  for (out = first; out != last; ++out) {
+    if (*out == value) return true;
+  }
+  return false;
+}
+
+template <>
+inline auto find<false, char>(const char* first, const char* last, char value,
+                              const char*& out) -> bool {
+  out =
+      static_cast<const char*>(memchr(first, value, to_unsigned(last - first)));
+  return out != nullptr;
+}
+
+// Parses the range [begin, end) as an unsigned integer. This function assumes
+// that the range is non-empty and the first character is a digit.
+template <typename Char>
+FMT_CONSTEXPR auto parse_nonnegative_int(const Char*& begin, const Char* end,
+                                         int error_value) noexcept -> int {
+  FMT_ASSERT(begin != end && '0' <= *begin && *begin <= '9', "");
+  unsigned value = 0, prev = 0;
+  auto p = begin;
+  do {
+    prev = value;
+    value = value * 10 + unsigned(*p - '0');
+    ++p;
+  } while (p != end && '0' <= *p && *p <= '9');
+  auto num_digits = p - begin;
+  begin = p;
+  int digits10 = static_cast<int>(sizeof(int) * CHAR_BIT * 3 / 10);
+  if (num_digits <= digits10) return static_cast<int>(value);
+  // Check for overflow.
+  unsigned max = INT_MAX;
+  return num_digits == digits10 + 1 &&
+                 prev * 10ull + unsigned(p[-1] - '0') <= max
+             ? static_cast<int>(value)
+             : error_value;
+}
+
+FMT_CONSTEXPR inline auto parse_align(char c) -> align_t {
+  switch (c) {
+  case '<':
+    return align::left;
+  case '>':
+    return align::right;
+  case '^':
+    return align::center;
+  }
+  return align::none;
+}
+
+template <typename Char> constexpr auto is_name_start(Char c) -> bool {
+  return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || c == '_';
+}
+
+template <typename Char, typename Handler>
+FMT_CONSTEXPR auto do_parse_arg_id(const Char* begin, const Char* end,
+                                   Handler&& handler) -> const Char* {
+  Char c = *begin;
+  if (c >= '0' && c <= '9') {
+    int index = 0;
+    if (c != '0')
+      index = parse_nonnegative_int(begin, end, INT_MAX);
+    else
+      ++begin;
+    if (begin == end || (*begin != '}' && *begin != ':'))
+      report_error("invalid format string");
+    else
+      handler.on_index(index);
+    return begin;
+  }
+  if (!is_name_start(c)) {
+    report_error("invalid format string");
+    return begin;
+  }
+  auto it = begin;
+  do {
+    ++it;
+  } while (it != end && (is_name_start(*it) || ('0' <= *it && *it <= '9')));
+  handler.on_name({begin, to_unsigned(it - begin)});
+  return it;
+}
+
+template <typename Char, typename Handler>
+FMT_CONSTEXPR auto parse_arg_id(const Char* begin, const Char* end,
+                                Handler&& handler) -> const Char* {
+  FMT_ASSERT(begin != end, "");
+  Char c = *begin;
+  if (c != '}' && c != ':') return do_parse_arg_id(begin, end, handler);
+  handler.on_auto();
+  return begin;
+}
+
+template <typename Char> struct dynamic_spec_id_handler {
+  basic_format_parse_context<Char>& ctx;
+  arg_ref<Char>& ref;
+
+  FMT_CONSTEXPR void on_auto() {
+    int id = ctx.next_arg_id();
+    ref = arg_ref<Char>(id);
+    ctx.check_dynamic_spec(id);
+  }
+  FMT_CONSTEXPR void on_index(int id) {
+    ref = arg_ref<Char>(id);
+    ctx.check_arg_id(id);
+    ctx.check_dynamic_spec(id);
+  }
+  FMT_CONSTEXPR void on_name(basic_string_view<Char> id) {
+    ref = arg_ref<Char>(id);
+    ctx.check_arg_id(id);
+  }
+};
+
+// Parses [integer | "{" [arg_id] "}"].
+template <typename Char>
+FMT_CONSTEXPR auto parse_dynamic_spec(const Char* begin, const Char* end,
+                                      int& value, arg_ref<Char>& ref,
+                                      basic_format_parse_context<Char>& ctx)
+    -> const Char* {
+  FMT_ASSERT(begin != end, "");
+  if ('0' <= *begin && *begin <= '9') {
+    int val = parse_nonnegative_int(begin, end, -1);
+    if (val != -1)
+      value = val;
+    else
+      report_error("number is too big");
+  } else if (*begin == '{') {
+    ++begin;
+    auto handler = dynamic_spec_id_handler<Char>{ctx, ref};
+    if (begin != end) begin = parse_arg_id(begin, end, handler);
+    if (begin != end && *begin == '}') return ++begin;
+    report_error("invalid format string");
+  }
+  return begin;
+}
+
+template <typename Char>
+FMT_CONSTEXPR auto parse_precision(const Char* begin, const Char* end,
+                                   int& value, arg_ref<Char>& ref,
+                                   basic_format_parse_context<Char>& ctx)
+    -> const Char* {
+  ++begin;
+  if (begin == end || *begin == '}') {
+    report_error("invalid precision");
+    return begin;
+  }
+  return parse_dynamic_spec(begin, end, value, ref, ctx);
+}
+
+enum class state { start, align, sign, hash, zero, width, precision, locale };
+
+// Parses standard format specifiers.
+template <typename Char>
+FMT_CONSTEXPR auto parse_format_specs(const Char* begin, const Char* end,
+                                      dynamic_format_specs<Char>& specs,
+                                      basic_format_parse_context<Char>& ctx,
+                                      type arg_type) -> const Char* {
+  auto c = '\0';
+  if (end - begin > 1) {
+    auto next = to_ascii(begin[1]);
+    c = parse_align(next) == align::none ? to_ascii(*begin) : '\0';
+  } else {
+    if (begin == end) return begin;
+    c = to_ascii(*begin);
+  }
+
+  struct {
+    state current_state = state::start;
+    FMT_CONSTEXPR void operator()(state s, bool valid = true) {
+      if (current_state >= s || !valid)
+        report_error("invalid format specifier");
+      current_state = s;
+    }
+  } enter_state;
+
+  using pres = presentation_type;
+  constexpr auto integral_set = sint_set | uint_set | bool_set | char_set;
+  struct {
+    const Char*& begin;
+    dynamic_format_specs<Char>& specs;
+    type arg_type;
+
+    FMT_CONSTEXPR auto operator()(pres pres_type, int set) -> const Char* {
+      if (!in(arg_type, set)) {
+        if (arg_type == type::none_type) return begin;
+        report_error("invalid format specifier");
+      }
+      specs.type = pres_type;
+      return begin + 1;
+    }
+  } parse_presentation_type{begin, specs, arg_type};
+
+  for (;;) {
+    switch (c) {
+    case '<':
+    case '>':
+    case '^':
+      enter_state(state::align);
+      specs.align = parse_align(c);
+      ++begin;
+      break;
+    case '+':
+    case '-':
+    case ' ':
+      if (arg_type == type::none_type) return begin;
+      enter_state(state::sign, in(arg_type, sint_set | float_set));
+      switch (c) {
+      case '+':
+        specs.sign = sign::plus;
+        break;
+      case '-':
+        specs.sign = sign::minus;
+        break;
+      case ' ':
+        specs.sign = sign::space;
+        break;
+      }
+      ++begin;
+      break;
+    case '#':
+      if (arg_type == type::none_type) return begin;
+      enter_state(state::hash, is_arithmetic_type(arg_type));
+      specs.alt = true;
+      ++begin;
+      break;
+    case '0':
+      enter_state(state::zero);
+      if (!is_arithmetic_type(arg_type)) {
+        if (arg_type == type::none_type) return begin;
+        report_error("format specifier requires numeric argument");
+      }
+      if (specs.align == align::none) {
+        // Ignore 0 if align is specified for compatibility with std::format.
+        specs.align = align::numeric;
+        specs.fill = '0';
+      }
+      ++begin;
+      break;
+    case '1':
+    case '2':
+    case '3':
+    case '4':
+    case '5':
+    case '6':
+    case '7':
+    case '8':
+    case '9':
+    case '{':
+      enter_state(state::width);
+      begin = parse_dynamic_spec(begin, end, specs.width, specs.width_ref, ctx);
+      break;
+    case '.':
+      if (arg_type == type::none_type) return begin;
+      enter_state(state::precision,
+                  in(arg_type, float_set | string_set | cstring_set));
+      begin = parse_precision(begin, end, specs.precision, specs.precision_ref,
+                              ctx);
+      break;
+    case 'L':
+      if (arg_type == type::none_type) return begin;
+      enter_state(state::locale, is_arithmetic_type(arg_type));
+      specs.localized = true;
+      ++begin;
+      break;
+    case 'd':
+      return parse_presentation_type(pres::dec, integral_set);
+    case 'X':
+      specs.upper = true;
+      FMT_FALLTHROUGH;
+    case 'x':
+      return parse_presentation_type(pres::hex, integral_set);
+    case 'o':
+      return parse_presentation_type(pres::oct, integral_set);
+    case 'B':
+      specs.upper = true;
+      FMT_FALLTHROUGH;
+    case 'b':
+      return parse_presentation_type(pres::bin, integral_set);
+    case 'E':
+      specs.upper = true;
+      FMT_FALLTHROUGH;
+    case 'e':
+      return parse_presentation_type(pres::exp, float_set);
+    case 'F':
+      specs.upper = true;
+      FMT_FALLTHROUGH;
+    case 'f':
+      return parse_presentation_type(pres::fixed, float_set);
+    case 'G':
+      specs.upper = true;
+      FMT_FALLTHROUGH;
+    case 'g':
+      return parse_presentation_type(pres::general, float_set);
+    case 'A':
+      specs.upper = true;
+      FMT_FALLTHROUGH;
+    case 'a':
+      return parse_presentation_type(pres::hexfloat, float_set);
+    case 'c':
+      if (arg_type == type::bool_type) report_error("invalid format specifier");
+      return parse_presentation_type(pres::chr, integral_set);
+    case 's':
+      return parse_presentation_type(pres::string,
+                                     bool_set | string_set | cstring_set);
+    case 'p':
+      return parse_presentation_type(pres::pointer, pointer_set | cstring_set);
+    case '?':
+      return parse_presentation_type(pres::debug,
+                                     char_set | string_set | cstring_set);
+    case '}':
+      return begin;
+    default: {
+      if (*begin == '}') return begin;
+      // Parse fill and alignment.
+      auto fill_end = begin + code_point_length(begin);
+      if (end - fill_end <= 0) {
+        report_error("invalid format specifier");
+        return begin;
+      }
+      if (*begin == '{') {
+        report_error("invalid fill character '{'");
+        return begin;
+      }
+      auto align = parse_align(to_ascii(*fill_end));
+      enter_state(state::align, align != align::none);
+      specs.fill =
+          basic_string_view<Char>(begin, to_unsigned(fill_end - begin));
+      specs.align = align;
+      begin = fill_end + 1;
+    }
+    }
+    if (begin == end) return begin;
+    c = to_ascii(*begin);
+  }
+}
+
+template <typename Char, typename Handler>
+FMT_CONSTEXPR auto parse_replacement_field(const Char* begin, const Char* end,
+                                           Handler&& handler) -> const Char* {
+  struct id_adapter {
+    Handler& handler;
+    int arg_id;
+
+    FMT_CONSTEXPR void on_auto() { arg_id = handler.on_arg_id(); }
+    FMT_CONSTEXPR void on_index(int id) { arg_id = handler.on_arg_id(id); }
+    FMT_CONSTEXPR void on_name(basic_string_view<Char> id) {
+      arg_id = handler.on_arg_id(id);
+    }
+  };
+
+  ++begin;
+  if (begin == end) return handler.on_error("invalid format string"), end;
+  if (*begin == '}') {
+    handler.on_replacement_field(handler.on_arg_id(), begin);
+  } else if (*begin == '{') {
+    handler.on_text(begin, begin + 1);
+  } else {
+    auto adapter = id_adapter{handler, 0};
+    begin = parse_arg_id(begin, end, adapter);
+    Char c = begin != end ? *begin : Char();
+    if (c == '}') {
+      handler.on_replacement_field(adapter.arg_id, begin);
+    } else if (c == ':') {
+      begin = handler.on_format_specs(adapter.arg_id, begin + 1, end);
+      if (begin == end || *begin != '}')
+        return handler.on_error("unknown format specifier"), end;
+    } else {
+      return handler.on_error("missing '}' in format string"), end;
+    }
+  }
+  return begin + 1;
+}
+
+template <bool IS_CONSTEXPR, typename Char, typename Handler>
+FMT_CONSTEXPR void parse_format_string(basic_string_view<Char> format_str,
+                                       Handler&& handler) {
+  auto begin = format_str.data();
+  auto end = begin + format_str.size();
+  if (end - begin < 32) {
+    // Use a simple loop instead of memchr for small strings.
+    const Char* p = begin;
+    while (p != end) {
+      auto c = *p++;
+      if (c == '{') {
+        handler.on_text(begin, p - 1);
+        begin = p = parse_replacement_field(p - 1, end, handler);
+      } else if (c == '}') {
+        if (p == end || *p != '}')
+          return handler.on_error("unmatched '}' in format string");
+        handler.on_text(begin, p);
+        begin = ++p;
+      }
+    }
+    handler.on_text(begin, end);
+    return;
+  }
+  struct writer {
+    FMT_CONSTEXPR void operator()(const Char* from, const Char* to) {
+      if (from == to) return;
+      for (;;) {
+        const Char* p = nullptr;
+        if (!find<IS_CONSTEXPR>(from, to, Char('}'), p))
+          return handler_.on_text(from, to);
+        ++p;
+        if (p == to || *p != '}')
+          return handler_.on_error("unmatched '}' in format string");
+        handler_.on_text(from, p);
+        from = p + 1;
+      }
+    }
+    Handler& handler_;
+  } write = {handler};
+  while (begin != end) {
+    // Doing two passes with memchr (one for '{' and another for '}') is up to
+    // 2.5x faster than the naive one-pass implementation on big format strings.
+    const Char* p = begin;
+    if (*begin != '{' && !find<IS_CONSTEXPR>(begin + 1, end, Char('{'), p))
+      return write(begin, end);
+    write(begin, p);
+    begin = parse_replacement_field(p, end, handler);
+  }
+}
+
+template <typename T, bool = is_named_arg<T>::value> struct strip_named_arg {
+  using type = T;
+};
+template <typename T> struct strip_named_arg<T, true> {
+  using type = remove_cvref_t<decltype(T::value)>;
+};
+
+template <typename T, typename ParseContext>
+FMT_VISIBILITY("hidden")  // Suppress an ld warning on macOS (#3769).
+FMT_CONSTEXPR auto parse_format_specs(ParseContext& ctx)
+    -> decltype(ctx.begin()) {
+  using char_type = typename ParseContext::char_type;
+  using context = buffered_context<char_type>;
+  using mapped_type = conditional_t<
+      mapped_type_constant<T, context>::value != type::custom_type,
+      decltype(arg_mapper<context>().map(std::declval<const T&>())),
+      typename strip_named_arg<T>::type>;
+#if defined(__cpp_if_constexpr)
+  if constexpr (std::is_default_constructible<
+                    formatter<mapped_type, char_type>>::value) {
+    return formatter<mapped_type, char_type>().parse(ctx);
+  } else {
+    type_is_unformattable_for<T, char_type> _;
+    return ctx.begin();
+  }
+#else
+  return formatter<mapped_type, char_type>().parse(ctx);
+#endif
+}
+
+// Checks char specs and returns true iff the presentation type is char-like.
+FMT_CONSTEXPR inline auto check_char_specs(const format_specs& specs) -> bool {
+  if (specs.type != presentation_type::none &&
+      specs.type != presentation_type::chr &&
+      specs.type != presentation_type::debug) {
+    return false;
+  }
+  if (specs.align == align::numeric || specs.sign != sign::none || specs.alt)
+    report_error("invalid format specifier for char");
+  return true;
+}
+
+#if FMT_USE_NONTYPE_TEMPLATE_ARGS
+template <int N, typename T, typename... Args, typename Char>
+constexpr auto get_arg_index_by_name(basic_string_view<Char> name) -> int {
+  if constexpr (is_statically_named_arg<T>()) {
+    if (name == T::name) return N;
+  }
+  if constexpr (sizeof...(Args) > 0)
+    return get_arg_index_by_name<N + 1, Args...>(name);
+  (void)name;  // Workaround an MSVC bug about "unused" parameter.
+  return -1;
+}
+#endif
+
+template <typename... Args, typename Char>
+FMT_CONSTEXPR auto get_arg_index_by_name(basic_string_view<Char> name) -> int {
+#if FMT_USE_NONTYPE_TEMPLATE_ARGS
+  if constexpr (sizeof...(Args) > 0)
+    return get_arg_index_by_name<0, Args...>(name);
+#endif
+  (void)name;
+  return -1;
+}
+
+template <typename Char, typename... Args> class format_string_checker {
+ private:
+  using parse_context_type = compile_parse_context<Char>;
+  static constexpr int num_args = sizeof...(Args);
+
+  // Format specifier parsing function.
+  // In the future basic_format_parse_context will replace compile_parse_context
+  // here and will use is_constant_evaluated and downcasting to access the data
+  // needed for compile-time checks: https://godbolt.org/z/GvWzcTjh1.
+  using parse_func = const Char* (*)(parse_context_type&);
+
+  type types_[num_args > 0 ? static_cast<size_t>(num_args) : 1];
+  parse_context_type context_;
+  parse_func parse_funcs_[num_args > 0 ? static_cast<size_t>(num_args) : 1];
+
+ public:
+  explicit FMT_CONSTEXPR format_string_checker(basic_string_view<Char> fmt)
+      : types_{mapped_type_constant<Args, buffered_context<Char>>::value...},
+        context_(fmt, num_args, types_),
+        parse_funcs_{&parse_format_specs<Args, parse_context_type>...} {}
+
+  FMT_CONSTEXPR void on_text(const Char*, const Char*) {}
+
+  FMT_CONSTEXPR auto on_arg_id() -> int { return context_.next_arg_id(); }
+  FMT_CONSTEXPR auto on_arg_id(int id) -> int {
+    return context_.check_arg_id(id), id;
+  }
+  FMT_CONSTEXPR auto on_arg_id(basic_string_view<Char> id) -> int {
+#if FMT_USE_NONTYPE_TEMPLATE_ARGS
+    auto index = get_arg_index_by_name<Args...>(id);
+    if (index < 0) on_error("named argument is not found");
+    return index;
+#else
+    (void)id;
+    on_error("compile-time checks for named arguments require C++20 support");
+    return 0;
+#endif
+  }
+
+  FMT_CONSTEXPR void on_replacement_field(int id, const Char* begin) {
+    on_format_specs(id, begin, begin);  // Call parse() on empty specs.
+  }
+
+  FMT_CONSTEXPR auto on_format_specs(int id, const Char* begin, const Char*)
+      -> const Char* {
+    context_.advance_to(begin);
+    // id >= 0 check is a workaround for gcc 10 bug (#2065).
+    return id >= 0 && id < num_args ? parse_funcs_[id](context_) : begin;
+  }
+
+  FMT_NORETURN FMT_CONSTEXPR void on_error(const char* message) {
+    report_error(message);
+  }
+};
+
+// A base class for compile-time strings.
+struct compile_string {};
+
+template <typename S>
+using is_compile_string = std::is_base_of<compile_string, S>;
+
+// Reports a compile-time error if S is not a valid format string.
+template <typename..., typename S, FMT_ENABLE_IF(!is_compile_string<S>::value)>
+FMT_ALWAYS_INLINE void check_format_string(const S&) {
+#ifdef FMT_ENFORCE_COMPILE_STRING
+  static_assert(is_compile_string<S>::value,
+                "FMT_ENFORCE_COMPILE_STRING requires all format strings to use "
+                "FMT_STRING.");
+#endif
+}
+template <typename... Args, typename S,
+          FMT_ENABLE_IF(is_compile_string<S>::value)>
+void check_format_string(S format_str) {
+  using char_t = typename S::char_type;
+  FMT_CONSTEXPR auto s = basic_string_view<char_t>(format_str);
+  using checker = format_string_checker<char_t, remove_cvref_t<Args>...>;
+  FMT_CONSTEXPR bool error = (parse_format_string<true>(s, checker(s)), true);
+  ignore_unused(error);
+}
+
+// Report truncation to prevent silent data loss.
+inline void report_truncation(bool truncated) {
+  if (truncated) report_error("output is truncated");
+}
+
+// Use vformat_args and avoid type_identity to keep symbols short and workaround
+// a GCC <= 4.8 bug.
+template <typename Char = char> struct vformat_args {
+  using type = basic_format_args<buffered_context<Char>>;
+};
+template <> struct vformat_args<char> {
+  using type = format_args;
+};
+
+template <typename Char>
+void vformat_to(buffer<Char>& buf, basic_string_view<Char> fmt,
+                typename vformat_args<Char>::type args, locale_ref loc = {});
+
+FMT_API void vprint_mojibake(FILE*, string_view, format_args, bool = false);
+#ifndef _WIN32
+inline void vprint_mojibake(FILE*, string_view, format_args, bool) {}
+#endif
+
+template <typename T, typename Char, type TYPE> struct native_formatter {
+ private:
+  dynamic_format_specs<Char> specs_;
+
+ public:
+  using nonlocking = void;
+
+  template <typename ParseContext>
+  FMT_CONSTEXPR auto parse(ParseContext& ctx) -> const Char* {
+    if (ctx.begin() == ctx.end() || *ctx.begin() == '}') return ctx.begin();
+    auto end = parse_format_specs(ctx.begin(), ctx.end(), specs_, ctx, TYPE);
+    if (const_check(TYPE == type::char_type)) check_char_specs(specs_);
+    return end;
+  }
+
+  template <type U = TYPE,
+            FMT_ENABLE_IF(U == type::string_type || U == type::cstring_type ||
+                          U == type::char_type)>
+  FMT_CONSTEXPR void set_debug_format(bool set = true) {
+    specs_.type = set ? presentation_type::debug : presentation_type::none;
+  }
+
+  template <typename FormatContext>
+  FMT_CONSTEXPR auto format(const T& val, FormatContext& ctx) const
+      -> decltype(ctx.out());
+};
+}  // namespace detail
+
+FMT_BEGIN_EXPORT
+
+// A formatter specialization for natively supported types.
+template <typename T, typename Char>
+struct formatter<T, Char,
+                 enable_if_t<detail::type_constant<T, Char>::value !=
+                             detail::type::custom_type>>
+    : detail::native_formatter<T, Char, detail::type_constant<T, Char>::value> {
+};
+
+template <typename Char = char> struct runtime_format_string {
+  basic_string_view<Char> str;
+};
+
+/// A compile-time format string.
+template <typename Char, typename... Args> class basic_format_string {
+ private:
+  basic_string_view<Char> str_;
+
+ public:
+  template <
+      typename S,
+      FMT_ENABLE_IF(
+          std::is_convertible<const S&, basic_string_view<Char>>::value ||
+          (detail::is_compile_string<S>::value &&
+           std::is_constructible<basic_string_view<Char>, const S&>::value))>
+  FMT_CONSTEVAL FMT_ALWAYS_INLINE basic_format_string(const S& s) : str_(s) {
+    static_assert(
+        detail::count<
+            (std::is_base_of<detail::view, remove_reference_t<Args>>::value &&
+             std::is_reference<Args>::value)...>() == 0,
+        "passing views as lvalues is disallowed");
+#if FMT_USE_CONSTEVAL
+    if constexpr (detail::count_named_args<Args...>() ==
+                  detail::count_statically_named_args<Args...>()) {
+      using checker =
+          detail::format_string_checker<Char, remove_cvref_t<Args>...>;
+      detail::parse_format_string<true>(str_, checker(s));
+    }
+#else
+    detail::check_format_string<Args...>(s);
+#endif
+  }
+  basic_format_string(runtime_format_string<Char> fmt) : str_(fmt.str) {}
+
+  FMT_ALWAYS_INLINE operator basic_string_view<Char>() const { return str_; }
+  auto get() const -> basic_string_view<Char> { return str_; }
+};
+
+#if FMT_GCC_VERSION && FMT_GCC_VERSION < 409
+// Workaround broken conversion on older gcc.
+template <typename...> using format_string = string_view;
+inline auto runtime(string_view s) -> string_view { return s; }
+#else
+template <typename... Args>
+using format_string = basic_format_string<char, type_identity_t<Args>...>;
+/**
+ * Creates a runtime format string.
+ *
+ * **Example**:
+ *
+ *     // Check format string at runtime instead of compile-time.
+ *     fmt::print(fmt::runtime("{:d}"), "I am not a number");
+ */
+inline auto runtime(string_view s) -> runtime_format_string<> { return {{s}}; }
+#endif
+
+/// Formats a string and writes the output to `out`.
+template <typename OutputIt,
+          FMT_ENABLE_IF(detail::is_output_iterator<remove_cvref_t<OutputIt>,
+                                                   char>::value)>
+auto vformat_to(OutputIt&& out, string_view fmt, format_args args)
+    -> remove_cvref_t<OutputIt> {
+  auto&& buf = detail::get_buffer<char>(out);
+  detail::vformat_to(buf, fmt, args, {});
+  return detail::get_iterator(buf, out);
+}
+
+/**
+ * Formats `args` according to specifications in `fmt`, writes the result to
+ * the output iterator `out` and returns the iterator past the end of the output
+ * range. `format_to` does not append a terminating null character.
+ *
+ * **Example**:
+ *
+ *     auto out = std::vector<char>();
+ *     fmt::format_to(std::back_inserter(out), "{}", 42);
+ */
+template <typename OutputIt, typename... T,
+          FMT_ENABLE_IF(detail::is_output_iterator<remove_cvref_t<OutputIt>,
+                                                   char>::value)>
+FMT_INLINE auto format_to(OutputIt&& out, format_string<T...> fmt, T&&... args)
+    -> remove_cvref_t<OutputIt> {
+  return vformat_to(FMT_FWD(out), fmt, fmt::make_format_args(args...));
+}
+
+template <typename OutputIt> struct format_to_n_result {
+  /// Iterator past the end of the output range.
+  OutputIt out;
+  /// Total (not truncated) output size.
+  size_t size;
+};
+
+template <typename OutputIt, typename... T,
+          FMT_ENABLE_IF(detail::is_output_iterator<OutputIt, char>::value)>
+auto vformat_to_n(OutputIt out, size_t n, string_view fmt, format_args args)
+    -> format_to_n_result<OutputIt> {
+  using traits = detail::fixed_buffer_traits;
+  auto buf = detail::iterator_buffer<OutputIt, char, traits>(out, n);
+  detail::vformat_to(buf, fmt, args, {});
+  return {buf.out(), buf.count()};
+}
+
+/**
+ * Formats `args` according to specifications in `fmt`, writes up to `n`
+ * characters of the result to the output iterator `out` and returns the total
+ * (not truncated) output size and the iterator past the end of the output
+ * range. `format_to_n` does not append a terminating null character.
+ */
+template <typename OutputIt, typename... T,
+          FMT_ENABLE_IF(detail::is_output_iterator<OutputIt, char>::value)>
+FMT_INLINE auto format_to_n(OutputIt out, size_t n, format_string<T...> fmt,
+                            T&&... args) -> format_to_n_result<OutputIt> {
+  return vformat_to_n(out, n, fmt, fmt::make_format_args(args...));
+}
+
+template <typename OutputIt, typename Sentinel = OutputIt>
+struct format_to_result {
+  /// Iterator pointing to just after the last successful write in the range.
+  OutputIt out;
+  /// Specifies if the output was truncated.
+  bool truncated;
+
+  FMT_CONSTEXPR operator OutputIt&() & {
+    detail::report_truncation(truncated);
+    return out;
+  }
+  FMT_CONSTEXPR operator const OutputIt&() const& {
+    detail::report_truncation(truncated);
+    return out;
+  }
+  FMT_CONSTEXPR operator OutputIt&&() && {
+    detail::report_truncation(truncated);
+    return static_cast<OutputIt&&>(out);
+  }
+};
+
+template <size_t N>
+auto vformat_to(char (&out)[N], string_view fmt, format_args args)
+    -> format_to_result<char*> {
+  auto result = vformat_to_n(out, N, fmt, args);
+  return {result.out, result.size > N};
+}
+
+template <size_t N, typename... T>
+FMT_INLINE auto format_to(char (&out)[N], format_string<T...> fmt, T&&... args)
+    -> format_to_result<char*> {
+  auto result = fmt::format_to_n(out, N, fmt, static_cast<T&&>(args)...);
+  return {result.out, result.size > N};
+}
+
+/// Returns the number of chars in the output of `format(fmt, args...)`.
+template <typename... T>
+FMT_NODISCARD FMT_INLINE auto formatted_size(format_string<T...> fmt,
+                                             T&&... args) -> size_t {
+  auto buf = detail::counting_buffer<>();
+  detail::vformat_to<char>(buf, fmt, fmt::make_format_args(args...), {});
+  return buf.count();
+}
+
+FMT_API void vprint(string_view fmt, format_args args);
+FMT_API void vprint(FILE* f, string_view fmt, format_args args);
+FMT_API void vprint_buffered(FILE* f, string_view fmt, format_args args);
+FMT_API void vprintln(FILE* f, string_view fmt, format_args args);
+
+/**
+ * Formats `args` according to specifications in `fmt` and writes the output
+ * to `stdout`.
+ *
+ * **Example**:
+ *
+ *     fmt::print("The answer is {}.", 42);
+ */
+template <typename... T>
+FMT_INLINE void print(format_string<T...> fmt, T&&... args) {
+  const auto& vargs = fmt::make_format_args(args...);
+  if (!detail::use_utf8()) return detail::vprint_mojibake(stdout, fmt, vargs);
+  return detail::is_locking<T...>() ? vprint_buffered(stdout, fmt, vargs)
+                                    : vprint(fmt, vargs);
+}
+
+/**
+ * Formats `args` according to specifications in `fmt` and writes the
+ * output to the file `f`.
+ *
+ * **Example**:
+ *
+ *     fmt::print(stderr, "Don't {}!", "panic");
+ */
+template <typename... T>
+FMT_INLINE void print(FILE* f, format_string<T...> fmt, T&&... args) {
+  const auto& vargs = fmt::make_format_args(args...);
+  if (!detail::use_utf8()) return detail::vprint_mojibake(f, fmt, vargs);
+  return detail::is_locking<T...>() ? vprint_buffered(f, fmt, vargs)
+                                    : vprint(f, fmt, vargs);
+}
+
+/// Formats `args` according to specifications in `fmt` and writes the output
+/// to the file `f` followed by a newline.
+template <typename... T>
+FMT_INLINE void println(FILE* f, format_string<T...> fmt, T&&... args) {
+  const auto& vargs = fmt::make_format_args(args...);
+  return detail::use_utf8() ? vprintln(f, fmt, vargs)
+                            : detail::vprint_mojibake(f, fmt, vargs, true);
+}
+
+/// Formats `args` according to specifications in `fmt` and writes the output
+/// to `stdout` followed by a newline.
+template <typename... T>
+FMT_INLINE void println(format_string<T...> fmt, T&&... args) {
+  return fmt::println(stdout, fmt, static_cast<T&&>(args)...);
+}
+
+FMT_END_EXPORT
+FMT_GCC_PRAGMA("GCC pop_options")
+FMT_END_NAMESPACE
+
+#ifdef FMT_HEADER_ONLY
+#  include "format.h"
+#endif
+#endif  // FMT_BASE_H_
diff --git a/ext/spdlog/include/spdlog/fmt/bundled/chrono.h b/ext/spdlog/include/spdlog/fmt/bundled/chrono.h
new file mode 100644
index 0000000..c93123f
--- /dev/null
+++ b/ext/spdlog/include/spdlog/fmt/bundled/chrono.h
@@ -0,0 +1,2432 @@
+// Formatting library for C++ - chrono support
+//
+// Copyright (c) 2012 - present, Victor Zverovich
+// All rights reserved.
+//
+// For the license information refer to format.h.
+
+#ifndef FMT_CHRONO_H_
+#define FMT_CHRONO_H_
+
+#ifndef FMT_MODULE
+#  include <algorithm>
+#  include <chrono>
+#  include <cmath>    // std::isfinite
+#  include <cstring>  // std::memcpy
+#  include <ctime>
+#  include <iterator>
+#  include <locale>
+#  include <ostream>
+#  include <type_traits>
+#endif
+
+#include "format.h"
+
+FMT_BEGIN_NAMESPACE
+
+// Check if std::chrono::local_t is available.
+#ifndef FMT_USE_LOCAL_TIME
+#  ifdef __cpp_lib_chrono
+#    define FMT_USE_LOCAL_TIME (__cpp_lib_chrono >= 201907L)
+#  else
+#    define FMT_USE_LOCAL_TIME 0
+#  endif
+#endif
+
+// Check if std::chrono::utc_timestamp is available.
+#ifndef FMT_USE_UTC_TIME
+#  ifdef __cpp_lib_chrono
+#    define FMT_USE_UTC_TIME (__cpp_lib_chrono >= 201907L)
+#  else
+#    define FMT_USE_UTC_TIME 0
+#  endif
+#endif
+
+// Enable tzset.
+#ifndef FMT_USE_TZSET
+// UWP doesn't provide _tzset.
+#  if FMT_HAS_INCLUDE("winapifamily.h")
+#    include <winapifamily.h>
+#  endif
+#  if defined(_WIN32) && (!defined(WINAPI_FAMILY) || \
+                          (WINAPI_FAMILY == WINAPI_FAMILY_DESKTOP_APP))
+#    define FMT_USE_TZSET 1
+#  else
+#    define FMT_USE_TZSET 0
+#  endif
+#endif
+
+// Enable safe chrono durations, unless explicitly disabled.
+#ifndef FMT_SAFE_DURATION_CAST
+#  define FMT_SAFE_DURATION_CAST 1
+#endif
+#if FMT_SAFE_DURATION_CAST
+
+// For conversion between std::chrono::durations without undefined
+// behaviour or erroneous results.
+// This is a stripped down version of duration_cast, for inclusion in fmt.
+// See https://github.com/pauldreik/safe_duration_cast
+//
+// Copyright Paul Dreik 2019
+namespace safe_duration_cast {
+
+template <typename To, typename From,
+          FMT_ENABLE_IF(!std::is_same<From, To>::value &&
+                        std::numeric_limits<From>::is_signed ==
+                            std::numeric_limits<To>::is_signed)>
+FMT_CONSTEXPR auto lossless_integral_conversion(const From from, int& ec)
+    -> To {
+  ec = 0;
+  using F = std::numeric_limits<From>;
+  using T = std::numeric_limits<To>;
+  static_assert(F::is_integer, "From must be integral");
+  static_assert(T::is_integer, "To must be integral");
+
+  // A and B are both signed, or both unsigned.
+  if (detail::const_check(F::digits <= T::digits)) {
+    // From fits in To without any problem.
+  } else {
+    // From does not always fit in To, resort to a dynamic check.
+    if (from < (T::min)() || from > (T::max)()) {
+      // outside range.
+      ec = 1;
+      return {};
+    }
+  }
+  return static_cast<To>(from);
+}
+
+/// Converts From to To, without loss. If the dynamic value of from
+/// can't be converted to To without loss, ec is set.
+template <typename To, typename From,
+          FMT_ENABLE_IF(!std::is_same<From, To>::value &&
+                        std::numeric_limits<From>::is_signed !=
+                            std::numeric_limits<To>::is_signed)>
+FMT_CONSTEXPR auto lossless_integral_conversion(const From from, int& ec)
+    -> To {
+  ec = 0;
+  using F = std::numeric_limits<From>;
+  using T = std::numeric_limits<To>;
+  static_assert(F::is_integer, "From must be integral");
+  static_assert(T::is_integer, "To must be integral");
+
+  if (detail::const_check(F::is_signed && !T::is_signed)) {
+    // From may be negative, not allowed!
+    if (fmt::detail::is_negative(from)) {
+      ec = 1;
+      return {};
+    }
+    // From is positive. Can it always fit in To?
+    if (detail::const_check(F::digits > T::digits) &&
+        from > static_cast<From>(detail::max_value<To>())) {
+      ec = 1;
+      return {};
+    }
+  }
+
+  if (detail::const_check(!F::is_signed && T::is_signed &&
+                          F::digits >= T::digits) &&
+      from > static_cast<From>(detail::max_value<To>())) {
+    ec = 1;
+    return {};
+  }
+  return static_cast<To>(from);  // Lossless conversion.
+}
+
+template <typename To, typename From,
+          FMT_ENABLE_IF(std::is_same<From, To>::value)>
+FMT_CONSTEXPR auto lossless_integral_conversion(const From from, int& ec)
+    -> To {
+  ec = 0;
+  return from;
+}  // function
+
+// clang-format off
+/**
+ * converts From to To if possible, otherwise ec is set.
+ *
+ * input                            |    output
+ * ---------------------------------|---------------
+ * NaN                              | NaN
+ * Inf                              | Inf
+ * normal, fits in output           | converted (possibly lossy)
+ * normal, does not fit in output   | ec is set
+ * subnormal                        | best effort
+ * -Inf                             | -Inf
+ */
+// clang-format on
+template <typename To, typename From,
+          FMT_ENABLE_IF(!std::is_same<From, To>::value)>
+FMT_CONSTEXPR auto safe_float_conversion(const From from, int& ec) -> To {
+  ec = 0;
+  using T = std::numeric_limits<To>;
+  static_assert(std::is_floating_point<From>::value, "From must be floating");
+  static_assert(std::is_floating_point<To>::value, "To must be floating");
+
+  // catch the only happy case
+  if (std::isfinite(from)) {
+    if (from >= T::lowest() && from <= (T::max)()) {
+      return static_cast<To>(from);
+    }
+    // not within range.
+    ec = 1;
+    return {};
+  }
+
+  // nan and inf will be preserved
+  return static_cast<To>(from);
+}  // function
+
+template <typename To, typename From,
+          FMT_ENABLE_IF(std::is_same<From, To>::value)>
+FMT_CONSTEXPR auto safe_float_conversion(const From from, int& ec) -> To {
+  ec = 0;
+  static_assert(std::is_floating_point<From>::value, "From must be floating");
+  return from;
+}
+
+/// Safe duration cast between integral durations
+template <typename To, typename FromRep, typename FromPeriod,
+          FMT_ENABLE_IF(std::is_integral<FromRep>::value),
+          FMT_ENABLE_IF(std::is_integral<typename To::rep>::value)>
+auto safe_duration_cast(std::chrono::duration<FromRep, FromPeriod> from,
+                        int& ec) -> To {
+  using From = std::chrono::duration<FromRep, FromPeriod>;
+  ec = 0;
+  // the basic idea is that we need to convert from count() in the from type
+  // to count() in the To type, by multiplying it with this:
+  struct Factor
+      : std::ratio_divide<typename From::period, typename To::period> {};
+
+  static_assert(Factor::num > 0, "num must be positive");
+  static_assert(Factor::den > 0, "den must be positive");
+
+  // the conversion is like this: multiply from.count() with Factor::num
+  // /Factor::den and convert it to To::rep, all this without
+  // overflow/underflow. let's start by finding a suitable type that can hold
+  // both To, From and Factor::num
+  using IntermediateRep =
+      typename std::common_type<typename From::rep, typename To::rep,
+                                decltype(Factor::num)>::type;
+
+  // safe conversion to IntermediateRep
+  IntermediateRep count =
+      lossless_integral_conversion<IntermediateRep>(from.count(), ec);
+  if (ec) return {};
+  // multiply with Factor::num without overflow or underflow
+  if (detail::const_check(Factor::num != 1)) {
+    const auto max1 = detail::max_value<IntermediateRep>() / Factor::num;
+    if (count > max1) {
+      ec = 1;
+      return {};
+    }
+    const auto min1 =
+        (std::numeric_limits<IntermediateRep>::min)() / Factor::num;
+    if (detail::const_check(!std::is_unsigned<IntermediateRep>::value) &&
+        count < min1) {
+      ec = 1;
+      return {};
+    }
+    count *= Factor::num;
+  }
+
+  if (detail::const_check(Factor::den != 1)) count /= Factor::den;
+  auto tocount = lossless_integral_conversion<typename To::rep>(count, ec);
+  return ec ? To() : To(tocount);
+}
+
+/// Safe duration_cast between floating point durations
+template <typename To, typename FromRep, typename FromPeriod,
+          FMT_ENABLE_IF(std::is_floating_point<FromRep>::value),
+          FMT_ENABLE_IF(std::is_floating_point<typename To::rep>::value)>
+auto safe_duration_cast(std::chrono::duration<FromRep, FromPeriod> from,
+                        int& ec) -> To {
+  using From = std::chrono::duration<FromRep, FromPeriod>;
+  ec = 0;
+  if (std::isnan(from.count())) {
+    // nan in, gives nan out. easy.
+    return To{std::numeric_limits<typename To::rep>::quiet_NaN()};
+  }
+  // maybe we should also check if from is denormal, and decide what to do about
+  // it.
+
+  // +-inf should be preserved.
+  if (std::isinf(from.count())) {
+    return To{from.count()};
+  }
+
+  // the basic idea is that we need to convert from count() in the from type
+  // to count() in the To type, by multiplying it with this:
+  struct Factor
+      : std::ratio_divide<typename From::period, typename To::period> {};
+
+  static_assert(Factor::num > 0, "num must be positive");
+  static_assert(Factor::den > 0, "den must be positive");
+
+  // the conversion is like this: multiply from.count() with Factor::num
+  // /Factor::den and convert it to To::rep, all this without
+  // overflow/underflow. let's start by finding a suitable type that can hold
+  // both To, From and Factor::num
+  using IntermediateRep =
+      typename std::common_type<typename From::rep, typename To::rep,
+                                decltype(Factor::num)>::type;
+
+  // force conversion of From::rep -> IntermediateRep to be safe,
+  // even if it will never happen be narrowing in this context.
+  IntermediateRep count =
+      safe_float_conversion<IntermediateRep>(from.count(), ec);
+  if (ec) {
+    return {};
+  }
+
+  // multiply with Factor::num without overflow or underflow
+  if (detail::const_check(Factor::num != 1)) {
+    constexpr auto max1 = detail::max_value<IntermediateRep>() /
+                          static_cast<IntermediateRep>(Factor::num);
+    if (count > max1) {
+      ec = 1;
+      return {};
+    }
+    constexpr auto min1 = std::numeric_limits<IntermediateRep>::lowest() /
+                          static_cast<IntermediateRep>(Factor::num);
+    if (count < min1) {
+      ec = 1;
+      return {};
+    }
+    count *= static_cast<IntermediateRep>(Factor::num);
+  }
+
+  // this can't go wrong, right? den>0 is checked earlier.
+  if (detail::const_check(Factor::den != 1)) {
+    using common_t = typename std::common_type<IntermediateRep, intmax_t>::type;
+    count /= static_cast<common_t>(Factor::den);
+  }
+
+  // convert to the to type, safely
+  using ToRep = typename To::rep;
+
+  const ToRep tocount = safe_float_conversion<ToRep>(count, ec);
+  if (ec) {
+    return {};
+  }
+  return To{tocount};
+}
+}  // namespace safe_duration_cast
+#endif
+
+// Prevents expansion of a preceding token as a function-style macro.
+// Usage: f FMT_NOMACRO()
+#define FMT_NOMACRO
+
+namespace detail {
+template <typename T = void> struct null {};
+inline auto localtime_r FMT_NOMACRO(...) -> null<> { return null<>(); }
+inline auto localtime_s(...) -> null<> { return null<>(); }
+inline auto gmtime_r(...) -> null<> { return null<>(); }
+inline auto gmtime_s(...) -> null<> { return null<>(); }
+
+// It is defined here and not in ostream.h because the latter has expensive
+// includes.
+template <typename Streambuf> class formatbuf : public Streambuf {
+ private:
+  using char_type = typename Streambuf::char_type;
+  using streamsize = decltype(std::declval<Streambuf>().sputn(nullptr, 0));
+  using int_type = typename Streambuf::int_type;
+  using traits_type = typename Streambuf::traits_type;
+
+  buffer<char_type>& buffer_;
+
+ public:
+  explicit formatbuf(buffer<char_type>& buf) : buffer_(buf) {}
+
+ protected:
+  // The put area is always empty. This makes the implementation simpler and has
+  // the advantage that the streambuf and the buffer are always in sync and
+  // sputc never writes into uninitialized memory. A disadvantage is that each
+  // call to sputc always results in a (virtual) call to overflow. There is no
+  // disadvantage here for sputn since this always results in a call to xsputn.
+
+  auto overflow(int_type ch) -> int_type override {
+    if (!traits_type::eq_int_type(ch, traits_type::eof()))
+      buffer_.push_back(static_cast<char_type>(ch));
+    return ch;
+  }
+
+  auto xsputn(const char_type* s, streamsize count) -> streamsize override {
+    buffer_.append(s, s + count);
+    return count;
+  }
+};
+
+inline auto get_classic_locale() -> const std::locale& {
+  static const auto& locale = std::locale::classic();
+  return locale;
+}
+
+template <typename CodeUnit> struct codecvt_result {
+  static constexpr const size_t max_size = 32;
+  CodeUnit buf[max_size];
+  CodeUnit* end;
+};
+
+template <typename CodeUnit>
+void write_codecvt(codecvt_result<CodeUnit>& out, string_view in_buf,
+                   const std::locale& loc) {
+#if FMT_CLANG_VERSION
+#  pragma clang diagnostic push
+#  pragma clang diagnostic ignored "-Wdeprecated"
+  auto& f = std::use_facet<std::codecvt<CodeUnit, char, std::mbstate_t>>(loc);
+#  pragma clang diagnostic pop
+#else
+  auto& f = std::use_facet<std::codecvt<CodeUnit, char, std::mbstate_t>>(loc);
+#endif
+  auto mb = std::mbstate_t();
+  const char* from_next = nullptr;
+  auto result = f.in(mb, in_buf.begin(), in_buf.end(), from_next,
+                     std::begin(out.buf), std::end(out.buf), out.end);
+  if (result != std::codecvt_base::ok)
+    FMT_THROW(format_error("failed to format time"));
+}
+
+template <typename OutputIt>
+auto write_encoded_tm_str(OutputIt out, string_view in, const std::locale& loc)
+    -> OutputIt {
+  if (detail::use_utf8() && loc != get_classic_locale()) {
+    // char16_t and char32_t codecvts are broken in MSVC (linkage errors) and
+    // gcc-4.
+#if FMT_MSC_VERSION != 0 ||  \
+    (defined(__GLIBCXX__) && \
+     (!defined(_GLIBCXX_USE_DUAL_ABI) || _GLIBCXX_USE_DUAL_ABI == 0))
+    // The _GLIBCXX_USE_DUAL_ABI macro is always defined in libstdc++ from gcc-5
+    // and newer.
+    using code_unit = wchar_t;
+#else
+    using code_unit = char32_t;
+#endif
+
+    using unit_t = codecvt_result<code_unit>;
+    unit_t unit;
+    write_codecvt(unit, in, loc);
+    // In UTF-8 is used one to four one-byte code units.
+    auto u =
+        to_utf8<code_unit, basic_memory_buffer<char, unit_t::max_size * 4>>();
+    if (!u.convert({unit.buf, to_unsigned(unit.end - unit.buf)}))
+      FMT_THROW(format_error("failed to format time"));
+    return copy<char>(u.c_str(), u.c_str() + u.size(), out);
+  }
+  return copy<char>(in.data(), in.data() + in.size(), out);
+}
+
+template <typename Char, typename OutputIt,
+          FMT_ENABLE_IF(!std::is_same<Char, char>::value)>
+auto write_tm_str(OutputIt out, string_view sv, const std::locale& loc)
+    -> OutputIt {
+  codecvt_result<Char> unit;
+  write_codecvt(unit, sv, loc);
+  return copy<Char>(unit.buf, unit.end, out);
+}
+
+template <typename Char, typename OutputIt,
+          FMT_ENABLE_IF(std::is_same<Char, char>::value)>
+auto write_tm_str(OutputIt out, string_view sv, const std::locale& loc)
+    -> OutputIt {
+  return write_encoded_tm_str(out, sv, loc);
+}
+
+template <typename Char>
+inline void do_write(buffer<Char>& buf, const std::tm& time,
+                     const std::locale& loc, char format, char modifier) {
+  auto&& format_buf = formatbuf<std::basic_streambuf<Char>>(buf);
+  auto&& os = std::basic_ostream<Char>(&format_buf);
+  os.imbue(loc);
+  const auto& facet = std::use_facet<std::time_put<Char>>(loc);
+  auto end = facet.put(os, os, Char(' '), &time, format, modifier);
+  if (end.failed()) FMT_THROW(format_error("failed to format time"));
+}
+
+template <typename Char, typename OutputIt,
+          FMT_ENABLE_IF(!std::is_same<Char, char>::value)>
+auto write(OutputIt out, const std::tm& time, const std::locale& loc,
+           char format, char modifier = 0) -> OutputIt {
+  auto&& buf = get_buffer<Char>(out);
+  do_write<Char>(buf, time, loc, format, modifier);
+  return get_iterator(buf, out);
+}
+
+template <typename Char, typename OutputIt,
+          FMT_ENABLE_IF(std::is_same<Char, char>::value)>
+auto write(OutputIt out, const std::tm& time, const std::locale& loc,
+           char format, char modifier = 0) -> OutputIt {
+  auto&& buf = basic_memory_buffer<Char>();
+  do_write<char>(buf, time, loc, format, modifier);
+  return write_encoded_tm_str(out, string_view(buf.data(), buf.size()), loc);
+}
+
+template <typename Rep1, typename Rep2>
+struct is_same_arithmetic_type
+    : public std::integral_constant<bool,
+                                    (std::is_integral<Rep1>::value &&
+                                     std::is_integral<Rep2>::value) ||
+                                        (std::is_floating_point<Rep1>::value &&
+                                         std::is_floating_point<Rep2>::value)> {
+};
+
+template <
+    typename To, typename FromRep, typename FromPeriod,
+    FMT_ENABLE_IF(is_same_arithmetic_type<FromRep, typename To::rep>::value)>
+auto fmt_duration_cast(std::chrono::duration<FromRep, FromPeriod> from) -> To {
+#if FMT_SAFE_DURATION_CAST
+  // Throwing version of safe_duration_cast is only available for
+  // integer to integer or float to float casts.
+  int ec;
+  To to = safe_duration_cast::safe_duration_cast<To>(from, ec);
+  if (ec) FMT_THROW(format_error("cannot format duration"));
+  return to;
+#else
+  // Standard duration cast, may overflow.
+  return std::chrono::duration_cast<To>(from);
+#endif
+}
+
+template <
+    typename To, typename FromRep, typename FromPeriod,
+    FMT_ENABLE_IF(!is_same_arithmetic_type<FromRep, typename To::rep>::value)>
+auto fmt_duration_cast(std::chrono::duration<FromRep, FromPeriod> from) -> To {
+  // Mixed integer <-> float cast is not supported by safe_duration_cast.
+  return std::chrono::duration_cast<To>(from);
+}
+
+template <typename Duration>
+auto to_time_t(
+    std::chrono::time_point<std::chrono::system_clock, Duration> time_point)
+    -> std::time_t {
+  // Cannot use std::chrono::system_clock::to_time_t since this would first
+  // require a cast to std::chrono::system_clock::time_point, which could
+  // overflow.
+  return fmt_duration_cast<std::chrono::duration<std::time_t>>(
+             time_point.time_since_epoch())
+      .count();
+}
+}  // namespace detail
+
+FMT_BEGIN_EXPORT
+
+/**
+ * Converts given time since epoch as `std::time_t` value into calendar time,
+ * expressed in local time. Unlike `std::localtime`, this function is
+ * thread-safe on most platforms.
+ */
+inline auto localtime(std::time_t time) -> std::tm {
+  struct dispatcher {
+    std::time_t time_;
+    std::tm tm_;
+
+    dispatcher(std::time_t t) : time_(t) {}
+
+    auto run() -> bool {
+      using namespace fmt::detail;
+      return handle(localtime_r(&time_, &tm_));
+    }
+
+    auto handle(std::tm* tm) -> bool { return tm != nullptr; }
+
+    auto handle(detail::null<>) -> bool {
+      using namespace fmt::detail;
+      return fallback(localtime_s(&tm_, &time_));
+    }
+
+    auto fallback(int res) -> bool { return res == 0; }
+
+#if !FMT_MSC_VERSION
+    auto fallback(detail::null<>) -> bool {
+      using namespace fmt::detail;
+      std::tm* tm = std::localtime(&time_);
+      if (tm) tm_ = *tm;
+      return tm != nullptr;
+    }
+#endif
+  };
+  dispatcher lt(time);
+  // Too big time values may be unsupported.
+  if (!lt.run()) FMT_THROW(format_error("time_t value out of range"));
+  return lt.tm_;
+}
+
+#if FMT_USE_LOCAL_TIME
+template <typename Duration>
+inline auto localtime(std::chrono::local_time<Duration> time) -> std::tm {
+  return localtime(
+      detail::to_time_t(std::chrono::current_zone()->to_sys(time)));
+}
+#endif
+
+/**
+ * Converts given time since epoch as `std::time_t` value into calendar time,
+ * expressed in Coordinated Universal Time (UTC). Unlike `std::gmtime`, this
+ * function is thread-safe on most platforms.
+ */
+inline auto gmtime(std::time_t time) -> std::tm {
+  struct dispatcher {
+    std::time_t time_;
+    std::tm tm_;
+
+    dispatcher(std::time_t t) : time_(t) {}
+
+    auto run() -> bool {
+      using namespace fmt::detail;
+      return handle(gmtime_r(&time_, &tm_));
+    }
+
+    auto handle(std::tm* tm) -> bool { return tm != nullptr; }
+
+    auto handle(detail::null<>) -> bool {
+      using namespace fmt::detail;
+      return fallback(gmtime_s(&tm_, &time_));
+    }
+
+    auto fallback(int res) -> bool { return res == 0; }
+
+#if !FMT_MSC_VERSION
+    auto fallback(detail::null<>) -> bool {
+      std::tm* tm = std::gmtime(&time_);
+      if (tm) tm_ = *tm;
+      return tm != nullptr;
+    }
+#endif
+  };
+  auto gt = dispatcher(time);
+  // Too big time values may be unsupported.
+  if (!gt.run()) FMT_THROW(format_error("time_t value out of range"));
+  return gt.tm_;
+}
+
+template <typename Duration>
+inline auto gmtime(
+    std::chrono::time_point<std::chrono::system_clock, Duration> time_point)
+    -> std::tm {
+  return gmtime(detail::to_time_t(time_point));
+}
+
+namespace detail {
+
+// Writes two-digit numbers a, b and c separated by sep to buf.
+// The method by Pavel Novikov based on
+// https://johnnylee-sde.github.io/Fast-unsigned-integer-to-time-string/.
+inline void write_digit2_separated(char* buf, unsigned a, unsigned b,
+                                   unsigned c, char sep) {
+  unsigned long long digits =
+      a | (b << 24) | (static_cast<unsigned long long>(c) << 48);
+  // Convert each value to BCD.
+  // We have x = a * 10 + b and we want to convert it to BCD y = a * 16 + b.
+  // The difference is
+  //   y - x = a * 6
+  // a can be found from x:
+  //   a = floor(x / 10)
+  // then
+  //   y = x + a * 6 = x + floor(x / 10) * 6
+  // floor(x / 10) is (x * 205) >> 11 (needs 16 bits).
+  digits += (((digits * 205) >> 11) & 0x000f00000f00000f) * 6;
+  // Put low nibbles to high bytes and high nibbles to low bytes.
+  digits = ((digits & 0x00f00000f00000f0) >> 4) |
+           ((digits & 0x000f00000f00000f) << 8);
+  auto usep = static_cast<unsigned long long>(sep);
+  // Add ASCII '0' to each digit byte and insert separators.
+  digits |= 0x3030003030003030 | (usep << 16) | (usep << 40);
+
+  constexpr const size_t len = 8;
+  if (const_check(is_big_endian())) {
+    char tmp[len];
+    std::memcpy(tmp, &digits, len);
+    std::reverse_copy(tmp, tmp + len, buf);
+  } else {
+    std::memcpy(buf, &digits, len);
+  }
+}
+
+template <typename Period>
+FMT_CONSTEXPR inline auto get_units() -> const char* {
+  if (std::is_same<Period, std::atto>::value) return "as";
+  if (std::is_same<Period, std::femto>::value) return "fs";
+  if (std::is_same<Period, std::pico>::value) return "ps";
+  if (std::is_same<Period, std::nano>::value) return "ns";
+  if (std::is_same<Period, std::micro>::value) return "µs";
+  if (std::is_same<Period, std::milli>::value) return "ms";
+  if (std::is_same<Period, std::centi>::value) return "cs";
+  if (std::is_same<Period, std::deci>::value) return "ds";
+  if (std::is_same<Period, std::ratio<1>>::value) return "s";
+  if (std::is_same<Period, std::deca>::value) return "das";
+  if (std::is_same<Period, std::hecto>::value) return "hs";
+  if (std::is_same<Period, std::kilo>::value) return "ks";
+  if (std::is_same<Period, std::mega>::value) return "Ms";
+  if (std::is_same<Period, std::giga>::value) return "Gs";
+  if (std::is_same<Period, std::tera>::value) return "Ts";
+  if (std::is_same<Period, std::peta>::value) return "Ps";
+  if (std::is_same<Period, std::exa>::value) return "Es";
+  if (std::is_same<Period, std::ratio<60>>::value) return "min";
+  if (std::is_same<Period, std::ratio<3600>>::value) return "h";
+  if (std::is_same<Period, std::ratio<86400>>::value) return "d";
+  return nullptr;
+}
+
+enum class numeric_system {
+  standard,
+  // Alternative numeric system, e.g. 十二 instead of 12 in ja_JP locale.
+  alternative
+};
+
+// Glibc extensions for formatting numeric values.
+enum class pad_type {
+  // Pad a numeric result string with zeros (the default).
+  zero,
+  // Do not pad a numeric result string.
+  none,
+  // Pad a numeric result string with spaces.
+  space,
+};
+
+template <typename OutputIt>
+auto write_padding(OutputIt out, pad_type pad, int width) -> OutputIt {
+  if (pad == pad_type::none) return out;
+  return detail::fill_n(out, width, pad == pad_type::space ? ' ' : '0');
+}
+
+template <typename OutputIt>
+auto write_padding(OutputIt out, pad_type pad) -> OutputIt {
+  if (pad != pad_type::none) *out++ = pad == pad_type::space ? ' ' : '0';
+  return out;
+}
+
+// Parses a put_time-like format string and invokes handler actions.
+template <typename Char, typename Handler>
+FMT_CONSTEXPR auto parse_chrono_format(const Char* begin, const Char* end,
+                                       Handler&& handler) -> const Char* {
+  if (begin == end || *begin == '}') return begin;
+  if (*begin != '%') FMT_THROW(format_error("invalid format"));
+  auto ptr = begin;
+  while (ptr != end) {
+    pad_type pad = pad_type::zero;
+    auto c = *ptr;
+    if (c == '}') break;
+    if (c != '%') {
+      ++ptr;
+      continue;
+    }
+    if (begin != ptr) handler.on_text(begin, ptr);
+    ++ptr;  // consume '%'
+    if (ptr == end) FMT_THROW(format_error("invalid format"));
+    c = *ptr;
+    switch (c) {
+    case '_':
+      pad = pad_type::space;
+      ++ptr;
+      break;
+    case '-':
+      pad = pad_type::none;
+      ++ptr;
+      break;
+    }
+    if (ptr == end) FMT_THROW(format_error("invalid format"));
+    c = *ptr++;
+    switch (c) {
+    case '%':
+      handler.on_text(ptr - 1, ptr);
+      break;
+    case 'n': {
+      const Char newline[] = {'\n'};
+      handler.on_text(newline, newline + 1);
+      break;
+    }
+    case 't': {
+      const Char tab[] = {'\t'};
+      handler.on_text(tab, tab + 1);
+      break;
+    }
+    // Year:
+    case 'Y':
+      handler.on_year(numeric_system::standard);
+      break;
+    case 'y':
+      handler.on_short_year(numeric_system::standard);
+      break;
+    case 'C':
+      handler.on_century(numeric_system::standard);
+      break;
+    case 'G':
+      handler.on_iso_week_based_year();
+      break;
+    case 'g':
+      handler.on_iso_week_based_short_year();
+      break;
+    // Day of the week:
+    case 'a':
+      handler.on_abbr_weekday();
+      break;
+    case 'A':
+      handler.on_full_weekday();
+      break;
+    case 'w':
+      handler.on_dec0_weekday(numeric_system::standard);
+      break;
+    case 'u':
+      handler.on_dec1_weekday(numeric_system::standard);
+      break;
+    // Month:
+    case 'b':
+    case 'h':
+      handler.on_abbr_month();
+      break;
+    case 'B':
+      handler.on_full_month();
+      break;
+    case 'm':
+      handler.on_dec_month(numeric_system::standard);
+      break;
+    // Day of the year/month:
+    case 'U':
+      handler.on_dec0_week_of_year(numeric_system::standard, pad);
+      break;
+    case 'W':
+      handler.on_dec1_week_of_year(numeric_system::standard, pad);
+      break;
+    case 'V':
+      handler.on_iso_week_of_year(numeric_system::standard, pad);
+      break;
+    case 'j':
+      handler.on_day_of_year();
+      break;
+    case 'd':
+      handler.on_day_of_month(numeric_system::standard, pad);
+      break;
+    case 'e':
+      handler.on_day_of_month(numeric_system::standard, pad_type::space);
+      break;
+    // Hour, minute, second:
+    case 'H':
+      handler.on_24_hour(numeric_system::standard, pad);
+      break;
+    case 'I':
+      handler.on_12_hour(numeric_system::standard, pad);
+      break;
+    case 'M':
+      handler.on_minute(numeric_system::standard, pad);
+      break;
+    case 'S':
+      handler.on_second(numeric_system::standard, pad);
+      break;
+    // Other:
+    case 'c':
+      handler.on_datetime(numeric_system::standard);
+      break;
+    case 'x':
+      handler.on_loc_date(numeric_system::standard);
+      break;
+    case 'X':
+      handler.on_loc_time(numeric_system::standard);
+      break;
+    case 'D':
+      handler.on_us_date();
+      break;
+    case 'F':
+      handler.on_iso_date();
+      break;
+    case 'r':
+      handler.on_12_hour_time();
+      break;
+    case 'R':
+      handler.on_24_hour_time();
+      break;
+    case 'T':
+      handler.on_iso_time();
+      break;
+    case 'p':
+      handler.on_am_pm();
+      break;
+    case 'Q':
+      handler.on_duration_value();
+      break;
+    case 'q':
+      handler.on_duration_unit();
+      break;
+    case 'z':
+      handler.on_utc_offset(numeric_system::standard);
+      break;
+    case 'Z':
+      handler.on_tz_name();
+      break;
+    // Alternative representation:
+    case 'E': {
+      if (ptr == end) FMT_THROW(format_error("invalid format"));
+      c = *ptr++;
+      switch (c) {
+      case 'Y':
+        handler.on_year(numeric_system::alternative);
+        break;
+      case 'y':
+        handler.on_offset_year();
+        break;
+      case 'C':
+        handler.on_century(numeric_system::alternative);
+        break;
+      case 'c':
+        handler.on_datetime(numeric_system::alternative);
+        break;
+      case 'x':
+        handler.on_loc_date(numeric_system::alternative);
+        break;
+      case 'X':
+        handler.on_loc_time(numeric_system::alternative);
+        break;
+      case 'z':
+        handler.on_utc_offset(numeric_system::alternative);
+        break;
+      default:
+        FMT_THROW(format_error("invalid format"));
+      }
+      break;
+    }
+    case 'O':
+      if (ptr == end) FMT_THROW(format_error("invalid format"));
+      c = *ptr++;
+      switch (c) {
+      case 'y':
+        handler.on_short_year(numeric_system::alternative);
+        break;
+      case 'm':
+        handler.on_dec_month(numeric_system::alternative);
+        break;
+      case 'U':
+        handler.on_dec0_week_of_year(numeric_system::alternative, pad);
+        break;
+      case 'W':
+        handler.on_dec1_week_of_year(numeric_system::alternative, pad);
+        break;
+      case 'V':
+        handler.on_iso_week_of_year(numeric_system::alternative, pad);
+        break;
+      case 'd':
+        handler.on_day_of_month(numeric_system::alternative, pad);
+        break;
+      case 'e':
+        handler.on_day_of_month(numeric_system::alternative, pad_type::space);
+        break;
+      case 'w':
+        handler.on_dec0_weekday(numeric_system::alternative);
+        break;
+      case 'u':
+        handler.on_dec1_weekday(numeric_system::alternative);
+        break;
+      case 'H':
+        handler.on_24_hour(numeric_system::alternative, pad);
+        break;
+      case 'I':
+        handler.on_12_hour(numeric_system::alternative, pad);
+        break;
+      case 'M':
+        handler.on_minute(numeric_system::alternative, pad);
+        break;
+      case 'S':
+        handler.on_second(numeric_system::alternative, pad);
+        break;
+      case 'z':
+        handler.on_utc_offset(numeric_system::alternative);
+        break;
+      default:
+        FMT_THROW(format_error("invalid format"));
+      }
+      break;
+    default:
+      FMT_THROW(format_error("invalid format"));
+    }
+    begin = ptr;
+  }
+  if (begin != ptr) handler.on_text(begin, ptr);
+  return ptr;
+}
+
+template <typename Derived> struct null_chrono_spec_handler {
+  FMT_CONSTEXPR void unsupported() {
+    static_cast<Derived*>(this)->unsupported();
+  }
+  FMT_CONSTEXPR void on_year(numeric_system) { unsupported(); }
+  FMT_CONSTEXPR void on_short_year(numeric_system) { unsupported(); }
+  FMT_CONSTEXPR void on_offset_year() { unsupported(); }
+  FMT_CONSTEXPR void on_century(numeric_system) { unsupported(); }
+  FMT_CONSTEXPR void on_iso_week_based_year() { unsupported(); }
+  FMT_CONSTEXPR void on_iso_week_based_short_year() { unsupported(); }
+  FMT_CONSTEXPR void on_abbr_weekday() { unsupported(); }
+  FMT_CONSTEXPR void on_full_weekday() { unsupported(); }
+  FMT_CONSTEXPR void on_dec0_weekday(numeric_system) { unsupported(); }
+  FMT_CONSTEXPR void on_dec1_weekday(numeric_system) { unsupported(); }
+  FMT_CONSTEXPR void on_abbr_month() { unsupported(); }
+  FMT_CONSTEXPR void on_full_month() { unsupported(); }
+  FMT_CONSTEXPR void on_dec_month(numeric_system) { unsupported(); }
+  FMT_CONSTEXPR void on_dec0_week_of_year(numeric_system, pad_type) {
+    unsupported();
+  }
+  FMT_CONSTEXPR void on_dec1_week_of_year(numeric_system, pad_type) {
+    unsupported();
+  }
+  FMT_CONSTEXPR void on_iso_week_of_year(numeric_system, pad_type) {
+    unsupported();
+  }
+  FMT_CONSTEXPR void on_day_of_year() { unsupported(); }
+  FMT_CONSTEXPR void on_day_of_month(numeric_system, pad_type) {
+    unsupported();
+  }
+  FMT_CONSTEXPR void on_24_hour(numeric_system) { unsupported(); }
+  FMT_CONSTEXPR void on_12_hour(numeric_system) { unsupported(); }
+  FMT_CONSTEXPR void on_minute(numeric_system) { unsupported(); }
+  FMT_CONSTEXPR void on_second(numeric_system) { unsupported(); }
+  FMT_CONSTEXPR void on_datetime(numeric_system) { unsupported(); }
+  FMT_CONSTEXPR void on_loc_date(numeric_system) { unsupported(); }
+  FMT_CONSTEXPR void on_loc_time(numeric_system) { unsupported(); }
+  FMT_CONSTEXPR void on_us_date() { unsupported(); }
+  FMT_CONSTEXPR void on_iso_date() { unsupported(); }
+  FMT_CONSTEXPR void on_12_hour_time() { unsupported(); }
+  FMT_CONSTEXPR void on_24_hour_time() { unsupported(); }
+  FMT_CONSTEXPR void on_iso_time() { unsupported(); }
+  FMT_CONSTEXPR void on_am_pm() { unsupported(); }
+  FMT_CONSTEXPR void on_duration_value() { unsupported(); }
+  FMT_CONSTEXPR void on_duration_unit() { unsupported(); }
+  FMT_CONSTEXPR void on_utc_offset(numeric_system) { unsupported(); }
+  FMT_CONSTEXPR void on_tz_name() { unsupported(); }
+};
+
+struct tm_format_checker : null_chrono_spec_handler<tm_format_checker> {
+  FMT_NORETURN void unsupported() { FMT_THROW(format_error("no format")); }
+
+  template <typename Char>
+  FMT_CONSTEXPR void on_text(const Char*, const Char*) {}
+  FMT_CONSTEXPR void on_year(numeric_system) {}
+  FMT_CONSTEXPR void on_short_year(numeric_system) {}
+  FMT_CONSTEXPR void on_offset_year() {}
+  FMT_CONSTEXPR void on_century(numeric_system) {}
+  FMT_CONSTEXPR void on_iso_week_based_year() {}
+  FMT_CONSTEXPR void on_iso_week_based_short_year() {}
+  FMT_CONSTEXPR void on_abbr_weekday() {}
+  FMT_CONSTEXPR void on_full_weekday() {}
+  FMT_CONSTEXPR void on_dec0_weekday(numeric_system) {}
+  FMT_CONSTEXPR void on_dec1_weekday(numeric_system) {}
+  FMT_CONSTEXPR void on_abbr_month() {}
+  FMT_CONSTEXPR void on_full_month() {}
+  FMT_CONSTEXPR void on_dec_month(numeric_system) {}
+  FMT_CONSTEXPR void on_dec0_week_of_year(numeric_system, pad_type) {}
+  FMT_CONSTEXPR void on_dec1_week_of_year(numeric_system, pad_type) {}
+  FMT_CONSTEXPR void on_iso_week_of_year(numeric_system, pad_type) {}
+  FMT_CONSTEXPR void on_day_of_year() {}
+  FMT_CONSTEXPR void on_day_of_month(numeric_system, pad_type) {}
+  FMT_CONSTEXPR void on_24_hour(numeric_system, pad_type) {}
+  FMT_CONSTEXPR void on_12_hour(numeric_system, pad_type) {}
+  FMT_CONSTEXPR void on_minute(numeric_system, pad_type) {}
+  FMT_CONSTEXPR void on_second(numeric_system, pad_type) {}
+  FMT_CONSTEXPR void on_datetime(numeric_system) {}
+  FMT_CONSTEXPR void on_loc_date(numeric_system) {}
+  FMT_CONSTEXPR void on_loc_time(numeric_system) {}
+  FMT_CONSTEXPR void on_us_date() {}
+  FMT_CONSTEXPR void on_iso_date() {}
+  FMT_CONSTEXPR void on_12_hour_time() {}
+  FMT_CONSTEXPR void on_24_hour_time() {}
+  FMT_CONSTEXPR void on_iso_time() {}
+  FMT_CONSTEXPR void on_am_pm() {}
+  FMT_CONSTEXPR void on_utc_offset(numeric_system) {}
+  FMT_CONSTEXPR void on_tz_name() {}
+};
+
+inline auto tm_wday_full_name(int wday) -> const char* {
+  static constexpr const char* full_name_list[] = {
+      "Sunday",   "Monday", "Tuesday", "Wednesday",
+      "Thursday", "Friday", "Saturday"};
+  return wday >= 0 && wday <= 6 ? full_name_list[wday] : "?";
+}
+inline auto tm_wday_short_name(int wday) -> const char* {
+  static constexpr const char* short_name_list[] = {"Sun", "Mon", "Tue", "Wed",
+                                                    "Thu", "Fri", "Sat"};
+  return wday >= 0 && wday <= 6 ? short_name_list[wday] : "???";
+}
+
+inline auto tm_mon_full_name(int mon) -> const char* {
+  static constexpr const char* full_name_list[] = {
+      "January", "February", "March",     "April",   "May",      "June",
+      "July",    "August",   "September", "October", "November", "December"};
+  return mon >= 0 && mon <= 11 ? full_name_list[mon] : "?";
+}
+inline auto tm_mon_short_name(int mon) -> const char* {
+  static constexpr const char* short_name_list[] = {
+      "Jan", "Feb", "Mar", "Apr", "May", "Jun",
+      "Jul", "Aug", "Sep", "Oct", "Nov", "Dec",
+  };
+  return mon >= 0 && mon <= 11 ? short_name_list[mon] : "???";
+}
+
+template <typename T, typename = void>
+struct has_member_data_tm_gmtoff : std::false_type {};
+template <typename T>
+struct has_member_data_tm_gmtoff<T, void_t<decltype(T::tm_gmtoff)>>
+    : std::true_type {};
+
+template <typename T, typename = void>
+struct has_member_data_tm_zone : std::false_type {};
+template <typename T>
+struct has_member_data_tm_zone<T, void_t<decltype(T::tm_zone)>>
+    : std::true_type {};
+
+#if FMT_USE_TZSET
+inline void tzset_once() {
+  static bool init = []() -> bool {
+    _tzset();
+    return true;
+  }();
+  ignore_unused(init);
+}
+#endif
+
+// Converts value to Int and checks that it's in the range [0, upper).
+template <typename T, typename Int, FMT_ENABLE_IF(std::is_integral<T>::value)>
+inline auto to_nonnegative_int(T value, Int upper) -> Int {
+  if (!std::is_unsigned<Int>::value &&
+      (value < 0 || to_unsigned(value) > to_unsigned(upper))) {
+    FMT_THROW(fmt::format_error("chrono value is out of range"));
+  }
+  return static_cast<Int>(value);
+}
+template <typename T, typename Int, FMT_ENABLE_IF(!std::is_integral<T>::value)>
+inline auto to_nonnegative_int(T value, Int upper) -> Int {
+  auto int_value = static_cast<Int>(value);
+  if (int_value < 0 || value > static_cast<T>(upper))
+    FMT_THROW(format_error("invalid value"));
+  return int_value;
+}
+
+constexpr auto pow10(std::uint32_t n) -> long long {
+  return n == 0 ? 1 : 10 * pow10(n - 1);
+}
+
+// Counts the number of fractional digits in the range [0, 18] according to the
+// C++20 spec. If more than 18 fractional digits are required then returns 6 for
+// microseconds precision.
+template <long long Num, long long Den, int N = 0,
+          bool Enabled = (N < 19) && (Num <= max_value<long long>() / 10)>
+struct count_fractional_digits {
+  static constexpr int value =
+      Num % Den == 0 ? N : count_fractional_digits<Num * 10, Den, N + 1>::value;
+};
+
+// Base case that doesn't instantiate any more templates
+// in order to avoid overflow.
+template <long long Num, long long Den, int N>
+struct count_fractional_digits<Num, Den, N, false> {
+  static constexpr int value = (Num % Den == 0) ? N : 6;
+};
+
+// Format subseconds which are given as an integer type with an appropriate
+// number of digits.
+template <typename Char, typename OutputIt, typename Duration>
+void write_fractional_seconds(OutputIt& out, Duration d, int precision = -1) {
+  constexpr auto num_fractional_digits =
+      count_fractional_digits<Duration::period::num,
+                              Duration::period::den>::value;
+
+  using subsecond_precision = std::chrono::duration<
+      typename std::common_type<typename Duration::rep,
+                                std::chrono::seconds::rep>::type,
+      std::ratio<1, detail::pow10(num_fractional_digits)>>;
+
+  const auto fractional = d - fmt_duration_cast<std::chrono::seconds>(d);
+  const auto subseconds =
+      std::chrono::treat_as_floating_point<
+          typename subsecond_precision::rep>::value
+          ? fractional.count()
+          : fmt_duration_cast<subsecond_precision>(fractional).count();
+  auto n = static_cast<uint32_or_64_or_128_t<long long>>(subseconds);
+  const int num_digits = detail::count_digits(n);
+
+  int leading_zeroes = (std::max)(0, num_fractional_digits - num_digits);
+  if (precision < 0) {
+    FMT_ASSERT(!std::is_floating_point<typename Duration::rep>::value, "");
+    if (std::ratio_less<typename subsecond_precision::period,
+                        std::chrono::seconds::period>::value) {
+      *out++ = '.';
+      out = detail::fill_n(out, leading_zeroes, '0');
+      out = format_decimal<Char>(out, n, num_digits).end;
+    }
+  } else if (precision > 0) {
+    *out++ = '.';
+    leading_zeroes = (std::min)(leading_zeroes, precision);
+    int remaining = precision - leading_zeroes;
+    out = detail::fill_n(out, leading_zeroes, '0');
+    if (remaining < num_digits) {
+      int num_truncated_digits = num_digits - remaining;
+      n /= to_unsigned(detail::pow10(to_unsigned(num_truncated_digits)));
+      if (n) {
+        out = format_decimal<Char>(out, n, remaining).end;
+      }
+      return;
+    }
+    if (n) {
+      out = format_decimal<Char>(out, n, num_digits).end;
+      remaining -= num_digits;
+    }
+    out = detail::fill_n(out, remaining, '0');
+  }
+}
+
+// Format subseconds which are given as a floating point type with an
+// appropriate number of digits. We cannot pass the Duration here, as we
+// explicitly need to pass the Rep value in the chrono_formatter.
+template <typename Duration>
+void write_floating_seconds(memory_buffer& buf, Duration duration,
+                            int num_fractional_digits = -1) {
+  using rep = typename Duration::rep;
+  FMT_ASSERT(std::is_floating_point<rep>::value, "");
+
+  auto val = duration.count();
+
+  if (num_fractional_digits < 0) {
+    // For `std::round` with fallback to `round`:
+    // On some toolchains `std::round` is not available (e.g. GCC 6).
+    using namespace std;
+    num_fractional_digits =
+        count_fractional_digits<Duration::period::num,
+                                Duration::period::den>::value;
+    if (num_fractional_digits < 6 && static_cast<rep>(round(val)) != val)
+      num_fractional_digits = 6;
+  }
+
+  fmt::format_to(std::back_inserter(buf), FMT_STRING("{:.{}f}"),
+                 std::fmod(val * static_cast<rep>(Duration::period::num) /
+                               static_cast<rep>(Duration::period::den),
+                           static_cast<rep>(60)),
+                 num_fractional_digits);
+}
+
+template <typename OutputIt, typename Char,
+          typename Duration = std::chrono::seconds>
+class tm_writer {
+ private:
+  static constexpr int days_per_week = 7;
+
+  const std::locale& loc_;
+  const bool is_classic_;
+  OutputIt out_;
+  const Duration* subsecs_;
+  const std::tm& tm_;
+
+  auto tm_sec() const noexcept -> int {
+    FMT_ASSERT(tm_.tm_sec >= 0 && tm_.tm_sec <= 61, "");
+    return tm_.tm_sec;
+  }
+  auto tm_min() const noexcept -> int {
+    FMT_ASSERT(tm_.tm_min >= 0 && tm_.tm_min <= 59, "");
+    return tm_.tm_min;
+  }
+  auto tm_hour() const noexcept -> int {
+    FMT_ASSERT(tm_.tm_hour >= 0 && tm_.tm_hour <= 23, "");
+    return tm_.tm_hour;
+  }
+  auto tm_mday() const noexcept -> int {
+    FMT_ASSERT(tm_.tm_mday >= 1 && tm_.tm_mday <= 31, "");
+    return tm_.tm_mday;
+  }
+  auto tm_mon() const noexcept -> int {
+    FMT_ASSERT(tm_.tm_mon >= 0 && tm_.tm_mon <= 11, "");
+    return tm_.tm_mon;
+  }
+  auto tm_year() const noexcept -> long long { return 1900ll + tm_.tm_year; }
+  auto tm_wday() const noexcept -> int {
+    FMT_ASSERT(tm_.tm_wday >= 0 && tm_.tm_wday <= 6, "");
+    return tm_.tm_wday;
+  }
+  auto tm_yday() const noexcept -> int {
+    FMT_ASSERT(tm_.tm_yday >= 0 && tm_.tm_yday <= 365, "");
+    return tm_.tm_yday;
+  }
+
+  auto tm_hour12() const noexcept -> int {
+    const auto h = tm_hour();
+    const auto z = h < 12 ? h : h - 12;
+    return z == 0 ? 12 : z;
+  }
+
+  // POSIX and the C Standard are unclear or inconsistent about what %C and %y
+  // do if the year is negative or exceeds 9999. Use the convention that %C
+  // concatenated with %y yields the same output as %Y, and that %Y contains at
+  // least 4 characters, with more only if necessary.
+  auto split_year_lower(long long year) const noexcept -> int {
+    auto l = year % 100;
+    if (l < 0) l = -l;  // l in [0, 99]
+    return static_cast<int>(l);
+  }
+
+  // Algorithm: https://en.wikipedia.org/wiki/ISO_week_date.
+  auto iso_year_weeks(long long curr_year) const noexcept -> int {
+    const auto prev_year = curr_year - 1;
+    const auto curr_p =
+        (curr_year + curr_year / 4 - curr_year / 100 + curr_year / 400) %
+        days_per_week;
+    const auto prev_p =
+        (prev_year + prev_year / 4 - prev_year / 100 + prev_year / 400) %
+        days_per_week;
+    return 52 + ((curr_p == 4 || prev_p == 3) ? 1 : 0);
+  }
+  auto iso_week_num(int tm_yday, int tm_wday) const noexcept -> int {
+    return (tm_yday + 11 - (tm_wday == 0 ? days_per_week : tm_wday)) /
+           days_per_week;
+  }
+  auto tm_iso_week_year() const noexcept -> long long {
+    const auto year = tm_year();
+    const auto w = iso_week_num(tm_yday(), tm_wday());
+    if (w < 1) return year - 1;
+    if (w > iso_year_weeks(year)) return year + 1;
+    return year;
+  }
+  auto tm_iso_week_of_year() const noexcept -> int {
+    const auto year = tm_year();
+    const auto w = iso_week_num(tm_yday(), tm_wday());
+    if (w < 1) return iso_year_weeks(year - 1);
+    if (w > iso_year_weeks(year)) return 1;
+    return w;
+  }
+
+  void write1(int value) {
+    *out_++ = static_cast<char>('0' + to_unsigned(value) % 10);
+  }
+  void write2(int value) {
+    const char* d = digits2(to_unsigned(value) % 100);
+    *out_++ = *d++;
+    *out_++ = *d;
+  }
+  void write2(int value, pad_type pad) {
+    unsigned int v = to_unsigned(value) % 100;
+    if (v >= 10) {
+      const char* d = digits2(v);
+      *out_++ = *d++;
+      *out_++ = *d;
+    } else {
+      out_ = detail::write_padding(out_, pad);
+      *out_++ = static_cast<char>('0' + v);
+    }
+  }
+
+  void write_year_extended(long long year) {
+    // At least 4 characters.
+    int width = 4;
+    if (year < 0) {
+      *out_++ = '-';
+      year = 0 - year;
+      --width;
+    }
+    uint32_or_64_or_128_t<long long> n = to_unsigned(year);
+    const int num_digits = count_digits(n);
+    if (width > num_digits)
+      out_ = detail::fill_n(out_, width - num_digits, '0');
+    out_ = format_decimal<Char>(out_, n, num_digits).end;
+  }
+  void write_year(long long year) {
+    if (year >= 0 && year < 10000) {
+      write2(static_cast<int>(year / 100));
+      write2(static_cast<int>(year % 100));
+    } else {
+      write_year_extended(year);
+    }
+  }
+
+  void write_utc_offset(long offset, numeric_system ns) {
+    if (offset < 0) {
+      *out_++ = '-';
+      offset = -offset;
+    } else {
+      *out_++ = '+';
+    }
+    offset /= 60;
+    write2(static_cast<int>(offset / 60));
+    if (ns != numeric_system::standard) *out_++ = ':';
+    write2(static_cast<int>(offset % 60));
+  }
+  template <typename T, FMT_ENABLE_IF(has_member_data_tm_gmtoff<T>::value)>
+  void format_utc_offset_impl(const T& tm, numeric_system ns) {
+    write_utc_offset(tm.tm_gmtoff, ns);
+  }
+  template <typename T, FMT_ENABLE_IF(!has_member_data_tm_gmtoff<T>::value)>
+  void format_utc_offset_impl(const T& tm, numeric_system ns) {
+#if defined(_WIN32) && defined(_UCRT)
+#  if FMT_USE_TZSET
+    tzset_once();
+#  endif
+    long offset = 0;
+    _get_timezone(&offset);
+    if (tm.tm_isdst) {
+      long dstbias = 0;
+      _get_dstbias(&dstbias);
+      offset += dstbias;
+    }
+    write_utc_offset(-offset, ns);
+#else
+    if (ns == numeric_system::standard) return format_localized('z');
+
+    // Extract timezone offset from timezone conversion functions.
+    std::tm gtm = tm;
+    std::time_t gt = std::mktime(&gtm);
+    std::tm ltm = gmtime(gt);
+    std::time_t lt = std::mktime(&ltm);
+    long offset = gt - lt;
+    write_utc_offset(offset, ns);
+#endif
+  }
+
+  template <typename T, FMT_ENABLE_IF(has_member_data_tm_zone<T>::value)>
+  void format_tz_name_impl(const T& tm) {
+    if (is_classic_)
+      out_ = write_tm_str<Char>(out_, tm.tm_zone, loc_);
+    else
+      format_localized('Z');
+  }
+  template <typename T, FMT_ENABLE_IF(!has_member_data_tm_zone<T>::value)>
+  void format_tz_name_impl(const T&) {
+    format_localized('Z');
+  }
+
+  void format_localized(char format, char modifier = 0) {
+    out_ = write<Char>(out_, tm_, loc_, format, modifier);
+  }
+
+ public:
+  tm_writer(const std::locale& loc, OutputIt out, const std::tm& tm,
+            const Duration* subsecs = nullptr)
+      : loc_(loc),
+        is_classic_(loc_ == get_classic_locale()),
+        out_(out),
+        subsecs_(subsecs),
+        tm_(tm) {}
+
+  auto out() const -> OutputIt { return out_; }
+
+  FMT_CONSTEXPR void on_text(const Char* begin, const Char* end) {
+    out_ = copy<Char>(begin, end, out_);
+  }
+
+  void on_abbr_weekday() {
+    if (is_classic_)
+      out_ = write(out_, tm_wday_short_name(tm_wday()));
+    else
+      format_localized('a');
+  }
+  void on_full_weekday() {
+    if (is_classic_)
+      out_ = write(out_, tm_wday_full_name(tm_wday()));
+    else
+      format_localized('A');
+  }
+  void on_dec0_weekday(numeric_system ns) {
+    if (is_classic_ || ns == numeric_system::standard) return write1(tm_wday());
+    format_localized('w', 'O');
+  }
+  void on_dec1_weekday(numeric_system ns) {
+    if (is_classic_ || ns == numeric_system::standard) {
+      auto wday = tm_wday();
+      write1(wday == 0 ? days_per_week : wday);
+    } else {
+      format_localized('u', 'O');
+    }
+  }
+
+  void on_abbr_month() {
+    if (is_classic_)
+      out_ = write(out_, tm_mon_short_name(tm_mon()));
+    else
+      format_localized('b');
+  }
+  void on_full_month() {
+    if (is_classic_)
+      out_ = write(out_, tm_mon_full_name(tm_mon()));
+    else
+      format_localized('B');
+  }
+
+  void on_datetime(numeric_system ns) {
+    if (is_classic_) {
+      on_abbr_weekday();
+      *out_++ = ' ';
+      on_abbr_month();
+      *out_++ = ' ';
+      on_day_of_month(numeric_system::standard, pad_type::space);
+      *out_++ = ' ';
+      on_iso_time();
+      *out_++ = ' ';
+      on_year(numeric_system::standard);
+    } else {
+      format_localized('c', ns == numeric_system::standard ? '\0' : 'E');
+    }
+  }
+  void on_loc_date(numeric_system ns) {
+    if (is_classic_)
+      on_us_date();
+    else
+      format_localized('x', ns == numeric_system::standard ? '\0' : 'E');
+  }
+  void on_loc_time(numeric_system ns) {
+    if (is_classic_)
+      on_iso_time();
+    else
+      format_localized('X', ns == numeric_system::standard ? '\0' : 'E');
+  }
+  void on_us_date() {
+    char buf[8];
+    write_digit2_separated(buf, to_unsigned(tm_mon() + 1),
+                           to_unsigned(tm_mday()),
+                           to_unsigned(split_year_lower(tm_year())), '/');
+    out_ = copy<Char>(std::begin(buf), std::end(buf), out_);
+  }
+  void on_iso_date() {
+    auto year = tm_year();
+    char buf[10];
+    size_t offset = 0;
+    if (year >= 0 && year < 10000) {
+      copy2(buf, digits2(static_cast<size_t>(year / 100)));
+    } else {
+      offset = 4;
+      write_year_extended(year);
+      year = 0;
+    }
+    write_digit2_separated(buf + 2, static_cast<unsigned>(year % 100),
+                           to_unsigned(tm_mon() + 1), to_unsigned(tm_mday()),
+                           '-');
+    out_ = copy<Char>(std::begin(buf) + offset, std::end(buf), out_);
+  }
+
+  void on_utc_offset(numeric_system ns) { format_utc_offset_impl(tm_, ns); }
+  void on_tz_name() { format_tz_name_impl(tm_); }
+
+  void on_year(numeric_system ns) {
+    if (is_classic_ || ns == numeric_system::standard)
+      return write_year(tm_year());
+    format_localized('Y', 'E');
+  }
+  void on_short_year(numeric_system ns) {
+    if (is_classic_ || ns == numeric_system::standard)
+      return write2(split_year_lower(tm_year()));
+    format_localized('y', 'O');
+  }
+  void on_offset_year() {
+    if (is_classic_) return write2(split_year_lower(tm_year()));
+    format_localized('y', 'E');
+  }
+
+  void on_century(numeric_system ns) {
+    if (is_classic_ || ns == numeric_system::standard) {
+      auto year = tm_year();
+      auto upper = year / 100;
+      if (year >= -99 && year < 0) {
+        // Zero upper on negative year.
+        *out_++ = '-';
+        *out_++ = '0';
+      } else if (upper >= 0 && upper < 100) {
+        write2(static_cast<int>(upper));
+      } else {
+        out_ = write<Char>(out_, upper);
+      }
+    } else {
+      format_localized('C', 'E');
+    }
+  }
+
+  void on_dec_month(numeric_system ns) {
+    if (is_classic_ || ns == numeric_system::standard)
+      return write2(tm_mon() + 1);
+    format_localized('m', 'O');
+  }
+
+  void on_dec0_week_of_year(numeric_system ns, pad_type pad) {
+    if (is_classic_ || ns == numeric_system::standard)
+      return write2((tm_yday() + days_per_week - tm_wday()) / days_per_week,
+                    pad);
+    format_localized('U', 'O');
+  }
+  void on_dec1_week_of_year(numeric_system ns, pad_type pad) {
+    if (is_classic_ || ns == numeric_system::standard) {
+      auto wday = tm_wday();
+      write2((tm_yday() + days_per_week -
+              (wday == 0 ? (days_per_week - 1) : (wday - 1))) /
+                 days_per_week,
+             pad);
+    } else {
+      format_localized('W', 'O');
+    }
+  }
+  void on_iso_week_of_year(numeric_system ns, pad_type pad) {
+    if (is_classic_ || ns == numeric_system::standard)
+      return write2(tm_iso_week_of_year(), pad);
+    format_localized('V', 'O');
+  }
+
+  void on_iso_week_based_year() { write_year(tm_iso_week_year()); }
+  void on_iso_week_based_short_year() {
+    write2(split_year_lower(tm_iso_week_year()));
+  }
+
+  void on_day_of_year() {
+    auto yday = tm_yday() + 1;
+    write1(yday / 100);
+    write2(yday % 100);
+  }
+  void on_day_of_month(numeric_system ns, pad_type pad) {
+    if (is_classic_ || ns == numeric_system::standard)
+      return write2(tm_mday(), pad);
+    format_localized('d', 'O');
+  }
+
+  void on_24_hour(numeric_system ns, pad_type pad) {
+    if (is_classic_ || ns == numeric_system::standard)
+      return write2(tm_hour(), pad);
+    format_localized('H', 'O');
+  }
+  void on_12_hour(numeric_system ns, pad_type pad) {
+    if (is_classic_ || ns == numeric_system::standard)
+      return write2(tm_hour12(), pad);
+    format_localized('I', 'O');
+  }
+  void on_minute(numeric_system ns, pad_type pad) {
+    if (is_classic_ || ns == numeric_system::standard)
+      return write2(tm_min(), pad);
+    format_localized('M', 'O');
+  }
+
+  void on_second(numeric_system ns, pad_type pad) {
+    if (is_classic_ || ns == numeric_system::standard) {
+      write2(tm_sec(), pad);
+      if (subsecs_) {
+        if (std::is_floating_point<typename Duration::rep>::value) {
+          auto buf = memory_buffer();
+          write_floating_seconds(buf, *subsecs_);
+          if (buf.size() > 1) {
+            // Remove the leading "0", write something like ".123".
+            out_ = std::copy(buf.begin() + 1, buf.end(), out_);
+          }
+        } else {
+          write_fractional_seconds<Char>(out_, *subsecs_);
+        }
+      }
+    } else {
+      // Currently no formatting of subseconds when a locale is set.
+      format_localized('S', 'O');
+    }
+  }
+
+  void on_12_hour_time() {
+    if (is_classic_) {
+      char buf[8];
+      write_digit2_separated(buf, to_unsigned(tm_hour12()),
+                             to_unsigned(tm_min()), to_unsigned(tm_sec()), ':');
+      out_ = copy<Char>(std::begin(buf), std::end(buf), out_);
+      *out_++ = ' ';
+      on_am_pm();
+    } else {
+      format_localized('r');
+    }
+  }
+  void on_24_hour_time() {
+    write2(tm_hour());
+    *out_++ = ':';
+    write2(tm_min());
+  }
+  void on_iso_time() {
+    on_24_hour_time();
+    *out_++ = ':';
+    on_second(numeric_system::standard, pad_type::zero);
+  }
+
+  void on_am_pm() {
+    if (is_classic_) {
+      *out_++ = tm_hour() < 12 ? 'A' : 'P';
+      *out_++ = 'M';
+    } else {
+      format_localized('p');
+    }
+  }
+
+  // These apply to chrono durations but not tm.
+  void on_duration_value() {}
+  void on_duration_unit() {}
+};
+
+struct chrono_format_checker : null_chrono_spec_handler<chrono_format_checker> {
+  bool has_precision_integral = false;
+
+  FMT_NORETURN void unsupported() { FMT_THROW(format_error("no date")); }
+
+  template <typename Char>
+  FMT_CONSTEXPR void on_text(const Char*, const Char*) {}
+  FMT_CONSTEXPR void on_day_of_year() {}
+  FMT_CONSTEXPR void on_24_hour(numeric_system, pad_type) {}
+  FMT_CONSTEXPR void on_12_hour(numeric_system, pad_type) {}
+  FMT_CONSTEXPR void on_minute(numeric_system, pad_type) {}
+  FMT_CONSTEXPR void on_second(numeric_system, pad_type) {}
+  FMT_CONSTEXPR void on_12_hour_time() {}
+  FMT_CONSTEXPR void on_24_hour_time() {}
+  FMT_CONSTEXPR void on_iso_time() {}
+  FMT_CONSTEXPR void on_am_pm() {}
+  FMT_CONSTEXPR void on_duration_value() const {
+    if (has_precision_integral) {
+      FMT_THROW(format_error("precision not allowed for this argument type"));
+    }
+  }
+  FMT_CONSTEXPR void on_duration_unit() {}
+};
+
+template <typename T,
+          FMT_ENABLE_IF(std::is_integral<T>::value&& has_isfinite<T>::value)>
+inline auto isfinite(T) -> bool {
+  return true;
+}
+
+template <typename T, FMT_ENABLE_IF(std::is_integral<T>::value)>
+inline auto mod(T x, int y) -> T {
+  return x % static_cast<T>(y);
+}
+template <typename T, FMT_ENABLE_IF(std::is_floating_point<T>::value)>
+inline auto mod(T x, int y) -> T {
+  return std::fmod(x, static_cast<T>(y));
+}
+
+// If T is an integral type, maps T to its unsigned counterpart, otherwise
+// leaves it unchanged (unlike std::make_unsigned).
+template <typename T, bool INTEGRAL = std::is_integral<T>::value>
+struct make_unsigned_or_unchanged {
+  using type = T;
+};
+
+template <typename T> struct make_unsigned_or_unchanged<T, true> {
+  using type = typename std::make_unsigned<T>::type;
+};
+
+template <typename Rep, typename Period,
+          FMT_ENABLE_IF(std::is_integral<Rep>::value)>
+inline auto get_milliseconds(std::chrono::duration<Rep, Period> d)
+    -> std::chrono::duration<Rep, std::milli> {
+  // this may overflow and/or the result may not fit in the
+  // target type.
+#if FMT_SAFE_DURATION_CAST
+  using CommonSecondsType =
+      typename std::common_type<decltype(d), std::chrono::seconds>::type;
+  const auto d_as_common = fmt_duration_cast<CommonSecondsType>(d);
+  const auto d_as_whole_seconds =
+      fmt_duration_cast<std::chrono::seconds>(d_as_common);
+  // this conversion should be nonproblematic
+  const auto diff = d_as_common - d_as_whole_seconds;
+  const auto ms =
+      fmt_duration_cast<std::chrono::duration<Rep, std::milli>>(diff);
+  return ms;
+#else
+  auto s = fmt_duration_cast<std::chrono::seconds>(d);
+  return fmt_duration_cast<std::chrono::milliseconds>(d - s);
+#endif
+}
+
+template <typename Char, typename Rep, typename OutputIt,
+          FMT_ENABLE_IF(std::is_integral<Rep>::value)>
+auto format_duration_value(OutputIt out, Rep val, int) -> OutputIt {
+  return write<Char>(out, val);
+}
+
+template <typename Char, typename Rep, typename OutputIt,
+          FMT_ENABLE_IF(std::is_floating_point<Rep>::value)>
+auto format_duration_value(OutputIt out, Rep val, int precision) -> OutputIt {
+  auto specs = format_specs();
+  specs.precision = precision;
+  specs.type =
+      precision >= 0 ? presentation_type::fixed : presentation_type::general;
+  return write<Char>(out, val, specs);
+}
+
+template <typename Char, typename OutputIt>
+auto copy_unit(string_view unit, OutputIt out, Char) -> OutputIt {
+  return std::copy(unit.begin(), unit.end(), out);
+}
+
+template <typename OutputIt>
+auto copy_unit(string_view unit, OutputIt out, wchar_t) -> OutputIt {
+  // This works when wchar_t is UTF-32 because units only contain characters
+  // that have the same representation in UTF-16 and UTF-32.
+  utf8_to_utf16 u(unit);
+  return std::copy(u.c_str(), u.c_str() + u.size(), out);
+}
+
+template <typename Char, typename Period, typename OutputIt>
+auto format_duration_unit(OutputIt out) -> OutputIt {
+  if (const char* unit = get_units<Period>())
+    return copy_unit(string_view(unit), out, Char());
+  *out++ = '[';
+  out = write<Char>(out, Period::num);
+  if (const_check(Period::den != 1)) {
+    *out++ = '/';
+    out = write<Char>(out, Period::den);
+  }
+  *out++ = ']';
+  *out++ = 's';
+  return out;
+}
+
+class get_locale {
+ private:
+  union {
+    std::locale locale_;
+  };
+  bool has_locale_ = false;
+
+ public:
+  get_locale(bool localized, locale_ref loc) : has_locale_(localized) {
+#ifndef FMT_STATIC_THOUSANDS_SEPARATOR
+    if (localized)
+      ::new (&locale_) std::locale(loc.template get<std::locale>());
+#endif
+  }
+  ~get_locale() {
+    if (has_locale_) locale_.~locale();
+  }
+  operator const std::locale&() const {
+    return has_locale_ ? locale_ : get_classic_locale();
+  }
+};
+
+template <typename FormatContext, typename OutputIt, typename Rep,
+          typename Period>
+struct chrono_formatter {
+  FormatContext& context;
+  OutputIt out;
+  int precision;
+  bool localized = false;
+  // rep is unsigned to avoid overflow.
+  using rep =
+      conditional_t<std::is_integral<Rep>::value && sizeof(Rep) < sizeof(int),
+                    unsigned, typename make_unsigned_or_unchanged<Rep>::type>;
+  rep val;
+  using seconds = std::chrono::duration<rep>;
+  seconds s;
+  using milliseconds = std::chrono::duration<rep, std::milli>;
+  bool negative;
+
+  using char_type = typename FormatContext::char_type;
+  using tm_writer_type = tm_writer<OutputIt, char_type>;
+
+  chrono_formatter(FormatContext& ctx, OutputIt o,
+                   std::chrono::duration<Rep, Period> d)
+      : context(ctx),
+        out(o),
+        val(static_cast<rep>(d.count())),
+        negative(false) {
+    if (d.count() < 0) {
+      val = 0 - val;
+      negative = true;
+    }
+
+    // this may overflow and/or the result may not fit in the
+    // target type.
+    // might need checked conversion (rep!=Rep)
+    s = fmt_duration_cast<seconds>(std::chrono::duration<rep, Period>(val));
+  }
+
+  // returns true if nan or inf, writes to out.
+  auto handle_nan_inf() -> bool {
+    if (isfinite(val)) {
+      return false;
+    }
+    if (isnan(val)) {
+      write_nan();
+      return true;
+    }
+    // must be +-inf
+    if (val > 0) {
+      write_pinf();
+    } else {
+      write_ninf();
+    }
+    return true;
+  }
+
+  auto days() const -> Rep { return static_cast<Rep>(s.count() / 86400); }
+  auto hour() const -> Rep {
+    return static_cast<Rep>(mod((s.count() / 3600), 24));
+  }
+
+  auto hour12() const -> Rep {
+    Rep hour = static_cast<Rep>(mod((s.count() / 3600), 12));
+    return hour <= 0 ? 12 : hour;
+  }
+
+  auto minute() const -> Rep {
+    return static_cast<Rep>(mod((s.count() / 60), 60));
+  }
+  auto second() const -> Rep { return static_cast<Rep>(mod(s.count(), 60)); }
+
+  auto time() const -> std::tm {
+    auto time = std::tm();
+    time.tm_hour = to_nonnegative_int(hour(), 24);
+    time.tm_min = to_nonnegative_int(minute(), 60);
+    time.tm_sec = to_nonnegative_int(second(), 60);
+    return time;
+  }
+
+  void write_sign() {
+    if (negative) {
+      *out++ = '-';
+      negative = false;
+    }
+  }
+
+  void write(Rep value, int width, pad_type pad = pad_type::zero) {
+    write_sign();
+    if (isnan(value)) return write_nan();
+    uint32_or_64_or_128_t<int> n =
+        to_unsigned(to_nonnegative_int(value, max_value<int>()));
+    int num_digits = detail::count_digits(n);
+    if (width > num_digits) {
+      out = detail::write_padding(out, pad, width - num_digits);
+    }
+    out = format_decimal<char_type>(out, n, num_digits).end;
+  }
+
+  void write_nan() { std::copy_n("nan", 3, out); }
+  void write_pinf() { std::copy_n("inf", 3, out); }
+  void write_ninf() { std::copy_n("-inf", 4, out); }
+
+  template <typename Callback, typename... Args>
+  void format_tm(const tm& time, Callback cb, Args... args) {
+    if (isnan(val)) return write_nan();
+    get_locale loc(localized, context.locale());
+    auto w = tm_writer_type(loc, out, time);
+    (w.*cb)(args...);
+    out = w.out();
+  }
+
+  void on_text(const char_type* begin, const char_type* end) {
+    std::copy(begin, end, out);
+  }
+
+  // These are not implemented because durations don't have date information.
+  void on_abbr_weekday() {}
+  void on_full_weekday() {}
+  void on_dec0_weekday(numeric_system) {}
+  void on_dec1_weekday(numeric_system) {}
+  void on_abbr_month() {}
+  void on_full_month() {}
+  void on_datetime(numeric_system) {}
+  void on_loc_date(numeric_system) {}
+  void on_loc_time(numeric_system) {}
+  void on_us_date() {}
+  void on_iso_date() {}
+  void on_utc_offset(numeric_system) {}
+  void on_tz_name() {}
+  void on_year(numeric_system) {}
+  void on_short_year(numeric_system) {}
+  void on_offset_year() {}
+  void on_century(numeric_system) {}
+  void on_iso_week_based_year() {}
+  void on_iso_week_based_short_year() {}
+  void on_dec_month(numeric_system) {}
+  void on_dec0_week_of_year(numeric_system, pad_type) {}
+  void on_dec1_week_of_year(numeric_system, pad_type) {}
+  void on_iso_week_of_year(numeric_system, pad_type) {}
+  void on_day_of_month(numeric_system, pad_type) {}
+
+  void on_day_of_year() {
+    if (handle_nan_inf()) return;
+    write(days(), 0);
+  }
+
+  void on_24_hour(numeric_system ns, pad_type pad) {
+    if (handle_nan_inf()) return;
+
+    if (ns == numeric_system::standard) return write(hour(), 2, pad);
+    auto time = tm();
+    time.tm_hour = to_nonnegative_int(hour(), 24);
+    format_tm(time, &tm_writer_type::on_24_hour, ns, pad);
+  }
+
+  void on_12_hour(numeric_system ns, pad_type pad) {
+    if (handle_nan_inf()) return;
+
+    if (ns == numeric_system::standard) return write(hour12(), 2, pad);
+    auto time = tm();
+    time.tm_hour = to_nonnegative_int(hour12(), 12);
+    format_tm(time, &tm_writer_type::on_12_hour, ns, pad);
+  }
+
+  void on_minute(numeric_system ns, pad_type pad) {
+    if (handle_nan_inf()) return;
+
+    if (ns == numeric_system::standard) return write(minute(), 2, pad);
+    auto time = tm();
+    time.tm_min = to_nonnegative_int(minute(), 60);
+    format_tm(time, &tm_writer_type::on_minute, ns, pad);
+  }
+
+  void on_second(numeric_system ns, pad_type pad) {
+    if (handle_nan_inf()) return;
+
+    if (ns == numeric_system::standard) {
+      if (std::is_floating_point<rep>::value) {
+        auto buf = memory_buffer();
+        write_floating_seconds(buf, std::chrono::duration<rep, Period>(val),
+                               precision);
+        if (negative) *out++ = '-';
+        if (buf.size() < 2 || buf[1] == '.') {
+          out = detail::write_padding(out, pad);
+        }
+        out = std::copy(buf.begin(), buf.end(), out);
+      } else {
+        write(second(), 2, pad);
+        write_fractional_seconds<char_type>(
+            out, std::chrono::duration<rep, Period>(val), precision);
+      }
+      return;
+    }
+    auto time = tm();
+    time.tm_sec = to_nonnegative_int(second(), 60);
+    format_tm(time, &tm_writer_type::on_second, ns, pad);
+  }
+
+  void on_12_hour_time() {
+    if (handle_nan_inf()) return;
+    format_tm(time(), &tm_writer_type::on_12_hour_time);
+  }
+
+  void on_24_hour_time() {
+    if (handle_nan_inf()) {
+      *out++ = ':';
+      handle_nan_inf();
+      return;
+    }
+
+    write(hour(), 2);
+    *out++ = ':';
+    write(minute(), 2);
+  }
+
+  void on_iso_time() {
+    on_24_hour_time();
+    *out++ = ':';
+    if (handle_nan_inf()) return;
+    on_second(numeric_system::standard, pad_type::zero);
+  }
+
+  void on_am_pm() {
+    if (handle_nan_inf()) return;
+    format_tm(time(), &tm_writer_type::on_am_pm);
+  }
+
+  void on_duration_value() {
+    if (handle_nan_inf()) return;
+    write_sign();
+    out = format_duration_value<char_type>(out, val, precision);
+  }
+
+  void on_duration_unit() {
+    out = format_duration_unit<char_type, Period>(out);
+  }
+};
+
+}  // namespace detail
+
+#if defined(__cpp_lib_chrono) && __cpp_lib_chrono >= 201907
+using weekday = std::chrono::weekday;
+using day = std::chrono::day;
+using month = std::chrono::month;
+using year = std::chrono::year;
+using year_month_day = std::chrono::year_month_day;
+#else
+// A fallback version of weekday.
+class weekday {
+ private:
+  unsigned char value_;
+
+ public:
+  weekday() = default;
+  constexpr explicit weekday(unsigned wd) noexcept
+      : value_(static_cast<unsigned char>(wd != 7 ? wd : 0)) {}
+  constexpr auto c_encoding() const noexcept -> unsigned { return value_; }
+};
+
+class day {
+ private:
+  unsigned char value_;
+
+ public:
+  day() = default;
+  constexpr explicit day(unsigned d) noexcept
+      : value_(static_cast<unsigned char>(d)) {}
+  constexpr explicit operator unsigned() const noexcept { return value_; }
+};
+
+class month {
+ private:
+  unsigned char value_;
+
+ public:
+  month() = default;
+  constexpr explicit month(unsigned m) noexcept
+      : value_(static_cast<unsigned char>(m)) {}
+  constexpr explicit operator unsigned() const noexcept { return value_; }
+};
+
+class year {
+ private:
+  int value_;
+
+ public:
+  year() = default;
+  constexpr explicit year(int y) noexcept : value_(y) {}
+  constexpr explicit operator int() const noexcept { return value_; }
+};
+
+class year_month_day {
+ private:
+  fmt::year year_;
+  fmt::month month_;
+  fmt::day day_;
+
+ public:
+  year_month_day() = default;
+  constexpr year_month_day(const year& y, const month& m, const day& d) noexcept
+      : year_(y), month_(m), day_(d) {}
+  constexpr auto year() const noexcept -> fmt::year { return year_; }
+  constexpr auto month() const noexcept -> fmt::month { return month_; }
+  constexpr auto day() const noexcept -> fmt::day { return day_; }
+};
+#endif
+
+template <typename Char>
+struct formatter<weekday, Char> : private formatter<std::tm, Char> {
+ private:
+  bool localized_ = false;
+  bool use_tm_formatter_ = false;
+
+ public:
+  FMT_CONSTEXPR auto parse(basic_format_parse_context<Char>& ctx)
+      -> decltype(ctx.begin()) {
+    auto it = ctx.begin(), end = ctx.end();
+    if (it != end && *it == 'L') {
+      ++it;
+      localized_ = true;
+      return it;
+    }
+    use_tm_formatter_ = it != end && *it != '}';
+    return use_tm_formatter_ ? formatter<std::tm, Char>::parse(ctx) : it;
+  }
+
+  template <typename FormatContext>
+  auto format(weekday wd, FormatContext& ctx) const -> decltype(ctx.out()) {
+    auto time = std::tm();
+    time.tm_wday = static_cast<int>(wd.c_encoding());
+    if (use_tm_formatter_) return formatter<std::tm, Char>::format(time, ctx);
+    detail::get_locale loc(localized_, ctx.locale());
+    auto w = detail::tm_writer<decltype(ctx.out()), Char>(loc, ctx.out(), time);
+    w.on_abbr_weekday();
+    return w.out();
+  }
+};
+
+template <typename Char>
+struct formatter<day, Char> : private formatter<std::tm, Char> {
+ private:
+  bool use_tm_formatter_ = false;
+
+ public:
+  FMT_CONSTEXPR auto parse(basic_format_parse_context<Char>& ctx)
+      -> decltype(ctx.begin()) {
+    auto it = ctx.begin(), end = ctx.end();
+    use_tm_formatter_ = it != end && *it != '}';
+    return use_tm_formatter_ ? formatter<std::tm, Char>::parse(ctx) : it;
+  }
+
+  template <typename FormatContext>
+  auto format(day d, FormatContext& ctx) const -> decltype(ctx.out()) {
+    auto time = std::tm();
+    time.tm_mday = static_cast<int>(static_cast<unsigned>(d));
+    if (use_tm_formatter_) return formatter<std::tm, Char>::format(time, ctx);
+    detail::get_locale loc(false, ctx.locale());
+    auto w = detail::tm_writer<decltype(ctx.out()), Char>(loc, ctx.out(), time);
+    w.on_day_of_month(detail::numeric_system::standard, detail::pad_type::zero);
+    return w.out();
+  }
+};
+
+template <typename Char>
+struct formatter<month, Char> : private formatter<std::tm, Char> {
+ private:
+  bool localized_ = false;
+  bool use_tm_formatter_ = false;
+
+ public:
+  FMT_CONSTEXPR auto parse(basic_format_parse_context<Char>& ctx)
+      -> decltype(ctx.begin()) {
+    auto it = ctx.begin(), end = ctx.end();
+    if (it != end && *it == 'L') {
+      ++it;
+      localized_ = true;
+      return it;
+    }
+    use_tm_formatter_ = it != end && *it != '}';
+    return use_tm_formatter_ ? formatter<std::tm, Char>::parse(ctx) : it;
+  }
+
+  template <typename FormatContext>
+  auto format(month m, FormatContext& ctx) const -> decltype(ctx.out()) {
+    auto time = std::tm();
+    time.tm_mon = static_cast<int>(static_cast<unsigned>(m)) - 1;
+    if (use_tm_formatter_) return formatter<std::tm, Char>::format(time, ctx);
+    detail::get_locale loc(localized_, ctx.locale());
+    auto w = detail::tm_writer<decltype(ctx.out()), Char>(loc, ctx.out(), time);
+    w.on_abbr_month();
+    return w.out();
+  }
+};
+
+template <typename Char>
+struct formatter<year, Char> : private formatter<std::tm, Char> {
+ private:
+  bool use_tm_formatter_ = false;
+
+ public:
+  FMT_CONSTEXPR auto parse(basic_format_parse_context<Char>& ctx)
+      -> decltype(ctx.begin()) {
+    auto it = ctx.begin(), end = ctx.end();
+    use_tm_formatter_ = it != end && *it != '}';
+    return use_tm_formatter_ ? formatter<std::tm, Char>::parse(ctx) : it;
+  }
+
+  template <typename FormatContext>
+  auto format(year y, FormatContext& ctx) const -> decltype(ctx.out()) {
+    auto time = std::tm();
+    time.tm_year = static_cast<int>(y) - 1900;
+    if (use_tm_formatter_) return formatter<std::tm, Char>::format(time, ctx);
+    detail::get_locale loc(false, ctx.locale());
+    auto w = detail::tm_writer<decltype(ctx.out()), Char>(loc, ctx.out(), time);
+    w.on_year(detail::numeric_system::standard);
+    return w.out();
+  }
+};
+
+template <typename Char>
+struct formatter<year_month_day, Char> : private formatter<std::tm, Char> {
+ private:
+  bool use_tm_formatter_ = false;
+
+ public:
+  FMT_CONSTEXPR auto parse(basic_format_parse_context<Char>& ctx)
+      -> decltype(ctx.begin()) {
+    auto it = ctx.begin(), end = ctx.end();
+    use_tm_formatter_ = it != end && *it != '}';
+    return use_tm_formatter_ ? formatter<std::tm, Char>::parse(ctx) : it;
+  }
+
+  template <typename FormatContext>
+  auto format(year_month_day val, FormatContext& ctx) const
+      -> decltype(ctx.out()) {
+    auto time = std::tm();
+    time.tm_year = static_cast<int>(val.year()) - 1900;
+    time.tm_mon = static_cast<int>(static_cast<unsigned>(val.month())) - 1;
+    time.tm_mday = static_cast<int>(static_cast<unsigned>(val.day()));
+    if (use_tm_formatter_) return formatter<std::tm, Char>::format(time, ctx);
+    detail::get_locale loc(true, ctx.locale());
+    auto w = detail::tm_writer<decltype(ctx.out()), Char>(loc, ctx.out(), time);
+    w.on_iso_date();
+    return w.out();
+  }
+};
+
+template <typename Rep, typename Period, typename Char>
+struct formatter<std::chrono::duration<Rep, Period>, Char> {
+ private:
+  format_specs specs_;
+  detail::arg_ref<Char> width_ref_;
+  detail::arg_ref<Char> precision_ref_;
+  bool localized_ = false;
+  basic_string_view<Char> format_str_;
+
+ public:
+  FMT_CONSTEXPR auto parse(basic_format_parse_context<Char>& ctx)
+      -> decltype(ctx.begin()) {
+    auto it = ctx.begin(), end = ctx.end();
+    if (it == end || *it == '}') return it;
+
+    it = detail::parse_align(it, end, specs_);
+    if (it == end) return it;
+
+    it = detail::parse_dynamic_spec(it, end, specs_.width, width_ref_, ctx);
+    if (it == end) return it;
+
+    auto checker = detail::chrono_format_checker();
+    if (*it == '.') {
+      checker.has_precision_integral = !std::is_floating_point<Rep>::value;
+      it = detail::parse_precision(it, end, specs_.precision, precision_ref_,
+                                   ctx);
+    }
+    if (it != end && *it == 'L') {
+      localized_ = true;
+      ++it;
+    }
+    end = detail::parse_chrono_format(it, end, checker);
+    format_str_ = {it, detail::to_unsigned(end - it)};
+    return end;
+  }
+
+  template <typename FormatContext>
+  auto format(std::chrono::duration<Rep, Period> d, FormatContext& ctx) const
+      -> decltype(ctx.out()) {
+    auto specs = specs_;
+    auto precision = specs.precision;
+    specs.precision = -1;
+    auto begin = format_str_.begin(), end = format_str_.end();
+    // As a possible future optimization, we could avoid extra copying if width
+    // is not specified.
+    auto buf = basic_memory_buffer<Char>();
+    auto out = std::back_inserter(buf);
+    detail::handle_dynamic_spec<detail::width_checker>(specs.width, width_ref_,
+                                                       ctx);
+    detail::handle_dynamic_spec<detail::precision_checker>(precision,
+                                                           precision_ref_, ctx);
+    if (begin == end || *begin == '}') {
+      out = detail::format_duration_value<Char>(out, d.count(), precision);
+      detail::format_duration_unit<Char, Period>(out);
+    } else {
+      using chrono_formatter =
+          detail::chrono_formatter<FormatContext, decltype(out), Rep, Period>;
+      auto f = chrono_formatter(ctx, out, d);
+      f.precision = precision;
+      f.localized = localized_;
+      detail::parse_chrono_format(begin, end, f);
+    }
+    return detail::write(
+        ctx.out(), basic_string_view<Char>(buf.data(), buf.size()), specs);
+  }
+};
+
+template <typename Char, typename Duration>
+struct formatter<std::chrono::time_point<std::chrono::system_clock, Duration>,
+                 Char> : formatter<std::tm, Char> {
+  FMT_CONSTEXPR formatter() {
+    this->format_str_ = detail::string_literal<Char, '%', 'F', ' ', '%', 'T'>{};
+  }
+
+  template <typename FormatContext>
+  auto format(std::chrono::time_point<std::chrono::system_clock, Duration> val,
+              FormatContext& ctx) const -> decltype(ctx.out()) {
+    std::tm tm = gmtime(val);
+    using period = typename Duration::period;
+    if (detail::const_check(
+            period::num == 1 && period::den == 1 &&
+            !std::is_floating_point<typename Duration::rep>::value)) {
+      return formatter<std::tm, Char>::format(tm, ctx);
+    }
+    Duration epoch = val.time_since_epoch();
+    Duration subsecs = detail::fmt_duration_cast<Duration>(
+        epoch - detail::fmt_duration_cast<std::chrono::seconds>(epoch));
+    if (subsecs.count() < 0) {
+      auto second =
+          detail::fmt_duration_cast<Duration>(std::chrono::seconds(1));
+      if (tm.tm_sec != 0)
+        --tm.tm_sec;
+      else
+        tm = gmtime(val - second);
+      subsecs += detail::fmt_duration_cast<Duration>(std::chrono::seconds(1));
+    }
+    return formatter<std::tm, Char>::do_format(tm, ctx, &subsecs);
+  }
+};
+
+#if FMT_USE_LOCAL_TIME
+template <typename Char, typename Duration>
+struct formatter<std::chrono::local_time<Duration>, Char>
+    : formatter<std::tm, Char> {
+  FMT_CONSTEXPR formatter() {
+    this->format_str_ = detail::string_literal<Char, '%', 'F', ' ', '%', 'T'>{};
+  }
+
+  template <typename FormatContext>
+  auto format(std::chrono::local_time<Duration> val, FormatContext& ctx) const
+      -> decltype(ctx.out()) {
+    using period = typename Duration::period;
+    if (period::num != 1 || period::den != 1 ||
+        std::is_floating_point<typename Duration::rep>::value) {
+      const auto epoch = val.time_since_epoch();
+      const auto subsecs = detail::fmt_duration_cast<Duration>(
+          epoch - detail::fmt_duration_cast<std::chrono::seconds>(epoch));
+
+      return formatter<std::tm, Char>::do_format(localtime(val), ctx, &subsecs);
+    }
+
+    return formatter<std::tm, Char>::format(localtime(val), ctx);
+  }
+};
+#endif
+
+#if FMT_USE_UTC_TIME
+template <typename Char, typename Duration>
+struct formatter<std::chrono::time_point<std::chrono::utc_clock, Duration>,
+                 Char>
+    : formatter<std::chrono::time_point<std::chrono::system_clock, Duration>,
+                Char> {
+  template <typename FormatContext>
+  auto format(std::chrono::time_point<std::chrono::utc_clock, Duration> val,
+              FormatContext& ctx) const -> decltype(ctx.out()) {
+    return formatter<
+        std::chrono::time_point<std::chrono::system_clock, Duration>,
+        Char>::format(std::chrono::utc_clock::to_sys(val), ctx);
+  }
+};
+#endif
+
+template <typename Char> struct formatter<std::tm, Char> {
+ private:
+  format_specs specs_;
+  detail::arg_ref<Char> width_ref_;
+
+ protected:
+  basic_string_view<Char> format_str_;
+
+  template <typename FormatContext, typename Duration>
+  auto do_format(const std::tm& tm, FormatContext& ctx,
+                 const Duration* subsecs) const -> decltype(ctx.out()) {
+    auto specs = specs_;
+    auto buf = basic_memory_buffer<Char>();
+    auto out = std::back_inserter(buf);
+    detail::handle_dynamic_spec<detail::width_checker>(specs.width, width_ref_,
+                                                       ctx);
+
+    auto loc_ref = ctx.locale();
+    detail::get_locale loc(static_cast<bool>(loc_ref), loc_ref);
+    auto w =
+        detail::tm_writer<decltype(out), Char, Duration>(loc, out, tm, subsecs);
+    detail::parse_chrono_format(format_str_.begin(), format_str_.end(), w);
+    return detail::write(
+        ctx.out(), basic_string_view<Char>(buf.data(), buf.size()), specs);
+  }
+
+ public:
+  FMT_CONSTEXPR auto parse(basic_format_parse_context<Char>& ctx)
+      -> decltype(ctx.begin()) {
+    auto it = ctx.begin(), end = ctx.end();
+    if (it == end || *it == '}') return it;
+
+    it = detail::parse_align(it, end, specs_);
+    if (it == end) return it;
+
+    it = detail::parse_dynamic_spec(it, end, specs_.width, width_ref_, ctx);
+    if (it == end) return it;
+
+    end = detail::parse_chrono_format(it, end, detail::tm_format_checker());
+    // Replace the default format_str only if the new spec is not empty.
+    if (end != it) format_str_ = {it, detail::to_unsigned(end - it)};
+    return end;
+  }
+
+  template <typename FormatContext>
+  auto format(const std::tm& tm, FormatContext& ctx) const
+      -> decltype(ctx.out()) {
+    return do_format<FormatContext, std::chrono::seconds>(tm, ctx, nullptr);
+  }
+};
+
+FMT_END_EXPORT
+FMT_END_NAMESPACE
+
+#endif  // FMT_CHRONO_H_
diff --git a/ext/spdlog/include/spdlog/fmt/bundled/color.h b/ext/spdlog/include/spdlog/fmt/bundled/color.h
new file mode 100644
index 0000000..f0e9dd9
--- /dev/null
+++ b/ext/spdlog/include/spdlog/fmt/bundled/color.h
@@ -0,0 +1,612 @@
+// Formatting library for C++ - color support
+//
+// Copyright (c) 2018 - present, Victor Zverovich and fmt contributors
+// All rights reserved.
+//
+// For the license information refer to format.h.
+
+#ifndef FMT_COLOR_H_
+#define FMT_COLOR_H_
+
+#include "format.h"
+
+FMT_BEGIN_NAMESPACE
+FMT_BEGIN_EXPORT
+
+enum class color : uint32_t {
+  alice_blue = 0xF0F8FF,               // rgb(240,248,255)
+  antique_white = 0xFAEBD7,            // rgb(250,235,215)
+  aqua = 0x00FFFF,                     // rgb(0,255,255)
+  aquamarine = 0x7FFFD4,               // rgb(127,255,212)
+  azure = 0xF0FFFF,                    // rgb(240,255,255)
+  beige = 0xF5F5DC,                    // rgb(245,245,220)
+  bisque = 0xFFE4C4,                   // rgb(255,228,196)
+  black = 0x000000,                    // rgb(0,0,0)
+  blanched_almond = 0xFFEBCD,          // rgb(255,235,205)
+  blue = 0x0000FF,                     // rgb(0,0,255)
+  blue_violet = 0x8A2BE2,              // rgb(138,43,226)
+  brown = 0xA52A2A,                    // rgb(165,42,42)
+  burly_wood = 0xDEB887,               // rgb(222,184,135)
+  cadet_blue = 0x5F9EA0,               // rgb(95,158,160)
+  chartreuse = 0x7FFF00,               // rgb(127,255,0)
+  chocolate = 0xD2691E,                // rgb(210,105,30)
+  coral = 0xFF7F50,                    // rgb(255,127,80)
+  cornflower_blue = 0x6495ED,          // rgb(100,149,237)
+  cornsilk = 0xFFF8DC,                 // rgb(255,248,220)
+  crimson = 0xDC143C,                  // rgb(220,20,60)
+  cyan = 0x00FFFF,                     // rgb(0,255,255)
+  dark_blue = 0x00008B,                // rgb(0,0,139)
+  dark_cyan = 0x008B8B,                // rgb(0,139,139)
+  dark_golden_rod = 0xB8860B,          // rgb(184,134,11)
+  dark_gray = 0xA9A9A9,                // rgb(169,169,169)
+  dark_green = 0x006400,               // rgb(0,100,0)
+  dark_khaki = 0xBDB76B,               // rgb(189,183,107)
+  dark_magenta = 0x8B008B,             // rgb(139,0,139)
+  dark_olive_green = 0x556B2F,         // rgb(85,107,47)
+  dark_orange = 0xFF8C00,              // rgb(255,140,0)
+  dark_orchid = 0x9932CC,              // rgb(153,50,204)
+  dark_red = 0x8B0000,                 // rgb(139,0,0)
+  dark_salmon = 0xE9967A,              // rgb(233,150,122)
+  dark_sea_green = 0x8FBC8F,           // rgb(143,188,143)
+  dark_slate_blue = 0x483D8B,          // rgb(72,61,139)
+  dark_slate_gray = 0x2F4F4F,          // rgb(47,79,79)
+  dark_turquoise = 0x00CED1,           // rgb(0,206,209)
+  dark_violet = 0x9400D3,              // rgb(148,0,211)
+  deep_pink = 0xFF1493,                // rgb(255,20,147)
+  deep_sky_blue = 0x00BFFF,            // rgb(0,191,255)
+  dim_gray = 0x696969,                 // rgb(105,105,105)
+  dodger_blue = 0x1E90FF,              // rgb(30,144,255)
+  fire_brick = 0xB22222,               // rgb(178,34,34)
+  floral_white = 0xFFFAF0,             // rgb(255,250,240)
+  forest_green = 0x228B22,             // rgb(34,139,34)
+  fuchsia = 0xFF00FF,                  // rgb(255,0,255)
+  gainsboro = 0xDCDCDC,                // rgb(220,220,220)
+  ghost_white = 0xF8F8FF,              // rgb(248,248,255)
+  gold = 0xFFD700,                     // rgb(255,215,0)
+  golden_rod = 0xDAA520,               // rgb(218,165,32)
+  gray = 0x808080,                     // rgb(128,128,128)
+  green = 0x008000,                    // rgb(0,128,0)
+  green_yellow = 0xADFF2F,             // rgb(173,255,47)
+  honey_dew = 0xF0FFF0,                // rgb(240,255,240)
+  hot_pink = 0xFF69B4,                 // rgb(255,105,180)
+  indian_red = 0xCD5C5C,               // rgb(205,92,92)
+  indigo = 0x4B0082,                   // rgb(75,0,130)
+  ivory = 0xFFFFF0,                    // rgb(255,255,240)
+  khaki = 0xF0E68C,                    // rgb(240,230,140)
+  lavender = 0xE6E6FA,                 // rgb(230,230,250)
+  lavender_blush = 0xFFF0F5,           // rgb(255,240,245)
+  lawn_green = 0x7CFC00,               // rgb(124,252,0)
+  lemon_chiffon = 0xFFFACD,            // rgb(255,250,205)
+  light_blue = 0xADD8E6,               // rgb(173,216,230)
+  light_coral = 0xF08080,              // rgb(240,128,128)
+  light_cyan = 0xE0FFFF,               // rgb(224,255,255)
+  light_golden_rod_yellow = 0xFAFAD2,  // rgb(250,250,210)
+  light_gray = 0xD3D3D3,               // rgb(211,211,211)
+  light_green = 0x90EE90,              // rgb(144,238,144)
+  light_pink = 0xFFB6C1,               // rgb(255,182,193)
+  light_salmon = 0xFFA07A,             // rgb(255,160,122)
+  light_sea_green = 0x20B2AA,          // rgb(32,178,170)
+  light_sky_blue = 0x87CEFA,           // rgb(135,206,250)
+  light_slate_gray = 0x778899,         // rgb(119,136,153)
+  light_steel_blue = 0xB0C4DE,         // rgb(176,196,222)
+  light_yellow = 0xFFFFE0,             // rgb(255,255,224)
+  lime = 0x00FF00,                     // rgb(0,255,0)
+  lime_green = 0x32CD32,               // rgb(50,205,50)
+  linen = 0xFAF0E6,                    // rgb(250,240,230)
+  magenta = 0xFF00FF,                  // rgb(255,0,255)
+  maroon = 0x800000,                   // rgb(128,0,0)
+  medium_aquamarine = 0x66CDAA,        // rgb(102,205,170)
+  medium_blue = 0x0000CD,              // rgb(0,0,205)
+  medium_orchid = 0xBA55D3,            // rgb(186,85,211)
+  medium_purple = 0x9370DB,            // rgb(147,112,219)
+  medium_sea_green = 0x3CB371,         // rgb(60,179,113)
+  medium_slate_blue = 0x7B68EE,        // rgb(123,104,238)
+  medium_spring_green = 0x00FA9A,      // rgb(0,250,154)
+  medium_turquoise = 0x48D1CC,         // rgb(72,209,204)
+  medium_violet_red = 0xC71585,        // rgb(199,21,133)
+  midnight_blue = 0x191970,            // rgb(25,25,112)
+  mint_cream = 0xF5FFFA,               // rgb(245,255,250)
+  misty_rose = 0xFFE4E1,               // rgb(255,228,225)
+  moccasin = 0xFFE4B5,                 // rgb(255,228,181)
+  navajo_white = 0xFFDEAD,             // rgb(255,222,173)
+  navy = 0x000080,                     // rgb(0,0,128)
+  old_lace = 0xFDF5E6,                 // rgb(253,245,230)
+  olive = 0x808000,                    // rgb(128,128,0)
+  olive_drab = 0x6B8E23,               // rgb(107,142,35)
+  orange = 0xFFA500,                   // rgb(255,165,0)
+  orange_red = 0xFF4500,               // rgb(255,69,0)
+  orchid = 0xDA70D6,                   // rgb(218,112,214)
+  pale_golden_rod = 0xEEE8AA,          // rgb(238,232,170)
+  pale_green = 0x98FB98,               // rgb(152,251,152)
+  pale_turquoise = 0xAFEEEE,           // rgb(175,238,238)
+  pale_violet_red = 0xDB7093,          // rgb(219,112,147)
+  papaya_whip = 0xFFEFD5,              // rgb(255,239,213)
+  peach_puff = 0xFFDAB9,               // rgb(255,218,185)
+  peru = 0xCD853F,                     // rgb(205,133,63)
+  pink = 0xFFC0CB,                     // rgb(255,192,203)
+  plum = 0xDDA0DD,                     // rgb(221,160,221)
+  powder_blue = 0xB0E0E6,              // rgb(176,224,230)
+  purple = 0x800080,                   // rgb(128,0,128)
+  rebecca_purple = 0x663399,           // rgb(102,51,153)
+  red = 0xFF0000,                      // rgb(255,0,0)
+  rosy_brown = 0xBC8F8F,               // rgb(188,143,143)
+  royal_blue = 0x4169E1,               // rgb(65,105,225)
+  saddle_brown = 0x8B4513,             // rgb(139,69,19)
+  salmon = 0xFA8072,                   // rgb(250,128,114)
+  sandy_brown = 0xF4A460,              // rgb(244,164,96)
+  sea_green = 0x2E8B57,                // rgb(46,139,87)
+  sea_shell = 0xFFF5EE,                // rgb(255,245,238)
+  sienna = 0xA0522D,                   // rgb(160,82,45)
+  silver = 0xC0C0C0,                   // rgb(192,192,192)
+  sky_blue = 0x87CEEB,                 // rgb(135,206,235)
+  slate_blue = 0x6A5ACD,               // rgb(106,90,205)
+  slate_gray = 0x708090,               // rgb(112,128,144)
+  snow = 0xFFFAFA,                     // rgb(255,250,250)
+  spring_green = 0x00FF7F,             // rgb(0,255,127)
+  steel_blue = 0x4682B4,               // rgb(70,130,180)
+  tan = 0xD2B48C,                      // rgb(210,180,140)
+  teal = 0x008080,                     // rgb(0,128,128)
+  thistle = 0xD8BFD8,                  // rgb(216,191,216)
+  tomato = 0xFF6347,                   // rgb(255,99,71)
+  turquoise = 0x40E0D0,                // rgb(64,224,208)
+  violet = 0xEE82EE,                   // rgb(238,130,238)
+  wheat = 0xF5DEB3,                    // rgb(245,222,179)
+  white = 0xFFFFFF,                    // rgb(255,255,255)
+  white_smoke = 0xF5F5F5,              // rgb(245,245,245)
+  yellow = 0xFFFF00,                   // rgb(255,255,0)
+  yellow_green = 0x9ACD32              // rgb(154,205,50)
+};                                     // enum class color
+
+enum class terminal_color : uint8_t {
+  black = 30,
+  red,
+  green,
+  yellow,
+  blue,
+  magenta,
+  cyan,
+  white,
+  bright_black = 90,
+  bright_red,
+  bright_green,
+  bright_yellow,
+  bright_blue,
+  bright_magenta,
+  bright_cyan,
+  bright_white
+};
+
+enum class emphasis : uint8_t {
+  bold = 1,
+  faint = 1 << 1,
+  italic = 1 << 2,
+  underline = 1 << 3,
+  blink = 1 << 4,
+  reverse = 1 << 5,
+  conceal = 1 << 6,
+  strikethrough = 1 << 7,
+};
+
+// rgb is a struct for red, green and blue colors.
+// Using the name "rgb" makes some editors show the color in a tooltip.
+struct rgb {
+  FMT_CONSTEXPR rgb() : r(0), g(0), b(0) {}
+  FMT_CONSTEXPR rgb(uint8_t r_, uint8_t g_, uint8_t b_) : r(r_), g(g_), b(b_) {}
+  FMT_CONSTEXPR rgb(uint32_t hex)
+      : r((hex >> 16) & 0xFF), g((hex >> 8) & 0xFF), b(hex & 0xFF) {}
+  FMT_CONSTEXPR rgb(color hex)
+      : r((uint32_t(hex) >> 16) & 0xFF),
+        g((uint32_t(hex) >> 8) & 0xFF),
+        b(uint32_t(hex) & 0xFF) {}
+  uint8_t r;
+  uint8_t g;
+  uint8_t b;
+};
+
+namespace detail {
+
+// color is a struct of either a rgb color or a terminal color.
+struct color_type {
+  FMT_CONSTEXPR color_type() noexcept : is_rgb(), value{} {}
+  FMT_CONSTEXPR color_type(color rgb_color) noexcept : is_rgb(true), value{} {
+    value.rgb_color = static_cast<uint32_t>(rgb_color);
+  }
+  FMT_CONSTEXPR color_type(rgb rgb_color) noexcept : is_rgb(true), value{} {
+    value.rgb_color = (static_cast<uint32_t>(rgb_color.r) << 16) |
+                      (static_cast<uint32_t>(rgb_color.g) << 8) | rgb_color.b;
+  }
+  FMT_CONSTEXPR color_type(terminal_color term_color) noexcept
+      : is_rgb(), value{} {
+    value.term_color = static_cast<uint8_t>(term_color);
+  }
+  bool is_rgb;
+  union color_union {
+    uint8_t term_color;
+    uint32_t rgb_color;
+  } value;
+};
+}  // namespace detail
+
+/// A text style consisting of foreground and background colors and emphasis.
+class text_style {
+ public:
+  FMT_CONSTEXPR text_style(emphasis em = emphasis()) noexcept
+      : set_foreground_color(), set_background_color(), ems(em) {}
+
+  FMT_CONSTEXPR auto operator|=(const text_style& rhs) -> text_style& {
+    if (!set_foreground_color) {
+      set_foreground_color = rhs.set_foreground_color;
+      foreground_color = rhs.foreground_color;
+    } else if (rhs.set_foreground_color) {
+      if (!foreground_color.is_rgb || !rhs.foreground_color.is_rgb)
+        report_error("can't OR a terminal color");
+      foreground_color.value.rgb_color |= rhs.foreground_color.value.rgb_color;
+    }
+
+    if (!set_background_color) {
+      set_background_color = rhs.set_background_color;
+      background_color = rhs.background_color;
+    } else if (rhs.set_background_color) {
+      if (!background_color.is_rgb || !rhs.background_color.is_rgb)
+        report_error("can't OR a terminal color");
+      background_color.value.rgb_color |= rhs.background_color.value.rgb_color;
+    }
+
+    ems = static_cast<emphasis>(static_cast<uint8_t>(ems) |
+                                static_cast<uint8_t>(rhs.ems));
+    return *this;
+  }
+
+  friend FMT_CONSTEXPR auto operator|(text_style lhs, const text_style& rhs)
+      -> text_style {
+    return lhs |= rhs;
+  }
+
+  FMT_CONSTEXPR auto has_foreground() const noexcept -> bool {
+    return set_foreground_color;
+  }
+  FMT_CONSTEXPR auto has_background() const noexcept -> bool {
+    return set_background_color;
+  }
+  FMT_CONSTEXPR auto has_emphasis() const noexcept -> bool {
+    return static_cast<uint8_t>(ems) != 0;
+  }
+  FMT_CONSTEXPR auto get_foreground() const noexcept -> detail::color_type {
+    FMT_ASSERT(has_foreground(), "no foreground specified for this style");
+    return foreground_color;
+  }
+  FMT_CONSTEXPR auto get_background() const noexcept -> detail::color_type {
+    FMT_ASSERT(has_background(), "no background specified for this style");
+    return background_color;
+  }
+  FMT_CONSTEXPR auto get_emphasis() const noexcept -> emphasis {
+    FMT_ASSERT(has_emphasis(), "no emphasis specified for this style");
+    return ems;
+  }
+
+ private:
+  FMT_CONSTEXPR text_style(bool is_foreground,
+                           detail::color_type text_color) noexcept
+      : set_foreground_color(), set_background_color(), ems() {
+    if (is_foreground) {
+      foreground_color = text_color;
+      set_foreground_color = true;
+    } else {
+      background_color = text_color;
+      set_background_color = true;
+    }
+  }
+
+  friend FMT_CONSTEXPR auto fg(detail::color_type foreground) noexcept
+      -> text_style;
+
+  friend FMT_CONSTEXPR auto bg(detail::color_type background) noexcept
+      -> text_style;
+
+  detail::color_type foreground_color;
+  detail::color_type background_color;
+  bool set_foreground_color;
+  bool set_background_color;
+  emphasis ems;
+};
+
+/// Creates a text style from the foreground (text) color.
+FMT_CONSTEXPR inline auto fg(detail::color_type foreground) noexcept
+    -> text_style {
+  return text_style(true, foreground);
+}
+
+/// Creates a text style from the background color.
+FMT_CONSTEXPR inline auto bg(detail::color_type background) noexcept
+    -> text_style {
+  return text_style(false, background);
+}
+
+FMT_CONSTEXPR inline auto operator|(emphasis lhs, emphasis rhs) noexcept
+    -> text_style {
+  return text_style(lhs) | rhs;
+}
+
+namespace detail {
+
+template <typename Char> struct ansi_color_escape {
+  FMT_CONSTEXPR ansi_color_escape(detail::color_type text_color,
+                                  const char* esc) noexcept {
+    // If we have a terminal color, we need to output another escape code
+    // sequence.
+    if (!text_color.is_rgb) {
+      bool is_background = esc == string_view("\x1b[48;2;");
+      uint32_t value = text_color.value.term_color;
+      // Background ASCII codes are the same as the foreground ones but with
+      // 10 more.
+      if (is_background) value += 10u;
+
+      size_t index = 0;
+      buffer[index++] = static_cast<Char>('\x1b');
+      buffer[index++] = static_cast<Char>('[');
+
+      if (value >= 100u) {
+        buffer[index++] = static_cast<Char>('1');
+        value %= 100u;
+      }
+      buffer[index++] = static_cast<Char>('0' + value / 10u);
+      buffer[index++] = static_cast<Char>('0' + value % 10u);
+
+      buffer[index++] = static_cast<Char>('m');
+      buffer[index++] = static_cast<Char>('\0');
+      return;
+    }
+
+    for (int i = 0; i < 7; i++) {
+      buffer[i] = static_cast<Char>(esc[i]);
+    }
+    rgb color(text_color.value.rgb_color);
+    to_esc(color.r, buffer + 7, ';');
+    to_esc(color.g, buffer + 11, ';');
+    to_esc(color.b, buffer + 15, 'm');
+    buffer[19] = static_cast<Char>(0);
+  }
+  FMT_CONSTEXPR ansi_color_escape(emphasis em) noexcept {
+    uint8_t em_codes[num_emphases] = {};
+    if (has_emphasis(em, emphasis::bold)) em_codes[0] = 1;
+    if (has_emphasis(em, emphasis::faint)) em_codes[1] = 2;
+    if (has_emphasis(em, emphasis::italic)) em_codes[2] = 3;
+    if (has_emphasis(em, emphasis::underline)) em_codes[3] = 4;
+    if (has_emphasis(em, emphasis::blink)) em_codes[4] = 5;
+    if (has_emphasis(em, emphasis::reverse)) em_codes[5] = 7;
+    if (has_emphasis(em, emphasis::conceal)) em_codes[6] = 8;
+    if (has_emphasis(em, emphasis::strikethrough)) em_codes[7] = 9;
+
+    size_t index = 0;
+    for (size_t i = 0; i < num_emphases; ++i) {
+      if (!em_codes[i]) continue;
+      buffer[index++] = static_cast<Char>('\x1b');
+      buffer[index++] = static_cast<Char>('[');
+      buffer[index++] = static_cast<Char>('0' + em_codes[i]);
+      buffer[index++] = static_cast<Char>('m');
+    }
+    buffer[index++] = static_cast<Char>(0);
+  }
+  FMT_CONSTEXPR operator const Char*() const noexcept { return buffer; }
+
+  FMT_CONSTEXPR auto begin() const noexcept -> const Char* { return buffer; }
+  FMT_CONSTEXPR20 auto end() const noexcept -> const Char* {
+    return buffer + basic_string_view<Char>(buffer).size();
+  }
+
+ private:
+  static constexpr size_t num_emphases = 8;
+  Char buffer[7u + 3u * num_emphases + 1u];
+
+  static FMT_CONSTEXPR void to_esc(uint8_t c, Char* out,
+                                   char delimiter) noexcept {
+    out[0] = static_cast<Char>('0' + c / 100);
+    out[1] = static_cast<Char>('0' + c / 10 % 10);
+    out[2] = static_cast<Char>('0' + c % 10);
+    out[3] = static_cast<Char>(delimiter);
+  }
+  static FMT_CONSTEXPR auto has_emphasis(emphasis em, emphasis mask) noexcept
+      -> bool {
+    return static_cast<uint8_t>(em) & static_cast<uint8_t>(mask);
+  }
+};
+
+template <typename Char>
+FMT_CONSTEXPR auto make_foreground_color(detail::color_type foreground) noexcept
+    -> ansi_color_escape<Char> {
+  return ansi_color_escape<Char>(foreground, "\x1b[38;2;");
+}
+
+template <typename Char>
+FMT_CONSTEXPR auto make_background_color(detail::color_type background) noexcept
+    -> ansi_color_escape<Char> {
+  return ansi_color_escape<Char>(background, "\x1b[48;2;");
+}
+
+template <typename Char>
+FMT_CONSTEXPR auto make_emphasis(emphasis em) noexcept
+    -> ansi_color_escape<Char> {
+  return ansi_color_escape<Char>(em);
+}
+
+template <typename Char> inline void reset_color(buffer<Char>& buffer) {
+  auto reset_color = string_view("\x1b[0m");
+  buffer.append(reset_color.begin(), reset_color.end());
+}
+
+template <typename T> struct styled_arg : detail::view {
+  const T& value;
+  text_style style;
+  styled_arg(const T& v, text_style s) : value(v), style(s) {}
+};
+
+template <typename Char>
+void vformat_to(
+    buffer<Char>& buf, const text_style& ts, basic_string_view<Char> format_str,
+    basic_format_args<buffered_context<type_identity_t<Char>>> args) {
+  bool has_style = false;
+  if (ts.has_emphasis()) {
+    has_style = true;
+    auto emphasis = detail::make_emphasis<Char>(ts.get_emphasis());
+    buf.append(emphasis.begin(), emphasis.end());
+  }
+  if (ts.has_foreground()) {
+    has_style = true;
+    auto foreground = detail::make_foreground_color<Char>(ts.get_foreground());
+    buf.append(foreground.begin(), foreground.end());
+  }
+  if (ts.has_background()) {
+    has_style = true;
+    auto background = detail::make_background_color<Char>(ts.get_background());
+    buf.append(background.begin(), background.end());
+  }
+  detail::vformat_to(buf, format_str, args, {});
+  if (has_style) detail::reset_color<Char>(buf);
+}
+
+}  // namespace detail
+
+inline void vprint(FILE* f, const text_style& ts, string_view fmt,
+                   format_args args) {
+  auto buf = memory_buffer();
+  detail::vformat_to(buf, ts, fmt, args);
+  print(f, FMT_STRING("{}"), string_view(buf.begin(), buf.size()));
+}
+
+/**
+ * Formats a string and prints it to the specified file stream using ANSI
+ * escape sequences to specify text formatting.
+ *
+ * **Example**:
+ *
+ *     fmt::print(fmt::emphasis::bold | fg(fmt::color::red),
+ *                "Elapsed time: {0:.2f} seconds", 1.23);
+ */
+template <typename... T>
+void print(FILE* f, const text_style& ts, format_string<T...> fmt,
+           T&&... args) {
+  vprint(f, ts, fmt, fmt::make_format_args(args...));
+}
+
+/**
+ * Formats a string and prints it to stdout using ANSI escape sequences to
+ * specify text formatting.
+ *
+ * **Example**:
+ *
+ *     fmt::print(fmt::emphasis::bold | fg(fmt::color::red),
+ *                "Elapsed time: {0:.2f} seconds", 1.23);
+ */
+template <typename... T>
+void print(const text_style& ts, format_string<T...> fmt, T&&... args) {
+  return print(stdout, ts, fmt, std::forward<T>(args)...);
+}
+
+inline auto vformat(const text_style& ts, string_view fmt, format_args args)
+    -> std::string {
+  auto buf = memory_buffer();
+  detail::vformat_to(buf, ts, fmt, args);
+  return fmt::to_string(buf);
+}
+
+/**
+ * Formats arguments and returns the result as a string using ANSI escape
+ * sequences to specify text formatting.
+ *
+ * **Example**:
+ *
+ * ```
+ * #include <fmt/color.h>
+ * std::string message = fmt::format(fmt::emphasis::bold | fg(fmt::color::red),
+ *                                   "The answer is {}", 42);
+ * ```
+ */
+template <typename... T>
+inline auto format(const text_style& ts, format_string<T...> fmt, T&&... args)
+    -> std::string {
+  return fmt::vformat(ts, fmt, fmt::make_format_args(args...));
+}
+
+/// Formats a string with the given text_style and writes the output to `out`.
+template <typename OutputIt,
+          FMT_ENABLE_IF(detail::is_output_iterator<OutputIt, char>::value)>
+auto vformat_to(OutputIt out, const text_style& ts, string_view fmt,
+                format_args args) -> OutputIt {
+  auto&& buf = detail::get_buffer<char>(out);
+  detail::vformat_to(buf, ts, fmt, args);
+  return detail::get_iterator(buf, out);
+}
+
+/**
+ * Formats arguments with the given text style, writes the result to the output
+ * iterator `out` and returns the iterator past the end of the output range.
+ *
+ * **Example**:
+ *
+ *     std::vector<char> out;
+ *     fmt::format_to(std::back_inserter(out),
+ *                    fmt::emphasis::bold | fg(fmt::color::red), "{}", 42);
+ */
+template <typename OutputIt, typename... T,
+          FMT_ENABLE_IF(detail::is_output_iterator<OutputIt, char>::value)>
+inline auto format_to(OutputIt out, const text_style& ts,
+                      format_string<T...> fmt, T&&... args) -> OutputIt {
+  return vformat_to(out, ts, fmt, fmt::make_format_args(args...));
+}
+
+template <typename T, typename Char>
+struct formatter<detail::styled_arg<T>, Char> : formatter<T, Char> {
+  template <typename FormatContext>
+  auto format(const detail::styled_arg<T>& arg, FormatContext& ctx) const
+      -> decltype(ctx.out()) {
+    const auto& ts = arg.style;
+    const auto& value = arg.value;
+    auto out = ctx.out();
+
+    bool has_style = false;
+    if (ts.has_emphasis()) {
+      has_style = true;
+      auto emphasis = detail::make_emphasis<Char>(ts.get_emphasis());
+      out = std::copy(emphasis.begin(), emphasis.end(), out);
+    }
+    if (ts.has_foreground()) {
+      has_style = true;
+      auto foreground =
+          detail::make_foreground_color<Char>(ts.get_foreground());
+      out = std::copy(foreground.begin(), foreground.end(), out);
+    }
+    if (ts.has_background()) {
+      has_style = true;
+      auto background =
+          detail::make_background_color<Char>(ts.get_background());
+      out = std::copy(background.begin(), background.end(), out);
+    }
+    out = formatter<T, Char>::format(value, ctx);
+    if (has_style) {
+      auto reset_color = string_view("\x1b[0m");
+      out = std::copy(reset_color.begin(), reset_color.end(), out);
+    }
+    return out;
+  }
+};
+
+/**
+ * Returns an argument that will be formatted using ANSI escape sequences,
+ * to be used in a formatting function.
+ *
+ * **Example**:
+ *
+ *     fmt::print("Elapsed time: {0:.2f} seconds",
+ *                fmt::styled(1.23, fmt::fg(fmt::color::green) |
+ *                                  fmt::bg(fmt::color::blue)));
+ */
+template <typename T>
+FMT_CONSTEXPR auto styled(const T& value, text_style ts)
+    -> detail::styled_arg<remove_cvref_t<T>> {
+  return detail::styled_arg<remove_cvref_t<T>>{value, ts};
+}
+
+FMT_END_EXPORT
+FMT_END_NAMESPACE
+
+#endif  // FMT_COLOR_H_
diff --git a/ext/spdlog/include/spdlog/fmt/bundled/compile.h b/ext/spdlog/include/spdlog/fmt/bundled/compile.h
new file mode 100644
index 0000000..b2afc2c
--- /dev/null
+++ b/ext/spdlog/include/spdlog/fmt/bundled/compile.h
@@ -0,0 +1,529 @@
+// Formatting library for C++ - experimental format string compilation
+//
+// Copyright (c) 2012 - present, Victor Zverovich and fmt contributors
+// All rights reserved.
+//
+// For the license information refer to format.h.
+
+#ifndef FMT_COMPILE_H_
+#define FMT_COMPILE_H_
+
+#ifndef FMT_MODULE
+#  include <iterator>  // std::back_inserter
+#endif
+
+#include "format.h"
+
+FMT_BEGIN_NAMESPACE
+
+// A compile-time string which is compiled into fast formatting code.
+FMT_EXPORT class compiled_string {};
+
+namespace detail {
+
+template <typename T, typename InputIt>
+FMT_CONSTEXPR inline auto copy(InputIt begin, InputIt end, counting_iterator it)
+    -> counting_iterator {
+  return it + (end - begin);
+}
+
+template <typename S>
+struct is_compiled_string : std::is_base_of<compiled_string, S> {};
+
+/**
+ * Converts a string literal `s` into a format string that will be parsed at
+ * compile time and converted into efficient formatting code. Requires C++17
+ * `constexpr if` compiler support.
+ *
+ * **Example**:
+ *
+ *     // Converts 42 into std::string using the most efficient method and no
+ *     // runtime format string processing.
+ *     std::string s = fmt::format(FMT_COMPILE("{}"), 42);
+ */
+#if defined(__cpp_if_constexpr) && defined(__cpp_return_type_deduction)
+#  define FMT_COMPILE(s) FMT_STRING_IMPL(s, fmt::compiled_string, explicit)
+#else
+#  define FMT_COMPILE(s) FMT_STRING(s)
+#endif
+
+#if FMT_USE_NONTYPE_TEMPLATE_ARGS
+template <typename Char, size_t N,
+          fmt::detail_exported::fixed_string<Char, N> Str>
+struct udl_compiled_string : compiled_string {
+  using char_type = Char;
+  explicit constexpr operator basic_string_view<char_type>() const {
+    return {Str.data, N - 1};
+  }
+};
+#endif
+
+template <typename T, typename... Tail>
+auto first(const T& value, const Tail&...) -> const T& {
+  return value;
+}
+
+#if defined(__cpp_if_constexpr) && defined(__cpp_return_type_deduction)
+template <typename... Args> struct type_list {};
+
+// Returns a reference to the argument at index N from [first, rest...].
+template <int N, typename T, typename... Args>
+constexpr const auto& get([[maybe_unused]] const T& first,
+                          [[maybe_unused]] const Args&... rest) {
+  static_assert(N < 1 + sizeof...(Args), "index is out of bounds");
+  if constexpr (N == 0)
+    return first;
+  else
+    return detail::get<N - 1>(rest...);
+}
+
+template <typename Char, typename... Args>
+constexpr int get_arg_index_by_name(basic_string_view<Char> name,
+                                    type_list<Args...>) {
+  return get_arg_index_by_name<Args...>(name);
+}
+
+template <int N, typename> struct get_type_impl;
+
+template <int N, typename... Args> struct get_type_impl<N, type_list<Args...>> {
+  using type =
+      remove_cvref_t<decltype(detail::get<N>(std::declval<Args>()...))>;
+};
+
+template <int N, typename T>
+using get_type = typename get_type_impl<N, T>::type;
+
+template <typename T> struct is_compiled_format : std::false_type {};
+
+template <typename Char> struct text {
+  basic_string_view<Char> data;
+  using char_type = Char;
+
+  template <typename OutputIt, typename... Args>
+  constexpr OutputIt format(OutputIt out, const Args&...) const {
+    return write<Char>(out, data);
+  }
+};
+
+template <typename Char>
+struct is_compiled_format<text<Char>> : std::true_type {};
+
+template <typename Char>
+constexpr text<Char> make_text(basic_string_view<Char> s, size_t pos,
+                               size_t size) {
+  return {{&s[pos], size}};
+}
+
+template <typename Char> struct code_unit {
+  Char value;
+  using char_type = Char;
+
+  template <typename OutputIt, typename... Args>
+  constexpr OutputIt format(OutputIt out, const Args&...) const {
+    *out++ = value;
+    return out;
+  }
+};
+
+// This ensures that the argument type is convertible to `const T&`.
+template <typename T, int N, typename... Args>
+constexpr const T& get_arg_checked(const Args&... args) {
+  const auto& arg = detail::get<N>(args...);
+  if constexpr (detail::is_named_arg<remove_cvref_t<decltype(arg)>>()) {
+    return arg.value;
+  } else {
+    return arg;
+  }
+}
+
+template <typename Char>
+struct is_compiled_format<code_unit<Char>> : std::true_type {};
+
+// A replacement field that refers to argument N.
+template <typename Char, typename T, int N> struct field {
+  using char_type = Char;
+
+  template <typename OutputIt, typename... Args>
+  constexpr OutputIt format(OutputIt out, const Args&... args) const {
+    const T& arg = get_arg_checked<T, N>(args...);
+    if constexpr (std::is_convertible<T, basic_string_view<Char>>::value) {
+      auto s = basic_string_view<Char>(arg);
+      return copy<Char>(s.begin(), s.end(), out);
+    }
+    return write<Char>(out, arg);
+  }
+};
+
+template <typename Char, typename T, int N>
+struct is_compiled_format<field<Char, T, N>> : std::true_type {};
+
+// A replacement field that refers to argument with name.
+template <typename Char> struct runtime_named_field {
+  using char_type = Char;
+  basic_string_view<Char> name;
+
+  template <typename OutputIt, typename T>
+  constexpr static bool try_format_argument(
+      OutputIt& out,
+      // [[maybe_unused]] due to unused-but-set-parameter warning in GCC 7,8,9
+      [[maybe_unused]] basic_string_view<Char> arg_name, const T& arg) {
+    if constexpr (is_named_arg<typename std::remove_cv<T>::type>::value) {
+      if (arg_name == arg.name) {
+        out = write<Char>(out, arg.value);
+        return true;
+      }
+    }
+    return false;
+  }
+
+  template <typename OutputIt, typename... Args>
+  constexpr OutputIt format(OutputIt out, const Args&... args) const {
+    bool found = (try_format_argument(out, name, args) || ...);
+    if (!found) {
+      FMT_THROW(format_error("argument with specified name is not found"));
+    }
+    return out;
+  }
+};
+
+template <typename Char>
+struct is_compiled_format<runtime_named_field<Char>> : std::true_type {};
+
+// A replacement field that refers to argument N and has format specifiers.
+template <typename Char, typename T, int N> struct spec_field {
+  using char_type = Char;
+  formatter<T, Char> fmt;
+
+  template <typename OutputIt, typename... Args>
+  constexpr FMT_INLINE OutputIt format(OutputIt out,
+                                       const Args&... args) const {
+    const auto& vargs =
+        fmt::make_format_args<basic_format_context<OutputIt, Char>>(args...);
+    basic_format_context<OutputIt, Char> ctx(out, vargs);
+    return fmt.format(get_arg_checked<T, N>(args...), ctx);
+  }
+};
+
+template <typename Char, typename T, int N>
+struct is_compiled_format<spec_field<Char, T, N>> : std::true_type {};
+
+template <typename L, typename R> struct concat {
+  L lhs;
+  R rhs;
+  using char_type = typename L::char_type;
+
+  template <typename OutputIt, typename... Args>
+  constexpr OutputIt format(OutputIt out, const Args&... args) const {
+    out = lhs.format(out, args...);
+    return rhs.format(out, args...);
+  }
+};
+
+template <typename L, typename R>
+struct is_compiled_format<concat<L, R>> : std::true_type {};
+
+template <typename L, typename R>
+constexpr concat<L, R> make_concat(L lhs, R rhs) {
+  return {lhs, rhs};
+}
+
+struct unknown_format {};
+
+template <typename Char>
+constexpr size_t parse_text(basic_string_view<Char> str, size_t pos) {
+  for (size_t size = str.size(); pos != size; ++pos) {
+    if (str[pos] == '{' || str[pos] == '}') break;
+  }
+  return pos;
+}
+
+template <typename Args, size_t POS, int ID, typename S>
+constexpr auto compile_format_string(S fmt);
+
+template <typename Args, size_t POS, int ID, typename T, typename S>
+constexpr auto parse_tail(T head, S fmt) {
+  if constexpr (POS != basic_string_view<typename S::char_type>(fmt).size()) {
+    constexpr auto tail = compile_format_string<Args, POS, ID>(fmt);
+    if constexpr (std::is_same<remove_cvref_t<decltype(tail)>,
+                               unknown_format>())
+      return tail;
+    else
+      return make_concat(head, tail);
+  } else {
+    return head;
+  }
+}
+
+template <typename T, typename Char> struct parse_specs_result {
+  formatter<T, Char> fmt;
+  size_t end;
+  int next_arg_id;
+};
+
+enum { manual_indexing_id = -1 };
+
+template <typename T, typename Char>
+constexpr parse_specs_result<T, Char> parse_specs(basic_string_view<Char> str,
+                                                  size_t pos, int next_arg_id) {
+  str.remove_prefix(pos);
+  auto ctx =
+      compile_parse_context<Char>(str, max_value<int>(), nullptr, next_arg_id);
+  auto f = formatter<T, Char>();
+  auto end = f.parse(ctx);
+  return {f, pos + fmt::detail::to_unsigned(end - str.data()),
+          next_arg_id == 0 ? manual_indexing_id : ctx.next_arg_id()};
+}
+
+template <typename Char> struct arg_id_handler {
+  arg_ref<Char> arg_id;
+
+  constexpr int on_auto() {
+    FMT_ASSERT(false, "handler cannot be used with automatic indexing");
+    return 0;
+  }
+  constexpr int on_index(int id) {
+    arg_id = arg_ref<Char>(id);
+    return 0;
+  }
+  constexpr int on_name(basic_string_view<Char> id) {
+    arg_id = arg_ref<Char>(id);
+    return 0;
+  }
+};
+
+template <typename Char> struct parse_arg_id_result {
+  arg_ref<Char> arg_id;
+  const Char* arg_id_end;
+};
+
+template <int ID, typename Char>
+constexpr auto parse_arg_id(const Char* begin, const Char* end) {
+  auto handler = arg_id_handler<Char>{arg_ref<Char>{}};
+  auto arg_id_end = parse_arg_id(begin, end, handler);
+  return parse_arg_id_result<Char>{handler.arg_id, arg_id_end};
+}
+
+template <typename T, typename Enable = void> struct field_type {
+  using type = remove_cvref_t<T>;
+};
+
+template <typename T>
+struct field_type<T, enable_if_t<detail::is_named_arg<T>::value>> {
+  using type = remove_cvref_t<decltype(T::value)>;
+};
+
+template <typename T, typename Args, size_t END_POS, int ARG_INDEX, int NEXT_ID,
+          typename S>
+constexpr auto parse_replacement_field_then_tail(S fmt) {
+  using char_type = typename S::char_type;
+  constexpr auto str = basic_string_view<char_type>(fmt);
+  constexpr char_type c = END_POS != str.size() ? str[END_POS] : char_type();
+  if constexpr (c == '}') {
+    return parse_tail<Args, END_POS + 1, NEXT_ID>(
+        field<char_type, typename field_type<T>::type, ARG_INDEX>(), fmt);
+  } else if constexpr (c != ':') {
+    FMT_THROW(format_error("expected ':'"));
+  } else {
+    constexpr auto result = parse_specs<typename field_type<T>::type>(
+        str, END_POS + 1, NEXT_ID == manual_indexing_id ? 0 : NEXT_ID);
+    if constexpr (result.end >= str.size() || str[result.end] != '}') {
+      FMT_THROW(format_error("expected '}'"));
+      return 0;
+    } else {
+      return parse_tail<Args, result.end + 1, result.next_arg_id>(
+          spec_field<char_type, typename field_type<T>::type, ARG_INDEX>{
+              result.fmt},
+          fmt);
+    }
+  }
+}
+
+// Compiles a non-empty format string and returns the compiled representation
+// or unknown_format() on unrecognized input.
+template <typename Args, size_t POS, int ID, typename S>
+constexpr auto compile_format_string(S fmt) {
+  using char_type = typename S::char_type;
+  constexpr auto str = basic_string_view<char_type>(fmt);
+  if constexpr (str[POS] == '{') {
+    if constexpr (POS + 1 == str.size())
+      FMT_THROW(format_error("unmatched '{' in format string"));
+    if constexpr (str[POS + 1] == '{') {
+      return parse_tail<Args, POS + 2, ID>(make_text(str, POS, 1), fmt);
+    } else if constexpr (str[POS + 1] == '}' || str[POS + 1] == ':') {
+      static_assert(ID != manual_indexing_id,
+                    "cannot switch from manual to automatic argument indexing");
+      constexpr auto next_id =
+          ID != manual_indexing_id ? ID + 1 : manual_indexing_id;
+      return parse_replacement_field_then_tail<get_type<ID, Args>, Args,
+                                               POS + 1, ID, next_id>(fmt);
+    } else {
+      constexpr auto arg_id_result =
+          parse_arg_id<ID>(str.data() + POS + 1, str.data() + str.size());
+      constexpr auto arg_id_end_pos = arg_id_result.arg_id_end - str.data();
+      constexpr char_type c =
+          arg_id_end_pos != str.size() ? str[arg_id_end_pos] : char_type();
+      static_assert(c == '}' || c == ':', "missing '}' in format string");
+      if constexpr (arg_id_result.arg_id.kind == arg_id_kind::index) {
+        static_assert(
+            ID == manual_indexing_id || ID == 0,
+            "cannot switch from automatic to manual argument indexing");
+        constexpr auto arg_index = arg_id_result.arg_id.val.index;
+        return parse_replacement_field_then_tail<get_type<arg_index, Args>,
+                                                 Args, arg_id_end_pos,
+                                                 arg_index, manual_indexing_id>(
+            fmt);
+      } else if constexpr (arg_id_result.arg_id.kind == arg_id_kind::name) {
+        constexpr auto arg_index =
+            get_arg_index_by_name(arg_id_result.arg_id.val.name, Args{});
+        if constexpr (arg_index >= 0) {
+          constexpr auto next_id =
+              ID != manual_indexing_id ? ID + 1 : manual_indexing_id;
+          return parse_replacement_field_then_tail<
+              decltype(get_type<arg_index, Args>::value), Args, arg_id_end_pos,
+              arg_index, next_id>(fmt);
+        } else if constexpr (c == '}') {
+          return parse_tail<Args, arg_id_end_pos + 1, ID>(
+              runtime_named_field<char_type>{arg_id_result.arg_id.val.name},
+              fmt);
+        } else if constexpr (c == ':') {
+          return unknown_format();  // no type info for specs parsing
+        }
+      }
+    }
+  } else if constexpr (str[POS] == '}') {
+    if constexpr (POS + 1 == str.size())
+      FMT_THROW(format_error("unmatched '}' in format string"));
+    return parse_tail<Args, POS + 2, ID>(make_text(str, POS, 1), fmt);
+  } else {
+    constexpr auto end = parse_text(str, POS + 1);
+    if constexpr (end - POS > 1) {
+      return parse_tail<Args, end, ID>(make_text(str, POS, end - POS), fmt);
+    } else {
+      return parse_tail<Args, end, ID>(code_unit<char_type>{str[POS]}, fmt);
+    }
+  }
+}
+
+template <typename... Args, typename S,
+          FMT_ENABLE_IF(detail::is_compiled_string<S>::value)>
+constexpr auto compile(S fmt) {
+  constexpr auto str = basic_string_view<typename S::char_type>(fmt);
+  if constexpr (str.size() == 0) {
+    return detail::make_text(str, 0, 0);
+  } else {
+    constexpr auto result =
+        detail::compile_format_string<detail::type_list<Args...>, 0, 0>(fmt);
+    return result;
+  }
+}
+#endif  // defined(__cpp_if_constexpr) && defined(__cpp_return_type_deduction)
+}  // namespace detail
+
+FMT_BEGIN_EXPORT
+
+#if defined(__cpp_if_constexpr) && defined(__cpp_return_type_deduction)
+
+template <typename CompiledFormat, typename... Args,
+          typename Char = typename CompiledFormat::char_type,
+          FMT_ENABLE_IF(detail::is_compiled_format<CompiledFormat>::value)>
+FMT_INLINE std::basic_string<Char> format(const CompiledFormat& cf,
+                                          const Args&... args) {
+  auto s = std::basic_string<Char>();
+  cf.format(std::back_inserter(s), args...);
+  return s;
+}
+
+template <typename OutputIt, typename CompiledFormat, typename... Args,
+          FMT_ENABLE_IF(detail::is_compiled_format<CompiledFormat>::value)>
+constexpr FMT_INLINE OutputIt format_to(OutputIt out, const CompiledFormat& cf,
+                                        const Args&... args) {
+  return cf.format(out, args...);
+}
+
+template <typename S, typename... Args,
+          FMT_ENABLE_IF(detail::is_compiled_string<S>::value)>
+FMT_INLINE std::basic_string<typename S::char_type> format(const S&,
+                                                           Args&&... args) {
+  if constexpr (std::is_same<typename S::char_type, char>::value) {
+    constexpr auto str = basic_string_view<typename S::char_type>(S());
+    if constexpr (str.size() == 2 && str[0] == '{' && str[1] == '}') {
+      const auto& first = detail::first(args...);
+      if constexpr (detail::is_named_arg<
+                        remove_cvref_t<decltype(first)>>::value) {
+        return fmt::to_string(first.value);
+      } else {
+        return fmt::to_string(first);
+      }
+    }
+  }
+  constexpr auto compiled = detail::compile<Args...>(S());
+  if constexpr (std::is_same<remove_cvref_t<decltype(compiled)>,
+                             detail::unknown_format>()) {
+    return fmt::format(
+        static_cast<basic_string_view<typename S::char_type>>(S()),
+        std::forward<Args>(args)...);
+  } else {
+    return fmt::format(compiled, std::forward<Args>(args)...);
+  }
+}
+
+template <typename OutputIt, typename S, typename... Args,
+          FMT_ENABLE_IF(detail::is_compiled_string<S>::value)>
+FMT_CONSTEXPR OutputIt format_to(OutputIt out, const S&, Args&&... args) {
+  constexpr auto compiled = detail::compile<Args...>(S());
+  if constexpr (std::is_same<remove_cvref_t<decltype(compiled)>,
+                             detail::unknown_format>()) {
+    return fmt::format_to(
+        out, static_cast<basic_string_view<typename S::char_type>>(S()),
+        std::forward<Args>(args)...);
+  } else {
+    return fmt::format_to(out, compiled, std::forward<Args>(args)...);
+  }
+}
+#endif
+
+template <typename OutputIt, typename S, typename... Args,
+          FMT_ENABLE_IF(detail::is_compiled_string<S>::value)>
+auto format_to_n(OutputIt out, size_t n, const S& fmt, Args&&... args)
+    -> format_to_n_result<OutputIt> {
+  using traits = detail::fixed_buffer_traits;
+  auto buf = detail::iterator_buffer<OutputIt, char, traits>(out, n);
+  fmt::format_to(std::back_inserter(buf), fmt, std::forward<Args>(args)...);
+  return {buf.out(), buf.count()};
+}
+
+template <typename S, typename... Args,
+          FMT_ENABLE_IF(detail::is_compiled_string<S>::value)>
+FMT_CONSTEXPR20 auto formatted_size(const S& fmt, const Args&... args)
+    -> size_t {
+  return fmt::format_to(detail::counting_iterator(), fmt, args...).count();
+}
+
+template <typename S, typename... Args,
+          FMT_ENABLE_IF(detail::is_compiled_string<S>::value)>
+void print(std::FILE* f, const S& fmt, const Args&... args) {
+  memory_buffer buffer;
+  fmt::format_to(std::back_inserter(buffer), fmt, args...);
+  detail::print(f, {buffer.data(), buffer.size()});
+}
+
+template <typename S, typename... Args,
+          FMT_ENABLE_IF(detail::is_compiled_string<S>::value)>
+void print(const S& fmt, const Args&... args) {
+  print(stdout, fmt, args...);
+}
+
+#if FMT_USE_NONTYPE_TEMPLATE_ARGS
+inline namespace literals {
+template <detail_exported::fixed_string Str> constexpr auto operator""_cf() {
+  using char_t = remove_cvref_t<decltype(Str.data[0])>;
+  return detail::udl_compiled_string<char_t, sizeof(Str.data) / sizeof(char_t),
+                                     Str>();
+}
+}  // namespace literals
+#endif
+
+FMT_END_EXPORT
+FMT_END_NAMESPACE
+
+#endif  // FMT_COMPILE_H_
diff --git a/ext/spdlog/include/spdlog/fmt/bundled/core.h b/ext/spdlog/include/spdlog/fmt/bundled/core.h
new file mode 100644
index 0000000..8ca735f
--- /dev/null
+++ b/ext/spdlog/include/spdlog/fmt/bundled/core.h
@@ -0,0 +1,5 @@
+// This file is only provided for compatibility and may be removed in future
+// versions. Use fmt/base.h if you don't need fmt::format and fmt/format.h
+// otherwise.
+
+#include "format.h"
diff --git a/ext/spdlog/include/spdlog/fmt/bundled/fmt.license.rst b/ext/spdlog/include/spdlog/fmt/bundled/fmt.license.rst
new file mode 100644
index 0000000..f0ec3db
--- /dev/null
+++ b/ext/spdlog/include/spdlog/fmt/bundled/fmt.license.rst
@@ -0,0 +1,27 @@
+Copyright (c) 2012 - present, Victor Zverovich
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+--- Optional exception to the license ---
+
+As an exception, if, as a result of your compiling your source code, portions
+of this Software are embedded into a machine-executable object form of such
+source code, you may redistribute such embedded portions in such object form
+without including the above copyright and permission notices.
diff --git a/ext/spdlog/include/spdlog/fmt/bundled/format-inl.h b/ext/spdlog/include/spdlog/fmt/bundled/format-inl.h
new file mode 100644
index 0000000..a887483
--- /dev/null
+++ b/ext/spdlog/include/spdlog/fmt/bundled/format-inl.h
@@ -0,0 +1,1928 @@
+// Formatting library for C++ - implementation
+//
+// Copyright (c) 2012 - 2016, Victor Zverovich
+// All rights reserved.
+//
+// For the license information refer to format.h.
+
+#ifndef FMT_FORMAT_INL_H_
+#define FMT_FORMAT_INL_H_
+
+#ifndef FMT_MODULE
+#  include <algorithm>
+#  include <cerrno>  // errno
+#  include <climits>
+#  include <cmath>
+#  include <exception>
+
+#  if !defined(FMT_STATIC_THOUSANDS_SEPARATOR)
+#    include <locale>
+#  endif
+#endif
+
+#if defined(_WIN32) && !defined(FMT_USE_WRITE_CONSOLE)
+#  include <io.h>  // _isatty
+#endif
+
+#include "format.h"
+
+FMT_BEGIN_NAMESPACE
+namespace detail {
+
+FMT_FUNC void assert_fail(const char* file, int line, const char* message) {
+  // Use unchecked std::fprintf to avoid triggering another assertion when
+  // writing to stderr fails
+  std::fprintf(stderr, "%s:%d: assertion failed: %s", file, line, message);
+  // Chosen instead of std::abort to satisfy Clang in CUDA mode during device
+  // code pass.
+  std::terminate();
+}
+
+FMT_FUNC void format_error_code(detail::buffer<char>& out, int error_code,
+                                string_view message) noexcept {
+  // Report error code making sure that the output fits into
+  // inline_buffer_size to avoid dynamic memory allocation and potential
+  // bad_alloc.
+  out.try_resize(0);
+  static const char SEP[] = ": ";
+  static const char ERROR_STR[] = "error ";
+  // Subtract 2 to account for terminating null characters in SEP and ERROR_STR.
+  size_t error_code_size = sizeof(SEP) + sizeof(ERROR_STR) - 2;
+  auto abs_value = static_cast<uint32_or_64_or_128_t<int>>(error_code);
+  if (detail::is_negative(error_code)) {
+    abs_value = 0 - abs_value;
+    ++error_code_size;
+  }
+  error_code_size += detail::to_unsigned(detail::count_digits(abs_value));
+  auto it = appender(out);
+  if (message.size() <= inline_buffer_size - error_code_size)
+    fmt::format_to(it, FMT_STRING("{}{}"), message, SEP);
+  fmt::format_to(it, FMT_STRING("{}{}"), ERROR_STR, error_code);
+  FMT_ASSERT(out.size() <= inline_buffer_size, "");
+}
+
+FMT_FUNC void report_error(format_func func, int error_code,
+                           const char* message) noexcept {
+  memory_buffer full_message;
+  func(full_message, error_code, message);
+  // Don't use fwrite_fully because the latter may throw.
+  if (std::fwrite(full_message.data(), full_message.size(), 1, stderr) > 0)
+    std::fputc('\n', stderr);
+}
+
+// A wrapper around fwrite that throws on error.
+inline void fwrite_fully(const void* ptr, size_t count, FILE* stream) {
+  size_t written = std::fwrite(ptr, 1, count, stream);
+  if (written < count)
+    FMT_THROW(system_error(errno, FMT_STRING("cannot write to file")));
+}
+
+#ifndef FMT_STATIC_THOUSANDS_SEPARATOR
+template <typename Locale>
+locale_ref::locale_ref(const Locale& loc) : locale_(&loc) {
+  static_assert(std::is_same<Locale, std::locale>::value, "");
+}
+
+template <typename Locale> auto locale_ref::get() const -> Locale {
+  static_assert(std::is_same<Locale, std::locale>::value, "");
+  return locale_ ? *static_cast<const std::locale*>(locale_) : std::locale();
+}
+
+template <typename Char>
+FMT_FUNC auto thousands_sep_impl(locale_ref loc) -> thousands_sep_result<Char> {
+  auto& facet = std::use_facet<std::numpunct<Char>>(loc.get<std::locale>());
+  auto grouping = facet.grouping();
+  auto thousands_sep = grouping.empty() ? Char() : facet.thousands_sep();
+  return {std::move(grouping), thousands_sep};
+}
+template <typename Char>
+FMT_FUNC auto decimal_point_impl(locale_ref loc) -> Char {
+  return std::use_facet<std::numpunct<Char>>(loc.get<std::locale>())
+      .decimal_point();
+}
+#else
+template <typename Char>
+FMT_FUNC auto thousands_sep_impl(locale_ref) -> thousands_sep_result<Char> {
+  return {"\03", FMT_STATIC_THOUSANDS_SEPARATOR};
+}
+template <typename Char> FMT_FUNC Char decimal_point_impl(locale_ref) {
+  return '.';
+}
+#endif
+
+FMT_FUNC auto write_loc(appender out, loc_value value,
+                        const format_specs& specs, locale_ref loc) -> bool {
+#ifdef FMT_STATIC_THOUSANDS_SEPARATOR
+  value.visit(loc_writer<>{
+      out, specs, std::string(1, FMT_STATIC_THOUSANDS_SEPARATOR), "\3", "."});
+  return true;
+#else
+  auto locale = loc.get<std::locale>();
+  // We cannot use the num_put<char> facet because it may produce output in
+  // a wrong encoding.
+  using facet = format_facet<std::locale>;
+  if (std::has_facet<facet>(locale))
+    return std::use_facet<facet>(locale).put(out, value, specs);
+  return facet(locale).put(out, value, specs);
+#endif
+}
+}  // namespace detail
+
+FMT_FUNC void report_error(const char* message) {
+  FMT_THROW(format_error(message));
+}
+
+template <typename Locale> typename Locale::id format_facet<Locale>::id;
+
+#ifndef FMT_STATIC_THOUSANDS_SEPARATOR
+template <typename Locale> format_facet<Locale>::format_facet(Locale& loc) {
+  auto& numpunct = std::use_facet<std::numpunct<char>>(loc);
+  grouping_ = numpunct.grouping();
+  if (!grouping_.empty()) separator_ = std::string(1, numpunct.thousands_sep());
+}
+
+template <>
+FMT_API FMT_FUNC auto format_facet<std::locale>::do_put(
+    appender out, loc_value val, const format_specs& specs) const -> bool {
+  return val.visit(
+      detail::loc_writer<>{out, specs, separator_, grouping_, decimal_point_});
+}
+#endif
+
+FMT_FUNC auto vsystem_error(int error_code, string_view fmt, format_args args)
+    -> std::system_error {
+  auto ec = std::error_code(error_code, std::generic_category());
+  return std::system_error(ec, vformat(fmt, args));
+}
+
+namespace detail {
+
+template <typename F>
+inline auto operator==(basic_fp<F> x, basic_fp<F> y) -> bool {
+  return x.f == y.f && x.e == y.e;
+}
+
+// Compilers should be able to optimize this into the ror instruction.
+FMT_CONSTEXPR inline auto rotr(uint32_t n, uint32_t r) noexcept -> uint32_t {
+  r &= 31;
+  return (n >> r) | (n << (32 - r));
+}
+FMT_CONSTEXPR inline auto rotr(uint64_t n, uint32_t r) noexcept -> uint64_t {
+  r &= 63;
+  return (n >> r) | (n << (64 - r));
+}
+
+// Implementation of Dragonbox algorithm: https://github.com/jk-jeon/dragonbox.
+namespace dragonbox {
+// Computes upper 64 bits of multiplication of a 32-bit unsigned integer and a
+// 64-bit unsigned integer.
+inline auto umul96_upper64(uint32_t x, uint64_t y) noexcept -> uint64_t {
+  return umul128_upper64(static_cast<uint64_t>(x) << 32, y);
+}
+
+// Computes lower 128 bits of multiplication of a 64-bit unsigned integer and a
+// 128-bit unsigned integer.
+inline auto umul192_lower128(uint64_t x, uint128_fallback y) noexcept
+    -> uint128_fallback {
+  uint64_t high = x * y.high();
+  uint128_fallback high_low = umul128(x, y.low());
+  return {high + high_low.high(), high_low.low()};
+}
+
+// Computes lower 64 bits of multiplication of a 32-bit unsigned integer and a
+// 64-bit unsigned integer.
+inline auto umul96_lower64(uint32_t x, uint64_t y) noexcept -> uint64_t {
+  return x * y;
+}
+
+// Various fast log computations.
+inline auto floor_log10_pow2_minus_log10_4_over_3(int e) noexcept -> int {
+  FMT_ASSERT(e <= 2936 && e >= -2985, "too large exponent");
+  return (e * 631305 - 261663) >> 21;
+}
+
+FMT_INLINE_VARIABLE constexpr struct {
+  uint32_t divisor;
+  int shift_amount;
+} div_small_pow10_infos[] = {{10, 16}, {100, 16}};
+
+// Replaces n by floor(n / pow(10, N)) returning true if and only if n is
+// divisible by pow(10, N).
+// Precondition: n <= pow(10, N + 1).
+template <int N>
+auto check_divisibility_and_divide_by_pow10(uint32_t& n) noexcept -> bool {
+  // The numbers below are chosen such that:
+  //   1. floor(n/d) = floor(nm / 2^k) where d=10 or d=100,
+  //   2. nm mod 2^k < m if and only if n is divisible by d,
+  // where m is magic_number, k is shift_amount
+  // and d is divisor.
+  //
+  // Item 1 is a common technique of replacing division by a constant with
+  // multiplication, see e.g. "Division by Invariant Integers Using
+  // Multiplication" by Granlund and Montgomery (1994). magic_number (m) is set
+  // to ceil(2^k/d) for large enough k.
+  // The idea for item 2 originates from Schubfach.
+  constexpr auto info = div_small_pow10_infos[N - 1];
+  FMT_ASSERT(n <= info.divisor * 10, "n is too large");
+  constexpr uint32_t magic_number =
+      (1u << info.shift_amount) / info.divisor + 1;
+  n *= magic_number;
+  const uint32_t comparison_mask = (1u << info.shift_amount) - 1;
+  bool result = (n & comparison_mask) < magic_number;
+  n >>= info.shift_amount;
+  return result;
+}
+
+// Computes floor(n / pow(10, N)) for small n and N.
+// Precondition: n <= pow(10, N + 1).
+template <int N> auto small_division_by_pow10(uint32_t n) noexcept -> uint32_t {
+  constexpr auto info = div_small_pow10_infos[N - 1];
+  FMT_ASSERT(n <= info.divisor * 10, "n is too large");
+  constexpr uint32_t magic_number =
+      (1u << info.shift_amount) / info.divisor + 1;
+  return (n * magic_number) >> info.shift_amount;
+}
+
+// Computes floor(n / 10^(kappa + 1)) (float)
+inline auto divide_by_10_to_kappa_plus_1(uint32_t n) noexcept -> uint32_t {
+  // 1374389535 = ceil(2^37/100)
+  return static_cast<uint32_t>((static_cast<uint64_t>(n) * 1374389535) >> 37);
+}
+// Computes floor(n / 10^(kappa + 1)) (double)
+inline auto divide_by_10_to_kappa_plus_1(uint64_t n) noexcept -> uint64_t {
+  // 2361183241434822607 = ceil(2^(64+7)/1000)
+  return umul128_upper64(n, 2361183241434822607ull) >> 7;
+}
+
+// Various subroutines using pow10 cache
+template <typename T> struct cache_accessor;
+
+template <> struct cache_accessor<float> {
+  using carrier_uint = float_info<float>::carrier_uint;
+  using cache_entry_type = uint64_t;
+
+  static auto get_cached_power(int k) noexcept -> uint64_t {
+    FMT_ASSERT(k >= float_info<float>::min_k && k <= float_info<float>::max_k,
+               "k is out of range");
+    static constexpr const uint64_t pow10_significands[] = {
+        0x81ceb32c4b43fcf5, 0xa2425ff75e14fc32, 0xcad2f7f5359a3b3f,
+        0xfd87b5f28300ca0e, 0x9e74d1b791e07e49, 0xc612062576589ddb,
+        0xf79687aed3eec552, 0x9abe14cd44753b53, 0xc16d9a0095928a28,
+        0xf1c90080baf72cb2, 0x971da05074da7bef, 0xbce5086492111aeb,
+        0xec1e4a7db69561a6, 0x9392ee8e921d5d08, 0xb877aa3236a4b44a,
+        0xe69594bec44de15c, 0x901d7cf73ab0acda, 0xb424dc35095cd810,
+        0xe12e13424bb40e14, 0x8cbccc096f5088cc, 0xafebff0bcb24aaff,
+        0xdbe6fecebdedd5bf, 0x89705f4136b4a598, 0xabcc77118461cefd,
+        0xd6bf94d5e57a42bd, 0x8637bd05af6c69b6, 0xa7c5ac471b478424,
+        0xd1b71758e219652c, 0x83126e978d4fdf3c, 0xa3d70a3d70a3d70b,
+        0xcccccccccccccccd, 0x8000000000000000, 0xa000000000000000,
+        0xc800000000000000, 0xfa00000000000000, 0x9c40000000000000,
+        0xc350000000000000, 0xf424000000000000, 0x9896800000000000,
+        0xbebc200000000000, 0xee6b280000000000, 0x9502f90000000000,
+        0xba43b74000000000, 0xe8d4a51000000000, 0x9184e72a00000000,
+        0xb5e620f480000000, 0xe35fa931a0000000, 0x8e1bc9bf04000000,
+        0xb1a2bc2ec5000000, 0xde0b6b3a76400000, 0x8ac7230489e80000,
+        0xad78ebc5ac620000, 0xd8d726b7177a8000, 0x878678326eac9000,
+        0xa968163f0a57b400, 0xd3c21bcecceda100, 0x84595161401484a0,
+        0xa56fa5b99019a5c8, 0xcecb8f27f4200f3a, 0x813f3978f8940985,
+        0xa18f07d736b90be6, 0xc9f2c9cd04674edf, 0xfc6f7c4045812297,
+        0x9dc5ada82b70b59e, 0xc5371912364ce306, 0xf684df56c3e01bc7,
+        0x9a130b963a6c115d, 0xc097ce7bc90715b4, 0xf0bdc21abb48db21,
+        0x96769950b50d88f5, 0xbc143fa4e250eb32, 0xeb194f8e1ae525fe,
+        0x92efd1b8d0cf37bf, 0xb7abc627050305ae, 0xe596b7b0c643c71a,
+        0x8f7e32ce7bea5c70, 0xb35dbf821ae4f38c, 0xe0352f62a19e306f};
+    return pow10_significands[k - float_info<float>::min_k];
+  }
+
+  struct compute_mul_result {
+    carrier_uint result;
+    bool is_integer;
+  };
+  struct compute_mul_parity_result {
+    bool parity;
+    bool is_integer;
+  };
+
+  static auto compute_mul(carrier_uint u,
+                          const cache_entry_type& cache) noexcept
+      -> compute_mul_result {
+    auto r = umul96_upper64(u, cache);
+    return {static_cast<carrier_uint>(r >> 32),
+            static_cast<carrier_uint>(r) == 0};
+  }
+
+  static auto compute_delta(const cache_entry_type& cache, int beta) noexcept
+      -> uint32_t {
+    return static_cast<uint32_t>(cache >> (64 - 1 - beta));
+  }
+
+  static auto compute_mul_parity(carrier_uint two_f,
+                                 const cache_entry_type& cache,
+                                 int beta) noexcept
+      -> compute_mul_parity_result {
+    FMT_ASSERT(beta >= 1, "");
+    FMT_ASSERT(beta < 64, "");
+
+    auto r = umul96_lower64(two_f, cache);
+    return {((r >> (64 - beta)) & 1) != 0,
+            static_cast<uint32_t>(r >> (32 - beta)) == 0};
+  }
+
+  static auto compute_left_endpoint_for_shorter_interval_case(
+      const cache_entry_type& cache, int beta) noexcept -> carrier_uint {
+    return static_cast<carrier_uint>(
+        (cache - (cache >> (num_significand_bits<float>() + 2))) >>
+        (64 - num_significand_bits<float>() - 1 - beta));
+  }
+
+  static auto compute_right_endpoint_for_shorter_interval_case(
+      const cache_entry_type& cache, int beta) noexcept -> carrier_uint {
+    return static_cast<carrier_uint>(
+        (cache + (cache >> (num_significand_bits<float>() + 1))) >>
+        (64 - num_significand_bits<float>() - 1 - beta));
+  }
+
+  static auto compute_round_up_for_shorter_interval_case(
+      const cache_entry_type& cache, int beta) noexcept -> carrier_uint {
+    return (static_cast<carrier_uint>(
+                cache >> (64 - num_significand_bits<float>() - 2 - beta)) +
+            1) /
+           2;
+  }
+};
+
+template <> struct cache_accessor<double> {
+  using carrier_uint = float_info<double>::carrier_uint;
+  using cache_entry_type = uint128_fallback;
+
+  static auto get_cached_power(int k) noexcept -> uint128_fallback {
+    FMT_ASSERT(k >= float_info<double>::min_k && k <= float_info<double>::max_k,
+               "k is out of range");
+
+    static constexpr const uint128_fallback pow10_significands[] = {
+#if FMT_USE_FULL_CACHE_DRAGONBOX
+      {0xff77b1fcbebcdc4f, 0x25e8e89c13bb0f7b},
+      {0x9faacf3df73609b1, 0x77b191618c54e9ad},
+      {0xc795830d75038c1d, 0xd59df5b9ef6a2418},
+      {0xf97ae3d0d2446f25, 0x4b0573286b44ad1e},
+      {0x9becce62836ac577, 0x4ee367f9430aec33},
+      {0xc2e801fb244576d5, 0x229c41f793cda740},
+      {0xf3a20279ed56d48a, 0x6b43527578c11110},
+      {0x9845418c345644d6, 0x830a13896b78aaaa},
+      {0xbe5691ef416bd60c, 0x23cc986bc656d554},
+      {0xedec366b11c6cb8f, 0x2cbfbe86b7ec8aa9},
+      {0x94b3a202eb1c3f39, 0x7bf7d71432f3d6aa},
+      {0xb9e08a83a5e34f07, 0xdaf5ccd93fb0cc54},
+      {0xe858ad248f5c22c9, 0xd1b3400f8f9cff69},
+      {0x91376c36d99995be, 0x23100809b9c21fa2},
+      {0xb58547448ffffb2d, 0xabd40a0c2832a78b},
+      {0xe2e69915b3fff9f9, 0x16c90c8f323f516d},
+      {0x8dd01fad907ffc3b, 0xae3da7d97f6792e4},
+      {0xb1442798f49ffb4a, 0x99cd11cfdf41779d},
+      {0xdd95317f31c7fa1d, 0x40405643d711d584},
+      {0x8a7d3eef7f1cfc52, 0x482835ea666b2573},
+      {0xad1c8eab5ee43b66, 0xda3243650005eed0},
+      {0xd863b256369d4a40, 0x90bed43e40076a83},
+      {0x873e4f75e2224e68, 0x5a7744a6e804a292},
+      {0xa90de3535aaae202, 0x711515d0a205cb37},
+      {0xd3515c2831559a83, 0x0d5a5b44ca873e04},
+      {0x8412d9991ed58091, 0xe858790afe9486c3},
+      {0xa5178fff668ae0b6, 0x626e974dbe39a873},
+      {0xce5d73ff402d98e3, 0xfb0a3d212dc81290},
+      {0x80fa687f881c7f8e, 0x7ce66634bc9d0b9a},
+      {0xa139029f6a239f72, 0x1c1fffc1ebc44e81},
+      {0xc987434744ac874e, 0xa327ffb266b56221},
+      {0xfbe9141915d7a922, 0x4bf1ff9f0062baa9},
+      {0x9d71ac8fada6c9b5, 0x6f773fc3603db4aa},
+      {0xc4ce17b399107c22, 0xcb550fb4384d21d4},
+      {0xf6019da07f549b2b, 0x7e2a53a146606a49},
+      {0x99c102844f94e0fb, 0x2eda7444cbfc426e},
+      {0xc0314325637a1939, 0xfa911155fefb5309},
+      {0xf03d93eebc589f88, 0x793555ab7eba27cb},
+      {0x96267c7535b763b5, 0x4bc1558b2f3458df},
+      {0xbbb01b9283253ca2, 0x9eb1aaedfb016f17},
+      {0xea9c227723ee8bcb, 0x465e15a979c1cadd},
+      {0x92a1958a7675175f, 0x0bfacd89ec191eca},
+      {0xb749faed14125d36, 0xcef980ec671f667c},
+      {0xe51c79a85916f484, 0x82b7e12780e7401b},
+      {0x8f31cc0937ae58d2, 0xd1b2ecb8b0908811},
+      {0xb2fe3f0b8599ef07, 0x861fa7e6dcb4aa16},
+      {0xdfbdcece67006ac9, 0x67a791e093e1d49b},
+      {0x8bd6a141006042bd, 0xe0c8bb2c5c6d24e1},
+      {0xaecc49914078536d, 0x58fae9f773886e19},
+      {0xda7f5bf590966848, 0xaf39a475506a899f},
+      {0x888f99797a5e012d, 0x6d8406c952429604},
+      {0xaab37fd7d8f58178, 0xc8e5087ba6d33b84},
+      {0xd5605fcdcf32e1d6, 0xfb1e4a9a90880a65},
+      {0x855c3be0a17fcd26, 0x5cf2eea09a550680},
+      {0xa6b34ad8c9dfc06f, 0xf42faa48c0ea481f},
+      {0xd0601d8efc57b08b, 0xf13b94daf124da27},
+      {0x823c12795db6ce57, 0x76c53d08d6b70859},
+      {0xa2cb1717b52481ed, 0x54768c4b0c64ca6f},
+      {0xcb7ddcdda26da268, 0xa9942f5dcf7dfd0a},
+      {0xfe5d54150b090b02, 0xd3f93b35435d7c4d},
+      {0x9efa548d26e5a6e1, 0xc47bc5014a1a6db0},
+      {0xc6b8e9b0709f109a, 0x359ab6419ca1091c},
+      {0xf867241c8cc6d4c0, 0xc30163d203c94b63},
+      {0x9b407691d7fc44f8, 0x79e0de63425dcf1e},
+      {0xc21094364dfb5636, 0x985915fc12f542e5},
+      {0xf294b943e17a2bc4, 0x3e6f5b7b17b2939e},
+      {0x979cf3ca6cec5b5a, 0xa705992ceecf9c43},
+      {0xbd8430bd08277231, 0x50c6ff782a838354},
+      {0xece53cec4a314ebd, 0xa4f8bf5635246429},
+      {0x940f4613ae5ed136, 0x871b7795e136be9a},
+      {0xb913179899f68584, 0x28e2557b59846e40},
+      {0xe757dd7ec07426e5, 0x331aeada2fe589d0},
+      {0x9096ea6f3848984f, 0x3ff0d2c85def7622},
+      {0xb4bca50b065abe63, 0x0fed077a756b53aa},
+      {0xe1ebce4dc7f16dfb, 0xd3e8495912c62895},
+      {0x8d3360f09cf6e4bd, 0x64712dd7abbbd95d},
+      {0xb080392cc4349dec, 0xbd8d794d96aacfb4},
+      {0xdca04777f541c567, 0xecf0d7a0fc5583a1},
+      {0x89e42caaf9491b60, 0xf41686c49db57245},
+      {0xac5d37d5b79b6239, 0x311c2875c522ced6},
+      {0xd77485cb25823ac7, 0x7d633293366b828c},
+      {0x86a8d39ef77164bc, 0xae5dff9c02033198},
+      {0xa8530886b54dbdeb, 0xd9f57f830283fdfd},
+      {0xd267caa862a12d66, 0xd072df63c324fd7c},
+      {0x8380dea93da4bc60, 0x4247cb9e59f71e6e},
+      {0xa46116538d0deb78, 0x52d9be85f074e609},
+      {0xcd795be870516656, 0x67902e276c921f8c},
+      {0x806bd9714632dff6, 0x00ba1cd8a3db53b7},
+      {0xa086cfcd97bf97f3, 0x80e8a40eccd228a5},
+      {0xc8a883c0fdaf7df0, 0x6122cd128006b2ce},
+      {0xfad2a4b13d1b5d6c, 0x796b805720085f82},
+      {0x9cc3a6eec6311a63, 0xcbe3303674053bb1},
+      {0xc3f490aa77bd60fc, 0xbedbfc4411068a9d},
+      {0xf4f1b4d515acb93b, 0xee92fb5515482d45},
+      {0x991711052d8bf3c5, 0x751bdd152d4d1c4b},
+      {0xbf5cd54678eef0b6, 0xd262d45a78a0635e},
+      {0xef340a98172aace4, 0x86fb897116c87c35},
+      {0x9580869f0e7aac0e, 0xd45d35e6ae3d4da1},
+      {0xbae0a846d2195712, 0x8974836059cca10a},
+      {0xe998d258869facd7, 0x2bd1a438703fc94c},
+      {0x91ff83775423cc06, 0x7b6306a34627ddd0},
+      {0xb67f6455292cbf08, 0x1a3bc84c17b1d543},
+      {0xe41f3d6a7377eeca, 0x20caba5f1d9e4a94},
+      {0x8e938662882af53e, 0x547eb47b7282ee9d},
+      {0xb23867fb2a35b28d, 0xe99e619a4f23aa44},
+      {0xdec681f9f4c31f31, 0x6405fa00e2ec94d5},
+      {0x8b3c113c38f9f37e, 0xde83bc408dd3dd05},
+      {0xae0b158b4738705e, 0x9624ab50b148d446},
+      {0xd98ddaee19068c76, 0x3badd624dd9b0958},
+      {0x87f8a8d4cfa417c9, 0xe54ca5d70a80e5d7},
+      {0xa9f6d30a038d1dbc, 0x5e9fcf4ccd211f4d},
+      {0xd47487cc8470652b, 0x7647c32000696720},
+      {0x84c8d4dfd2c63f3b, 0x29ecd9f40041e074},
+      {0xa5fb0a17c777cf09, 0xf468107100525891},
+      {0xcf79cc9db955c2cc, 0x7182148d4066eeb5},
+      {0x81ac1fe293d599bf, 0xc6f14cd848405531},
+      {0xa21727db38cb002f, 0xb8ada00e5a506a7d},
+      {0xca9cf1d206fdc03b, 0xa6d90811f0e4851d},
+      {0xfd442e4688bd304a, 0x908f4a166d1da664},
+      {0x9e4a9cec15763e2e, 0x9a598e4e043287ff},
+      {0xc5dd44271ad3cdba, 0x40eff1e1853f29fe},
+      {0xf7549530e188c128, 0xd12bee59e68ef47d},
+      {0x9a94dd3e8cf578b9, 0x82bb74f8301958cf},
+      {0xc13a148e3032d6e7, 0xe36a52363c1faf02},
+      {0xf18899b1bc3f8ca1, 0xdc44e6c3cb279ac2},
+      {0x96f5600f15a7b7e5, 0x29ab103a5ef8c0ba},
+      {0xbcb2b812db11a5de, 0x7415d448f6b6f0e8},
+      {0xebdf661791d60f56, 0x111b495b3464ad22},
+      {0x936b9fcebb25c995, 0xcab10dd900beec35},
+      {0xb84687c269ef3bfb, 0x3d5d514f40eea743},
+      {0xe65829b3046b0afa, 0x0cb4a5a3112a5113},
+      {0x8ff71a0fe2c2e6dc, 0x47f0e785eaba72ac},
+      {0xb3f4e093db73a093, 0x59ed216765690f57},
+      {0xe0f218b8d25088b8, 0x306869c13ec3532d},
+      {0x8c974f7383725573, 0x1e414218c73a13fc},
+      {0xafbd2350644eeacf, 0xe5d1929ef90898fb},
+      {0xdbac6c247d62a583, 0xdf45f746b74abf3a},
+      {0x894bc396ce5da772, 0x6b8bba8c328eb784},
+      {0xab9eb47c81f5114f, 0x066ea92f3f326565},
+      {0xd686619ba27255a2, 0xc80a537b0efefebe},
+      {0x8613fd0145877585, 0xbd06742ce95f5f37},
+      {0xa798fc4196e952e7, 0x2c48113823b73705},
+      {0xd17f3b51fca3a7a0, 0xf75a15862ca504c6},
+      {0x82ef85133de648c4, 0x9a984d73dbe722fc},
+      {0xa3ab66580d5fdaf5, 0xc13e60d0d2e0ebbb},
+      {0xcc963fee10b7d1b3, 0x318df905079926a9},
+      {0xffbbcfe994e5c61f, 0xfdf17746497f7053},
+      {0x9fd561f1fd0f9bd3, 0xfeb6ea8bedefa634},
+      {0xc7caba6e7c5382c8, 0xfe64a52ee96b8fc1},
+      {0xf9bd690a1b68637b, 0x3dfdce7aa3c673b1},
+      {0x9c1661a651213e2d, 0x06bea10ca65c084f},
+      {0xc31bfa0fe5698db8, 0x486e494fcff30a63},
+      {0xf3e2f893dec3f126, 0x5a89dba3c3efccfb},
+      {0x986ddb5c6b3a76b7, 0xf89629465a75e01d},
+      {0xbe89523386091465, 0xf6bbb397f1135824},
+      {0xee2ba6c0678b597f, 0x746aa07ded582e2d},
+      {0x94db483840b717ef, 0xa8c2a44eb4571cdd},
+      {0xba121a4650e4ddeb, 0x92f34d62616ce414},
+      {0xe896a0d7e51e1566, 0x77b020baf9c81d18},
+      {0x915e2486ef32cd60, 0x0ace1474dc1d122f},
+      {0xb5b5ada8aaff80b8, 0x0d819992132456bb},
+      {0xe3231912d5bf60e6, 0x10e1fff697ed6c6a},
+      {0x8df5efabc5979c8f, 0xca8d3ffa1ef463c2},
+      {0xb1736b96b6fd83b3, 0xbd308ff8a6b17cb3},
+      {0xddd0467c64bce4a0, 0xac7cb3f6d05ddbdf},
+      {0x8aa22c0dbef60ee4, 0x6bcdf07a423aa96c},
+      {0xad4ab7112eb3929d, 0x86c16c98d2c953c7},
+      {0xd89d64d57a607744, 0xe871c7bf077ba8b8},
+      {0x87625f056c7c4a8b, 0x11471cd764ad4973},
+      {0xa93af6c6c79b5d2d, 0xd598e40d3dd89bd0},
+      {0xd389b47879823479, 0x4aff1d108d4ec2c4},
+      {0x843610cb4bf160cb, 0xcedf722a585139bb},
+      {0xa54394fe1eedb8fe, 0xc2974eb4ee658829},
+      {0xce947a3da6a9273e, 0x733d226229feea33},
+      {0x811ccc668829b887, 0x0806357d5a3f5260},
+      {0xa163ff802a3426a8, 0xca07c2dcb0cf26f8},
+      {0xc9bcff6034c13052, 0xfc89b393dd02f0b6},
+      {0xfc2c3f3841f17c67, 0xbbac2078d443ace3},
+      {0x9d9ba7832936edc0, 0xd54b944b84aa4c0e},
+      {0xc5029163f384a931, 0x0a9e795e65d4df12},
+      {0xf64335bcf065d37d, 0x4d4617b5ff4a16d6},
+      {0x99ea0196163fa42e, 0x504bced1bf8e4e46},
+      {0xc06481fb9bcf8d39, 0xe45ec2862f71e1d7},
+      {0xf07da27a82c37088, 0x5d767327bb4e5a4d},
+      {0x964e858c91ba2655, 0x3a6a07f8d510f870},
+      {0xbbe226efb628afea, 0x890489f70a55368c},
+      {0xeadab0aba3b2dbe5, 0x2b45ac74ccea842f},
+      {0x92c8ae6b464fc96f, 0x3b0b8bc90012929e},
+      {0xb77ada0617e3bbcb, 0x09ce6ebb40173745},
+      {0xe55990879ddcaabd, 0xcc420a6a101d0516},
+      {0x8f57fa54c2a9eab6, 0x9fa946824a12232e},
+      {0xb32df8e9f3546564, 0x47939822dc96abfa},
+      {0xdff9772470297ebd, 0x59787e2b93bc56f8},
+      {0x8bfbea76c619ef36, 0x57eb4edb3c55b65b},
+      {0xaefae51477a06b03, 0xede622920b6b23f2},
+      {0xdab99e59958885c4, 0xe95fab368e45ecee},
+      {0x88b402f7fd75539b, 0x11dbcb0218ebb415},
+      {0xaae103b5fcd2a881, 0xd652bdc29f26a11a},
+      {0xd59944a37c0752a2, 0x4be76d3346f04960},
+      {0x857fcae62d8493a5, 0x6f70a4400c562ddc},
+      {0xa6dfbd9fb8e5b88e, 0xcb4ccd500f6bb953},
+      {0xd097ad07a71f26b2, 0x7e2000a41346a7a8},
+      {0x825ecc24c873782f, 0x8ed400668c0c28c9},
+      {0xa2f67f2dfa90563b, 0x728900802f0f32fb},
+      {0xcbb41ef979346bca, 0x4f2b40a03ad2ffba},
+      {0xfea126b7d78186bc, 0xe2f610c84987bfa9},
+      {0x9f24b832e6b0f436, 0x0dd9ca7d2df4d7ca},
+      {0xc6ede63fa05d3143, 0x91503d1c79720dbc},
+      {0xf8a95fcf88747d94, 0x75a44c6397ce912b},
+      {0x9b69dbe1b548ce7c, 0xc986afbe3ee11abb},
+      {0xc24452da229b021b, 0xfbe85badce996169},
+      {0xf2d56790ab41c2a2, 0xfae27299423fb9c4},
+      {0x97c560ba6b0919a5, 0xdccd879fc967d41b},
+      {0xbdb6b8e905cb600f, 0x5400e987bbc1c921},
+      {0xed246723473e3813, 0x290123e9aab23b69},
+      {0x9436c0760c86e30b, 0xf9a0b6720aaf6522},
+      {0xb94470938fa89bce, 0xf808e40e8d5b3e6a},
+      {0xe7958cb87392c2c2, 0xb60b1d1230b20e05},
+      {0x90bd77f3483bb9b9, 0xb1c6f22b5e6f48c3},
+      {0xb4ecd5f01a4aa828, 0x1e38aeb6360b1af4},
+      {0xe2280b6c20dd5232, 0x25c6da63c38de1b1},
+      {0x8d590723948a535f, 0x579c487e5a38ad0f},
+      {0xb0af48ec79ace837, 0x2d835a9df0c6d852},
+      {0xdcdb1b2798182244, 0xf8e431456cf88e66},
+      {0x8a08f0f8bf0f156b, 0x1b8e9ecb641b5900},
+      {0xac8b2d36eed2dac5, 0xe272467e3d222f40},
+      {0xd7adf884aa879177, 0x5b0ed81dcc6abb10},
+      {0x86ccbb52ea94baea, 0x98e947129fc2b4ea},
+      {0xa87fea27a539e9a5, 0x3f2398d747b36225},
+      {0xd29fe4b18e88640e, 0x8eec7f0d19a03aae},
+      {0x83a3eeeef9153e89, 0x1953cf68300424ad},
+      {0xa48ceaaab75a8e2b, 0x5fa8c3423c052dd8},
+      {0xcdb02555653131b6, 0x3792f412cb06794e},
+      {0x808e17555f3ebf11, 0xe2bbd88bbee40bd1},
+      {0xa0b19d2ab70e6ed6, 0x5b6aceaeae9d0ec5},
+      {0xc8de047564d20a8b, 0xf245825a5a445276},
+      {0xfb158592be068d2e, 0xeed6e2f0f0d56713},
+      {0x9ced737bb6c4183d, 0x55464dd69685606c},
+      {0xc428d05aa4751e4c, 0xaa97e14c3c26b887},
+      {0xf53304714d9265df, 0xd53dd99f4b3066a9},
+      {0x993fe2c6d07b7fab, 0xe546a8038efe402a},
+      {0xbf8fdb78849a5f96, 0xde98520472bdd034},
+      {0xef73d256a5c0f77c, 0x963e66858f6d4441},
+      {0x95a8637627989aad, 0xdde7001379a44aa9},
+      {0xbb127c53b17ec159, 0x5560c018580d5d53},
+      {0xe9d71b689dde71af, 0xaab8f01e6e10b4a7},
+      {0x9226712162ab070d, 0xcab3961304ca70e9},
+      {0xb6b00d69bb55c8d1, 0x3d607b97c5fd0d23},
+      {0xe45c10c42a2b3b05, 0x8cb89a7db77c506b},
+      {0x8eb98a7a9a5b04e3, 0x77f3608e92adb243},
+      {0xb267ed1940f1c61c, 0x55f038b237591ed4},
+      {0xdf01e85f912e37a3, 0x6b6c46dec52f6689},
+      {0x8b61313bbabce2c6, 0x2323ac4b3b3da016},
+      {0xae397d8aa96c1b77, 0xabec975e0a0d081b},
+      {0xd9c7dced53c72255, 0x96e7bd358c904a22},
+      {0x881cea14545c7575, 0x7e50d64177da2e55},
+      {0xaa242499697392d2, 0xdde50bd1d5d0b9ea},
+      {0xd4ad2dbfc3d07787, 0x955e4ec64b44e865},
+      {0x84ec3c97da624ab4, 0xbd5af13bef0b113f},
+      {0xa6274bbdd0fadd61, 0xecb1ad8aeacdd58f},
+      {0xcfb11ead453994ba, 0x67de18eda5814af3},
+      {0x81ceb32c4b43fcf4, 0x80eacf948770ced8},
+      {0xa2425ff75e14fc31, 0xa1258379a94d028e},
+      {0xcad2f7f5359a3b3e, 0x096ee45813a04331},
+      {0xfd87b5f28300ca0d, 0x8bca9d6e188853fd},
+      {0x9e74d1b791e07e48, 0x775ea264cf55347e},
+      {0xc612062576589dda, 0x95364afe032a819e},
+      {0xf79687aed3eec551, 0x3a83ddbd83f52205},
+      {0x9abe14cd44753b52, 0xc4926a9672793543},
+      {0xc16d9a0095928a27, 0x75b7053c0f178294},
+      {0xf1c90080baf72cb1, 0x5324c68b12dd6339},
+      {0x971da05074da7bee, 0xd3f6fc16ebca5e04},
+      {0xbce5086492111aea, 0x88f4bb1ca6bcf585},
+      {0xec1e4a7db69561a5, 0x2b31e9e3d06c32e6},
+      {0x9392ee8e921d5d07, 0x3aff322e62439fd0},
+      {0xb877aa3236a4b449, 0x09befeb9fad487c3},
+      {0xe69594bec44de15b, 0x4c2ebe687989a9b4},
+      {0x901d7cf73ab0acd9, 0x0f9d37014bf60a11},
+      {0xb424dc35095cd80f, 0x538484c19ef38c95},
+      {0xe12e13424bb40e13, 0x2865a5f206b06fba},
+      {0x8cbccc096f5088cb, 0xf93f87b7442e45d4},
+      {0xafebff0bcb24aafe, 0xf78f69a51539d749},
+      {0xdbe6fecebdedd5be, 0xb573440e5a884d1c},
+      {0x89705f4136b4a597, 0x31680a88f8953031},
+      {0xabcc77118461cefc, 0xfdc20d2b36ba7c3e},
+      {0xd6bf94d5e57a42bc, 0x3d32907604691b4d},
+      {0x8637bd05af6c69b5, 0xa63f9a49c2c1b110},
+      {0xa7c5ac471b478423, 0x0fcf80dc33721d54},
+      {0xd1b71758e219652b, 0xd3c36113404ea4a9},
+      {0x83126e978d4fdf3b, 0x645a1cac083126ea},
+      {0xa3d70a3d70a3d70a, 0x3d70a3d70a3d70a4},
+      {0xcccccccccccccccc, 0xcccccccccccccccd},
+      {0x8000000000000000, 0x0000000000000000},
+      {0xa000000000000000, 0x0000000000000000},
+      {0xc800000000000000, 0x0000000000000000},
+      {0xfa00000000000000, 0x0000000000000000},
+      {0x9c40000000000000, 0x0000000000000000},
+      {0xc350000000000000, 0x0000000000000000},
+      {0xf424000000000000, 0x0000000000000000},
+      {0x9896800000000000, 0x0000000000000000},
+      {0xbebc200000000000, 0x0000000000000000},
+      {0xee6b280000000000, 0x0000000000000000},
+      {0x9502f90000000000, 0x0000000000000000},
+      {0xba43b74000000000, 0x0000000000000000},
+      {0xe8d4a51000000000, 0x0000000000000000},
+      {0x9184e72a00000000, 0x0000000000000000},
+      {0xb5e620f480000000, 0x0000000000000000},
+      {0xe35fa931a0000000, 0x0000000000000000},
+      {0x8e1bc9bf04000000, 0x0000000000000000},
+      {0xb1a2bc2ec5000000, 0x0000000000000000},
+      {0xde0b6b3a76400000, 0x0000000000000000},
+      {0x8ac7230489e80000, 0x0000000000000000},
+      {0xad78ebc5ac620000, 0x0000000000000000},
+      {0xd8d726b7177a8000, 0x0000000000000000},
+      {0x878678326eac9000, 0x0000000000000000},
+      {0xa968163f0a57b400, 0x0000000000000000},
+      {0xd3c21bcecceda100, 0x0000000000000000},
+      {0x84595161401484a0, 0x0000000000000000},
+      {0xa56fa5b99019a5c8, 0x0000000000000000},
+      {0xcecb8f27f4200f3a, 0x0000000000000000},
+      {0x813f3978f8940984, 0x4000000000000000},
+      {0xa18f07d736b90be5, 0x5000000000000000},
+      {0xc9f2c9cd04674ede, 0xa400000000000000},
+      {0xfc6f7c4045812296, 0x4d00000000000000},
+      {0x9dc5ada82b70b59d, 0xf020000000000000},
+      {0xc5371912364ce305, 0x6c28000000000000},
+      {0xf684df56c3e01bc6, 0xc732000000000000},
+      {0x9a130b963a6c115c, 0x3c7f400000000000},
+      {0xc097ce7bc90715b3, 0x4b9f100000000000},
+      {0xf0bdc21abb48db20, 0x1e86d40000000000},
+      {0x96769950b50d88f4, 0x1314448000000000},
+      {0xbc143fa4e250eb31, 0x17d955a000000000},
+      {0xeb194f8e1ae525fd, 0x5dcfab0800000000},
+      {0x92efd1b8d0cf37be, 0x5aa1cae500000000},
+      {0xb7abc627050305ad, 0xf14a3d9e40000000},
+      {0xe596b7b0c643c719, 0x6d9ccd05d0000000},
+      {0x8f7e32ce7bea5c6f, 0xe4820023a2000000},
+      {0xb35dbf821ae4f38b, 0xdda2802c8a800000},
+      {0xe0352f62a19e306e, 0xd50b2037ad200000},
+      {0x8c213d9da502de45, 0x4526f422cc340000},
+      {0xaf298d050e4395d6, 0x9670b12b7f410000},
+      {0xdaf3f04651d47b4c, 0x3c0cdd765f114000},
+      {0x88d8762bf324cd0f, 0xa5880a69fb6ac800},
+      {0xab0e93b6efee0053, 0x8eea0d047a457a00},
+      {0xd5d238a4abe98068, 0x72a4904598d6d880},
+      {0x85a36366eb71f041, 0x47a6da2b7f864750},
+      {0xa70c3c40a64e6c51, 0x999090b65f67d924},
+      {0xd0cf4b50cfe20765, 0xfff4b4e3f741cf6d},
+      {0x82818f1281ed449f, 0xbff8f10e7a8921a5},
+      {0xa321f2d7226895c7, 0xaff72d52192b6a0e},
+      {0xcbea6f8ceb02bb39, 0x9bf4f8a69f764491},
+      {0xfee50b7025c36a08, 0x02f236d04753d5b5},
+      {0x9f4f2726179a2245, 0x01d762422c946591},
+      {0xc722f0ef9d80aad6, 0x424d3ad2b7b97ef6},
+      {0xf8ebad2b84e0d58b, 0xd2e0898765a7deb3},
+      {0x9b934c3b330c8577, 0x63cc55f49f88eb30},
+      {0xc2781f49ffcfa6d5, 0x3cbf6b71c76b25fc},
+      {0xf316271c7fc3908a, 0x8bef464e3945ef7b},
+      {0x97edd871cfda3a56, 0x97758bf0e3cbb5ad},
+      {0xbde94e8e43d0c8ec, 0x3d52eeed1cbea318},
+      {0xed63a231d4c4fb27, 0x4ca7aaa863ee4bde},
+      {0x945e455f24fb1cf8, 0x8fe8caa93e74ef6b},
+      {0xb975d6b6ee39e436, 0xb3e2fd538e122b45},
+      {0xe7d34c64a9c85d44, 0x60dbbca87196b617},
+      {0x90e40fbeea1d3a4a, 0xbc8955e946fe31ce},
+      {0xb51d13aea4a488dd, 0x6babab6398bdbe42},
+      {0xe264589a4dcdab14, 0xc696963c7eed2dd2},
+      {0x8d7eb76070a08aec, 0xfc1e1de5cf543ca3},
+      {0xb0de65388cc8ada8, 0x3b25a55f43294bcc},
+      {0xdd15fe86affad912, 0x49ef0eb713f39ebf},
+      {0x8a2dbf142dfcc7ab, 0x6e3569326c784338},
+      {0xacb92ed9397bf996, 0x49c2c37f07965405},
+      {0xd7e77a8f87daf7fb, 0xdc33745ec97be907},
+      {0x86f0ac99b4e8dafd, 0x69a028bb3ded71a4},
+      {0xa8acd7c0222311bc, 0xc40832ea0d68ce0d},
+      {0xd2d80db02aabd62b, 0xf50a3fa490c30191},
+      {0x83c7088e1aab65db, 0x792667c6da79e0fb},
+      {0xa4b8cab1a1563f52, 0x577001b891185939},
+      {0xcde6fd5e09abcf26, 0xed4c0226b55e6f87},
+      {0x80b05e5ac60b6178, 0x544f8158315b05b5},
+      {0xa0dc75f1778e39d6, 0x696361ae3db1c722},
+      {0xc913936dd571c84c, 0x03bc3a19cd1e38ea},
+      {0xfb5878494ace3a5f, 0x04ab48a04065c724},
+      {0x9d174b2dcec0e47b, 0x62eb0d64283f9c77},
+      {0xc45d1df942711d9a, 0x3ba5d0bd324f8395},
+      {0xf5746577930d6500, 0xca8f44ec7ee3647a},
+      {0x9968bf6abbe85f20, 0x7e998b13cf4e1ecc},
+      {0xbfc2ef456ae276e8, 0x9e3fedd8c321a67f},
+      {0xefb3ab16c59b14a2, 0xc5cfe94ef3ea101f},
+      {0x95d04aee3b80ece5, 0xbba1f1d158724a13},
+      {0xbb445da9ca61281f, 0x2a8a6e45ae8edc98},
+      {0xea1575143cf97226, 0xf52d09d71a3293be},
+      {0x924d692ca61be758, 0x593c2626705f9c57},
+      {0xb6e0c377cfa2e12e, 0x6f8b2fb00c77836d},
+      {0xe498f455c38b997a, 0x0b6dfb9c0f956448},
+      {0x8edf98b59a373fec, 0x4724bd4189bd5ead},
+      {0xb2977ee300c50fe7, 0x58edec91ec2cb658},
+      {0xdf3d5e9bc0f653e1, 0x2f2967b66737e3ee},
+      {0x8b865b215899f46c, 0xbd79e0d20082ee75},
+      {0xae67f1e9aec07187, 0xecd8590680a3aa12},
+      {0xda01ee641a708de9, 0xe80e6f4820cc9496},
+      {0x884134fe908658b2, 0x3109058d147fdcde},
+      {0xaa51823e34a7eede, 0xbd4b46f0599fd416},
+      {0xd4e5e2cdc1d1ea96, 0x6c9e18ac7007c91b},
+      {0x850fadc09923329e, 0x03e2cf6bc604ddb1},
+      {0xa6539930bf6bff45, 0x84db8346b786151d},
+      {0xcfe87f7cef46ff16, 0xe612641865679a64},
+      {0x81f14fae158c5f6e, 0x4fcb7e8f3f60c07f},
+      {0xa26da3999aef7749, 0xe3be5e330f38f09e},
+      {0xcb090c8001ab551c, 0x5cadf5bfd3072cc6},
+      {0xfdcb4fa002162a63, 0x73d9732fc7c8f7f7},
+      {0x9e9f11c4014dda7e, 0x2867e7fddcdd9afb},
+      {0xc646d63501a1511d, 0xb281e1fd541501b9},
+      {0xf7d88bc24209a565, 0x1f225a7ca91a4227},
+      {0x9ae757596946075f, 0x3375788de9b06959},
+      {0xc1a12d2fc3978937, 0x0052d6b1641c83af},
+      {0xf209787bb47d6b84, 0xc0678c5dbd23a49b},
+      {0x9745eb4d50ce6332, 0xf840b7ba963646e1},
+      {0xbd176620a501fbff, 0xb650e5a93bc3d899},
+      {0xec5d3fa8ce427aff, 0xa3e51f138ab4cebf},
+      {0x93ba47c980e98cdf, 0xc66f336c36b10138},
+      {0xb8a8d9bbe123f017, 0xb80b0047445d4185},
+      {0xe6d3102ad96cec1d, 0xa60dc059157491e6},
+      {0x9043ea1ac7e41392, 0x87c89837ad68db30},
+      {0xb454e4a179dd1877, 0x29babe4598c311fc},
+      {0xe16a1dc9d8545e94, 0xf4296dd6fef3d67b},
+      {0x8ce2529e2734bb1d, 0x1899e4a65f58660d},
+      {0xb01ae745b101e9e4, 0x5ec05dcff72e7f90},
+      {0xdc21a1171d42645d, 0x76707543f4fa1f74},
+      {0x899504ae72497eba, 0x6a06494a791c53a9},
+      {0xabfa45da0edbde69, 0x0487db9d17636893},
+      {0xd6f8d7509292d603, 0x45a9d2845d3c42b7},
+      {0x865b86925b9bc5c2, 0x0b8a2392ba45a9b3},
+      {0xa7f26836f282b732, 0x8e6cac7768d7141f},
+      {0xd1ef0244af2364ff, 0x3207d795430cd927},
+      {0x8335616aed761f1f, 0x7f44e6bd49e807b9},
+      {0xa402b9c5a8d3a6e7, 0x5f16206c9c6209a7},
+      {0xcd036837130890a1, 0x36dba887c37a8c10},
+      {0x802221226be55a64, 0xc2494954da2c978a},
+      {0xa02aa96b06deb0fd, 0xf2db9baa10b7bd6d},
+      {0xc83553c5c8965d3d, 0x6f92829494e5acc8},
+      {0xfa42a8b73abbf48c, 0xcb772339ba1f17fa},
+      {0x9c69a97284b578d7, 0xff2a760414536efc},
+      {0xc38413cf25e2d70d, 0xfef5138519684abb},
+      {0xf46518c2ef5b8cd1, 0x7eb258665fc25d6a},
+      {0x98bf2f79d5993802, 0xef2f773ffbd97a62},
+      {0xbeeefb584aff8603, 0xaafb550ffacfd8fb},
+      {0xeeaaba2e5dbf6784, 0x95ba2a53f983cf39},
+      {0x952ab45cfa97a0b2, 0xdd945a747bf26184},
+      {0xba756174393d88df, 0x94f971119aeef9e5},
+      {0xe912b9d1478ceb17, 0x7a37cd5601aab85e},
+      {0x91abb422ccb812ee, 0xac62e055c10ab33b},
+      {0xb616a12b7fe617aa, 0x577b986b314d600a},
+      {0xe39c49765fdf9d94, 0xed5a7e85fda0b80c},
+      {0x8e41ade9fbebc27d, 0x14588f13be847308},
+      {0xb1d219647ae6b31c, 0x596eb2d8ae258fc9},
+      {0xde469fbd99a05fe3, 0x6fca5f8ed9aef3bc},
+      {0x8aec23d680043bee, 0x25de7bb9480d5855},
+      {0xada72ccc20054ae9, 0xaf561aa79a10ae6b},
+      {0xd910f7ff28069da4, 0x1b2ba1518094da05},
+      {0x87aa9aff79042286, 0x90fb44d2f05d0843},
+      {0xa99541bf57452b28, 0x353a1607ac744a54},
+      {0xd3fa922f2d1675f2, 0x42889b8997915ce9},
+      {0x847c9b5d7c2e09b7, 0x69956135febada12},
+      {0xa59bc234db398c25, 0x43fab9837e699096},
+      {0xcf02b2c21207ef2e, 0x94f967e45e03f4bc},
+      {0x8161afb94b44f57d, 0x1d1be0eebac278f6},
+      {0xa1ba1ba79e1632dc, 0x6462d92a69731733},
+      {0xca28a291859bbf93, 0x7d7b8f7503cfdcff},
+      {0xfcb2cb35e702af78, 0x5cda735244c3d43f},
+      {0x9defbf01b061adab, 0x3a0888136afa64a8},
+      {0xc56baec21c7a1916, 0x088aaa1845b8fdd1},
+      {0xf6c69a72a3989f5b, 0x8aad549e57273d46},
+      {0x9a3c2087a63f6399, 0x36ac54e2f678864c},
+      {0xc0cb28a98fcf3c7f, 0x84576a1bb416a7de},
+      {0xf0fdf2d3f3c30b9f, 0x656d44a2a11c51d6},
+      {0x969eb7c47859e743, 0x9f644ae5a4b1b326},
+      {0xbc4665b596706114, 0x873d5d9f0dde1fef},
+      {0xeb57ff22fc0c7959, 0xa90cb506d155a7eb},
+      {0x9316ff75dd87cbd8, 0x09a7f12442d588f3},
+      {0xb7dcbf5354e9bece, 0x0c11ed6d538aeb30},
+      {0xe5d3ef282a242e81, 0x8f1668c8a86da5fb},
+      {0x8fa475791a569d10, 0xf96e017d694487bd},
+      {0xb38d92d760ec4455, 0x37c981dcc395a9ad},
+      {0xe070f78d3927556a, 0x85bbe253f47b1418},
+      {0x8c469ab843b89562, 0x93956d7478ccec8f},
+      {0xaf58416654a6babb, 0x387ac8d1970027b3},
+      {0xdb2e51bfe9d0696a, 0x06997b05fcc0319f},
+      {0x88fcf317f22241e2, 0x441fece3bdf81f04},
+      {0xab3c2fddeeaad25a, 0xd527e81cad7626c4},
+      {0xd60b3bd56a5586f1, 0x8a71e223d8d3b075},
+      {0x85c7056562757456, 0xf6872d5667844e4a},
+      {0xa738c6bebb12d16c, 0xb428f8ac016561dc},
+      {0xd106f86e69d785c7, 0xe13336d701beba53},
+      {0x82a45b450226b39c, 0xecc0024661173474},
+      {0xa34d721642b06084, 0x27f002d7f95d0191},
+      {0xcc20ce9bd35c78a5, 0x31ec038df7b441f5},
+      {0xff290242c83396ce, 0x7e67047175a15272},
+      {0x9f79a169bd203e41, 0x0f0062c6e984d387},
+      {0xc75809c42c684dd1, 0x52c07b78a3e60869},
+      {0xf92e0c3537826145, 0xa7709a56ccdf8a83},
+      {0x9bbcc7a142b17ccb, 0x88a66076400bb692},
+      {0xc2abf989935ddbfe, 0x6acff893d00ea436},
+      {0xf356f7ebf83552fe, 0x0583f6b8c4124d44},
+      {0x98165af37b2153de, 0xc3727a337a8b704b},
+      {0xbe1bf1b059e9a8d6, 0x744f18c0592e4c5d},
+      {0xeda2ee1c7064130c, 0x1162def06f79df74},
+      {0x9485d4d1c63e8be7, 0x8addcb5645ac2ba9},
+      {0xb9a74a0637ce2ee1, 0x6d953e2bd7173693},
+      {0xe8111c87c5c1ba99, 0xc8fa8db6ccdd0438},
+      {0x910ab1d4db9914a0, 0x1d9c9892400a22a3},
+      {0xb54d5e4a127f59c8, 0x2503beb6d00cab4c},
+      {0xe2a0b5dc971f303a, 0x2e44ae64840fd61e},
+      {0x8da471a9de737e24, 0x5ceaecfed289e5d3},
+      {0xb10d8e1456105dad, 0x7425a83e872c5f48},
+      {0xdd50f1996b947518, 0xd12f124e28f7771a},
+      {0x8a5296ffe33cc92f, 0x82bd6b70d99aaa70},
+      {0xace73cbfdc0bfb7b, 0x636cc64d1001550c},
+      {0xd8210befd30efa5a, 0x3c47f7e05401aa4f},
+      {0x8714a775e3e95c78, 0x65acfaec34810a72},
+      {0xa8d9d1535ce3b396, 0x7f1839a741a14d0e},
+      {0xd31045a8341ca07c, 0x1ede48111209a051},
+      {0x83ea2b892091e44d, 0x934aed0aab460433},
+      {0xa4e4b66b68b65d60, 0xf81da84d56178540},
+      {0xce1de40642e3f4b9, 0x36251260ab9d668f},
+      {0x80d2ae83e9ce78f3, 0xc1d72b7c6b42601a},
+      {0xa1075a24e4421730, 0xb24cf65b8612f820},
+      {0xc94930ae1d529cfc, 0xdee033f26797b628},
+      {0xfb9b7cd9a4a7443c, 0x169840ef017da3b2},
+      {0x9d412e0806e88aa5, 0x8e1f289560ee864f},
+      {0xc491798a08a2ad4e, 0xf1a6f2bab92a27e3},
+      {0xf5b5d7ec8acb58a2, 0xae10af696774b1dc},
+      {0x9991a6f3d6bf1765, 0xacca6da1e0a8ef2a},
+      {0xbff610b0cc6edd3f, 0x17fd090a58d32af4},
+      {0xeff394dcff8a948e, 0xddfc4b4cef07f5b1},
+      {0x95f83d0a1fb69cd9, 0x4abdaf101564f98f},
+      {0xbb764c4ca7a4440f, 0x9d6d1ad41abe37f2},
+      {0xea53df5fd18d5513, 0x84c86189216dc5ee},
+      {0x92746b9be2f8552c, 0x32fd3cf5b4e49bb5},
+      {0xb7118682dbb66a77, 0x3fbc8c33221dc2a2},
+      {0xe4d5e82392a40515, 0x0fabaf3feaa5334b},
+      {0x8f05b1163ba6832d, 0x29cb4d87f2a7400f},
+      {0xb2c71d5bca9023f8, 0x743e20e9ef511013},
+      {0xdf78e4b2bd342cf6, 0x914da9246b255417},
+      {0x8bab8eefb6409c1a, 0x1ad089b6c2f7548f},
+      {0xae9672aba3d0c320, 0xa184ac2473b529b2},
+      {0xda3c0f568cc4f3e8, 0xc9e5d72d90a2741f},
+      {0x8865899617fb1871, 0x7e2fa67c7a658893},
+      {0xaa7eebfb9df9de8d, 0xddbb901b98feeab8},
+      {0xd51ea6fa85785631, 0x552a74227f3ea566},
+      {0x8533285c936b35de, 0xd53a88958f872760},
+      {0xa67ff273b8460356, 0x8a892abaf368f138},
+      {0xd01fef10a657842c, 0x2d2b7569b0432d86},
+      {0x8213f56a67f6b29b, 0x9c3b29620e29fc74},
+      {0xa298f2c501f45f42, 0x8349f3ba91b47b90},
+      {0xcb3f2f7642717713, 0x241c70a936219a74},
+      {0xfe0efb53d30dd4d7, 0xed238cd383aa0111},
+      {0x9ec95d1463e8a506, 0xf4363804324a40ab},
+      {0xc67bb4597ce2ce48, 0xb143c6053edcd0d6},
+      {0xf81aa16fdc1b81da, 0xdd94b7868e94050b},
+      {0x9b10a4e5e9913128, 0xca7cf2b4191c8327},
+      {0xc1d4ce1f63f57d72, 0xfd1c2f611f63a3f1},
+      {0xf24a01a73cf2dccf, 0xbc633b39673c8ced},
+      {0x976e41088617ca01, 0xd5be0503e085d814},
+      {0xbd49d14aa79dbc82, 0x4b2d8644d8a74e19},
+      {0xec9c459d51852ba2, 0xddf8e7d60ed1219f},
+      {0x93e1ab8252f33b45, 0xcabb90e5c942b504},
+      {0xb8da1662e7b00a17, 0x3d6a751f3b936244},
+      {0xe7109bfba19c0c9d, 0x0cc512670a783ad5},
+      {0x906a617d450187e2, 0x27fb2b80668b24c6},
+      {0xb484f9dc9641e9da, 0xb1f9f660802dedf7},
+      {0xe1a63853bbd26451, 0x5e7873f8a0396974},
+      {0x8d07e33455637eb2, 0xdb0b487b6423e1e9},
+      {0xb049dc016abc5e5f, 0x91ce1a9a3d2cda63},
+      {0xdc5c5301c56b75f7, 0x7641a140cc7810fc},
+      {0x89b9b3e11b6329ba, 0xa9e904c87fcb0a9e},
+      {0xac2820d9623bf429, 0x546345fa9fbdcd45},
+      {0xd732290fbacaf133, 0xa97c177947ad4096},
+      {0x867f59a9d4bed6c0, 0x49ed8eabcccc485e},
+      {0xa81f301449ee8c70, 0x5c68f256bfff5a75},
+      {0xd226fc195c6a2f8c, 0x73832eec6fff3112},
+      {0x83585d8fd9c25db7, 0xc831fd53c5ff7eac},
+      {0xa42e74f3d032f525, 0xba3e7ca8b77f5e56},
+      {0xcd3a1230c43fb26f, 0x28ce1bd2e55f35ec},
+      {0x80444b5e7aa7cf85, 0x7980d163cf5b81b4},
+      {0xa0555e361951c366, 0xd7e105bcc3326220},
+      {0xc86ab5c39fa63440, 0x8dd9472bf3fefaa8},
+      {0xfa856334878fc150, 0xb14f98f6f0feb952},
+      {0x9c935e00d4b9d8d2, 0x6ed1bf9a569f33d4},
+      {0xc3b8358109e84f07, 0x0a862f80ec4700c9},
+      {0xf4a642e14c6262c8, 0xcd27bb612758c0fb},
+      {0x98e7e9cccfbd7dbd, 0x8038d51cb897789d},
+      {0xbf21e44003acdd2c, 0xe0470a63e6bd56c4},
+      {0xeeea5d5004981478, 0x1858ccfce06cac75},
+      {0x95527a5202df0ccb, 0x0f37801e0c43ebc9},
+      {0xbaa718e68396cffd, 0xd30560258f54e6bb},
+      {0xe950df20247c83fd, 0x47c6b82ef32a206a},
+      {0x91d28b7416cdd27e, 0x4cdc331d57fa5442},
+      {0xb6472e511c81471d, 0xe0133fe4adf8e953},
+      {0xe3d8f9e563a198e5, 0x58180fddd97723a7},
+      {0x8e679c2f5e44ff8f, 0x570f09eaa7ea7649},
+      {0xb201833b35d63f73, 0x2cd2cc6551e513db},
+      {0xde81e40a034bcf4f, 0xf8077f7ea65e58d2},
+      {0x8b112e86420f6191, 0xfb04afaf27faf783},
+      {0xadd57a27d29339f6, 0x79c5db9af1f9b564},
+      {0xd94ad8b1c7380874, 0x18375281ae7822bd},
+      {0x87cec76f1c830548, 0x8f2293910d0b15b6},
+      {0xa9c2794ae3a3c69a, 0xb2eb3875504ddb23},
+      {0xd433179d9c8cb841, 0x5fa60692a46151ec},
+      {0x849feec281d7f328, 0xdbc7c41ba6bcd334},
+      {0xa5c7ea73224deff3, 0x12b9b522906c0801},
+      {0xcf39e50feae16bef, 0xd768226b34870a01},
+      {0x81842f29f2cce375, 0xe6a1158300d46641},
+      {0xa1e53af46f801c53, 0x60495ae3c1097fd1},
+      {0xca5e89b18b602368, 0x385bb19cb14bdfc5},
+      {0xfcf62c1dee382c42, 0x46729e03dd9ed7b6},
+      {0x9e19db92b4e31ba9, 0x6c07a2c26a8346d2},
+      {0xc5a05277621be293, 0xc7098b7305241886},
+      {0xf70867153aa2db38, 0xb8cbee4fc66d1ea8},
+      {0x9a65406d44a5c903, 0x737f74f1dc043329},
+      {0xc0fe908895cf3b44, 0x505f522e53053ff3},
+      {0xf13e34aabb430a15, 0x647726b9e7c68ff0},
+      {0x96c6e0eab509e64d, 0x5eca783430dc19f6},
+      {0xbc789925624c5fe0, 0xb67d16413d132073},
+      {0xeb96bf6ebadf77d8, 0xe41c5bd18c57e890},
+      {0x933e37a534cbaae7, 0x8e91b962f7b6f15a},
+      {0xb80dc58e81fe95a1, 0x723627bbb5a4adb1},
+      {0xe61136f2227e3b09, 0xcec3b1aaa30dd91d},
+      {0x8fcac257558ee4e6, 0x213a4f0aa5e8a7b2},
+      {0xb3bd72ed2af29e1f, 0xa988e2cd4f62d19e},
+      {0xe0accfa875af45a7, 0x93eb1b80a33b8606},
+      {0x8c6c01c9498d8b88, 0xbc72f130660533c4},
+      {0xaf87023b9bf0ee6a, 0xeb8fad7c7f8680b5},
+      {0xdb68c2ca82ed2a05, 0xa67398db9f6820e2},
+#else
+      {0xff77b1fcbebcdc4f, 0x25e8e89c13bb0f7b},
+      {0xce5d73ff402d98e3, 0xfb0a3d212dc81290},
+      {0xa6b34ad8c9dfc06f, 0xf42faa48c0ea481f},
+      {0x86a8d39ef77164bc, 0xae5dff9c02033198},
+      {0xd98ddaee19068c76, 0x3badd624dd9b0958},
+      {0xafbd2350644eeacf, 0xe5d1929ef90898fb},
+      {0x8df5efabc5979c8f, 0xca8d3ffa1ef463c2},
+      {0xe55990879ddcaabd, 0xcc420a6a101d0516},
+      {0xb94470938fa89bce, 0xf808e40e8d5b3e6a},
+      {0x95a8637627989aad, 0xdde7001379a44aa9},
+      {0xf1c90080baf72cb1, 0x5324c68b12dd6339},
+      {0xc350000000000000, 0x0000000000000000},
+      {0x9dc5ada82b70b59d, 0xf020000000000000},
+      {0xfee50b7025c36a08, 0x02f236d04753d5b5},
+      {0xcde6fd5e09abcf26, 0xed4c0226b55e6f87},
+      {0xa6539930bf6bff45, 0x84db8346b786151d},
+      {0x865b86925b9bc5c2, 0x0b8a2392ba45a9b3},
+      {0xd910f7ff28069da4, 0x1b2ba1518094da05},
+      {0xaf58416654a6babb, 0x387ac8d1970027b3},
+      {0x8da471a9de737e24, 0x5ceaecfed289e5d3},
+      {0xe4d5e82392a40515, 0x0fabaf3feaa5334b},
+      {0xb8da1662e7b00a17, 0x3d6a751f3b936244},
+      {0x95527a5202df0ccb, 0x0f37801e0c43ebc9},
+      {0xf13e34aabb430a15, 0x647726b9e7c68ff0}
+#endif
+    };
+
+#if FMT_USE_FULL_CACHE_DRAGONBOX
+    return pow10_significands[k - float_info<double>::min_k];
+#else
+    static constexpr const uint64_t powers_of_5_64[] = {
+        0x0000000000000001, 0x0000000000000005, 0x0000000000000019,
+        0x000000000000007d, 0x0000000000000271, 0x0000000000000c35,
+        0x0000000000003d09, 0x000000000001312d, 0x000000000005f5e1,
+        0x00000000001dcd65, 0x00000000009502f9, 0x0000000002e90edd,
+        0x000000000e8d4a51, 0x0000000048c27395, 0x000000016bcc41e9,
+        0x000000071afd498d, 0x0000002386f26fc1, 0x000000b1a2bc2ec5,
+        0x000003782dace9d9, 0x00001158e460913d, 0x000056bc75e2d631,
+        0x0001b1ae4d6e2ef5, 0x000878678326eac9, 0x002a5a058fc295ed,
+        0x00d3c21bcecceda1, 0x0422ca8b0a00a425, 0x14adf4b7320334b9};
+
+    static const int compression_ratio = 27;
+
+    // Compute base index.
+    int cache_index = (k - float_info<double>::min_k) / compression_ratio;
+    int kb = cache_index * compression_ratio + float_info<double>::min_k;
+    int offset = k - kb;
+
+    // Get base cache.
+    uint128_fallback base_cache = pow10_significands[cache_index];
+    if (offset == 0) return base_cache;
+
+    // Compute the required amount of bit-shift.
+    int alpha = floor_log2_pow10(kb + offset) - floor_log2_pow10(kb) - offset;
+    FMT_ASSERT(alpha > 0 && alpha < 64, "shifting error detected");
+
+    // Try to recover the real cache.
+    uint64_t pow5 = powers_of_5_64[offset];
+    uint128_fallback recovered_cache = umul128(base_cache.high(), pow5);
+    uint128_fallback middle_low = umul128(base_cache.low(), pow5);
+
+    recovered_cache += middle_low.high();
+
+    uint64_t high_to_middle = recovered_cache.high() << (64 - alpha);
+    uint64_t middle_to_low = recovered_cache.low() << (64 - alpha);
+
+    recovered_cache =
+        uint128_fallback{(recovered_cache.low() >> alpha) | high_to_middle,
+                         ((middle_low.low() >> alpha) | middle_to_low)};
+    FMT_ASSERT(recovered_cache.low() + 1 != 0, "");
+    return {recovered_cache.high(), recovered_cache.low() + 1};
+#endif
+  }
+
+  struct compute_mul_result {
+    carrier_uint result;
+    bool is_integer;
+  };
+  struct compute_mul_parity_result {
+    bool parity;
+    bool is_integer;
+  };
+
+  static auto compute_mul(carrier_uint u,
+                          const cache_entry_type& cache) noexcept
+      -> compute_mul_result {
+    auto r = umul192_upper128(u, cache);
+    return {r.high(), r.low() == 0};
+  }
+
+  static auto compute_delta(cache_entry_type const& cache, int beta) noexcept
+      -> uint32_t {
+    return static_cast<uint32_t>(cache.high() >> (64 - 1 - beta));
+  }
+
+  static auto compute_mul_parity(carrier_uint two_f,
+                                 const cache_entry_type& cache,
+                                 int beta) noexcept
+      -> compute_mul_parity_result {
+    FMT_ASSERT(beta >= 1, "");
+    FMT_ASSERT(beta < 64, "");
+
+    auto r = umul192_lower128(two_f, cache);
+    return {((r.high() >> (64 - beta)) & 1) != 0,
+            ((r.high() << beta) | (r.low() >> (64 - beta))) == 0};
+  }
+
+  static auto compute_left_endpoint_for_shorter_interval_case(
+      const cache_entry_type& cache, int beta) noexcept -> carrier_uint {
+    return (cache.high() -
+            (cache.high() >> (num_significand_bits<double>() + 2))) >>
+           (64 - num_significand_bits<double>() - 1 - beta);
+  }
+
+  static auto compute_right_endpoint_for_shorter_interval_case(
+      const cache_entry_type& cache, int beta) noexcept -> carrier_uint {
+    return (cache.high() +
+            (cache.high() >> (num_significand_bits<double>() + 1))) >>
+           (64 - num_significand_bits<double>() - 1 - beta);
+  }
+
+  static auto compute_round_up_for_shorter_interval_case(
+      const cache_entry_type& cache, int beta) noexcept -> carrier_uint {
+    return ((cache.high() >> (64 - num_significand_bits<double>() - 2 - beta)) +
+            1) /
+           2;
+  }
+};
+
+FMT_FUNC auto get_cached_power(int k) noexcept -> uint128_fallback {
+  return cache_accessor<double>::get_cached_power(k);
+}
+
+// Various integer checks
+template <typename T>
+auto is_left_endpoint_integer_shorter_interval(int exponent) noexcept -> bool {
+  const int case_shorter_interval_left_endpoint_lower_threshold = 2;
+  const int case_shorter_interval_left_endpoint_upper_threshold = 3;
+  return exponent >= case_shorter_interval_left_endpoint_lower_threshold &&
+         exponent <= case_shorter_interval_left_endpoint_upper_threshold;
+}
+
+// Remove trailing zeros from n and return the number of zeros removed (float)
+FMT_INLINE int remove_trailing_zeros(uint32_t& n, int s = 0) noexcept {
+  FMT_ASSERT(n != 0, "");
+  // Modular inverse of 5 (mod 2^32): (mod_inv_5 * 5) mod 2^32 = 1.
+  constexpr uint32_t mod_inv_5 = 0xcccccccd;
+  constexpr uint32_t mod_inv_25 = 0xc28f5c29;  // = mod_inv_5 * mod_inv_5
+
+  while (true) {
+    auto q = rotr(n * mod_inv_25, 2);
+    if (q > max_value<uint32_t>() / 100) break;
+    n = q;
+    s += 2;
+  }
+  auto q = rotr(n * mod_inv_5, 1);
+  if (q <= max_value<uint32_t>() / 10) {
+    n = q;
+    s |= 1;
+  }
+  return s;
+}
+
+// Removes trailing zeros and returns the number of zeros removed (double)
+FMT_INLINE int remove_trailing_zeros(uint64_t& n) noexcept {
+  FMT_ASSERT(n != 0, "");
+
+  // This magic number is ceil(2^90 / 10^8).
+  constexpr uint64_t magic_number = 12379400392853802749ull;
+  auto nm = umul128(n, magic_number);
+
+  // Is n is divisible by 10^8?
+  if ((nm.high() & ((1ull << (90 - 64)) - 1)) == 0 && nm.low() < magic_number) {
+    // If yes, work with the quotient...
+    auto n32 = static_cast<uint32_t>(nm.high() >> (90 - 64));
+    // ... and use the 32 bit variant of the function
+    int s = remove_trailing_zeros(n32, 8);
+    n = n32;
+    return s;
+  }
+
+  // If n is not divisible by 10^8, work with n itself.
+  constexpr uint64_t mod_inv_5 = 0xcccccccccccccccd;
+  constexpr uint64_t mod_inv_25 = 0x8f5c28f5c28f5c29;  // mod_inv_5 * mod_inv_5
+
+  int s = 0;
+  while (true) {
+    auto q = rotr(n * mod_inv_25, 2);
+    if (q > max_value<uint64_t>() / 100) break;
+    n = q;
+    s += 2;
+  }
+  auto q = rotr(n * mod_inv_5, 1);
+  if (q <= max_value<uint64_t>() / 10) {
+    n = q;
+    s |= 1;
+  }
+
+  return s;
+}
+
+// The main algorithm for shorter interval case
+template <typename T>
+FMT_INLINE decimal_fp<T> shorter_interval_case(int exponent) noexcept {
+  decimal_fp<T> ret_value;
+  // Compute k and beta
+  const int minus_k = floor_log10_pow2_minus_log10_4_over_3(exponent);
+  const int beta = exponent + floor_log2_pow10(-minus_k);
+
+  // Compute xi and zi
+  using cache_entry_type = typename cache_accessor<T>::cache_entry_type;
+  const cache_entry_type cache = cache_accessor<T>::get_cached_power(-minus_k);
+
+  auto xi = cache_accessor<T>::compute_left_endpoint_for_shorter_interval_case(
+      cache, beta);
+  auto zi = cache_accessor<T>::compute_right_endpoint_for_shorter_interval_case(
+      cache, beta);
+
+  // If the left endpoint is not an integer, increase it
+  if (!is_left_endpoint_integer_shorter_interval<T>(exponent)) ++xi;
+
+  // Try bigger divisor
+  ret_value.significand = zi / 10;
+
+  // If succeed, remove trailing zeros if necessary and return
+  if (ret_value.significand * 10 >= xi) {
+    ret_value.exponent = minus_k + 1;
+    ret_value.exponent += remove_trailing_zeros(ret_value.significand);
+    return ret_value;
+  }
+
+  // Otherwise, compute the round-up of y
+  ret_value.significand =
+      cache_accessor<T>::compute_round_up_for_shorter_interval_case(cache,
+                                                                    beta);
+  ret_value.exponent = minus_k;
+
+  // When tie occurs, choose one of them according to the rule
+  if (exponent >= float_info<T>::shorter_interval_tie_lower_threshold &&
+      exponent <= float_info<T>::shorter_interval_tie_upper_threshold) {
+    ret_value.significand = ret_value.significand % 2 == 0
+                                ? ret_value.significand
+                                : ret_value.significand - 1;
+  } else if (ret_value.significand < xi) {
+    ++ret_value.significand;
+  }
+  return ret_value;
+}
+
+template <typename T> auto to_decimal(T x) noexcept -> decimal_fp<T> {
+  // Step 1: integer promotion & Schubfach multiplier calculation.
+
+  using carrier_uint = typename float_info<T>::carrier_uint;
+  using cache_entry_type = typename cache_accessor<T>::cache_entry_type;
+  auto br = bit_cast<carrier_uint>(x);
+
+  // Extract significand bits and exponent bits.
+  const carrier_uint significand_mask =
+      (static_cast<carrier_uint>(1) << num_significand_bits<T>()) - 1;
+  carrier_uint significand = (br & significand_mask);
+  int exponent =
+      static_cast<int>((br & exponent_mask<T>()) >> num_significand_bits<T>());
+
+  if (exponent != 0) {  // Check if normal.
+    exponent -= exponent_bias<T>() + num_significand_bits<T>();
+
+    // Shorter interval case; proceed like Schubfach.
+    // In fact, when exponent == 1 and significand == 0, the interval is
+    // regular. However, it can be shown that the end-results are anyway same.
+    if (significand == 0) return shorter_interval_case<T>(exponent);
+
+    significand |= (static_cast<carrier_uint>(1) << num_significand_bits<T>());
+  } else {
+    // Subnormal case; the interval is always regular.
+    if (significand == 0) return {0, 0};
+    exponent =
+        std::numeric_limits<T>::min_exponent - num_significand_bits<T>() - 1;
+  }
+
+  const bool include_left_endpoint = (significand % 2 == 0);
+  const bool include_right_endpoint = include_left_endpoint;
+
+  // Compute k and beta.
+  const int minus_k = floor_log10_pow2(exponent) - float_info<T>::kappa;
+  const cache_entry_type cache = cache_accessor<T>::get_cached_power(-minus_k);
+  const int beta = exponent + floor_log2_pow10(-minus_k);
+
+  // Compute zi and deltai.
+  // 10^kappa <= deltai < 10^(kappa + 1)
+  const uint32_t deltai = cache_accessor<T>::compute_delta(cache, beta);
+  const carrier_uint two_fc = significand << 1;
+
+  // For the case of binary32, the result of integer check is not correct for
+  // 29711844 * 2^-82
+  // = 6.1442653300000000008655037797566933477355632930994033813476... * 10^-18
+  // and 29711844 * 2^-81
+  // = 1.2288530660000000001731007559513386695471126586198806762695... * 10^-17,
+  // and they are the unique counterexamples. However, since 29711844 is even,
+  // this does not cause any problem for the endpoints calculations; it can only
+  // cause a problem when we need to perform integer check for the center.
+  // Fortunately, with these inputs, that branch is never executed, so we are
+  // fine.
+  const typename cache_accessor<T>::compute_mul_result z_mul =
+      cache_accessor<T>::compute_mul((two_fc | 1) << beta, cache);
+
+  // Step 2: Try larger divisor; remove trailing zeros if necessary.
+
+  // Using an upper bound on zi, we might be able to optimize the division
+  // better than the compiler; we are computing zi / big_divisor here.
+  decimal_fp<T> ret_value;
+  ret_value.significand = divide_by_10_to_kappa_plus_1(z_mul.result);
+  uint32_t r = static_cast<uint32_t>(z_mul.result - float_info<T>::big_divisor *
+                                                        ret_value.significand);
+
+  if (r < deltai) {
+    // Exclude the right endpoint if necessary.
+    if (r == 0 && (z_mul.is_integer & !include_right_endpoint)) {
+      --ret_value.significand;
+      r = float_info<T>::big_divisor;
+      goto small_divisor_case_label;
+    }
+  } else if (r > deltai) {
+    goto small_divisor_case_label;
+  } else {
+    // r == deltai; compare fractional parts.
+    const typename cache_accessor<T>::compute_mul_parity_result x_mul =
+        cache_accessor<T>::compute_mul_parity(two_fc - 1, cache, beta);
+
+    if (!(x_mul.parity | (x_mul.is_integer & include_left_endpoint)))
+      goto small_divisor_case_label;
+  }
+  ret_value.exponent = minus_k + float_info<T>::kappa + 1;
+
+  // We may need to remove trailing zeros.
+  ret_value.exponent += remove_trailing_zeros(ret_value.significand);
+  return ret_value;
+
+  // Step 3: Find the significand with the smaller divisor.
+
+small_divisor_case_label:
+  ret_value.significand *= 10;
+  ret_value.exponent = minus_k + float_info<T>::kappa;
+
+  uint32_t dist = r - (deltai / 2) + (float_info<T>::small_divisor / 2);
+  const bool approx_y_parity =
+      ((dist ^ (float_info<T>::small_divisor / 2)) & 1) != 0;
+
+  // Is dist divisible by 10^kappa?
+  const bool divisible_by_small_divisor =
+      check_divisibility_and_divide_by_pow10<float_info<T>::kappa>(dist);
+
+  // Add dist / 10^kappa to the significand.
+  ret_value.significand += dist;
+
+  if (!divisible_by_small_divisor) return ret_value;
+
+  // Check z^(f) >= epsilon^(f).
+  // We have either yi == zi - epsiloni or yi == (zi - epsiloni) - 1,
+  // where yi == zi - epsiloni if and only if z^(f) >= epsilon^(f).
+  // Since there are only 2 possibilities, we only need to care about the
+  // parity. Also, zi and r should have the same parity since the divisor
+  // is an even number.
+  const auto y_mul = cache_accessor<T>::compute_mul_parity(two_fc, cache, beta);
+
+  // If z^(f) >= epsilon^(f), we might have a tie when z^(f) == epsilon^(f),
+  // or equivalently, when y is an integer.
+  if (y_mul.parity != approx_y_parity)
+    --ret_value.significand;
+  else if (y_mul.is_integer & (ret_value.significand % 2 != 0))
+    --ret_value.significand;
+  return ret_value;
+}
+}  // namespace dragonbox
+}  // namespace detail
+
+template <> struct formatter<detail::bigint> {
+  FMT_CONSTEXPR auto parse(format_parse_context& ctx)
+      -> format_parse_context::iterator {
+    return ctx.begin();
+  }
+
+  auto format(const detail::bigint& n, format_context& ctx) const
+      -> format_context::iterator {
+    auto out = ctx.out();
+    bool first = true;
+    for (auto i = n.bigits_.size(); i > 0; --i) {
+      auto value = n.bigits_[i - 1u];
+      if (first) {
+        out = fmt::format_to(out, FMT_STRING("{:x}"), value);
+        first = false;
+        continue;
+      }
+      out = fmt::format_to(out, FMT_STRING("{:08x}"), value);
+    }
+    if (n.exp_ > 0)
+      out = fmt::format_to(out, FMT_STRING("p{}"),
+                           n.exp_ * detail::bigint::bigit_bits);
+    return out;
+  }
+};
+
+FMT_FUNC detail::utf8_to_utf16::utf8_to_utf16(string_view s) {
+  for_each_codepoint(s, [this](uint32_t cp, string_view) {
+    if (cp == invalid_code_point) FMT_THROW(std::runtime_error("invalid utf8"));
+    if (cp <= 0xFFFF) {
+      buffer_.push_back(static_cast<wchar_t>(cp));
+    } else {
+      cp -= 0x10000;
+      buffer_.push_back(static_cast<wchar_t>(0xD800 + (cp >> 10)));
+      buffer_.push_back(static_cast<wchar_t>(0xDC00 + (cp & 0x3FF)));
+    }
+    return true;
+  });
+  buffer_.push_back(0);
+}
+
+FMT_FUNC void format_system_error(detail::buffer<char>& out, int error_code,
+                                  const char* message) noexcept {
+  FMT_TRY {
+    auto ec = std::error_code(error_code, std::generic_category());
+    detail::write(appender(out), std::system_error(ec, message).what());
+    return;
+  }
+  FMT_CATCH(...) {}
+  format_error_code(out, error_code, message);
+}
+
+FMT_FUNC void report_system_error(int error_code,
+                                  const char* message) noexcept {
+  report_error(format_system_error, error_code, message);
+}
+
+FMT_FUNC auto vformat(string_view fmt, format_args args) -> std::string {
+  // Don't optimize the "{}" case to keep the binary size small and because it
+  // can be better optimized in fmt::format anyway.
+  auto buffer = memory_buffer();
+  detail::vformat_to(buffer, fmt, args);
+  return to_string(buffer);
+}
+
+namespace detail {
+
+template <typename T> struct span {
+  T* data;
+  size_t size;
+};
+
+template <typename F> auto flockfile(F* f) -> decltype(_lock_file(f)) {
+  _lock_file(f);
+}
+template <typename F> auto funlockfile(F* f) -> decltype(_unlock_file(f)) {
+  _unlock_file(f);
+}
+
+#ifndef getc_unlocked
+template <typename F> auto getc_unlocked(F* f) -> decltype(_fgetc_nolock(f)) {
+  return _fgetc_nolock(f);
+}
+#endif
+
+template <typename F = FILE, typename Enable = void>
+struct has_flockfile : std::false_type {};
+
+template <typename F>
+struct has_flockfile<F, void_t<decltype(flockfile(&std::declval<F&>()))>>
+    : std::true_type {};
+
+// A FILE wrapper. F is FILE defined as a template parameter to make system API
+// detection work.
+template <typename F> class file_base {
+ public:
+  F* file_;
+
+ public:
+  file_base(F* file) : file_(file) {}
+  operator F*() const { return file_; }
+
+  // Reads a code unit from the stream.
+  auto get() -> int {
+    int result = getc_unlocked(file_);
+    if (result == EOF && ferror(file_) != 0)
+      FMT_THROW(system_error(errno, FMT_STRING("getc failed")));
+    return result;
+  }
+
+  // Puts the code unit back into the stream buffer.
+  void unget(char c) {
+    if (ungetc(c, file_) == EOF)
+      FMT_THROW(system_error(errno, FMT_STRING("ungetc failed")));
+  }
+
+  void flush() { fflush(this->file_); }
+};
+
+// A FILE wrapper for glibc.
+template <typename F> class glibc_file : public file_base<F> {
+ private:
+  enum {
+    line_buffered = 0x200,  // _IO_LINE_BUF
+    unbuffered = 2          // _IO_UNBUFFERED
+  };
+
+ public:
+  using file_base<F>::file_base;
+
+  auto is_buffered() const -> bool {
+    return (this->file_->_flags & unbuffered) == 0;
+  }
+
+  void init_buffer() {
+    if (this->file_->_IO_write_ptr) return;
+    // Force buffer initialization by placing and removing a char in a buffer.
+    putc_unlocked(0, this->file_);
+    --this->file_->_IO_write_ptr;
+  }
+
+  // Returns the file's read buffer.
+  auto get_read_buffer() const -> span<const char> {
+    auto ptr = this->file_->_IO_read_ptr;
+    return {ptr, to_unsigned(this->file_->_IO_read_end - ptr)};
+  }
+
+  // Returns the file's write buffer.
+  auto get_write_buffer() const -> span<char> {
+    auto ptr = this->file_->_IO_write_ptr;
+    return {ptr, to_unsigned(this->file_->_IO_buf_end - ptr)};
+  }
+
+  void advance_write_buffer(size_t size) { this->file_->_IO_write_ptr += size; }
+
+  bool needs_flush() const {
+    if ((this->file_->_flags & line_buffered) == 0) return false;
+    char* end = this->file_->_IO_write_end;
+    return memchr(end, '\n', to_unsigned(this->file_->_IO_write_ptr - end));
+  }
+
+  void flush() { fflush_unlocked(this->file_); }
+};
+
+// A FILE wrapper for Apple's libc.
+template <typename F> class apple_file : public file_base<F> {
+ private:
+  enum {
+    line_buffered = 1,  // __SNBF
+    unbuffered = 2      // __SLBF
+  };
+
+ public:
+  using file_base<F>::file_base;
+
+  auto is_buffered() const -> bool {
+    return (this->file_->_flags & unbuffered) == 0;
+  }
+
+  void init_buffer() {
+    if (this->file_->_p) return;
+    // Force buffer initialization by placing and removing a char in a buffer.
+    putc_unlocked(0, this->file_);
+    --this->file_->_p;
+    ++this->file_->_w;
+  }
+
+  auto get_read_buffer() const -> span<const char> {
+    return {reinterpret_cast<char*>(this->file_->_p),
+            to_unsigned(this->file_->_r)};
+  }
+
+  auto get_write_buffer() const -> span<char> {
+    return {reinterpret_cast<char*>(this->file_->_p),
+            to_unsigned(this->file_->_bf._base + this->file_->_bf._size -
+                        this->file_->_p)};
+  }
+
+  void advance_write_buffer(size_t size) {
+    this->file_->_p += size;
+    this->file_->_w -= size;
+  }
+
+  bool needs_flush() const {
+    if ((this->file_->_flags & line_buffered) == 0) return false;
+    return memchr(this->file_->_p + this->file_->_w, '\n',
+                  to_unsigned(-this->file_->_w));
+  }
+};
+
+// A fallback FILE wrapper.
+template <typename F> class fallback_file : public file_base<F> {
+ private:
+  char next_;  // The next unconsumed character in the buffer.
+  bool has_next_ = false;
+
+ public:
+  using file_base<F>::file_base;
+
+  auto is_buffered() const -> bool { return false; }
+  auto needs_flush() const -> bool { return false; }
+  void init_buffer() {}
+
+  auto get_read_buffer() const -> span<const char> {
+    return {&next_, has_next_ ? 1u : 0u};
+  }
+
+  auto get_write_buffer() const -> span<char> { return {nullptr, 0}; }
+
+  void advance_write_buffer(size_t) {}
+
+  auto get() -> int {
+    has_next_ = false;
+    return file_base<F>::get();
+  }
+
+  void unget(char c) {
+    file_base<F>::unget(c);
+    next_ = c;
+    has_next_ = true;
+  }
+};
+
+#ifndef FMT_USE_FALLBACK_FILE
+#  define FMT_USE_FALLBACK_FILE 1
+#endif
+
+template <typename F,
+          FMT_ENABLE_IF(sizeof(F::_p) != 0 && !FMT_USE_FALLBACK_FILE)>
+auto get_file(F* f, int) -> apple_file<F> {
+  return f;
+}
+template <typename F,
+          FMT_ENABLE_IF(sizeof(F::_IO_read_ptr) != 0 && !FMT_USE_FALLBACK_FILE)>
+inline auto get_file(F* f, int) -> glibc_file<F> {
+  return f;
+}
+
+inline auto get_file(FILE* f, ...) -> fallback_file<FILE> { return f; }
+
+using file_ref = decltype(get_file(static_cast<FILE*>(nullptr), 0));
+
+template <typename F = FILE, typename Enable = void>
+class file_print_buffer : public buffer<char> {
+ public:
+  explicit file_print_buffer(F*) : buffer(nullptr, size_t()) {}
+};
+
+template <typename F>
+class file_print_buffer<F, enable_if_t<has_flockfile<F>::value>>
+    : public buffer<char> {
+ private:
+  file_ref file_;
+
+  static void grow(buffer<char>& base, size_t) {
+    auto& self = static_cast<file_print_buffer&>(base);
+    self.file_.advance_write_buffer(self.size());
+    if (self.file_.get_write_buffer().size == 0) self.file_.flush();
+    auto buf = self.file_.get_write_buffer();
+    FMT_ASSERT(buf.size > 0, "");
+    self.set(buf.data, buf.size);
+    self.clear();
+  }
+
+ public:
+  explicit file_print_buffer(F* f) : buffer(grow, size_t()), file_(f) {
+    flockfile(f);
+    file_.init_buffer();
+    auto buf = file_.get_write_buffer();
+    set(buf.data, buf.size);
+  }
+  ~file_print_buffer() {
+    file_.advance_write_buffer(size());
+    bool flush = file_.needs_flush();
+    F* f = file_;    // Make funlockfile depend on the template parameter F
+    funlockfile(f);  // for the system API detection to work.
+    if (flush) fflush(file_);
+  }
+};
+
+#if !defined(_WIN32) || defined(FMT_USE_WRITE_CONSOLE)
+FMT_FUNC auto write_console(int, string_view) -> bool { return false; }
+#else
+using dword = conditional_t<sizeof(long) == 4, unsigned long, unsigned>;
+extern "C" __declspec(dllimport) int __stdcall WriteConsoleW(  //
+    void*, const void*, dword, dword*, void*);
+
+FMT_FUNC bool write_console(int fd, string_view text) {
+  auto u16 = utf8_to_utf16(text);
+  return WriteConsoleW(reinterpret_cast<void*>(_get_osfhandle(fd)), u16.c_str(),
+                       static_cast<dword>(u16.size()), nullptr, nullptr) != 0;
+}
+#endif
+
+#ifdef _WIN32
+// Print assuming legacy (non-Unicode) encoding.
+FMT_FUNC void vprint_mojibake(std::FILE* f, string_view fmt, format_args args,
+                              bool newline) {
+  auto buffer = memory_buffer();
+  detail::vformat_to(buffer, fmt, args);
+  if (newline) buffer.push_back('\n');
+  fwrite_fully(buffer.data(), buffer.size(), f);
+}
+#endif
+
+FMT_FUNC void print(std::FILE* f, string_view text) {
+#if defined(_WIN32) && !defined(FMT_USE_WRITE_CONSOLE)
+  int fd = _fileno(f);
+  if (_isatty(fd)) {
+    std::fflush(f);
+    if (write_console(fd, text)) return;
+  }
+#endif
+  fwrite_fully(text.data(), text.size(), f);
+}
+}  // namespace detail
+
+FMT_FUNC void vprint_buffered(std::FILE* f, string_view fmt, format_args args) {
+  auto buffer = memory_buffer();
+  detail::vformat_to(buffer, fmt, args);
+  detail::print(f, {buffer.data(), buffer.size()});
+}
+
+FMT_FUNC void vprint(std::FILE* f, string_view fmt, format_args args) {
+  if (!detail::file_ref(f).is_buffered() || !detail::has_flockfile<>())
+    return vprint_buffered(f, fmt, args);
+  auto&& buffer = detail::file_print_buffer<>(f);
+  return detail::vformat_to(buffer, fmt, args);
+}
+
+FMT_FUNC void vprintln(std::FILE* f, string_view fmt, format_args args) {
+  auto buffer = memory_buffer();
+  detail::vformat_to(buffer, fmt, args);
+  buffer.push_back('\n');
+  detail::print(f, {buffer.data(), buffer.size()});
+}
+
+FMT_FUNC void vprint(string_view fmt, format_args args) {
+  vprint(stdout, fmt, args);
+}
+
+namespace detail {
+
+struct singleton {
+  unsigned char upper;
+  unsigned char lower_count;
+};
+
+inline auto is_printable(uint16_t x, const singleton* singletons,
+                         size_t singletons_size,
+                         const unsigned char* singleton_lowers,
+                         const unsigned char* normal, size_t normal_size)
+    -> bool {
+  auto upper = x >> 8;
+  auto lower_start = 0;
+  for (size_t i = 0; i < singletons_size; ++i) {
+    auto s = singletons[i];
+    auto lower_end = lower_start + s.lower_count;
+    if (upper < s.upper) break;
+    if (upper == s.upper) {
+      for (auto j = lower_start; j < lower_end; ++j) {
+        if (singleton_lowers[j] == (x & 0xff)) return false;
+      }
+    }
+    lower_start = lower_end;
+  }
+
+  auto xsigned = static_cast<int>(x);
+  auto current = true;
+  for (size_t i = 0; i < normal_size; ++i) {
+    auto v = static_cast<int>(normal[i]);
+    auto len = (v & 0x80) != 0 ? (v & 0x7f) << 8 | normal[++i] : v;
+    xsigned -= len;
+    if (xsigned < 0) break;
+    current = !current;
+  }
+  return current;
+}
+
+// This code is generated by support/printable.py.
+FMT_FUNC auto is_printable(uint32_t cp) -> bool {
+  static constexpr singleton singletons0[] = {
+      {0x00, 1},  {0x03, 5},  {0x05, 6},  {0x06, 3},  {0x07, 6},  {0x08, 8},
+      {0x09, 17}, {0x0a, 28}, {0x0b, 25}, {0x0c, 20}, {0x0d, 16}, {0x0e, 13},
+      {0x0f, 4},  {0x10, 3},  {0x12, 18}, {0x13, 9},  {0x16, 1},  {0x17, 5},
+      {0x18, 2},  {0x19, 3},  {0x1a, 7},  {0x1c, 2},  {0x1d, 1},  {0x1f, 22},
+      {0x20, 3},  {0x2b, 3},  {0x2c, 2},  {0x2d, 11}, {0x2e, 1},  {0x30, 3},
+      {0x31, 2},  {0x32, 1},  {0xa7, 2},  {0xa9, 2},  {0xaa, 4},  {0xab, 8},
+      {0xfa, 2},  {0xfb, 5},  {0xfd, 4},  {0xfe, 3},  {0xff, 9},
+  };
+  static constexpr unsigned char singletons0_lower[] = {
+      0xad, 0x78, 0x79, 0x8b, 0x8d, 0xa2, 0x30, 0x57, 0x58, 0x8b, 0x8c, 0x90,
+      0x1c, 0x1d, 0xdd, 0x0e, 0x0f, 0x4b, 0x4c, 0xfb, 0xfc, 0x2e, 0x2f, 0x3f,
+      0x5c, 0x5d, 0x5f, 0xb5, 0xe2, 0x84, 0x8d, 0x8e, 0x91, 0x92, 0xa9, 0xb1,
+      0xba, 0xbb, 0xc5, 0xc6, 0xc9, 0xca, 0xde, 0xe4, 0xe5, 0xff, 0x00, 0x04,
+      0x11, 0x12, 0x29, 0x31, 0x34, 0x37, 0x3a, 0x3b, 0x3d, 0x49, 0x4a, 0x5d,
+      0x84, 0x8e, 0x92, 0xa9, 0xb1, 0xb4, 0xba, 0xbb, 0xc6, 0xca, 0xce, 0xcf,
+      0xe4, 0xe5, 0x00, 0x04, 0x0d, 0x0e, 0x11, 0x12, 0x29, 0x31, 0x34, 0x3a,
+      0x3b, 0x45, 0x46, 0x49, 0x4a, 0x5e, 0x64, 0x65, 0x84, 0x91, 0x9b, 0x9d,
+      0xc9, 0xce, 0xcf, 0x0d, 0x11, 0x29, 0x45, 0x49, 0x57, 0x64, 0x65, 0x8d,
+      0x91, 0xa9, 0xb4, 0xba, 0xbb, 0xc5, 0xc9, 0xdf, 0xe4, 0xe5, 0xf0, 0x0d,
+      0x11, 0x45, 0x49, 0x64, 0x65, 0x80, 0x84, 0xb2, 0xbc, 0xbe, 0xbf, 0xd5,
+      0xd7, 0xf0, 0xf1, 0x83, 0x85, 0x8b, 0xa4, 0xa6, 0xbe, 0xbf, 0xc5, 0xc7,
+      0xce, 0xcf, 0xda, 0xdb, 0x48, 0x98, 0xbd, 0xcd, 0xc6, 0xce, 0xcf, 0x49,
+      0x4e, 0x4f, 0x57, 0x59, 0x5e, 0x5f, 0x89, 0x8e, 0x8f, 0xb1, 0xb6, 0xb7,
+      0xbf, 0xc1, 0xc6, 0xc7, 0xd7, 0x11, 0x16, 0x17, 0x5b, 0x5c, 0xf6, 0xf7,
+      0xfe, 0xff, 0x80, 0x0d, 0x6d, 0x71, 0xde, 0xdf, 0x0e, 0x0f, 0x1f, 0x6e,
+      0x6f, 0x1c, 0x1d, 0x5f, 0x7d, 0x7e, 0xae, 0xaf, 0xbb, 0xbc, 0xfa, 0x16,
+      0x17, 0x1e, 0x1f, 0x46, 0x47, 0x4e, 0x4f, 0x58, 0x5a, 0x5c, 0x5e, 0x7e,
+      0x7f, 0xb5, 0xc5, 0xd4, 0xd5, 0xdc, 0xf0, 0xf1, 0xf5, 0x72, 0x73, 0x8f,
+      0x74, 0x75, 0x96, 0x2f, 0x5f, 0x26, 0x2e, 0x2f, 0xa7, 0xaf, 0xb7, 0xbf,
+      0xc7, 0xcf, 0xd7, 0xdf, 0x9a, 0x40, 0x97, 0x98, 0x30, 0x8f, 0x1f, 0xc0,
+      0xc1, 0xce, 0xff, 0x4e, 0x4f, 0x5a, 0x5b, 0x07, 0x08, 0x0f, 0x10, 0x27,
+      0x2f, 0xee, 0xef, 0x6e, 0x6f, 0x37, 0x3d, 0x3f, 0x42, 0x45, 0x90, 0x91,
+      0xfe, 0xff, 0x53, 0x67, 0x75, 0xc8, 0xc9, 0xd0, 0xd1, 0xd8, 0xd9, 0xe7,
+      0xfe, 0xff,
+  };
+  static constexpr singleton singletons1[] = {
+      {0x00, 6},  {0x01, 1}, {0x03, 1},  {0x04, 2}, {0x08, 8},  {0x09, 2},
+      {0x0a, 5},  {0x0b, 2}, {0x0e, 4},  {0x10, 1}, {0x11, 2},  {0x12, 5},
+      {0x13, 17}, {0x14, 1}, {0x15, 2},  {0x17, 2}, {0x19, 13}, {0x1c, 5},
+      {0x1d, 8},  {0x24, 1}, {0x6a, 3},  {0x6b, 2}, {0xbc, 2},  {0xd1, 2},
+      {0xd4, 12}, {0xd5, 9}, {0xd6, 2},  {0xd7, 2}, {0xda, 1},  {0xe0, 5},
+      {0xe1, 2},  {0xe8, 2}, {0xee, 32}, {0xf0, 4}, {0xf8, 2},  {0xf9, 2},
+      {0xfa, 2},  {0xfb, 1},
+  };
+  static constexpr unsigned char singletons1_lower[] = {
+      0x0c, 0x27, 0x3b, 0x3e, 0x4e, 0x4f, 0x8f, 0x9e, 0x9e, 0x9f, 0x06, 0x07,
+      0x09, 0x36, 0x3d, 0x3e, 0x56, 0xf3, 0xd0, 0xd1, 0x04, 0x14, 0x18, 0x36,
+      0x37, 0x56, 0x57, 0x7f, 0xaa, 0xae, 0xaf, 0xbd, 0x35, 0xe0, 0x12, 0x87,
+      0x89, 0x8e, 0x9e, 0x04, 0x0d, 0x0e, 0x11, 0x12, 0x29, 0x31, 0x34, 0x3a,
+      0x45, 0x46, 0x49, 0x4a, 0x4e, 0x4f, 0x64, 0x65, 0x5c, 0xb6, 0xb7, 0x1b,
+      0x1c, 0x07, 0x08, 0x0a, 0x0b, 0x14, 0x17, 0x36, 0x39, 0x3a, 0xa8, 0xa9,
+      0xd8, 0xd9, 0x09, 0x37, 0x90, 0x91, 0xa8, 0x07, 0x0a, 0x3b, 0x3e, 0x66,
+      0x69, 0x8f, 0x92, 0x6f, 0x5f, 0xee, 0xef, 0x5a, 0x62, 0x9a, 0x9b, 0x27,
+      0x28, 0x55, 0x9d, 0xa0, 0xa1, 0xa3, 0xa4, 0xa7, 0xa8, 0xad, 0xba, 0xbc,
+      0xc4, 0x06, 0x0b, 0x0c, 0x15, 0x1d, 0x3a, 0x3f, 0x45, 0x51, 0xa6, 0xa7,
+      0xcc, 0xcd, 0xa0, 0x07, 0x19, 0x1a, 0x22, 0x25, 0x3e, 0x3f, 0xc5, 0xc6,
+      0x04, 0x20, 0x23, 0x25, 0x26, 0x28, 0x33, 0x38, 0x3a, 0x48, 0x4a, 0x4c,
+      0x50, 0x53, 0x55, 0x56, 0x58, 0x5a, 0x5c, 0x5e, 0x60, 0x63, 0x65, 0x66,
+      0x6b, 0x73, 0x78, 0x7d, 0x7f, 0x8a, 0xa4, 0xaa, 0xaf, 0xb0, 0xc0, 0xd0,
+      0xae, 0xaf, 0x79, 0xcc, 0x6e, 0x6f, 0x93,
+  };
+  static constexpr unsigned char normal0[] = {
+      0x00, 0x20, 0x5f, 0x22, 0x82, 0xdf, 0x04, 0x82, 0x44, 0x08, 0x1b, 0x04,
+      0x06, 0x11, 0x81, 0xac, 0x0e, 0x80, 0xab, 0x35, 0x28, 0x0b, 0x80, 0xe0,
+      0x03, 0x19, 0x08, 0x01, 0x04, 0x2f, 0x04, 0x34, 0x04, 0x07, 0x03, 0x01,
+      0x07, 0x06, 0x07, 0x11, 0x0a, 0x50, 0x0f, 0x12, 0x07, 0x55, 0x07, 0x03,
+      0x04, 0x1c, 0x0a, 0x09, 0x03, 0x08, 0x03, 0x07, 0x03, 0x02, 0x03, 0x03,
+      0x03, 0x0c, 0x04, 0x05, 0x03, 0x0b, 0x06, 0x01, 0x0e, 0x15, 0x05, 0x3a,
+      0x03, 0x11, 0x07, 0x06, 0x05, 0x10, 0x07, 0x57, 0x07, 0x02, 0x07, 0x15,
+      0x0d, 0x50, 0x04, 0x43, 0x03, 0x2d, 0x03, 0x01, 0x04, 0x11, 0x06, 0x0f,
+      0x0c, 0x3a, 0x04, 0x1d, 0x25, 0x5f, 0x20, 0x6d, 0x04, 0x6a, 0x25, 0x80,
+      0xc8, 0x05, 0x82, 0xb0, 0x03, 0x1a, 0x06, 0x82, 0xfd, 0x03, 0x59, 0x07,
+      0x15, 0x0b, 0x17, 0x09, 0x14, 0x0c, 0x14, 0x0c, 0x6a, 0x06, 0x0a, 0x06,
+      0x1a, 0x06, 0x59, 0x07, 0x2b, 0x05, 0x46, 0x0a, 0x2c, 0x04, 0x0c, 0x04,
+      0x01, 0x03, 0x31, 0x0b, 0x2c, 0x04, 0x1a, 0x06, 0x0b, 0x03, 0x80, 0xac,
+      0x06, 0x0a, 0x06, 0x21, 0x3f, 0x4c, 0x04, 0x2d, 0x03, 0x74, 0x08, 0x3c,
+      0x03, 0x0f, 0x03, 0x3c, 0x07, 0x38, 0x08, 0x2b, 0x05, 0x82, 0xff, 0x11,
+      0x18, 0x08, 0x2f, 0x11, 0x2d, 0x03, 0x20, 0x10, 0x21, 0x0f, 0x80, 0x8c,
+      0x04, 0x82, 0x97, 0x19, 0x0b, 0x15, 0x88, 0x94, 0x05, 0x2f, 0x05, 0x3b,
+      0x07, 0x02, 0x0e, 0x18, 0x09, 0x80, 0xb3, 0x2d, 0x74, 0x0c, 0x80, 0xd6,
+      0x1a, 0x0c, 0x05, 0x80, 0xff, 0x05, 0x80, 0xdf, 0x0c, 0xee, 0x0d, 0x03,
+      0x84, 0x8d, 0x03, 0x37, 0x09, 0x81, 0x5c, 0x14, 0x80, 0xb8, 0x08, 0x80,
+      0xcb, 0x2a, 0x38, 0x03, 0x0a, 0x06, 0x38, 0x08, 0x46, 0x08, 0x0c, 0x06,
+      0x74, 0x0b, 0x1e, 0x03, 0x5a, 0x04, 0x59, 0x09, 0x80, 0x83, 0x18, 0x1c,
+      0x0a, 0x16, 0x09, 0x4c, 0x04, 0x80, 0x8a, 0x06, 0xab, 0xa4, 0x0c, 0x17,
+      0x04, 0x31, 0xa1, 0x04, 0x81, 0xda, 0x26, 0x07, 0x0c, 0x05, 0x05, 0x80,
+      0xa5, 0x11, 0x81, 0x6d, 0x10, 0x78, 0x28, 0x2a, 0x06, 0x4c, 0x04, 0x80,
+      0x8d, 0x04, 0x80, 0xbe, 0x03, 0x1b, 0x03, 0x0f, 0x0d,
+  };
+  static constexpr unsigned char normal1[] = {
+      0x5e, 0x22, 0x7b, 0x05, 0x03, 0x04, 0x2d, 0x03, 0x66, 0x03, 0x01, 0x2f,
+      0x2e, 0x80, 0x82, 0x1d, 0x03, 0x31, 0x0f, 0x1c, 0x04, 0x24, 0x09, 0x1e,
+      0x05, 0x2b, 0x05, 0x44, 0x04, 0x0e, 0x2a, 0x80, 0xaa, 0x06, 0x24, 0x04,
+      0x24, 0x04, 0x28, 0x08, 0x34, 0x0b, 0x01, 0x80, 0x90, 0x81, 0x37, 0x09,
+      0x16, 0x0a, 0x08, 0x80, 0x98, 0x39, 0x03, 0x63, 0x08, 0x09, 0x30, 0x16,
+      0x05, 0x21, 0x03, 0x1b, 0x05, 0x01, 0x40, 0x38, 0x04, 0x4b, 0x05, 0x2f,
+      0x04, 0x0a, 0x07, 0x09, 0x07, 0x40, 0x20, 0x27, 0x04, 0x0c, 0x09, 0x36,
+      0x03, 0x3a, 0x05, 0x1a, 0x07, 0x04, 0x0c, 0x07, 0x50, 0x49, 0x37, 0x33,
+      0x0d, 0x33, 0x07, 0x2e, 0x08, 0x0a, 0x81, 0x26, 0x52, 0x4e, 0x28, 0x08,
+      0x2a, 0x56, 0x1c, 0x14, 0x17, 0x09, 0x4e, 0x04, 0x1e, 0x0f, 0x43, 0x0e,
+      0x19, 0x07, 0x0a, 0x06, 0x48, 0x08, 0x27, 0x09, 0x75, 0x0b, 0x3f, 0x41,
+      0x2a, 0x06, 0x3b, 0x05, 0x0a, 0x06, 0x51, 0x06, 0x01, 0x05, 0x10, 0x03,
+      0x05, 0x80, 0x8b, 0x62, 0x1e, 0x48, 0x08, 0x0a, 0x80, 0xa6, 0x5e, 0x22,
+      0x45, 0x0b, 0x0a, 0x06, 0x0d, 0x13, 0x39, 0x07, 0x0a, 0x36, 0x2c, 0x04,
+      0x10, 0x80, 0xc0, 0x3c, 0x64, 0x53, 0x0c, 0x48, 0x09, 0x0a, 0x46, 0x45,
+      0x1b, 0x48, 0x08, 0x53, 0x1d, 0x39, 0x81, 0x07, 0x46, 0x0a, 0x1d, 0x03,
+      0x47, 0x49, 0x37, 0x03, 0x0e, 0x08, 0x0a, 0x06, 0x39, 0x07, 0x0a, 0x81,
+      0x36, 0x19, 0x80, 0xb7, 0x01, 0x0f, 0x32, 0x0d, 0x83, 0x9b, 0x66, 0x75,
+      0x0b, 0x80, 0xc4, 0x8a, 0xbc, 0x84, 0x2f, 0x8f, 0xd1, 0x82, 0x47, 0xa1,
+      0xb9, 0x82, 0x39, 0x07, 0x2a, 0x04, 0x02, 0x60, 0x26, 0x0a, 0x46, 0x0a,
+      0x28, 0x05, 0x13, 0x82, 0xb0, 0x5b, 0x65, 0x4b, 0x04, 0x39, 0x07, 0x11,
+      0x40, 0x05, 0x0b, 0x02, 0x0e, 0x97, 0xf8, 0x08, 0x84, 0xd6, 0x2a, 0x09,
+      0xa2, 0xf7, 0x81, 0x1f, 0x31, 0x03, 0x11, 0x04, 0x08, 0x81, 0x8c, 0x89,
+      0x04, 0x6b, 0x05, 0x0d, 0x03, 0x09, 0x07, 0x10, 0x93, 0x60, 0x80, 0xf6,
+      0x0a, 0x73, 0x08, 0x6e, 0x17, 0x46, 0x80, 0x9a, 0x14, 0x0c, 0x57, 0x09,
+      0x19, 0x80, 0x87, 0x81, 0x47, 0x03, 0x85, 0x42, 0x0f, 0x15, 0x85, 0x50,
+      0x2b, 0x80, 0xd5, 0x2d, 0x03, 0x1a, 0x04, 0x02, 0x81, 0x70, 0x3a, 0x05,
+      0x01, 0x85, 0x00, 0x80, 0xd7, 0x29, 0x4c, 0x04, 0x0a, 0x04, 0x02, 0x83,
+      0x11, 0x44, 0x4c, 0x3d, 0x80, 0xc2, 0x3c, 0x06, 0x01, 0x04, 0x55, 0x05,
+      0x1b, 0x34, 0x02, 0x81, 0x0e, 0x2c, 0x04, 0x64, 0x0c, 0x56, 0x0a, 0x80,
+      0xae, 0x38, 0x1d, 0x0d, 0x2c, 0x04, 0x09, 0x07, 0x02, 0x0e, 0x06, 0x80,
+      0x9a, 0x83, 0xd8, 0x08, 0x0d, 0x03, 0x0d, 0x03, 0x74, 0x0c, 0x59, 0x07,
+      0x0c, 0x14, 0x0c, 0x04, 0x38, 0x08, 0x0a, 0x06, 0x28, 0x08, 0x22, 0x4e,
+      0x81, 0x54, 0x0c, 0x15, 0x03, 0x03, 0x05, 0x07, 0x09, 0x19, 0x07, 0x07,
+      0x09, 0x03, 0x0d, 0x07, 0x29, 0x80, 0xcb, 0x25, 0x0a, 0x84, 0x06,
+  };
+  auto lower = static_cast<uint16_t>(cp);
+  if (cp < 0x10000) {
+    return is_printable(lower, singletons0,
+                        sizeof(singletons0) / sizeof(*singletons0),
+                        singletons0_lower, normal0, sizeof(normal0));
+  }
+  if (cp < 0x20000) {
+    return is_printable(lower, singletons1,
+                        sizeof(singletons1) / sizeof(*singletons1),
+                        singletons1_lower, normal1, sizeof(normal1));
+  }
+  if (0x2a6de <= cp && cp < 0x2a700) return false;
+  if (0x2b735 <= cp && cp < 0x2b740) return false;
+  if (0x2b81e <= cp && cp < 0x2b820) return false;
+  if (0x2cea2 <= cp && cp < 0x2ceb0) return false;
+  if (0x2ebe1 <= cp && cp < 0x2f800) return false;
+  if (0x2fa1e <= cp && cp < 0x30000) return false;
+  if (0x3134b <= cp && cp < 0xe0100) return false;
+  if (0xe01f0 <= cp && cp < 0x110000) return false;
+  return cp < 0x110000;
+}
+
+}  // namespace detail
+
+FMT_END_NAMESPACE
+
+#endif  // FMT_FORMAT_INL_H_
diff --git a/ext/spdlog/include/spdlog/fmt/bundled/format.h b/ext/spdlog/include/spdlog/fmt/bundled/format.h
new file mode 100644
index 0000000..67f0ab7
--- /dev/null
+++ b/ext/spdlog/include/spdlog/fmt/bundled/format.h
@@ -0,0 +1,4427 @@
+/*
+  Formatting library for C++
+
+  Copyright (c) 2012 - present, Victor Zverovich
+
+  Permission is hereby granted, free of charge, to any person obtaining
+  a copy of this software and associated documentation files (the
+  "Software"), to deal in the Software without restriction, including
+  without limitation the rights to use, copy, modify, merge, publish,
+  distribute, sublicense, and/or sell copies of the Software, and to
+  permit persons to whom the Software is furnished to do so, subject to
+  the following conditions:
+
+  The above copyright notice and this permission notice shall be
+  included in all copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+  --- Optional exception to the license ---
+
+  As an exception, if, as a result of your compiling your source code, portions
+  of this Software are embedded into a machine-executable object form of such
+  source code, you may redistribute such embedded portions in such object form
+  without including the above copyright and permission notices.
+ */
+
+#ifndef FMT_FORMAT_H_
+#define FMT_FORMAT_H_
+
+#ifndef _LIBCPP_REMOVE_TRANSITIVE_INCLUDES
+#  define _LIBCPP_REMOVE_TRANSITIVE_INCLUDES
+#  define FMT_REMOVE_TRANSITIVE_INCLUDES
+#endif
+
+#include "base.h"
+
+#ifndef FMT_MODULE
+#  include <cmath>             // std::signbit
+#  include <cstdint>           // uint32_t
+#  include <cstring>           // std::memcpy
+#  include <initializer_list>  // std::initializer_list
+#  include <limits>            // std::numeric_limits
+#  if defined(__GLIBCXX__) && !defined(_GLIBCXX_USE_DUAL_ABI)
+// Workaround for pre gcc 5 libstdc++.
+#    include <memory>  // std::allocator_traits
+#  endif
+#  include <stdexcept>     // std::runtime_error
+#  include <string>        // std::string
+#  include <system_error>  // std::system_error
+
+// Checking FMT_CPLUSPLUS for warning suppression in MSVC.
+#  if FMT_HAS_INCLUDE(<bit>) && FMT_CPLUSPLUS > 201703L
+#    include <bit>  // std::bit_cast
+#  endif
+
+// libc++ supports string_view in pre-c++17.
+#  if FMT_HAS_INCLUDE(<string_view>) && \
+      (FMT_CPLUSPLUS >= 201703L || defined(_LIBCPP_VERSION))
+#    include <string_view>
+#    define FMT_USE_STRING_VIEW
+#  endif
+#endif  // FMT_MODULE
+
+#if defined __cpp_inline_variables && __cpp_inline_variables >= 201606L
+#  define FMT_INLINE_VARIABLE inline
+#else
+#  define FMT_INLINE_VARIABLE
+#endif
+
+#ifndef FMT_NO_UNIQUE_ADDRESS
+#  if FMT_CPLUSPLUS >= 202002L
+#    if FMT_HAS_CPP_ATTRIBUTE(no_unique_address)
+#      define FMT_NO_UNIQUE_ADDRESS [[no_unique_address]]
+// VS2019 v16.10 and later except clang-cl (https://reviews.llvm.org/D110485).
+#    elif (FMT_MSC_VERSION >= 1929) && !FMT_CLANG_VERSION
+#      define FMT_NO_UNIQUE_ADDRESS [[msvc::no_unique_address]]
+#    endif
+#  endif
+#endif
+#ifndef FMT_NO_UNIQUE_ADDRESS
+#  define FMT_NO_UNIQUE_ADDRESS
+#endif
+
+// Visibility when compiled as a shared library/object.
+#if defined(FMT_LIB_EXPORT) || defined(FMT_SHARED)
+#  define FMT_SO_VISIBILITY(value) FMT_VISIBILITY(value)
+#else
+#  define FMT_SO_VISIBILITY(value)
+#endif
+
+#ifdef __has_builtin
+#  define FMT_HAS_BUILTIN(x) __has_builtin(x)
+#else
+#  define FMT_HAS_BUILTIN(x) 0
+#endif
+
+#if FMT_GCC_VERSION || FMT_CLANG_VERSION
+#  define FMT_NOINLINE __attribute__((noinline))
+#else
+#  define FMT_NOINLINE
+#endif
+
+namespace std {
+template <> struct iterator_traits<fmt::appender> {
+  using iterator_category = output_iterator_tag;
+  using value_type = char;
+};
+}  // namespace std
+
+#ifndef FMT_THROW
+#  if FMT_EXCEPTIONS
+#    if FMT_MSC_VERSION || defined(__NVCC__)
+FMT_BEGIN_NAMESPACE
+namespace detail {
+template <typename Exception> inline void do_throw(const Exception& x) {
+  // Silence unreachable code warnings in MSVC and NVCC because these
+  // are nearly impossible to fix in a generic code.
+  volatile bool b = true;
+  if (b) throw x;
+}
+}  // namespace detail
+FMT_END_NAMESPACE
+#      define FMT_THROW(x) detail::do_throw(x)
+#    else
+#      define FMT_THROW(x) throw x
+#    endif
+#  else
+#    define FMT_THROW(x) \
+      ::fmt::detail::assert_fail(__FILE__, __LINE__, (x).what())
+#  endif
+#endif
+
+#ifndef FMT_MAYBE_UNUSED
+#  if FMT_HAS_CPP17_ATTRIBUTE(maybe_unused)
+#    define FMT_MAYBE_UNUSED [[maybe_unused]]
+#  else
+#    define FMT_MAYBE_UNUSED
+#  endif
+#endif
+
+#ifndef FMT_USE_USER_DEFINED_LITERALS
+// EDG based compilers (Intel, NVIDIA, Elbrus, etc), GCC and MSVC support UDLs.
+//
+// GCC before 4.9 requires a space in `operator"" _a` which is invalid in later
+// compiler versions.
+#  if (FMT_HAS_FEATURE(cxx_user_literals) || FMT_GCC_VERSION >= 409 || \
+       FMT_MSC_VERSION >= 1900) &&                                     \
+      (!defined(__EDG_VERSION__) || __EDG_VERSION__ >= /* UDL feature */ 480)
+#    define FMT_USE_USER_DEFINED_LITERALS 1
+#  else
+#    define FMT_USE_USER_DEFINED_LITERALS 0
+#  endif
+#endif
+
+// Defining FMT_REDUCE_INT_INSTANTIATIONS to 1, will reduce the number of
+// integer formatter template instantiations to just one by only using the
+// largest integer type. This results in a reduction in binary size but will
+// cause a decrease in integer formatting performance.
+#if !defined(FMT_REDUCE_INT_INSTANTIATIONS)
+#  define FMT_REDUCE_INT_INSTANTIATIONS 0
+#endif
+
+// __builtin_clz is broken in clang with Microsoft CodeGen:
+// https://github.com/fmtlib/fmt/issues/519.
+#if !FMT_MSC_VERSION
+#  if FMT_HAS_BUILTIN(__builtin_clz) || FMT_GCC_VERSION || FMT_ICC_VERSION
+#    define FMT_BUILTIN_CLZ(n) __builtin_clz(n)
+#  endif
+#  if FMT_HAS_BUILTIN(__builtin_clzll) || FMT_GCC_VERSION || FMT_ICC_VERSION
+#    define FMT_BUILTIN_CLZLL(n) __builtin_clzll(n)
+#  endif
+#endif
+
+// __builtin_ctz is broken in Intel Compiler Classic on Windows:
+// https://github.com/fmtlib/fmt/issues/2510.
+#ifndef __ICL
+#  if FMT_HAS_BUILTIN(__builtin_ctz) || FMT_GCC_VERSION || FMT_ICC_VERSION || \
+      defined(__NVCOMPILER)
+#    define FMT_BUILTIN_CTZ(n) __builtin_ctz(n)
+#  endif
+#  if FMT_HAS_BUILTIN(__builtin_ctzll) || FMT_GCC_VERSION || \
+      FMT_ICC_VERSION || defined(__NVCOMPILER)
+#    define FMT_BUILTIN_CTZLL(n) __builtin_ctzll(n)
+#  endif
+#endif
+
+#if FMT_MSC_VERSION
+#  include <intrin.h>  // _BitScanReverse[64], _BitScanForward[64], _umul128
+#endif
+
+// Some compilers masquerade as both MSVC and GCC-likes or otherwise support
+// __builtin_clz and __builtin_clzll, so only define FMT_BUILTIN_CLZ using the
+// MSVC intrinsics if the clz and clzll builtins are not available.
+#if FMT_MSC_VERSION && !defined(FMT_BUILTIN_CLZLL) && \
+    !defined(FMT_BUILTIN_CTZLL)
+FMT_BEGIN_NAMESPACE
+namespace detail {
+// Avoid Clang with Microsoft CodeGen's -Wunknown-pragmas warning.
+#  if !defined(__clang__)
+#    pragma intrinsic(_BitScanForward)
+#    pragma intrinsic(_BitScanReverse)
+#    if defined(_WIN64)
+#      pragma intrinsic(_BitScanForward64)
+#      pragma intrinsic(_BitScanReverse64)
+#    endif
+#  endif
+
+inline auto clz(uint32_t x) -> int {
+  unsigned long r = 0;
+  _BitScanReverse(&r, x);
+  FMT_ASSERT(x != 0, "");
+  // Static analysis complains about using uninitialized data
+  // "r", but the only way that can happen is if "x" is 0,
+  // which the callers guarantee to not happen.
+  FMT_MSC_WARNING(suppress : 6102)
+  return 31 ^ static_cast<int>(r);
+}
+#  define FMT_BUILTIN_CLZ(n) detail::clz(n)
+
+inline auto clzll(uint64_t x) -> int {
+  unsigned long r = 0;
+#  ifdef _WIN64
+  _BitScanReverse64(&r, x);
+#  else
+  // Scan the high 32 bits.
+  if (_BitScanReverse(&r, static_cast<uint32_t>(x >> 32)))
+    return 63 ^ static_cast<int>(r + 32);
+  // Scan the low 32 bits.
+  _BitScanReverse(&r, static_cast<uint32_t>(x));
+#  endif
+  FMT_ASSERT(x != 0, "");
+  FMT_MSC_WARNING(suppress : 6102)  // Suppress a bogus static analysis warning.
+  return 63 ^ static_cast<int>(r);
+}
+#  define FMT_BUILTIN_CLZLL(n) detail::clzll(n)
+
+inline auto ctz(uint32_t x) -> int {
+  unsigned long r = 0;
+  _BitScanForward(&r, x);
+  FMT_ASSERT(x != 0, "");
+  FMT_MSC_WARNING(suppress : 6102)  // Suppress a bogus static analysis warning.
+  return static_cast<int>(r);
+}
+#  define FMT_BUILTIN_CTZ(n) detail::ctz(n)
+
+inline auto ctzll(uint64_t x) -> int {
+  unsigned long r = 0;
+  FMT_ASSERT(x != 0, "");
+  FMT_MSC_WARNING(suppress : 6102)  // Suppress a bogus static analysis warning.
+#  ifdef _WIN64
+  _BitScanForward64(&r, x);
+#  else
+  // Scan the low 32 bits.
+  if (_BitScanForward(&r, static_cast<uint32_t>(x))) return static_cast<int>(r);
+  // Scan the high 32 bits.
+  _BitScanForward(&r, static_cast<uint32_t>(x >> 32));
+  r += 32;
+#  endif
+  return static_cast<int>(r);
+}
+#  define FMT_BUILTIN_CTZLL(n) detail::ctzll(n)
+}  // namespace detail
+FMT_END_NAMESPACE
+#endif
+
+FMT_BEGIN_NAMESPACE
+
+template <typename Char, typename Traits, typename Allocator>
+struct is_contiguous<std::basic_string<Char, Traits, Allocator>>
+    : std::true_type {};
+
+namespace detail {
+
+FMT_CONSTEXPR inline void abort_fuzzing_if(bool condition) {
+  ignore_unused(condition);
+#ifdef FMT_FUZZ
+  if (condition) throw std::runtime_error("fuzzing limit reached");
+#endif
+}
+
+#if defined(FMT_USE_STRING_VIEW)
+template <typename Char> using std_string_view = std::basic_string_view<Char>;
+#else
+template <typename T> struct std_string_view {};
+#endif
+
+// Implementation of std::bit_cast for pre-C++20.
+template <typename To, typename From, FMT_ENABLE_IF(sizeof(To) == sizeof(From))>
+FMT_CONSTEXPR20 auto bit_cast(const From& from) -> To {
+#ifdef __cpp_lib_bit_cast
+  if (is_constant_evaluated()) return std::bit_cast<To>(from);
+#endif
+  auto to = To();
+  // The cast suppresses a bogus -Wclass-memaccess on GCC.
+  std::memcpy(static_cast<void*>(&to), &from, sizeof(to));
+  return to;
+}
+
+inline auto is_big_endian() -> bool {
+#ifdef _WIN32
+  return false;
+#elif defined(__BIG_ENDIAN__)
+  return true;
+#elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__)
+  return __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__;
+#else
+  struct bytes {
+    char data[sizeof(int)];
+  };
+  return bit_cast<bytes>(1).data[0] == 0;
+#endif
+}
+
+class uint128_fallback {
+ private:
+  uint64_t lo_, hi_;
+
+ public:
+  constexpr uint128_fallback(uint64_t hi, uint64_t lo) : lo_(lo), hi_(hi) {}
+  constexpr uint128_fallback(uint64_t value = 0) : lo_(value), hi_(0) {}
+
+  constexpr auto high() const noexcept -> uint64_t { return hi_; }
+  constexpr auto low() const noexcept -> uint64_t { return lo_; }
+
+  template <typename T, FMT_ENABLE_IF(std::is_integral<T>::value)>
+  constexpr explicit operator T() const {
+    return static_cast<T>(lo_);
+  }
+
+  friend constexpr auto operator==(const uint128_fallback& lhs,
+                                   const uint128_fallback& rhs) -> bool {
+    return lhs.hi_ == rhs.hi_ && lhs.lo_ == rhs.lo_;
+  }
+  friend constexpr auto operator!=(const uint128_fallback& lhs,
+                                   const uint128_fallback& rhs) -> bool {
+    return !(lhs == rhs);
+  }
+  friend constexpr auto operator>(const uint128_fallback& lhs,
+                                  const uint128_fallback& rhs) -> bool {
+    return lhs.hi_ != rhs.hi_ ? lhs.hi_ > rhs.hi_ : lhs.lo_ > rhs.lo_;
+  }
+  friend constexpr auto operator|(const uint128_fallback& lhs,
+                                  const uint128_fallback& rhs)
+      -> uint128_fallback {
+    return {lhs.hi_ | rhs.hi_, lhs.lo_ | rhs.lo_};
+  }
+  friend constexpr auto operator&(const uint128_fallback& lhs,
+                                  const uint128_fallback& rhs)
+      -> uint128_fallback {
+    return {lhs.hi_ & rhs.hi_, lhs.lo_ & rhs.lo_};
+  }
+  friend constexpr auto operator~(const uint128_fallback& n)
+      -> uint128_fallback {
+    return {~n.hi_, ~n.lo_};
+  }
+  friend auto operator+(const uint128_fallback& lhs,
+                        const uint128_fallback& rhs) -> uint128_fallback {
+    auto result = uint128_fallback(lhs);
+    result += rhs;
+    return result;
+  }
+  friend auto operator*(const uint128_fallback& lhs, uint32_t rhs)
+      -> uint128_fallback {
+    FMT_ASSERT(lhs.hi_ == 0, "");
+    uint64_t hi = (lhs.lo_ >> 32) * rhs;
+    uint64_t lo = (lhs.lo_ & ~uint32_t()) * rhs;
+    uint64_t new_lo = (hi << 32) + lo;
+    return {(hi >> 32) + (new_lo < lo ? 1 : 0), new_lo};
+  }
+  friend auto operator-(const uint128_fallback& lhs, uint64_t rhs)
+      -> uint128_fallback {
+    return {lhs.hi_ - (lhs.lo_ < rhs ? 1 : 0), lhs.lo_ - rhs};
+  }
+  FMT_CONSTEXPR auto operator>>(int shift) const -> uint128_fallback {
+    if (shift == 64) return {0, hi_};
+    if (shift > 64) return uint128_fallback(0, hi_) >> (shift - 64);
+    return {hi_ >> shift, (hi_ << (64 - shift)) | (lo_ >> shift)};
+  }
+  FMT_CONSTEXPR auto operator<<(int shift) const -> uint128_fallback {
+    if (shift == 64) return {lo_, 0};
+    if (shift > 64) return uint128_fallback(lo_, 0) << (shift - 64);
+    return {hi_ << shift | (lo_ >> (64 - shift)), (lo_ << shift)};
+  }
+  FMT_CONSTEXPR auto operator>>=(int shift) -> uint128_fallback& {
+    return *this = *this >> shift;
+  }
+  FMT_CONSTEXPR void operator+=(uint128_fallback n) {
+    uint64_t new_lo = lo_ + n.lo_;
+    uint64_t new_hi = hi_ + n.hi_ + (new_lo < lo_ ? 1 : 0);
+    FMT_ASSERT(new_hi >= hi_, "");
+    lo_ = new_lo;
+    hi_ = new_hi;
+  }
+  FMT_CONSTEXPR void operator&=(uint128_fallback n) {
+    lo_ &= n.lo_;
+    hi_ &= n.hi_;
+  }
+
+  FMT_CONSTEXPR20 auto operator+=(uint64_t n) noexcept -> uint128_fallback& {
+    if (is_constant_evaluated()) {
+      lo_ += n;
+      hi_ += (lo_ < n ? 1 : 0);
+      return *this;
+    }
+#if FMT_HAS_BUILTIN(__builtin_addcll) && !defined(__ibmxl__)
+    unsigned long long carry;
+    lo_ = __builtin_addcll(lo_, n, 0, &carry);
+    hi_ += carry;
+#elif FMT_HAS_BUILTIN(__builtin_ia32_addcarryx_u64) && !defined(__ibmxl__)
+    unsigned long long result;
+    auto carry = __builtin_ia32_addcarryx_u64(0, lo_, n, &result);
+    lo_ = result;
+    hi_ += carry;
+#elif defined(_MSC_VER) && defined(_M_X64)
+    auto carry = _addcarry_u64(0, lo_, n, &lo_);
+    _addcarry_u64(carry, hi_, 0, &hi_);
+#else
+    lo_ += n;
+    hi_ += (lo_ < n ? 1 : 0);
+#endif
+    return *this;
+  }
+};
+
+using uint128_t = conditional_t<FMT_USE_INT128, uint128_opt, uint128_fallback>;
+
+#ifdef UINTPTR_MAX
+using uintptr_t = ::uintptr_t;
+#else
+using uintptr_t = uint128_t;
+#endif
+
+// Returns the largest possible value for type T. Same as
+// std::numeric_limits<T>::max() but shorter and not affected by the max macro.
+template <typename T> constexpr auto max_value() -> T {
+  return (std::numeric_limits<T>::max)();
+}
+template <typename T> constexpr auto num_bits() -> int {
+  return std::numeric_limits<T>::digits;
+}
+// std::numeric_limits<T>::digits may return 0 for 128-bit ints.
+template <> constexpr auto num_bits<int128_opt>() -> int { return 128; }
+template <> constexpr auto num_bits<uint128_opt>() -> int { return 128; }
+template <> constexpr auto num_bits<uint128_fallback>() -> int { return 128; }
+
+// A heterogeneous bit_cast used for converting 96-bit long double to uint128_t
+// and 128-bit pointers to uint128_fallback.
+template <typename To, typename From, FMT_ENABLE_IF(sizeof(To) > sizeof(From))>
+inline auto bit_cast(const From& from) -> To {
+  constexpr auto size = static_cast<int>(sizeof(From) / sizeof(unsigned));
+  struct data_t {
+    unsigned value[static_cast<unsigned>(size)];
+  } data = bit_cast<data_t>(from);
+  auto result = To();
+  if (const_check(is_big_endian())) {
+    for (int i = 0; i < size; ++i)
+      result = (result << num_bits<unsigned>()) | data.value[i];
+  } else {
+    for (int i = size - 1; i >= 0; --i)
+      result = (result << num_bits<unsigned>()) | data.value[i];
+  }
+  return result;
+}
+
+template <typename UInt>
+FMT_CONSTEXPR20 inline auto countl_zero_fallback(UInt n) -> int {
+  int lz = 0;
+  constexpr UInt msb_mask = static_cast<UInt>(1) << (num_bits<UInt>() - 1);
+  for (; (n & msb_mask) == 0; n <<= 1) lz++;
+  return lz;
+}
+
+FMT_CONSTEXPR20 inline auto countl_zero(uint32_t n) -> int {
+#ifdef FMT_BUILTIN_CLZ
+  if (!is_constant_evaluated()) return FMT_BUILTIN_CLZ(n);
+#endif
+  return countl_zero_fallback(n);
+}
+
+FMT_CONSTEXPR20 inline auto countl_zero(uint64_t n) -> int {
+#ifdef FMT_BUILTIN_CLZLL
+  if (!is_constant_evaluated()) return FMT_BUILTIN_CLZLL(n);
+#endif
+  return countl_zero_fallback(n);
+}
+
+FMT_INLINE void assume(bool condition) {
+  (void)condition;
+#if FMT_HAS_BUILTIN(__builtin_assume) && !FMT_ICC_VERSION
+  __builtin_assume(condition);
+#elif FMT_GCC_VERSION
+  if (!condition) __builtin_unreachable();
+#endif
+}
+
+// An approximation of iterator_t for pre-C++20 systems.
+template <typename T>
+using iterator_t = decltype(std::begin(std::declval<T&>()));
+template <typename T> using sentinel_t = decltype(std::end(std::declval<T&>()));
+
+// A workaround for std::string not having mutable data() until C++17.
+template <typename Char>
+inline auto get_data(std::basic_string<Char>& s) -> Char* {
+  return &s[0];
+}
+template <typename Container>
+inline auto get_data(Container& c) -> typename Container::value_type* {
+  return c.data();
+}
+
+// Attempts to reserve space for n extra characters in the output range.
+// Returns a pointer to the reserved range or a reference to it.
+template <typename OutputIt,
+          FMT_ENABLE_IF(is_back_insert_iterator<OutputIt>::value&&
+                            is_contiguous<typename OutputIt::container>::value)>
+#if FMT_CLANG_VERSION >= 307 && !FMT_ICC_VERSION
+__attribute__((no_sanitize("undefined")))
+#endif
+inline auto
+reserve(OutputIt it, size_t n) -> typename OutputIt::value_type* {
+  auto& c = get_container(it);
+  size_t size = c.size();
+  c.resize(size + n);
+  return get_data(c) + size;
+}
+
+template <typename T>
+inline auto reserve(basic_appender<T> it, size_t n) -> basic_appender<T> {
+  buffer<T>& buf = get_container(it);
+  buf.try_reserve(buf.size() + n);
+  return it;
+}
+
+template <typename Iterator>
+constexpr auto reserve(Iterator& it, size_t) -> Iterator& {
+  return it;
+}
+
+template <typename OutputIt>
+using reserve_iterator =
+    remove_reference_t<decltype(reserve(std::declval<OutputIt&>(), 0))>;
+
+template <typename T, typename OutputIt>
+constexpr auto to_pointer(OutputIt, size_t) -> T* {
+  return nullptr;
+}
+template <typename T> auto to_pointer(basic_appender<T> it, size_t n) -> T* {
+  buffer<T>& buf = get_container(it);
+  auto size = buf.size();
+  buf.try_reserve(size + n);
+  if (buf.capacity() < size + n) return nullptr;
+  buf.try_resize(size + n);
+  return buf.data() + size;
+}
+
+template <typename OutputIt,
+          FMT_ENABLE_IF(is_back_insert_iterator<OutputIt>::value&&
+                            is_contiguous<typename OutputIt::container>::value)>
+inline auto base_iterator(OutputIt it,
+                          typename OutputIt::container_type::value_type*)
+    -> OutputIt {
+  return it;
+}
+
+template <typename Iterator>
+constexpr auto base_iterator(Iterator, Iterator it) -> Iterator {
+  return it;
+}
+
+// <algorithm> is spectacularly slow to compile in C++20 so use a simple fill_n
+// instead (#1998).
+template <typename OutputIt, typename Size, typename T>
+FMT_CONSTEXPR auto fill_n(OutputIt out, Size count, const T& value)
+    -> OutputIt {
+  for (Size i = 0; i < count; ++i) *out++ = value;
+  return out;
+}
+template <typename T, typename Size>
+FMT_CONSTEXPR20 auto fill_n(T* out, Size count, char value) -> T* {
+  if (is_constant_evaluated()) {
+    return fill_n<T*, Size, T>(out, count, value);
+  }
+  std::memset(out, value, to_unsigned(count));
+  return out + count;
+}
+
+template <typename OutChar, typename InputIt, typename OutputIt>
+FMT_CONSTEXPR FMT_NOINLINE auto copy_noinline(InputIt begin, InputIt end,
+                                              OutputIt out) -> OutputIt {
+  return copy<OutChar>(begin, end, out);
+}
+
+// A public domain branchless UTF-8 decoder by Christopher Wellons:
+// https://github.com/skeeto/branchless-utf8
+/* Decode the next character, c, from s, reporting errors in e.
+ *
+ * Since this is a branchless decoder, four bytes will be read from the
+ * buffer regardless of the actual length of the next character. This
+ * means the buffer _must_ have at least three bytes of zero padding
+ * following the end of the data stream.
+ *
+ * Errors are reported in e, which will be non-zero if the parsed
+ * character was somehow invalid: invalid byte sequence, non-canonical
+ * encoding, or a surrogate half.
+ *
+ * The function returns a pointer to the next character. When an error
+ * occurs, this pointer will be a guess that depends on the particular
+ * error, but it will always advance at least one byte.
+ */
+FMT_CONSTEXPR inline auto utf8_decode(const char* s, uint32_t* c, int* e)
+    -> const char* {
+  constexpr const int masks[] = {0x00, 0x7f, 0x1f, 0x0f, 0x07};
+  constexpr const uint32_t mins[] = {4194304, 0, 128, 2048, 65536};
+  constexpr const int shiftc[] = {0, 18, 12, 6, 0};
+  constexpr const int shifte[] = {0, 6, 4, 2, 0};
+
+  int len = "\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\1\0\0\0\0\0\0\0\0\2\2\2\2\3\3\4"
+      [static_cast<unsigned char>(*s) >> 3];
+  // Compute the pointer to the next character early so that the next
+  // iteration can start working on the next character. Neither Clang
+  // nor GCC figure out this reordering on their own.
+  const char* next = s + len + !len;
+
+  using uchar = unsigned char;
+
+  // Assume a four-byte character and load four bytes. Unused bits are
+  // shifted out.
+  *c = uint32_t(uchar(s[0]) & masks[len]) << 18;
+  *c |= uint32_t(uchar(s[1]) & 0x3f) << 12;
+  *c |= uint32_t(uchar(s[2]) & 0x3f) << 6;
+  *c |= uint32_t(uchar(s[3]) & 0x3f) << 0;
+  *c >>= shiftc[len];
+
+  // Accumulate the various error conditions.
+  *e = (*c < mins[len]) << 6;       // non-canonical encoding
+  *e |= ((*c >> 11) == 0x1b) << 7;  // surrogate half?
+  *e |= (*c > 0x10FFFF) << 8;       // out of range?
+  *e |= (uchar(s[1]) & 0xc0) >> 2;
+  *e |= (uchar(s[2]) & 0xc0) >> 4;
+  *e |= uchar(s[3]) >> 6;
+  *e ^= 0x2a;  // top two bits of each tail byte correct?
+  *e >>= shifte[len];
+
+  return next;
+}
+
+constexpr FMT_INLINE_VARIABLE uint32_t invalid_code_point = ~uint32_t();
+
+// Invokes f(cp, sv) for every code point cp in s with sv being the string view
+// corresponding to the code point. cp is invalid_code_point on error.
+template <typename F>
+FMT_CONSTEXPR void for_each_codepoint(string_view s, F f) {
+  auto decode = [f](const char* buf_ptr, const char* ptr) {
+    auto cp = uint32_t();
+    auto error = 0;
+    auto end = utf8_decode(buf_ptr, &cp, &error);
+    bool result = f(error ? invalid_code_point : cp,
+                    string_view(ptr, error ? 1 : to_unsigned(end - buf_ptr)));
+    return result ? (error ? buf_ptr + 1 : end) : nullptr;
+  };
+  auto p = s.data();
+  const size_t block_size = 4;  // utf8_decode always reads blocks of 4 chars.
+  if (s.size() >= block_size) {
+    for (auto end = p + s.size() - block_size + 1; p < end;) {
+      p = decode(p, p);
+      if (!p) return;
+    }
+  }
+  if (auto num_chars_left = s.data() + s.size() - p) {
+    char buf[2 * block_size - 1] = {};
+    copy<char>(p, p + num_chars_left, buf);
+    const char* buf_ptr = buf;
+    do {
+      auto end = decode(buf_ptr, p);
+      if (!end) return;
+      p += end - buf_ptr;
+      buf_ptr = end;
+    } while (buf_ptr - buf < num_chars_left);
+  }
+}
+
+template <typename Char>
+inline auto compute_width(basic_string_view<Char> s) -> size_t {
+  return s.size();
+}
+
+// Computes approximate display width of a UTF-8 string.
+FMT_CONSTEXPR inline auto compute_width(string_view s) -> size_t {
+  size_t num_code_points = 0;
+  // It is not a lambda for compatibility with C++14.
+  struct count_code_points {
+    size_t* count;
+    FMT_CONSTEXPR auto operator()(uint32_t cp, string_view) const -> bool {
+      *count += detail::to_unsigned(
+          1 +
+          (cp >= 0x1100 &&
+           (cp <= 0x115f ||  // Hangul Jamo init. consonants
+            cp == 0x2329 ||  // LEFT-POINTING ANGLE BRACKET
+            cp == 0x232a ||  // RIGHT-POINTING ANGLE BRACKET
+            // CJK ... Yi except IDEOGRAPHIC HALF FILL SPACE:
+            (cp >= 0x2e80 && cp <= 0xa4cf && cp != 0x303f) ||
+            (cp >= 0xac00 && cp <= 0xd7a3) ||    // Hangul Syllables
+            (cp >= 0xf900 && cp <= 0xfaff) ||    // CJK Compatibility Ideographs
+            (cp >= 0xfe10 && cp <= 0xfe19) ||    // Vertical Forms
+            (cp >= 0xfe30 && cp <= 0xfe6f) ||    // CJK Compatibility Forms
+            (cp >= 0xff00 && cp <= 0xff60) ||    // Fullwidth Forms
+            (cp >= 0xffe0 && cp <= 0xffe6) ||    // Fullwidth Forms
+            (cp >= 0x20000 && cp <= 0x2fffd) ||  // CJK
+            (cp >= 0x30000 && cp <= 0x3fffd) ||
+            // Miscellaneous Symbols and Pictographs + Emoticons:
+            (cp >= 0x1f300 && cp <= 0x1f64f) ||
+            // Supplemental Symbols and Pictographs:
+            (cp >= 0x1f900 && cp <= 0x1f9ff))));
+      return true;
+    }
+  };
+  // We could avoid branches by using utf8_decode directly.
+  for_each_codepoint(s, count_code_points{&num_code_points});
+  return num_code_points;
+}
+
+template <typename Char>
+inline auto code_point_index(basic_string_view<Char> s, size_t n) -> size_t {
+  size_t size = s.size();
+  return n < size ? n : size;
+}
+
+// Calculates the index of the nth code point in a UTF-8 string.
+inline auto code_point_index(string_view s, size_t n) -> size_t {
+  size_t result = s.size();
+  const char* begin = s.begin();
+  for_each_codepoint(s, [begin, &n, &result](uint32_t, string_view sv) {
+    if (n != 0) {
+      --n;
+      return true;
+    }
+    result = to_unsigned(sv.begin() - begin);
+    return false;
+  });
+  return result;
+}
+
+template <typename T> struct is_integral : std::is_integral<T> {};
+template <> struct is_integral<int128_opt> : std::true_type {};
+template <> struct is_integral<uint128_t> : std::true_type {};
+
+template <typename T>
+using is_signed =
+    std::integral_constant<bool, std::numeric_limits<T>::is_signed ||
+                                     std::is_same<T, int128_opt>::value>;
+
+template <typename T>
+using is_integer =
+    bool_constant<is_integral<T>::value && !std::is_same<T, bool>::value &&
+                  !std::is_same<T, char>::value &&
+                  !std::is_same<T, wchar_t>::value>;
+
+#ifndef FMT_USE_FLOAT
+#  define FMT_USE_FLOAT 1
+#endif
+#ifndef FMT_USE_DOUBLE
+#  define FMT_USE_DOUBLE 1
+#endif
+#ifndef FMT_USE_LONG_DOUBLE
+#  define FMT_USE_LONG_DOUBLE 1
+#endif
+
+#if defined(FMT_USE_FLOAT128)
+// Use the provided definition.
+#elif FMT_CLANG_VERSION && FMT_HAS_INCLUDE(<quadmath.h>)
+#  define FMT_USE_FLOAT128 1
+#elif FMT_GCC_VERSION && defined(_GLIBCXX_USE_FLOAT128) && \
+    !defined(__STRICT_ANSI__)
+#  define FMT_USE_FLOAT128 1
+#else
+#  define FMT_USE_FLOAT128 0
+#endif
+#if FMT_USE_FLOAT128
+using float128 = __float128;
+#else
+using float128 = void;
+#endif
+
+template <typename T> using is_float128 = std::is_same<T, float128>;
+
+template <typename T>
+using is_floating_point =
+    bool_constant<std::is_floating_point<T>::value || is_float128<T>::value>;
+
+template <typename T, bool = std::is_floating_point<T>::value>
+struct is_fast_float : bool_constant<std::numeric_limits<T>::is_iec559 &&
+                                     sizeof(T) <= sizeof(double)> {};
+template <typename T> struct is_fast_float<T, false> : std::false_type {};
+
+template <typename T>
+using is_double_double = bool_constant<std::numeric_limits<T>::digits == 106>;
+
+#ifndef FMT_USE_FULL_CACHE_DRAGONBOX
+#  define FMT_USE_FULL_CACHE_DRAGONBOX 0
+#endif
+
+template <typename T, typename Enable = void>
+struct is_locale : std::false_type {};
+template <typename T>
+struct is_locale<T, void_t<decltype(T::classic())>> : std::true_type {};
+}  // namespace detail
+
+FMT_BEGIN_EXPORT
+
+// The number of characters to store in the basic_memory_buffer object itself
+// to avoid dynamic memory allocation.
+enum { inline_buffer_size = 500 };
+
+/**
+ * A dynamically growing memory buffer for trivially copyable/constructible
+ * types with the first `SIZE` elements stored in the object itself. Most
+ * commonly used via the `memory_buffer` alias for `char`.
+ *
+ * **Example**:
+ *
+ *     auto out = fmt::memory_buffer();
+ *     fmt::format_to(std::back_inserter(out), "The answer is {}.", 42);
+ *
+ * This will append "The answer is 42." to `out`. The buffer content can be
+ * converted to `std::string` with `to_string(out)`.
+ */
+template <typename T, size_t SIZE = inline_buffer_size,
+          typename Allocator = std::allocator<T>>
+class basic_memory_buffer : public detail::buffer<T> {
+ private:
+  T store_[SIZE];
+
+  // Don't inherit from Allocator to avoid generating type_info for it.
+  FMT_NO_UNIQUE_ADDRESS Allocator alloc_;
+
+  // Deallocate memory allocated by the buffer.
+  FMT_CONSTEXPR20 void deallocate() {
+    T* data = this->data();
+    if (data != store_) alloc_.deallocate(data, this->capacity());
+  }
+
+  static FMT_CONSTEXPR20 void grow(detail::buffer<T>& buf, size_t size) {
+    detail::abort_fuzzing_if(size > 5000);
+    auto& self = static_cast<basic_memory_buffer&>(buf);
+    const size_t max_size =
+        std::allocator_traits<Allocator>::max_size(self.alloc_);
+    size_t old_capacity = buf.capacity();
+    size_t new_capacity = old_capacity + old_capacity / 2;
+    if (size > new_capacity)
+      new_capacity = size;
+    else if (new_capacity > max_size)
+      new_capacity = size > max_size ? size : max_size;
+    T* old_data = buf.data();
+    T* new_data = self.alloc_.allocate(new_capacity);
+    // Suppress a bogus -Wstringop-overflow in gcc 13.1 (#3481).
+    detail::assume(buf.size() <= new_capacity);
+    // The following code doesn't throw, so the raw pointer above doesn't leak.
+    memcpy(new_data, old_data, buf.size() * sizeof(T));
+    self.set(new_data, new_capacity);
+    // deallocate must not throw according to the standard, but even if it does,
+    // the buffer already uses the new storage and will deallocate it in
+    // destructor.
+    if (old_data != self.store_) self.alloc_.deallocate(old_data, old_capacity);
+  }
+
+ public:
+  using value_type = T;
+  using const_reference = const T&;
+
+  FMT_CONSTEXPR20 explicit basic_memory_buffer(
+      const Allocator& alloc = Allocator())
+      : detail::buffer<T>(grow), alloc_(alloc) {
+    this->set(store_, SIZE);
+    if (detail::is_constant_evaluated()) detail::fill_n(store_, SIZE, T());
+  }
+  FMT_CONSTEXPR20 ~basic_memory_buffer() { deallocate(); }
+
+ private:
+  // Move data from other to this buffer.
+  FMT_CONSTEXPR20 void move(basic_memory_buffer& other) {
+    alloc_ = std::move(other.alloc_);
+    T* data = other.data();
+    size_t size = other.size(), capacity = other.capacity();
+    if (data == other.store_) {
+      this->set(store_, capacity);
+      detail::copy<T>(other.store_, other.store_ + size, store_);
+    } else {
+      this->set(data, capacity);
+      // Set pointer to the inline array so that delete is not called
+      // when deallocating.
+      other.set(other.store_, 0);
+      other.clear();
+    }
+    this->resize(size);
+  }
+
+ public:
+  /// Constructs a `basic_memory_buffer` object moving the content of the other
+  /// object to it.
+  FMT_CONSTEXPR20 basic_memory_buffer(basic_memory_buffer&& other) noexcept
+      : detail::buffer<T>(grow) {
+    move(other);
+  }
+
+  /// Moves the content of the other `basic_memory_buffer` object to this one.
+  auto operator=(basic_memory_buffer&& other) noexcept -> basic_memory_buffer& {
+    FMT_ASSERT(this != &other, "");
+    deallocate();
+    move(other);
+    return *this;
+  }
+
+  // Returns a copy of the allocator associated with this buffer.
+  auto get_allocator() const -> Allocator { return alloc_; }
+
+  /// Resizes the buffer to contain `count` elements. If T is a POD type new
+  /// elements may not be initialized.
+  FMT_CONSTEXPR20 void resize(size_t count) { this->try_resize(count); }
+
+  /// Increases the buffer capacity to `new_capacity`.
+  void reserve(size_t new_capacity) { this->try_reserve(new_capacity); }
+
+  using detail::buffer<T>::append;
+  template <typename ContiguousRange>
+  void append(const ContiguousRange& range) {
+    append(range.data(), range.data() + range.size());
+  }
+};
+
+using memory_buffer = basic_memory_buffer<char>;
+
+template <typename T, size_t SIZE, typename Allocator>
+struct is_contiguous<basic_memory_buffer<T, SIZE, Allocator>> : std::true_type {
+};
+
+FMT_END_EXPORT
+namespace detail {
+FMT_API auto write_console(int fd, string_view text) -> bool;
+FMT_API void print(std::FILE*, string_view);
+}  // namespace detail
+
+FMT_BEGIN_EXPORT
+
+// Suppress a misleading warning in older versions of clang.
+#if FMT_CLANG_VERSION
+#  pragma clang diagnostic ignored "-Wweak-vtables"
+#endif
+
+/// An error reported from a formatting function.
+class FMT_SO_VISIBILITY("default") format_error : public std::runtime_error {
+ public:
+  using std::runtime_error::runtime_error;
+};
+
+namespace detail_exported {
+#if FMT_USE_NONTYPE_TEMPLATE_ARGS
+template <typename Char, size_t N> struct fixed_string {
+  constexpr fixed_string(const Char (&str)[N]) {
+    detail::copy<Char, const Char*, Char*>(static_cast<const Char*>(str),
+                                           str + N, data);
+  }
+  Char data[N] = {};
+};
+#endif
+
+// Converts a compile-time string to basic_string_view.
+template <typename Char, size_t N>
+constexpr auto compile_string_to_view(const Char (&s)[N])
+    -> basic_string_view<Char> {
+  // Remove trailing NUL character if needed. Won't be present if this is used
+  // with a raw character array (i.e. not defined as a string).
+  return {s, N - (std::char_traits<Char>::to_int_type(s[N - 1]) == 0 ? 1 : 0)};
+}
+template <typename Char>
+constexpr auto compile_string_to_view(basic_string_view<Char> s)
+    -> basic_string_view<Char> {
+  return s;
+}
+}  // namespace detail_exported
+
+// A generic formatting context with custom output iterator and character
+// (code unit) support. Char is the format string code unit type which can be
+// different from OutputIt::value_type.
+template <typename OutputIt, typename Char> class generic_context {
+ private:
+  OutputIt out_;
+  basic_format_args<generic_context> args_;
+  detail::locale_ref loc_;
+
+ public:
+  using char_type = Char;
+  using iterator = OutputIt;
+  using parse_context_type = basic_format_parse_context<Char>;
+  template <typename T> using formatter_type = formatter<T, Char>;
+
+  constexpr generic_context(OutputIt out,
+                            basic_format_args<generic_context> ctx_args,
+                            detail::locale_ref loc = {})
+      : out_(out), args_(ctx_args), loc_(loc) {}
+  generic_context(generic_context&&) = default;
+  generic_context(const generic_context&) = delete;
+  void operator=(const generic_context&) = delete;
+
+  constexpr auto arg(int id) const -> basic_format_arg<generic_context> {
+    return args_.get(id);
+  }
+  auto arg(basic_string_view<Char> name) -> basic_format_arg<generic_context> {
+    return args_.get(name);
+  }
+  FMT_CONSTEXPR auto arg_id(basic_string_view<Char> name) -> int {
+    return args_.get_id(name);
+  }
+  auto args() const -> const basic_format_args<generic_context>& {
+    return args_;
+  }
+
+  FMT_CONSTEXPR auto out() -> iterator { return out_; }
+
+  void advance_to(iterator it) {
+    if (!detail::is_back_insert_iterator<iterator>()) out_ = it;
+  }
+
+  FMT_CONSTEXPR auto locale() -> detail::locale_ref { return loc_; }
+};
+
+class loc_value {
+ private:
+  basic_format_arg<format_context> value_;
+
+ public:
+  template <typename T, FMT_ENABLE_IF(!detail::is_float128<T>::value)>
+  loc_value(T value) : value_(detail::make_arg<format_context>(value)) {}
+
+  template <typename T, FMT_ENABLE_IF(detail::is_float128<T>::value)>
+  loc_value(T) {}
+
+  template <typename Visitor> auto visit(Visitor&& vis) -> decltype(vis(0)) {
+    return value_.visit(vis);
+  }
+};
+
+// A locale facet that formats values in UTF-8.
+// It is parameterized on the locale to avoid the heavy <locale> include.
+template <typename Locale> class format_facet : public Locale::facet {
+ private:
+  std::string separator_;
+  std::string grouping_;
+  std::string decimal_point_;
+
+ protected:
+  virtual auto do_put(appender out, loc_value val,
+                      const format_specs& specs) const -> bool;
+
+ public:
+  static FMT_API typename Locale::id id;
+
+  explicit format_facet(Locale& loc);
+  explicit format_facet(string_view sep = "",
+                        std::initializer_list<unsigned char> g = {3},
+                        std::string decimal_point = ".")
+      : separator_(sep.data(), sep.size()),
+        grouping_(g.begin(), g.end()),
+        decimal_point_(decimal_point) {}
+
+  auto put(appender out, loc_value val, const format_specs& specs) const
+      -> bool {
+    return do_put(out, val, specs);
+  }
+};
+
+FMT_END_EXPORT
+
+namespace detail {
+
+// Returns true if value is negative, false otherwise.
+// Same as `value < 0` but doesn't produce warnings if T is an unsigned type.
+template <typename T, FMT_ENABLE_IF(is_signed<T>::value)>
+constexpr auto is_negative(T value) -> bool {
+  return value < 0;
+}
+template <typename T, FMT_ENABLE_IF(!is_signed<T>::value)>
+constexpr auto is_negative(T) -> bool {
+  return false;
+}
+
+template <typename T>
+FMT_CONSTEXPR auto is_supported_floating_point(T) -> bool {
+  if (std::is_same<T, float>()) return FMT_USE_FLOAT;
+  if (std::is_same<T, double>()) return FMT_USE_DOUBLE;
+  if (std::is_same<T, long double>()) return FMT_USE_LONG_DOUBLE;
+  return true;
+}
+
+// Smallest of uint32_t, uint64_t, uint128_t that is large enough to
+// represent all values of an integral type T.
+template <typename T>
+using uint32_or_64_or_128_t =
+    conditional_t<num_bits<T>() <= 32 && !FMT_REDUCE_INT_INSTANTIATIONS,
+                  uint32_t,
+                  conditional_t<num_bits<T>() <= 64, uint64_t, uint128_t>>;
+template <typename T>
+using uint64_or_128_t = conditional_t<num_bits<T>() <= 64, uint64_t, uint128_t>;
+
+#define FMT_POWERS_OF_10(factor)                                  \
+  factor * 10, (factor) * 100, (factor) * 1000, (factor) * 10000, \
+      (factor) * 100000, (factor) * 1000000, (factor) * 10000000, \
+      (factor) * 100000000, (factor) * 1000000000
+
+// Converts value in the range [0, 100) to a string.
+constexpr auto digits2(size_t value) -> const char* {
+  // GCC generates slightly better code when value is pointer-size.
+  return &"0001020304050607080910111213141516171819"
+         "2021222324252627282930313233343536373839"
+         "4041424344454647484950515253545556575859"
+         "6061626364656667686970717273747576777879"
+         "8081828384858687888990919293949596979899"[value * 2];
+}
+
+// Sign is a template parameter to workaround a bug in gcc 4.8.
+template <typename Char, typename Sign> constexpr auto sign(Sign s) -> Char {
+#if !FMT_GCC_VERSION || FMT_GCC_VERSION >= 604
+  static_assert(std::is_same<Sign, sign_t>::value, "");
+#endif
+  return static_cast<char>(((' ' << 24) | ('+' << 16) | ('-' << 8)) >> (s * 8));
+}
+
+template <typename T> FMT_CONSTEXPR auto count_digits_fallback(T n) -> int {
+  int count = 1;
+  for (;;) {
+    // Integer division is slow so do it for a group of four digits instead
+    // of for every digit. The idea comes from the talk by Alexandrescu
+    // "Three Optimization Tips for C++". See speed-test for a comparison.
+    if (n < 10) return count;
+    if (n < 100) return count + 1;
+    if (n < 1000) return count + 2;
+    if (n < 10000) return count + 3;
+    n /= 10000u;
+    count += 4;
+  }
+}
+#if FMT_USE_INT128
+FMT_CONSTEXPR inline auto count_digits(uint128_opt n) -> int {
+  return count_digits_fallback(n);
+}
+#endif
+
+#ifdef FMT_BUILTIN_CLZLL
+// It is a separate function rather than a part of count_digits to workaround
+// the lack of static constexpr in constexpr functions.
+inline auto do_count_digits(uint64_t n) -> int {
+  // This has comparable performance to the version by Kendall Willets
+  // (https://github.com/fmtlib/format-benchmark/blob/master/digits10)
+  // but uses smaller tables.
+  // Maps bsr(n) to ceil(log10(pow(2, bsr(n) + 1) - 1)).
+  static constexpr uint8_t bsr2log10[] = {
+      1,  1,  1,  2,  2,  2,  3,  3,  3,  4,  4,  4,  4,  5,  5,  5,
+      6,  6,  6,  7,  7,  7,  7,  8,  8,  8,  9,  9,  9,  10, 10, 10,
+      10, 11, 11, 11, 12, 12, 12, 13, 13, 13, 13, 14, 14, 14, 15, 15,
+      15, 16, 16, 16, 16, 17, 17, 17, 18, 18, 18, 19, 19, 19, 19, 20};
+  auto t = bsr2log10[FMT_BUILTIN_CLZLL(n | 1) ^ 63];
+  static constexpr const uint64_t zero_or_powers_of_10[] = {
+      0, 0, FMT_POWERS_OF_10(1U), FMT_POWERS_OF_10(1000000000ULL),
+      10000000000000000000ULL};
+  return t - (n < zero_or_powers_of_10[t]);
+}
+#endif
+
+// Returns the number of decimal digits in n. Leading zeros are not counted
+// except for n == 0 in which case count_digits returns 1.
+FMT_CONSTEXPR20 inline auto count_digits(uint64_t n) -> int {
+#ifdef FMT_BUILTIN_CLZLL
+  if (!is_constant_evaluated()) return do_count_digits(n);
+#endif
+  return count_digits_fallback(n);
+}
+
+// Counts the number of digits in n. BITS = log2(radix).
+template <int BITS, typename UInt>
+FMT_CONSTEXPR auto count_digits(UInt n) -> int {
+#ifdef FMT_BUILTIN_CLZ
+  if (!is_constant_evaluated() && num_bits<UInt>() == 32)
+    return (FMT_BUILTIN_CLZ(static_cast<uint32_t>(n) | 1) ^ 31) / BITS + 1;
+#endif
+  // Lambda avoids unreachable code warnings from NVHPC.
+  return [](UInt m) {
+    int num_digits = 0;
+    do {
+      ++num_digits;
+    } while ((m >>= BITS) != 0);
+    return num_digits;
+  }(n);
+}
+
+#ifdef FMT_BUILTIN_CLZ
+// It is a separate function rather than a part of count_digits to workaround
+// the lack of static constexpr in constexpr functions.
+FMT_INLINE auto do_count_digits(uint32_t n) -> int {
+// An optimization by Kendall Willets from https://bit.ly/3uOIQrB.
+// This increments the upper 32 bits (log10(T) - 1) when >= T is added.
+#  define FMT_INC(T) (((sizeof(#T) - 1ull) << 32) - T)
+  static constexpr uint64_t table[] = {
+      FMT_INC(0),          FMT_INC(0),          FMT_INC(0),           // 8
+      FMT_INC(10),         FMT_INC(10),         FMT_INC(10),          // 64
+      FMT_INC(100),        FMT_INC(100),        FMT_INC(100),         // 512
+      FMT_INC(1000),       FMT_INC(1000),       FMT_INC(1000),        // 4096
+      FMT_INC(10000),      FMT_INC(10000),      FMT_INC(10000),       // 32k
+      FMT_INC(100000),     FMT_INC(100000),     FMT_INC(100000),      // 256k
+      FMT_INC(1000000),    FMT_INC(1000000),    FMT_INC(1000000),     // 2048k
+      FMT_INC(10000000),   FMT_INC(10000000),   FMT_INC(10000000),    // 16M
+      FMT_INC(100000000),  FMT_INC(100000000),  FMT_INC(100000000),   // 128M
+      FMT_INC(1000000000), FMT_INC(1000000000), FMT_INC(1000000000),  // 1024M
+      FMT_INC(1000000000), FMT_INC(1000000000)                        // 4B
+  };
+  auto inc = table[FMT_BUILTIN_CLZ(n | 1) ^ 31];
+  return static_cast<int>((n + inc) >> 32);
+}
+#endif
+
+// Optional version of count_digits for better performance on 32-bit platforms.
+FMT_CONSTEXPR20 inline auto count_digits(uint32_t n) -> int {
+#ifdef FMT_BUILTIN_CLZ
+  if (!is_constant_evaluated()) {
+    return do_count_digits(n);
+  }
+#endif
+  return count_digits_fallback(n);
+}
+
+template <typename Int> constexpr auto digits10() noexcept -> int {
+  return std::numeric_limits<Int>::digits10;
+}
+template <> constexpr auto digits10<int128_opt>() noexcept -> int { return 38; }
+template <> constexpr auto digits10<uint128_t>() noexcept -> int { return 38; }
+
+template <typename Char> struct thousands_sep_result {
+  std::string grouping;
+  Char thousands_sep;
+};
+
+template <typename Char>
+FMT_API auto thousands_sep_impl(locale_ref loc) -> thousands_sep_result<Char>;
+template <typename Char>
+inline auto thousands_sep(locale_ref loc) -> thousands_sep_result<Char> {
+  auto result = thousands_sep_impl<char>(loc);
+  return {result.grouping, Char(result.thousands_sep)};
+}
+template <>
+inline auto thousands_sep(locale_ref loc) -> thousands_sep_result<wchar_t> {
+  return thousands_sep_impl<wchar_t>(loc);
+}
+
+template <typename Char>
+FMT_API auto decimal_point_impl(locale_ref loc) -> Char;
+template <typename Char> inline auto decimal_point(locale_ref loc) -> Char {
+  return Char(decimal_point_impl<char>(loc));
+}
+template <> inline auto decimal_point(locale_ref loc) -> wchar_t {
+  return decimal_point_impl<wchar_t>(loc);
+}
+
+// Compares two characters for equality.
+template <typename Char> auto equal2(const Char* lhs, const char* rhs) -> bool {
+  return lhs[0] == Char(rhs[0]) && lhs[1] == Char(rhs[1]);
+}
+inline auto equal2(const char* lhs, const char* rhs) -> bool {
+  return memcmp(lhs, rhs, 2) == 0;
+}
+
+// Copies two characters from src to dst.
+template <typename Char>
+FMT_CONSTEXPR20 FMT_INLINE void copy2(Char* dst, const char* src) {
+  if (!is_constant_evaluated() && sizeof(Char) == sizeof(char)) {
+    memcpy(dst, src, 2);
+    return;
+  }
+  *dst++ = static_cast<Char>(*src++);
+  *dst = static_cast<Char>(*src);
+}
+
+template <typename Iterator> struct format_decimal_result {
+  Iterator begin;
+  Iterator end;
+};
+
+// Formats a decimal unsigned integer value writing into out pointing to a
+// buffer of specified size. The caller must ensure that the buffer is large
+// enough.
+template <typename Char, typename UInt>
+FMT_CONSTEXPR20 auto format_decimal(Char* out, UInt value, int size)
+    -> format_decimal_result<Char*> {
+  FMT_ASSERT(size >= count_digits(value), "invalid digit count");
+  out += size;
+  Char* end = out;
+  while (value >= 100) {
+    // Integer division is slow so do it for a group of two digits instead
+    // of for every digit. The idea comes from the talk by Alexandrescu
+    // "Three Optimization Tips for C++". See speed-test for a comparison.
+    out -= 2;
+    copy2(out, digits2(static_cast<size_t>(value % 100)));
+    value /= 100;
+  }
+  if (value < 10) {
+    *--out = static_cast<Char>('0' + value);
+    return {out, end};
+  }
+  out -= 2;
+  copy2(out, digits2(static_cast<size_t>(value)));
+  return {out, end};
+}
+
+template <typename Char, typename UInt, typename Iterator,
+          FMT_ENABLE_IF(!std::is_pointer<remove_cvref_t<Iterator>>::value)>
+FMT_CONSTEXPR inline auto format_decimal(Iterator out, UInt value, int size)
+    -> format_decimal_result<Iterator> {
+  // Buffer is large enough to hold all digits (digits10 + 1).
+  Char buffer[digits10<UInt>() + 1] = {};
+  auto end = format_decimal(buffer, value, size).end;
+  return {out, detail::copy_noinline<Char>(buffer, end, out)};
+}
+
+template <unsigned BASE_BITS, typename Char, typename UInt>
+FMT_CONSTEXPR auto format_uint(Char* buffer, UInt value, int num_digits,
+                               bool upper = false) -> Char* {
+  buffer += num_digits;
+  Char* end = buffer;
+  do {
+    const char* digits = upper ? "0123456789ABCDEF" : "0123456789abcdef";
+    unsigned digit = static_cast<unsigned>(value & ((1 << BASE_BITS) - 1));
+    *--buffer = static_cast<Char>(BASE_BITS < 4 ? static_cast<char>('0' + digit)
+                                                : digits[digit]);
+  } while ((value >>= BASE_BITS) != 0);
+  return end;
+}
+
+template <unsigned BASE_BITS, typename Char, typename It, typename UInt>
+FMT_CONSTEXPR inline auto format_uint(It out, UInt value, int num_digits,
+                                      bool upper = false) -> It {
+  if (auto ptr = to_pointer<Char>(out, to_unsigned(num_digits))) {
+    format_uint<BASE_BITS>(ptr, value, num_digits, upper);
+    return out;
+  }
+  // Buffer should be large enough to hold all digits (digits / BASE_BITS + 1).
+  char buffer[num_bits<UInt>() / BASE_BITS + 1] = {};
+  format_uint<BASE_BITS>(buffer, value, num_digits, upper);
+  return detail::copy_noinline<Char>(buffer, buffer + num_digits, out);
+}
+
+// A converter from UTF-8 to UTF-16.
+class utf8_to_utf16 {
+ private:
+  basic_memory_buffer<wchar_t> buffer_;
+
+ public:
+  FMT_API explicit utf8_to_utf16(string_view s);
+  operator basic_string_view<wchar_t>() const { return {&buffer_[0], size()}; }
+  auto size() const -> size_t { return buffer_.size() - 1; }
+  auto c_str() const -> const wchar_t* { return &buffer_[0]; }
+  auto str() const -> std::wstring { return {&buffer_[0], size()}; }
+};
+
+enum class to_utf8_error_policy { abort, replace };
+
+// A converter from UTF-16/UTF-32 (host endian) to UTF-8.
+template <typename WChar, typename Buffer = memory_buffer> class to_utf8 {
+ private:
+  Buffer buffer_;
+
+ public:
+  to_utf8() {}
+  explicit to_utf8(basic_string_view<WChar> s,
+                   to_utf8_error_policy policy = to_utf8_error_policy::abort) {
+    static_assert(sizeof(WChar) == 2 || sizeof(WChar) == 4,
+                  "Expect utf16 or utf32");
+    if (!convert(s, policy))
+      FMT_THROW(std::runtime_error(sizeof(WChar) == 2 ? "invalid utf16"
+                                                      : "invalid utf32"));
+  }
+  operator string_view() const { return string_view(&buffer_[0], size()); }
+  auto size() const -> size_t { return buffer_.size() - 1; }
+  auto c_str() const -> const char* { return &buffer_[0]; }
+  auto str() const -> std::string { return std::string(&buffer_[0], size()); }
+
+  // Performs conversion returning a bool instead of throwing exception on
+  // conversion error. This method may still throw in case of memory allocation
+  // error.
+  auto convert(basic_string_view<WChar> s,
+               to_utf8_error_policy policy = to_utf8_error_policy::abort)
+      -> bool {
+    if (!convert(buffer_, s, policy)) return false;
+    buffer_.push_back(0);
+    return true;
+  }
+  static auto convert(Buffer& buf, basic_string_view<WChar> s,
+                      to_utf8_error_policy policy = to_utf8_error_policy::abort)
+      -> bool {
+    for (auto p = s.begin(); p != s.end(); ++p) {
+      uint32_t c = static_cast<uint32_t>(*p);
+      if (sizeof(WChar) == 2 && c >= 0xd800 && c <= 0xdfff) {
+        // Handle a surrogate pair.
+        ++p;
+        if (p == s.end() || (c & 0xfc00) != 0xd800 || (*p & 0xfc00) != 0xdc00) {
+          if (policy == to_utf8_error_policy::abort) return false;
+          buf.append(string_view("\xEF\xBF\xBD"));
+          --p;
+        } else {
+          c = (c << 10) + static_cast<uint32_t>(*p) - 0x35fdc00;
+        }
+      } else if (c < 0x80) {
+        buf.push_back(static_cast<char>(c));
+      } else if (c < 0x800) {
+        buf.push_back(static_cast<char>(0xc0 | (c >> 6)));
+        buf.push_back(static_cast<char>(0x80 | (c & 0x3f)));
+      } else if ((c >= 0x800 && c <= 0xd7ff) || (c >= 0xe000 && c <= 0xffff)) {
+        buf.push_back(static_cast<char>(0xe0 | (c >> 12)));
+        buf.push_back(static_cast<char>(0x80 | ((c & 0xfff) >> 6)));
+        buf.push_back(static_cast<char>(0x80 | (c & 0x3f)));
+      } else if (c >= 0x10000 && c <= 0x10ffff) {
+        buf.push_back(static_cast<char>(0xf0 | (c >> 18)));
+        buf.push_back(static_cast<char>(0x80 | ((c & 0x3ffff) >> 12)));
+        buf.push_back(static_cast<char>(0x80 | ((c & 0xfff) >> 6)));
+        buf.push_back(static_cast<char>(0x80 | (c & 0x3f)));
+      } else {
+        return false;
+      }
+    }
+    return true;
+  }
+};
+
+// Computes 128-bit result of multiplication of two 64-bit unsigned integers.
+inline auto umul128(uint64_t x, uint64_t y) noexcept -> uint128_fallback {
+#if FMT_USE_INT128
+  auto p = static_cast<uint128_opt>(x) * static_cast<uint128_opt>(y);
+  return {static_cast<uint64_t>(p >> 64), static_cast<uint64_t>(p)};
+#elif defined(_MSC_VER) && defined(_M_X64)
+  auto hi = uint64_t();
+  auto lo = _umul128(x, y, &hi);
+  return {hi, lo};
+#else
+  const uint64_t mask = static_cast<uint64_t>(max_value<uint32_t>());
+
+  uint64_t a = x >> 32;
+  uint64_t b = x & mask;
+  uint64_t c = y >> 32;
+  uint64_t d = y & mask;
+
+  uint64_t ac = a * c;
+  uint64_t bc = b * c;
+  uint64_t ad = a * d;
+  uint64_t bd = b * d;
+
+  uint64_t intermediate = (bd >> 32) + (ad & mask) + (bc & mask);
+
+  return {ac + (intermediate >> 32) + (ad >> 32) + (bc >> 32),
+          (intermediate << 32) + (bd & mask)};
+#endif
+}
+
+namespace dragonbox {
+// Computes floor(log10(pow(2, e))) for e in [-2620, 2620] using the method from
+// https://fmt.dev/papers/Dragonbox.pdf#page=28, section 6.1.
+inline auto floor_log10_pow2(int e) noexcept -> int {
+  FMT_ASSERT(e <= 2620 && e >= -2620, "too large exponent");
+  static_assert((-1 >> 1) == -1, "right shift is not arithmetic");
+  return (e * 315653) >> 20;
+}
+
+inline auto floor_log2_pow10(int e) noexcept -> int {
+  FMT_ASSERT(e <= 1233 && e >= -1233, "too large exponent");
+  return (e * 1741647) >> 19;
+}
+
+// Computes upper 64 bits of multiplication of two 64-bit unsigned integers.
+inline auto umul128_upper64(uint64_t x, uint64_t y) noexcept -> uint64_t {
+#if FMT_USE_INT128
+  auto p = static_cast<uint128_opt>(x) * static_cast<uint128_opt>(y);
+  return static_cast<uint64_t>(p >> 64);
+#elif defined(_MSC_VER) && defined(_M_X64)
+  return __umulh(x, y);
+#else
+  return umul128(x, y).high();
+#endif
+}
+
+// Computes upper 128 bits of multiplication of a 64-bit unsigned integer and a
+// 128-bit unsigned integer.
+inline auto umul192_upper128(uint64_t x, uint128_fallback y) noexcept
+    -> uint128_fallback {
+  uint128_fallback r = umul128(x, y.high());
+  r += umul128_upper64(x, y.low());
+  return r;
+}
+
+FMT_API auto get_cached_power(int k) noexcept -> uint128_fallback;
+
+// Type-specific information that Dragonbox uses.
+template <typename T, typename Enable = void> struct float_info;
+
+template <> struct float_info<float> {
+  using carrier_uint = uint32_t;
+  static const int exponent_bits = 8;
+  static const int kappa = 1;
+  static const int big_divisor = 100;
+  static const int small_divisor = 10;
+  static const int min_k = -31;
+  static const int max_k = 46;
+  static const int shorter_interval_tie_lower_threshold = -35;
+  static const int shorter_interval_tie_upper_threshold = -35;
+};
+
+template <> struct float_info<double> {
+  using carrier_uint = uint64_t;
+  static const int exponent_bits = 11;
+  static const int kappa = 2;
+  static const int big_divisor = 1000;
+  static const int small_divisor = 100;
+  static const int min_k = -292;
+  static const int max_k = 341;
+  static const int shorter_interval_tie_lower_threshold = -77;
+  static const int shorter_interval_tie_upper_threshold = -77;
+};
+
+// An 80- or 128-bit floating point number.
+template <typename T>
+struct float_info<T, enable_if_t<std::numeric_limits<T>::digits == 64 ||
+                                 std::numeric_limits<T>::digits == 113 ||
+                                 is_float128<T>::value>> {
+  using carrier_uint = detail::uint128_t;
+  static const int exponent_bits = 15;
+};
+
+// A double-double floating point number.
+template <typename T>
+struct float_info<T, enable_if_t<is_double_double<T>::value>> {
+  using carrier_uint = detail::uint128_t;
+};
+
+template <typename T> struct decimal_fp {
+  using significand_type = typename float_info<T>::carrier_uint;
+  significand_type significand;
+  int exponent;
+};
+
+template <typename T> FMT_API auto to_decimal(T x) noexcept -> decimal_fp<T>;
+}  // namespace dragonbox
+
+// Returns true iff Float has the implicit bit which is not stored.
+template <typename Float> constexpr auto has_implicit_bit() -> bool {
+  // An 80-bit FP number has a 64-bit significand an no implicit bit.
+  return std::numeric_limits<Float>::digits != 64;
+}
+
+// Returns the number of significand bits stored in Float. The implicit bit is
+// not counted since it is not stored.
+template <typename Float> constexpr auto num_significand_bits() -> int {
+  // std::numeric_limits may not support __float128.
+  return is_float128<Float>() ? 112
+                              : (std::numeric_limits<Float>::digits -
+                                 (has_implicit_bit<Float>() ? 1 : 0));
+}
+
+template <typename Float>
+constexpr auto exponent_mask() ->
+    typename dragonbox::float_info<Float>::carrier_uint {
+  using float_uint = typename dragonbox::float_info<Float>::carrier_uint;
+  return ((float_uint(1) << dragonbox::float_info<Float>::exponent_bits) - 1)
+         << num_significand_bits<Float>();
+}
+template <typename Float> constexpr auto exponent_bias() -> int {
+  // std::numeric_limits may not support __float128.
+  return is_float128<Float>() ? 16383
+                              : std::numeric_limits<Float>::max_exponent - 1;
+}
+
+// Writes the exponent exp in the form "[+-]d{2,3}" to buffer.
+template <typename Char, typename It>
+FMT_CONSTEXPR auto write_exponent(int exp, It it) -> It {
+  FMT_ASSERT(-10000 < exp && exp < 10000, "exponent out of range");
+  if (exp < 0) {
+    *it++ = static_cast<Char>('-');
+    exp = -exp;
+  } else {
+    *it++ = static_cast<Char>('+');
+  }
+  if (exp >= 100) {
+    const char* top = digits2(to_unsigned(exp / 100));
+    if (exp >= 1000) *it++ = static_cast<Char>(top[0]);
+    *it++ = static_cast<Char>(top[1]);
+    exp %= 100;
+  }
+  const char* d = digits2(to_unsigned(exp));
+  *it++ = static_cast<Char>(d[0]);
+  *it++ = static_cast<Char>(d[1]);
+  return it;
+}
+
+// A floating-point number f * pow(2, e) where F is an unsigned type.
+template <typename F> struct basic_fp {
+  F f;
+  int e;
+
+  static constexpr const int num_significand_bits =
+      static_cast<int>(sizeof(F) * num_bits<unsigned char>());
+
+  constexpr basic_fp() : f(0), e(0) {}
+  constexpr basic_fp(uint64_t f_val, int e_val) : f(f_val), e(e_val) {}
+
+  // Constructs fp from an IEEE754 floating-point number.
+  template <typename Float> FMT_CONSTEXPR basic_fp(Float n) { assign(n); }
+
+  // Assigns n to this and return true iff predecessor is closer than successor.
+  template <typename Float, FMT_ENABLE_IF(!is_double_double<Float>::value)>
+  FMT_CONSTEXPR auto assign(Float n) -> bool {
+    static_assert(std::numeric_limits<Float>::digits <= 113, "unsupported FP");
+    // Assume Float is in the format [sign][exponent][significand].
+    using carrier_uint = typename dragonbox::float_info<Float>::carrier_uint;
+    const auto num_float_significand_bits =
+        detail::num_significand_bits<Float>();
+    const auto implicit_bit = carrier_uint(1) << num_float_significand_bits;
+    const auto significand_mask = implicit_bit - 1;
+    auto u = bit_cast<carrier_uint>(n);
+    f = static_cast<F>(u & significand_mask);
+    auto biased_e = static_cast<int>((u & exponent_mask<Float>()) >>
+                                     num_float_significand_bits);
+    // The predecessor is closer if n is a normalized power of 2 (f == 0)
+    // other than the smallest normalized number (biased_e > 1).
+    auto is_predecessor_closer = f == 0 && biased_e > 1;
+    if (biased_e == 0)
+      biased_e = 1;  // Subnormals use biased exponent 1 (min exponent).
+    else if (has_implicit_bit<Float>())
+      f += static_cast<F>(implicit_bit);
+    e = biased_e - exponent_bias<Float>() - num_float_significand_bits;
+    if (!has_implicit_bit<Float>()) ++e;
+    return is_predecessor_closer;
+  }
+
+  template <typename Float, FMT_ENABLE_IF(is_double_double<Float>::value)>
+  FMT_CONSTEXPR auto assign(Float n) -> bool {
+    static_assert(std::numeric_limits<double>::is_iec559, "unsupported FP");
+    return assign(static_cast<double>(n));
+  }
+};
+
+using fp = basic_fp<unsigned long long>;
+
+// Normalizes the value converted from double and multiplied by (1 << SHIFT).
+template <int SHIFT = 0, typename F>
+FMT_CONSTEXPR auto normalize(basic_fp<F> value) -> basic_fp<F> {
+  // Handle subnormals.
+  const auto implicit_bit = F(1) << num_significand_bits<double>();
+  const auto shifted_implicit_bit = implicit_bit << SHIFT;
+  while ((value.f & shifted_implicit_bit) == 0) {
+    value.f <<= 1;
+    --value.e;
+  }
+  // Subtract 1 to account for hidden bit.
+  const auto offset = basic_fp<F>::num_significand_bits -
+                      num_significand_bits<double>() - SHIFT - 1;
+  value.f <<= offset;
+  value.e -= offset;
+  return value;
+}
+
+// Computes lhs * rhs / pow(2, 64) rounded to nearest with half-up tie breaking.
+FMT_CONSTEXPR inline auto multiply(uint64_t lhs, uint64_t rhs) -> uint64_t {
+#if FMT_USE_INT128
+  auto product = static_cast<__uint128_t>(lhs) * rhs;
+  auto f = static_cast<uint64_t>(product >> 64);
+  return (static_cast<uint64_t>(product) & (1ULL << 63)) != 0 ? f + 1 : f;
+#else
+  // Multiply 32-bit parts of significands.
+  uint64_t mask = (1ULL << 32) - 1;
+  uint64_t a = lhs >> 32, b = lhs & mask;
+  uint64_t c = rhs >> 32, d = rhs & mask;
+  uint64_t ac = a * c, bc = b * c, ad = a * d, bd = b * d;
+  // Compute mid 64-bit of result and round.
+  uint64_t mid = (bd >> 32) + (ad & mask) + (bc & mask) + (1U << 31);
+  return ac + (ad >> 32) + (bc >> 32) + (mid >> 32);
+#endif
+}
+
+FMT_CONSTEXPR inline auto operator*(fp x, fp y) -> fp {
+  return {multiply(x.f, y.f), x.e + y.e + 64};
+}
+
+template <typename T, bool doublish = num_bits<T>() == num_bits<double>()>
+using convert_float_result =
+    conditional_t<std::is_same<T, float>::value || doublish, double, T>;
+
+template <typename T>
+constexpr auto convert_float(T value) -> convert_float_result<T> {
+  return static_cast<convert_float_result<T>>(value);
+}
+
+template <typename Char, typename OutputIt>
+FMT_NOINLINE FMT_CONSTEXPR auto fill(OutputIt it, size_t n, const fill_t& fill)
+    -> OutputIt {
+  auto fill_size = fill.size();
+  if (fill_size == 1) return detail::fill_n(it, n, fill.template get<Char>());
+  if (const Char* data = fill.template data<Char>()) {
+    for (size_t i = 0; i < n; ++i) it = copy<Char>(data, data + fill_size, it);
+  }
+  return it;
+}
+
+// Writes the output of f, padded according to format specifications in specs.
+// size: output size in code units.
+// width: output display width in (terminal) column positions.
+template <typename Char, align::type align = align::left, typename OutputIt,
+          typename F>
+FMT_CONSTEXPR auto write_padded(OutputIt out, const format_specs& specs,
+                                size_t size, size_t width, F&& f) -> OutputIt {
+  static_assert(align == align::left || align == align::right, "");
+  unsigned spec_width = to_unsigned(specs.width);
+  size_t padding = spec_width > width ? spec_width - width : 0;
+  // Shifts are encoded as string literals because static constexpr is not
+  // supported in constexpr functions.
+  auto* shifts = align == align::left ? "\x1f\x1f\x00\x01" : "\x00\x1f\x00\x01";
+  size_t left_padding = padding >> shifts[specs.align];
+  size_t right_padding = padding - left_padding;
+  auto it = reserve(out, size + padding * specs.fill.size());
+  if (left_padding != 0) it = fill<Char>(it, left_padding, specs.fill);
+  it = f(it);
+  if (right_padding != 0) it = fill<Char>(it, right_padding, specs.fill);
+  return base_iterator(out, it);
+}
+
+template <typename Char, align::type align = align::left, typename OutputIt,
+          typename F>
+constexpr auto write_padded(OutputIt out, const format_specs& specs,
+                            size_t size, F&& f) -> OutputIt {
+  return write_padded<Char, align>(out, specs, size, size, f);
+}
+
+template <typename Char, align::type align = align::left, typename OutputIt>
+FMT_CONSTEXPR auto write_bytes(OutputIt out, string_view bytes,
+                               const format_specs& specs = {}) -> OutputIt {
+  return write_padded<Char, align>(
+      out, specs, bytes.size(), [bytes](reserve_iterator<OutputIt> it) {
+        const char* data = bytes.data();
+        return copy<Char>(data, data + bytes.size(), it);
+      });
+}
+
+template <typename Char, typename OutputIt, typename UIntPtr>
+auto write_ptr(OutputIt out, UIntPtr value, const format_specs* specs)
+    -> OutputIt {
+  int num_digits = count_digits<4>(value);
+  auto size = to_unsigned(num_digits) + size_t(2);
+  auto write = [=](reserve_iterator<OutputIt> it) {
+    *it++ = static_cast<Char>('0');
+    *it++ = static_cast<Char>('x');
+    return format_uint<4, Char>(it, value, num_digits);
+  };
+  return specs ? write_padded<Char, align::right>(out, *specs, size, write)
+               : base_iterator(out, write(reserve(out, size)));
+}
+
+// Returns true iff the code point cp is printable.
+FMT_API auto is_printable(uint32_t cp) -> bool;
+
+inline auto needs_escape(uint32_t cp) -> bool {
+  return cp < 0x20 || cp == 0x7f || cp == '"' || cp == '\\' ||
+         !is_printable(cp);
+}
+
+template <typename Char> struct find_escape_result {
+  const Char* begin;
+  const Char* end;
+  uint32_t cp;
+};
+
+template <typename Char>
+auto find_escape(const Char* begin, const Char* end)
+    -> find_escape_result<Char> {
+  for (; begin != end; ++begin) {
+    uint32_t cp = static_cast<unsigned_char<Char>>(*begin);
+    if (const_check(sizeof(Char) == 1) && cp >= 0x80) continue;
+    if (needs_escape(cp)) return {begin, begin + 1, cp};
+  }
+  return {begin, nullptr, 0};
+}
+
+inline auto find_escape(const char* begin, const char* end)
+    -> find_escape_result<char> {
+  if (!use_utf8()) return find_escape<char>(begin, end);
+  auto result = find_escape_result<char>{end, nullptr, 0};
+  for_each_codepoint(string_view(begin, to_unsigned(end - begin)),
+                     [&](uint32_t cp, string_view sv) {
+                       if (needs_escape(cp)) {
+                         result = {sv.begin(), sv.end(), cp};
+                         return false;
+                       }
+                       return true;
+                     });
+  return result;
+}
+
+#define FMT_STRING_IMPL(s, base, explicit)                                    \
+  [] {                                                                        \
+    /* Use the hidden visibility as a workaround for a GCC bug (#1973). */    \
+    /* Use a macro-like name to avoid shadowing warnings. */                  \
+    struct FMT_VISIBILITY("hidden") FMT_COMPILE_STRING : base {               \
+      using char_type FMT_MAYBE_UNUSED = fmt::remove_cvref_t<decltype(s[0])>; \
+      FMT_MAYBE_UNUSED FMT_CONSTEXPR explicit                                 \
+      operator fmt::basic_string_view<char_type>() const {                    \
+        return fmt::detail_exported::compile_string_to_view<char_type>(s);    \
+      }                                                                       \
+    };                                                                        \
+    return FMT_COMPILE_STRING();                                              \
+  }()
+
+/**
+ * Constructs a compile-time format string from a string literal `s`.
+ *
+ * **Example**:
+ *
+ *     // A compile-time error because 'd' is an invalid specifier for strings.
+ *     std::string s = fmt::format(FMT_STRING("{:d}"), "foo");
+ */
+#define FMT_STRING(s) FMT_STRING_IMPL(s, fmt::detail::compile_string, )
+
+template <size_t width, typename Char, typename OutputIt>
+auto write_codepoint(OutputIt out, char prefix, uint32_t cp) -> OutputIt {
+  *out++ = static_cast<Char>('\\');
+  *out++ = static_cast<Char>(prefix);
+  Char buf[width];
+  fill_n(buf, width, static_cast<Char>('0'));
+  format_uint<4>(buf, cp, width);
+  return copy<Char>(buf, buf + width, out);
+}
+
+template <typename OutputIt, typename Char>
+auto write_escaped_cp(OutputIt out, const find_escape_result<Char>& escape)
+    -> OutputIt {
+  auto c = static_cast<Char>(escape.cp);
+  switch (escape.cp) {
+  case '\n':
+    *out++ = static_cast<Char>('\\');
+    c = static_cast<Char>('n');
+    break;
+  case '\r':
+    *out++ = static_cast<Char>('\\');
+    c = static_cast<Char>('r');
+    break;
+  case '\t':
+    *out++ = static_cast<Char>('\\');
+    c = static_cast<Char>('t');
+    break;
+  case '"':
+    FMT_FALLTHROUGH;
+  case '\'':
+    FMT_FALLTHROUGH;
+  case '\\':
+    *out++ = static_cast<Char>('\\');
+    break;
+  default:
+    if (escape.cp < 0x100) return write_codepoint<2, Char>(out, 'x', escape.cp);
+    if (escape.cp < 0x10000)
+      return write_codepoint<4, Char>(out, 'u', escape.cp);
+    if (escape.cp < 0x110000)
+      return write_codepoint<8, Char>(out, 'U', escape.cp);
+    for (Char escape_char : basic_string_view<Char>(
+             escape.begin, to_unsigned(escape.end - escape.begin))) {
+      out = write_codepoint<2, Char>(out, 'x',
+                                     static_cast<uint32_t>(escape_char) & 0xFF);
+    }
+    return out;
+  }
+  *out++ = c;
+  return out;
+}
+
+template <typename Char, typename OutputIt>
+auto write_escaped_string(OutputIt out, basic_string_view<Char> str)
+    -> OutputIt {
+  *out++ = static_cast<Char>('"');
+  auto begin = str.begin(), end = str.end();
+  do {
+    auto escape = find_escape(begin, end);
+    out = copy<Char>(begin, escape.begin, out);
+    begin = escape.end;
+    if (!begin) break;
+    out = write_escaped_cp<OutputIt, Char>(out, escape);
+  } while (begin != end);
+  *out++ = static_cast<Char>('"');
+  return out;
+}
+
+template <typename Char, typename OutputIt>
+auto write_escaped_char(OutputIt out, Char v) -> OutputIt {
+  Char v_array[1] = {v};
+  *out++ = static_cast<Char>('\'');
+  if ((needs_escape(static_cast<uint32_t>(v)) && v != static_cast<Char>('"')) ||
+      v == static_cast<Char>('\'')) {
+    out = write_escaped_cp(out,
+                           find_escape_result<Char>{v_array, v_array + 1,
+                                                    static_cast<uint32_t>(v)});
+  } else {
+    *out++ = v;
+  }
+  *out++ = static_cast<Char>('\'');
+  return out;
+}
+
+template <typename Char, typename OutputIt>
+FMT_CONSTEXPR auto write_char(OutputIt out, Char value,
+                              const format_specs& specs) -> OutputIt {
+  bool is_debug = specs.type == presentation_type::debug;
+  return write_padded<Char>(out, specs, 1, [=](reserve_iterator<OutputIt> it) {
+    if (is_debug) return write_escaped_char(it, value);
+    *it++ = value;
+    return it;
+  });
+}
+template <typename Char, typename OutputIt>
+FMT_CONSTEXPR auto write(OutputIt out, Char value, const format_specs& specs,
+                         locale_ref loc = {}) -> OutputIt {
+  // char is formatted as unsigned char for consistency across platforms.
+  using unsigned_type =
+      conditional_t<std::is_same<Char, char>::value, unsigned char, unsigned>;
+  return check_char_specs(specs)
+             ? write_char<Char>(out, value, specs)
+             : write<Char>(out, static_cast<unsigned_type>(value), specs, loc);
+}
+
+// Data for write_int that doesn't depend on output iterator type. It is used to
+// avoid template code bloat.
+template <typename Char> struct write_int_data {
+  size_t size;
+  size_t padding;
+
+  FMT_CONSTEXPR write_int_data(int num_digits, unsigned prefix,
+                               const format_specs& specs)
+      : size((prefix >> 24) + to_unsigned(num_digits)), padding(0) {
+    if (specs.align == align::numeric) {
+      auto width = to_unsigned(specs.width);
+      if (width > size) {
+        padding = width - size;
+        size = width;
+      }
+    } else if (specs.precision > num_digits) {
+      size = (prefix >> 24) + to_unsigned(specs.precision);
+      padding = to_unsigned(specs.precision - num_digits);
+    }
+  }
+};
+
+// Writes an integer in the format
+//   <left-padding><prefix><numeric-padding><digits><right-padding>
+// where <digits> are written by write_digits(it).
+// prefix contains chars in three lower bytes and the size in the fourth byte.
+template <typename Char, typename OutputIt, typename W>
+FMT_CONSTEXPR FMT_INLINE auto write_int(OutputIt out, int num_digits,
+                                        unsigned prefix,
+                                        const format_specs& specs,
+                                        W write_digits) -> OutputIt {
+  // Slightly faster check for specs.width == 0 && specs.precision == -1.
+  if ((specs.width | (specs.precision + 1)) == 0) {
+    auto it = reserve(out, to_unsigned(num_digits) + (prefix >> 24));
+    if (prefix != 0) {
+      for (unsigned p = prefix & 0xffffff; p != 0; p >>= 8)
+        *it++ = static_cast<Char>(p & 0xff);
+    }
+    return base_iterator(out, write_digits(it));
+  }
+  auto data = write_int_data<Char>(num_digits, prefix, specs);
+  return write_padded<Char, align::right>(
+      out, specs, data.size, [=](reserve_iterator<OutputIt> it) {
+        for (unsigned p = prefix & 0xffffff; p != 0; p >>= 8)
+          *it++ = static_cast<Char>(p & 0xff);
+        it = detail::fill_n(it, data.padding, static_cast<Char>('0'));
+        return write_digits(it);
+      });
+}
+
+template <typename Char> class digit_grouping {
+ private:
+  std::string grouping_;
+  std::basic_string<Char> thousands_sep_;
+
+  struct next_state {
+    std::string::const_iterator group;
+    int pos;
+  };
+  auto initial_state() const -> next_state { return {grouping_.begin(), 0}; }
+
+  // Returns the next digit group separator position.
+  auto next(next_state& state) const -> int {
+    if (thousands_sep_.empty()) return max_value<int>();
+    if (state.group == grouping_.end()) return state.pos += grouping_.back();
+    if (*state.group <= 0 || *state.group == max_value<char>())
+      return max_value<int>();
+    state.pos += *state.group++;
+    return state.pos;
+  }
+
+ public:
+  explicit digit_grouping(locale_ref loc, bool localized = true) {
+    if (!localized) return;
+    auto sep = thousands_sep<Char>(loc);
+    grouping_ = sep.grouping;
+    if (sep.thousands_sep) thousands_sep_.assign(1, sep.thousands_sep);
+  }
+  digit_grouping(std::string grouping, std::basic_string<Char> sep)
+      : grouping_(std::move(grouping)), thousands_sep_(std::move(sep)) {}
+
+  auto has_separator() const -> bool { return !thousands_sep_.empty(); }
+
+  auto count_separators(int num_digits) const -> int {
+    int count = 0;
+    auto state = initial_state();
+    while (num_digits > next(state)) ++count;
+    return count;
+  }
+
+  // Applies grouping to digits and write the output to out.
+  template <typename Out, typename C>
+  auto apply(Out out, basic_string_view<C> digits) const -> Out {
+    auto num_digits = static_cast<int>(digits.size());
+    auto separators = basic_memory_buffer<int>();
+    separators.push_back(0);
+    auto state = initial_state();
+    while (int i = next(state)) {
+      if (i >= num_digits) break;
+      separators.push_back(i);
+    }
+    for (int i = 0, sep_index = static_cast<int>(separators.size() - 1);
+         i < num_digits; ++i) {
+      if (num_digits - i == separators[sep_index]) {
+        out = copy<Char>(thousands_sep_.data(),
+                         thousands_sep_.data() + thousands_sep_.size(), out);
+        --sep_index;
+      }
+      *out++ = static_cast<Char>(digits[to_unsigned(i)]);
+    }
+    return out;
+  }
+};
+
+FMT_CONSTEXPR inline void prefix_append(unsigned& prefix, unsigned value) {
+  prefix |= prefix != 0 ? value << 8 : value;
+  prefix += (1u + (value > 0xff ? 1 : 0)) << 24;
+}
+
+// Writes a decimal integer with digit grouping.
+template <typename OutputIt, typename UInt, typename Char>
+auto write_int(OutputIt out, UInt value, unsigned prefix,
+               const format_specs& specs, const digit_grouping<Char>& grouping)
+    -> OutputIt {
+  static_assert(std::is_same<uint64_or_128_t<UInt>, UInt>::value, "");
+  int num_digits = 0;
+  auto buffer = memory_buffer();
+  switch (specs.type) {
+  default:
+    FMT_ASSERT(false, "");
+    FMT_FALLTHROUGH;
+  case presentation_type::none:
+  case presentation_type::dec:
+    num_digits = count_digits(value);
+    format_decimal<char>(appender(buffer), value, num_digits);
+    break;
+  case presentation_type::hex:
+    if (specs.alt)
+      prefix_append(prefix, unsigned(specs.upper ? 'X' : 'x') << 8 | '0');
+    num_digits = count_digits<4>(value);
+    format_uint<4, char>(appender(buffer), value, num_digits, specs.upper);
+    break;
+  case presentation_type::oct:
+    num_digits = count_digits<3>(value);
+    // Octal prefix '0' is counted as a digit, so only add it if precision
+    // is not greater than the number of digits.
+    if (specs.alt && specs.precision <= num_digits && value != 0)
+      prefix_append(prefix, '0');
+    format_uint<3, char>(appender(buffer), value, num_digits);
+    break;
+  case presentation_type::bin:
+    if (specs.alt)
+      prefix_append(prefix, unsigned(specs.upper ? 'B' : 'b') << 8 | '0');
+    num_digits = count_digits<1>(value);
+    format_uint<1, char>(appender(buffer), value, num_digits);
+    break;
+  case presentation_type::chr:
+    return write_char<Char>(out, static_cast<Char>(value), specs);
+  }
+
+  unsigned size = (prefix != 0 ? prefix >> 24 : 0) + to_unsigned(num_digits) +
+                  to_unsigned(grouping.count_separators(num_digits));
+  return write_padded<Char, align::right>(
+      out, specs, size, size, [&](reserve_iterator<OutputIt> it) {
+        for (unsigned p = prefix & 0xffffff; p != 0; p >>= 8)
+          *it++ = static_cast<Char>(p & 0xff);
+        return grouping.apply(it, string_view(buffer.data(), buffer.size()));
+      });
+}
+
+// Writes a localized value.
+FMT_API auto write_loc(appender out, loc_value value, const format_specs& specs,
+                       locale_ref loc) -> bool;
+template <typename OutputIt>
+inline auto write_loc(OutputIt, loc_value, const format_specs&, locale_ref)
+    -> bool {
+  return false;
+}
+
+template <typename UInt> struct write_int_arg {
+  UInt abs_value;
+  unsigned prefix;
+};
+
+template <typename T>
+FMT_CONSTEXPR auto make_write_int_arg(T value, sign_t sign)
+    -> write_int_arg<uint32_or_64_or_128_t<T>> {
+  auto prefix = 0u;
+  auto abs_value = static_cast<uint32_or_64_or_128_t<T>>(value);
+  if (is_negative(value)) {
+    prefix = 0x01000000 | '-';
+    abs_value = 0 - abs_value;
+  } else {
+    constexpr const unsigned prefixes[4] = {0, 0, 0x1000000u | '+',
+                                            0x1000000u | ' '};
+    prefix = prefixes[sign];
+  }
+  return {abs_value, prefix};
+}
+
+template <typename Char = char> struct loc_writer {
+  basic_appender<Char> out;
+  const format_specs& specs;
+  std::basic_string<Char> sep;
+  std::string grouping;
+  std::basic_string<Char> decimal_point;
+
+  template <typename T, FMT_ENABLE_IF(is_integer<T>::value)>
+  auto operator()(T value) -> bool {
+    auto arg = make_write_int_arg(value, specs.sign);
+    write_int(out, static_cast<uint64_or_128_t<T>>(arg.abs_value), arg.prefix,
+              specs, digit_grouping<Char>(grouping, sep));
+    return true;
+  }
+
+  template <typename T, FMT_ENABLE_IF(!is_integer<T>::value)>
+  auto operator()(T) -> bool {
+    return false;
+  }
+};
+
+template <typename Char, typename OutputIt, typename T>
+FMT_CONSTEXPR FMT_INLINE auto write_int(OutputIt out, write_int_arg<T> arg,
+                                        const format_specs& specs, locale_ref)
+    -> OutputIt {
+  static_assert(std::is_same<T, uint32_or_64_or_128_t<T>>::value, "");
+  auto abs_value = arg.abs_value;
+  auto prefix = arg.prefix;
+  switch (specs.type) {
+  default:
+    FMT_ASSERT(false, "");
+    FMT_FALLTHROUGH;
+  case presentation_type::none:
+  case presentation_type::dec: {
+    int num_digits = count_digits(abs_value);
+    return write_int<Char>(
+        out, num_digits, prefix, specs, [=](reserve_iterator<OutputIt> it) {
+          return format_decimal<Char>(it, abs_value, num_digits).end;
+        });
+  }
+  case presentation_type::hex: {
+    if (specs.alt)
+      prefix_append(prefix, unsigned(specs.upper ? 'X' : 'x') << 8 | '0');
+    int num_digits = count_digits<4>(abs_value);
+    return write_int<Char>(
+        out, num_digits, prefix, specs, [=](reserve_iterator<OutputIt> it) {
+          return format_uint<4, Char>(it, abs_value, num_digits, specs.upper);
+        });
+  }
+  case presentation_type::oct: {
+    int num_digits = count_digits<3>(abs_value);
+    // Octal prefix '0' is counted as a digit, so only add it if precision
+    // is not greater than the number of digits.
+    if (specs.alt && specs.precision <= num_digits && abs_value != 0)
+      prefix_append(prefix, '0');
+    return write_int<Char>(
+        out, num_digits, prefix, specs, [=](reserve_iterator<OutputIt> it) {
+          return format_uint<3, Char>(it, abs_value, num_digits);
+        });
+  }
+  case presentation_type::bin: {
+    if (specs.alt)
+      prefix_append(prefix, unsigned(specs.upper ? 'B' : 'b') << 8 | '0');
+    int num_digits = count_digits<1>(abs_value);
+    return write_int<Char>(
+        out, num_digits, prefix, specs, [=](reserve_iterator<OutputIt> it) {
+          return format_uint<1, Char>(it, abs_value, num_digits);
+        });
+  }
+  case presentation_type::chr:
+    return write_char<Char>(out, static_cast<Char>(abs_value), specs);
+  }
+}
+template <typename Char, typename OutputIt, typename T>
+FMT_CONSTEXPR FMT_NOINLINE auto write_int_noinline(OutputIt out,
+                                                   write_int_arg<T> arg,
+                                                   const format_specs& specs,
+                                                   locale_ref loc) -> OutputIt {
+  return write_int<Char>(out, arg, specs, loc);
+}
+template <typename Char, typename T,
+          FMT_ENABLE_IF(is_integral<T>::value &&
+                        !std::is_same<T, bool>::value &&
+                        !std::is_same<T, Char>::value)>
+FMT_CONSTEXPR FMT_INLINE auto write(basic_appender<Char> out, T value,
+                                    const format_specs& specs, locale_ref loc)
+    -> basic_appender<Char> {
+  if (specs.localized && write_loc(out, value, specs, loc)) return out;
+  return write_int_noinline<Char>(out, make_write_int_arg(value, specs.sign),
+                                  specs, loc);
+}
+// An inlined version of write used in format string compilation.
+template <typename Char, typename OutputIt, typename T,
+          FMT_ENABLE_IF(is_integral<T>::value &&
+                        !std::is_same<T, bool>::value &&
+                        !std::is_same<T, Char>::value &&
+                        !std::is_same<OutputIt, basic_appender<Char>>::value)>
+FMT_CONSTEXPR FMT_INLINE auto write(OutputIt out, T value,
+                                    const format_specs& specs, locale_ref loc)
+    -> OutputIt {
+  if (specs.localized && write_loc(out, value, specs, loc)) return out;
+  return write_int<Char>(out, make_write_int_arg(value, specs.sign), specs,
+                         loc);
+}
+
+// An output iterator that counts the number of objects written to it and
+// discards them.
+class counting_iterator {
+ private:
+  size_t count_;
+
+ public:
+  using iterator_category = std::output_iterator_tag;
+  using difference_type = std::ptrdiff_t;
+  using pointer = void;
+  using reference = void;
+  FMT_UNCHECKED_ITERATOR(counting_iterator);
+
+  struct value_type {
+    template <typename T> FMT_CONSTEXPR void operator=(const T&) {}
+  };
+
+  FMT_CONSTEXPR counting_iterator() : count_(0) {}
+
+  FMT_CONSTEXPR auto count() const -> size_t { return count_; }
+
+  FMT_CONSTEXPR auto operator++() -> counting_iterator& {
+    ++count_;
+    return *this;
+  }
+  FMT_CONSTEXPR auto operator++(int) -> counting_iterator {
+    auto it = *this;
+    ++*this;
+    return it;
+  }
+
+  FMT_CONSTEXPR friend auto operator+(counting_iterator it, difference_type n)
+      -> counting_iterator {
+    it.count_ += static_cast<size_t>(n);
+    return it;
+  }
+
+  FMT_CONSTEXPR auto operator*() const -> value_type { return {}; }
+};
+
+template <typename Char, typename OutputIt>
+FMT_CONSTEXPR auto write(OutputIt out, basic_string_view<Char> s,
+                         const format_specs& specs) -> OutputIt {
+  auto data = s.data();
+  auto size = s.size();
+  if (specs.precision >= 0 && to_unsigned(specs.precision) < size)
+    size = code_point_index(s, to_unsigned(specs.precision));
+  bool is_debug = specs.type == presentation_type::debug;
+  size_t width = 0;
+
+  if (is_debug) size = write_escaped_string(counting_iterator{}, s).count();
+
+  if (specs.width != 0) {
+    if (is_debug)
+      width = size;
+    else
+      width = compute_width(basic_string_view<Char>(data, size));
+  }
+  return write_padded<Char>(out, specs, size, width,
+                            [=](reserve_iterator<OutputIt> it) {
+                              if (is_debug) return write_escaped_string(it, s);
+                              return copy<Char>(data, data + size, it);
+                            });
+}
+template <typename Char, typename OutputIt>
+FMT_CONSTEXPR auto write(OutputIt out,
+                         basic_string_view<type_identity_t<Char>> s,
+                         const format_specs& specs, locale_ref) -> OutputIt {
+  return write<Char>(out, s, specs);
+}
+template <typename Char, typename OutputIt>
+FMT_CONSTEXPR auto write(OutputIt out, const Char* s, const format_specs& specs,
+                         locale_ref) -> OutputIt {
+  if (specs.type == presentation_type::pointer)
+    return write_ptr<Char>(out, bit_cast<uintptr_t>(s), &specs);
+  if (!s) report_error("string pointer is null");
+  return write<Char>(out, basic_string_view<Char>(s), specs, {});
+}
+
+template <typename Char, typename OutputIt, typename T,
+          FMT_ENABLE_IF(is_integral<T>::value &&
+                        !std::is_same<T, bool>::value &&
+                        !std::is_same<T, Char>::value)>
+FMT_CONSTEXPR auto write(OutputIt out, T value) -> OutputIt {
+  auto abs_value = static_cast<uint32_or_64_or_128_t<T>>(value);
+  bool negative = is_negative(value);
+  // Don't do -abs_value since it trips unsigned-integer-overflow sanitizer.
+  if (negative) abs_value = ~abs_value + 1;
+  int num_digits = count_digits(abs_value);
+  auto size = (negative ? 1 : 0) + static_cast<size_t>(num_digits);
+  if (auto ptr = to_pointer<Char>(out, size)) {
+    if (negative) *ptr++ = static_cast<Char>('-');
+    format_decimal<Char>(ptr, abs_value, num_digits);
+    return out;
+  }
+  if (negative) *out++ = static_cast<Char>('-');
+  return format_decimal<Char>(out, abs_value, num_digits).end;
+}
+
+// DEPRECATED!
+template <typename Char>
+FMT_CONSTEXPR auto parse_align(const Char* begin, const Char* end,
+                               format_specs& specs) -> const Char* {
+  FMT_ASSERT(begin != end, "");
+  auto align = align::none;
+  auto p = begin + code_point_length(begin);
+  if (end - p <= 0) p = begin;
+  for (;;) {
+    switch (to_ascii(*p)) {
+    case '<':
+      align = align::left;
+      break;
+    case '>':
+      align = align::right;
+      break;
+    case '^':
+      align = align::center;
+      break;
+    }
+    if (align != align::none) {
+      if (p != begin) {
+        auto c = *begin;
+        if (c == '}') return begin;
+        if (c == '{') {
+          report_error("invalid fill character '{'");
+          return begin;
+        }
+        specs.fill = basic_string_view<Char>(begin, to_unsigned(p - begin));
+        begin = p + 1;
+      } else {
+        ++begin;
+      }
+      break;
+    } else if (p == begin) {
+      break;
+    }
+    p = begin;
+  }
+  specs.align = align;
+  return begin;
+}
+
+// A floating-point presentation format.
+enum class float_format : unsigned char {
+  general,  // General: exponent notation or fixed point based on magnitude.
+  exp,      // Exponent notation with the default precision of 6, e.g. 1.2e-3.
+  fixed     // Fixed point with the default precision of 6, e.g. 0.0012.
+};
+
+struct float_specs {
+  int precision;
+  float_format format : 8;
+  sign_t sign : 8;
+  bool locale : 1;
+  bool binary32 : 1;
+  bool showpoint : 1;
+};
+
+// DEPRECATED!
+FMT_CONSTEXPR inline auto parse_float_type_spec(const format_specs& specs)
+    -> float_specs {
+  auto result = float_specs();
+  result.showpoint = specs.alt;
+  result.locale = specs.localized;
+  switch (specs.type) {
+  default:
+    FMT_FALLTHROUGH;
+  case presentation_type::none:
+    result.format = float_format::general;
+    break;
+  case presentation_type::exp:
+    result.format = float_format::exp;
+    result.showpoint |= specs.precision != 0;
+    break;
+  case presentation_type::fixed:
+    result.format = float_format::fixed;
+    result.showpoint |= specs.precision != 0;
+    break;
+  case presentation_type::general:
+    result.format = float_format::general;
+    break;
+  }
+  return result;
+}
+
+template <typename Char, typename OutputIt>
+FMT_CONSTEXPR20 auto write_nonfinite(OutputIt out, bool isnan,
+                                     format_specs specs, sign_t sign)
+    -> OutputIt {
+  auto str =
+      isnan ? (specs.upper ? "NAN" : "nan") : (specs.upper ? "INF" : "inf");
+  constexpr size_t str_size = 3;
+  auto size = str_size + (sign ? 1 : 0);
+  // Replace '0'-padding with space for non-finite values.
+  const bool is_zero_fill =
+      specs.fill.size() == 1 && specs.fill.template get<Char>() == '0';
+  if (is_zero_fill) specs.fill = ' ';
+  return write_padded<Char>(out, specs, size,
+                            [=](reserve_iterator<OutputIt> it) {
+                              if (sign) *it++ = detail::sign<Char>(sign);
+                              return copy<Char>(str, str + str_size, it);
+                            });
+}
+
+// A decimal floating-point number significand * pow(10, exp).
+struct big_decimal_fp {
+  const char* significand;
+  int significand_size;
+  int exponent;
+};
+
+constexpr auto get_significand_size(const big_decimal_fp& f) -> int {
+  return f.significand_size;
+}
+template <typename T>
+inline auto get_significand_size(const dragonbox::decimal_fp<T>& f) -> int {
+  return count_digits(f.significand);
+}
+
+template <typename Char, typename OutputIt>
+constexpr auto write_significand(OutputIt out, const char* significand,
+                                 int significand_size) -> OutputIt {
+  return copy<Char>(significand, significand + significand_size, out);
+}
+template <typename Char, typename OutputIt, typename UInt>
+inline auto write_significand(OutputIt out, UInt significand,
+                              int significand_size) -> OutputIt {
+  return format_decimal<Char>(out, significand, significand_size).end;
+}
+template <typename Char, typename OutputIt, typename T, typename Grouping>
+FMT_CONSTEXPR20 auto write_significand(OutputIt out, T significand,
+                                       int significand_size, int exponent,
+                                       const Grouping& grouping) -> OutputIt {
+  if (!grouping.has_separator()) {
+    out = write_significand<Char>(out, significand, significand_size);
+    return detail::fill_n(out, exponent, static_cast<Char>('0'));
+  }
+  auto buffer = memory_buffer();
+  write_significand<char>(appender(buffer), significand, significand_size);
+  detail::fill_n(appender(buffer), exponent, '0');
+  return grouping.apply(out, string_view(buffer.data(), buffer.size()));
+}
+
+template <typename Char, typename UInt,
+          FMT_ENABLE_IF(std::is_integral<UInt>::value)>
+inline auto write_significand(Char* out, UInt significand, int significand_size,
+                              int integral_size, Char decimal_point) -> Char* {
+  if (!decimal_point)
+    return format_decimal(out, significand, significand_size).end;
+  out += significand_size + 1;
+  Char* end = out;
+  int floating_size = significand_size - integral_size;
+  for (int i = floating_size / 2; i > 0; --i) {
+    out -= 2;
+    copy2(out, digits2(static_cast<std::size_t>(significand % 100)));
+    significand /= 100;
+  }
+  if (floating_size % 2 != 0) {
+    *--out = static_cast<Char>('0' + significand % 10);
+    significand /= 10;
+  }
+  *--out = decimal_point;
+  format_decimal(out - integral_size, significand, integral_size);
+  return end;
+}
+
+template <typename OutputIt, typename UInt, typename Char,
+          FMT_ENABLE_IF(!std::is_pointer<remove_cvref_t<OutputIt>>::value)>
+inline auto write_significand(OutputIt out, UInt significand,
+                              int significand_size, int integral_size,
+                              Char decimal_point) -> OutputIt {
+  // Buffer is large enough to hold digits (digits10 + 1) and a decimal point.
+  Char buffer[digits10<UInt>() + 2];
+  auto end = write_significand(buffer, significand, significand_size,
+                               integral_size, decimal_point);
+  return detail::copy_noinline<Char>(buffer, end, out);
+}
+
+template <typename OutputIt, typename Char>
+FMT_CONSTEXPR auto write_significand(OutputIt out, const char* significand,
+                                     int significand_size, int integral_size,
+                                     Char decimal_point) -> OutputIt {
+  out = detail::copy_noinline<Char>(significand, significand + integral_size,
+                                    out);
+  if (!decimal_point) return out;
+  *out++ = decimal_point;
+  return detail::copy_noinline<Char>(significand + integral_size,
+                                     significand + significand_size, out);
+}
+
+template <typename OutputIt, typename Char, typename T, typename Grouping>
+FMT_CONSTEXPR20 auto write_significand(OutputIt out, T significand,
+                                       int significand_size, int integral_size,
+                                       Char decimal_point,
+                                       const Grouping& grouping) -> OutputIt {
+  if (!grouping.has_separator()) {
+    return write_significand(out, significand, significand_size, integral_size,
+                             decimal_point);
+  }
+  auto buffer = basic_memory_buffer<Char>();
+  write_significand(basic_appender<Char>(buffer), significand, significand_size,
+                    integral_size, decimal_point);
+  grouping.apply(
+      out, basic_string_view<Char>(buffer.data(), to_unsigned(integral_size)));
+  return detail::copy_noinline<Char>(buffer.data() + integral_size,
+                                     buffer.end(), out);
+}
+
+template <typename Char, typename OutputIt, typename DecimalFP,
+          typename Grouping = digit_grouping<Char>>
+FMT_CONSTEXPR20 auto do_write_float(OutputIt out, const DecimalFP& f,
+                                    const format_specs& specs,
+                                    float_specs fspecs, locale_ref loc)
+    -> OutputIt {
+  auto significand = f.significand;
+  int significand_size = get_significand_size(f);
+  const Char zero = static_cast<Char>('0');
+  auto sign = fspecs.sign;
+  size_t size = to_unsigned(significand_size) + (sign ? 1 : 0);
+  using iterator = reserve_iterator<OutputIt>;
+
+  Char decimal_point =
+      fspecs.locale ? detail::decimal_point<Char>(loc) : static_cast<Char>('.');
+
+  int output_exp = f.exponent + significand_size - 1;
+  auto use_exp_format = [=]() {
+    if (fspecs.format == float_format::exp) return true;
+    if (fspecs.format != float_format::general) return false;
+    // Use the fixed notation if the exponent is in [exp_lower, exp_upper),
+    // e.g. 0.0001 instead of 1e-04. Otherwise use the exponent notation.
+    const int exp_lower = -4, exp_upper = 16;
+    return output_exp < exp_lower ||
+           output_exp >= (fspecs.precision > 0 ? fspecs.precision : exp_upper);
+  };
+  if (use_exp_format()) {
+    int num_zeros = 0;
+    if (fspecs.showpoint) {
+      num_zeros = fspecs.precision - significand_size;
+      if (num_zeros < 0) num_zeros = 0;
+      size += to_unsigned(num_zeros);
+    } else if (significand_size == 1) {
+      decimal_point = Char();
+    }
+    auto abs_output_exp = output_exp >= 0 ? output_exp : -output_exp;
+    int exp_digits = 2;
+    if (abs_output_exp >= 100) exp_digits = abs_output_exp >= 1000 ? 4 : 3;
+
+    size += to_unsigned((decimal_point ? 1 : 0) + 2 + exp_digits);
+    char exp_char = specs.upper ? 'E' : 'e';
+    auto write = [=](iterator it) {
+      if (sign) *it++ = detail::sign<Char>(sign);
+      // Insert a decimal point after the first digit and add an exponent.
+      it = write_significand(it, significand, significand_size, 1,
+                             decimal_point);
+      if (num_zeros > 0) it = detail::fill_n(it, num_zeros, zero);
+      *it++ = static_cast<Char>(exp_char);
+      return write_exponent<Char>(output_exp, it);
+    };
+    return specs.width > 0
+               ? write_padded<Char, align::right>(out, specs, size, write)
+               : base_iterator(out, write(reserve(out, size)));
+  }
+
+  int exp = f.exponent + significand_size;
+  if (f.exponent >= 0) {
+    // 1234e5 -> 123400000[.0+]
+    size += to_unsigned(f.exponent);
+    int num_zeros = fspecs.precision - exp;
+    abort_fuzzing_if(num_zeros > 5000);
+    if (fspecs.showpoint) {
+      ++size;
+      if (num_zeros <= 0 && fspecs.format != float_format::fixed) num_zeros = 0;
+      if (num_zeros > 0) size += to_unsigned(num_zeros);
+    }
+    auto grouping = Grouping(loc, fspecs.locale);
+    size += to_unsigned(grouping.count_separators(exp));
+    return write_padded<Char, align::right>(out, specs, size, [&](iterator it) {
+      if (sign) *it++ = detail::sign<Char>(sign);
+      it = write_significand<Char>(it, significand, significand_size,
+                                   f.exponent, grouping);
+      if (!fspecs.showpoint) return it;
+      *it++ = decimal_point;
+      return num_zeros > 0 ? detail::fill_n(it, num_zeros, zero) : it;
+    });
+  } else if (exp > 0) {
+    // 1234e-2 -> 12.34[0+]
+    int num_zeros = fspecs.showpoint ? fspecs.precision - significand_size : 0;
+    size += 1 + to_unsigned(num_zeros > 0 ? num_zeros : 0);
+    auto grouping = Grouping(loc, fspecs.locale);
+    size += to_unsigned(grouping.count_separators(exp));
+    return write_padded<Char, align::right>(out, specs, size, [&](iterator it) {
+      if (sign) *it++ = detail::sign<Char>(sign);
+      it = write_significand(it, significand, significand_size, exp,
+                             decimal_point, grouping);
+      return num_zeros > 0 ? detail::fill_n(it, num_zeros, zero) : it;
+    });
+  }
+  // 1234e-6 -> 0.001234
+  int num_zeros = -exp;
+  if (significand_size == 0 && fspecs.precision >= 0 &&
+      fspecs.precision < num_zeros) {
+    num_zeros = fspecs.precision;
+  }
+  bool pointy = num_zeros != 0 || significand_size != 0 || fspecs.showpoint;
+  size += 1 + (pointy ? 1 : 0) + to_unsigned(num_zeros);
+  return write_padded<Char, align::right>(out, specs, size, [&](iterator it) {
+    if (sign) *it++ = detail::sign<Char>(sign);
+    *it++ = zero;
+    if (!pointy) return it;
+    *it++ = decimal_point;
+    it = detail::fill_n(it, num_zeros, zero);
+    return write_significand<Char>(it, significand, significand_size);
+  });
+}
+
+template <typename Char> class fallback_digit_grouping {
+ public:
+  constexpr fallback_digit_grouping(locale_ref, bool) {}
+
+  constexpr auto has_separator() const -> bool { return false; }
+
+  constexpr auto count_separators(int) const -> int { return 0; }
+
+  template <typename Out, typename C>
+  constexpr auto apply(Out out, basic_string_view<C>) const -> Out {
+    return out;
+  }
+};
+
+template <typename Char, typename OutputIt, typename DecimalFP>
+FMT_CONSTEXPR20 auto write_float(OutputIt out, const DecimalFP& f,
+                                 const format_specs& specs, float_specs fspecs,
+                                 locale_ref loc) -> OutputIt {
+  if (is_constant_evaluated()) {
+    return do_write_float<Char, OutputIt, DecimalFP,
+                          fallback_digit_grouping<Char>>(out, f, specs, fspecs,
+                                                         loc);
+  } else {
+    return do_write_float<Char>(out, f, specs, fspecs, loc);
+  }
+}
+
+template <typename T> constexpr auto isnan(T value) -> bool {
+  return value != value;  // std::isnan doesn't support __float128.
+}
+
+template <typename T, typename Enable = void>
+struct has_isfinite : std::false_type {};
+
+template <typename T>
+struct has_isfinite<T, enable_if_t<sizeof(std::isfinite(T())) != 0>>
+    : std::true_type {};
+
+template <typename T, FMT_ENABLE_IF(std::is_floating_point<T>::value&&
+                                        has_isfinite<T>::value)>
+FMT_CONSTEXPR20 auto isfinite(T value) -> bool {
+  constexpr T inf = T(std::numeric_limits<double>::infinity());
+  if (is_constant_evaluated())
+    return !detail::isnan(value) && value < inf && value > -inf;
+  return std::isfinite(value);
+}
+template <typename T, FMT_ENABLE_IF(!has_isfinite<T>::value)>
+FMT_CONSTEXPR auto isfinite(T value) -> bool {
+  T inf = T(std::numeric_limits<double>::infinity());
+  // std::isfinite doesn't support __float128.
+  return !detail::isnan(value) && value < inf && value > -inf;
+}
+
+template <typename T, FMT_ENABLE_IF(is_floating_point<T>::value)>
+FMT_INLINE FMT_CONSTEXPR bool signbit(T value) {
+  if (is_constant_evaluated()) {
+#ifdef __cpp_if_constexpr
+    if constexpr (std::numeric_limits<double>::is_iec559) {
+      auto bits = detail::bit_cast<uint64_t>(static_cast<double>(value));
+      return (bits >> (num_bits<uint64_t>() - 1)) != 0;
+    }
+#endif
+  }
+  return std::signbit(static_cast<double>(value));
+}
+
+inline FMT_CONSTEXPR20 void adjust_precision(int& precision, int exp10) {
+  // Adjust fixed precision by exponent because it is relative to decimal
+  // point.
+  if (exp10 > 0 && precision > max_value<int>() - exp10)
+    FMT_THROW(format_error("number is too big"));
+  precision += exp10;
+}
+
+class bigint {
+ private:
+  // A bigint is stored as an array of bigits (big digits), with bigit at index
+  // 0 being the least significant one.
+  using bigit = uint32_t;
+  using double_bigit = uint64_t;
+  enum { bigits_capacity = 32 };
+  basic_memory_buffer<bigit, bigits_capacity> bigits_;
+  int exp_;
+
+  FMT_CONSTEXPR20 auto operator[](int index) const -> bigit {
+    return bigits_[to_unsigned(index)];
+  }
+  FMT_CONSTEXPR20 auto operator[](int index) -> bigit& {
+    return bigits_[to_unsigned(index)];
+  }
+
+  static constexpr const int bigit_bits = num_bits<bigit>();
+
+  friend struct formatter<bigint>;
+
+  FMT_CONSTEXPR20 void subtract_bigits(int index, bigit other, bigit& borrow) {
+    auto result = static_cast<double_bigit>((*this)[index]) - other - borrow;
+    (*this)[index] = static_cast<bigit>(result);
+    borrow = static_cast<bigit>(result >> (bigit_bits * 2 - 1));
+  }
+
+  FMT_CONSTEXPR20 void remove_leading_zeros() {
+    int num_bigits = static_cast<int>(bigits_.size()) - 1;
+    while (num_bigits > 0 && (*this)[num_bigits] == 0) --num_bigits;
+    bigits_.resize(to_unsigned(num_bigits + 1));
+  }
+
+  // Computes *this -= other assuming aligned bigints and *this >= other.
+  FMT_CONSTEXPR20 void subtract_aligned(const bigint& other) {
+    FMT_ASSERT(other.exp_ >= exp_, "unaligned bigints");
+    FMT_ASSERT(compare(*this, other) >= 0, "");
+    bigit borrow = 0;
+    int i = other.exp_ - exp_;
+    for (size_t j = 0, n = other.bigits_.size(); j != n; ++i, ++j)
+      subtract_bigits(i, other.bigits_[j], borrow);
+    while (borrow > 0) subtract_bigits(i, 0, borrow);
+    remove_leading_zeros();
+  }
+
+  FMT_CONSTEXPR20 void multiply(uint32_t value) {
+    const double_bigit wide_value = value;
+    bigit carry = 0;
+    for (size_t i = 0, n = bigits_.size(); i < n; ++i) {
+      double_bigit result = bigits_[i] * wide_value + carry;
+      bigits_[i] = static_cast<bigit>(result);
+      carry = static_cast<bigit>(result >> bigit_bits);
+    }
+    if (carry != 0) bigits_.push_back(carry);
+  }
+
+  template <typename UInt, FMT_ENABLE_IF(std::is_same<UInt, uint64_t>::value ||
+                                         std::is_same<UInt, uint128_t>::value)>
+  FMT_CONSTEXPR20 void multiply(UInt value) {
+    using half_uint =
+        conditional_t<std::is_same<UInt, uint128_t>::value, uint64_t, uint32_t>;
+    const int shift = num_bits<half_uint>() - bigit_bits;
+    const UInt lower = static_cast<half_uint>(value);
+    const UInt upper = value >> num_bits<half_uint>();
+    UInt carry = 0;
+    for (size_t i = 0, n = bigits_.size(); i < n; ++i) {
+      UInt result = lower * bigits_[i] + static_cast<bigit>(carry);
+      carry = (upper * bigits_[i] << shift) + (result >> bigit_bits) +
+              (carry >> bigit_bits);
+      bigits_[i] = static_cast<bigit>(result);
+    }
+    while (carry != 0) {
+      bigits_.push_back(static_cast<bigit>(carry));
+      carry >>= bigit_bits;
+    }
+  }
+
+  template <typename UInt, FMT_ENABLE_IF(std::is_same<UInt, uint64_t>::value ||
+                                         std::is_same<UInt, uint128_t>::value)>
+  FMT_CONSTEXPR20 void assign(UInt n) {
+    size_t num_bigits = 0;
+    do {
+      bigits_[num_bigits++] = static_cast<bigit>(n);
+      n >>= bigit_bits;
+    } while (n != 0);
+    bigits_.resize(num_bigits);
+    exp_ = 0;
+  }
+
+ public:
+  FMT_CONSTEXPR20 bigint() : exp_(0) {}
+  explicit bigint(uint64_t n) { assign(n); }
+
+  bigint(const bigint&) = delete;
+  void operator=(const bigint&) = delete;
+
+  FMT_CONSTEXPR20 void assign(const bigint& other) {
+    auto size = other.bigits_.size();
+    bigits_.resize(size);
+    auto data = other.bigits_.data();
+    copy<bigit>(data, data + size, bigits_.data());
+    exp_ = other.exp_;
+  }
+
+  template <typename Int> FMT_CONSTEXPR20 void operator=(Int n) {
+    FMT_ASSERT(n > 0, "");
+    assign(uint64_or_128_t<Int>(n));
+  }
+
+  FMT_CONSTEXPR20 auto num_bigits() const -> int {
+    return static_cast<int>(bigits_.size()) + exp_;
+  }
+
+  FMT_NOINLINE FMT_CONSTEXPR20 auto operator<<=(int shift) -> bigint& {
+    FMT_ASSERT(shift >= 0, "");
+    exp_ += shift / bigit_bits;
+    shift %= bigit_bits;
+    if (shift == 0) return *this;
+    bigit carry = 0;
+    for (size_t i = 0, n = bigits_.size(); i < n; ++i) {
+      bigit c = bigits_[i] >> (bigit_bits - shift);
+      bigits_[i] = (bigits_[i] << shift) + carry;
+      carry = c;
+    }
+    if (carry != 0) bigits_.push_back(carry);
+    return *this;
+  }
+
+  template <typename Int>
+  FMT_CONSTEXPR20 auto operator*=(Int value) -> bigint& {
+    FMT_ASSERT(value > 0, "");
+    multiply(uint32_or_64_or_128_t<Int>(value));
+    return *this;
+  }
+
+  friend FMT_CONSTEXPR20 auto compare(const bigint& lhs, const bigint& rhs)
+      -> int {
+    int num_lhs_bigits = lhs.num_bigits(), num_rhs_bigits = rhs.num_bigits();
+    if (num_lhs_bigits != num_rhs_bigits)
+      return num_lhs_bigits > num_rhs_bigits ? 1 : -1;
+    int i = static_cast<int>(lhs.bigits_.size()) - 1;
+    int j = static_cast<int>(rhs.bigits_.size()) - 1;
+    int end = i - j;
+    if (end < 0) end = 0;
+    for (; i >= end; --i, --j) {
+      bigit lhs_bigit = lhs[i], rhs_bigit = rhs[j];
+      if (lhs_bigit != rhs_bigit) return lhs_bigit > rhs_bigit ? 1 : -1;
+    }
+    if (i != j) return i > j ? 1 : -1;
+    return 0;
+  }
+
+  // Returns compare(lhs1 + lhs2, rhs).
+  friend FMT_CONSTEXPR20 auto add_compare(const bigint& lhs1,
+                                          const bigint& lhs2, const bigint& rhs)
+      -> int {
+    auto minimum = [](int a, int b) { return a < b ? a : b; };
+    auto maximum = [](int a, int b) { return a > b ? a : b; };
+    int max_lhs_bigits = maximum(lhs1.num_bigits(), lhs2.num_bigits());
+    int num_rhs_bigits = rhs.num_bigits();
+    if (max_lhs_bigits + 1 < num_rhs_bigits) return -1;
+    if (max_lhs_bigits > num_rhs_bigits) return 1;
+    auto get_bigit = [](const bigint& n, int i) -> bigit {
+      return i >= n.exp_ && i < n.num_bigits() ? n[i - n.exp_] : 0;
+    };
+    double_bigit borrow = 0;
+    int min_exp = minimum(minimum(lhs1.exp_, lhs2.exp_), rhs.exp_);
+    for (int i = num_rhs_bigits - 1; i >= min_exp; --i) {
+      double_bigit sum =
+          static_cast<double_bigit>(get_bigit(lhs1, i)) + get_bigit(lhs2, i);
+      bigit rhs_bigit = get_bigit(rhs, i);
+      if (sum > rhs_bigit + borrow) return 1;
+      borrow = rhs_bigit + borrow - sum;
+      if (borrow > 1) return -1;
+      borrow <<= bigit_bits;
+    }
+    return borrow != 0 ? -1 : 0;
+  }
+
+  // Assigns pow(10, exp) to this bigint.
+  FMT_CONSTEXPR20 void assign_pow10(int exp) {
+    FMT_ASSERT(exp >= 0, "");
+    if (exp == 0) return *this = 1;
+    // Find the top bit.
+    int bitmask = 1;
+    while (exp >= bitmask) bitmask <<= 1;
+    bitmask >>= 1;
+    // pow(10, exp) = pow(5, exp) * pow(2, exp). First compute pow(5, exp) by
+    // repeated squaring and multiplication.
+    *this = 5;
+    bitmask >>= 1;
+    while (bitmask != 0) {
+      square();
+      if ((exp & bitmask) != 0) *this *= 5;
+      bitmask >>= 1;
+    }
+    *this <<= exp;  // Multiply by pow(2, exp) by shifting.
+  }
+
+  FMT_CONSTEXPR20 void square() {
+    int num_bigits = static_cast<int>(bigits_.size());
+    int num_result_bigits = 2 * num_bigits;
+    basic_memory_buffer<bigit, bigits_capacity> n(std::move(bigits_));
+    bigits_.resize(to_unsigned(num_result_bigits));
+    auto sum = uint128_t();
+    for (int bigit_index = 0; bigit_index < num_bigits; ++bigit_index) {
+      // Compute bigit at position bigit_index of the result by adding
+      // cross-product terms n[i] * n[j] such that i + j == bigit_index.
+      for (int i = 0, j = bigit_index; j >= 0; ++i, --j) {
+        // Most terms are multiplied twice which can be optimized in the future.
+        sum += static_cast<double_bigit>(n[i]) * n[j];
+      }
+      (*this)[bigit_index] = static_cast<bigit>(sum);
+      sum >>= num_bits<bigit>();  // Compute the carry.
+    }
+    // Do the same for the top half.
+    for (int bigit_index = num_bigits; bigit_index < num_result_bigits;
+         ++bigit_index) {
+      for (int j = num_bigits - 1, i = bigit_index - j; i < num_bigits;)
+        sum += static_cast<double_bigit>(n[i++]) * n[j--];
+      (*this)[bigit_index] = static_cast<bigit>(sum);
+      sum >>= num_bits<bigit>();
+    }
+    remove_leading_zeros();
+    exp_ *= 2;
+  }
+
+  // If this bigint has a bigger exponent than other, adds trailing zero to make
+  // exponents equal. This simplifies some operations such as subtraction.
+  FMT_CONSTEXPR20 void align(const bigint& other) {
+    int exp_difference = exp_ - other.exp_;
+    if (exp_difference <= 0) return;
+    int num_bigits = static_cast<int>(bigits_.size());
+    bigits_.resize(to_unsigned(num_bigits + exp_difference));
+    for (int i = num_bigits - 1, j = i + exp_difference; i >= 0; --i, --j)
+      bigits_[j] = bigits_[i];
+    memset(bigits_.data(), 0, to_unsigned(exp_difference) * sizeof(bigit));
+    exp_ -= exp_difference;
+  }
+
+  // Divides this bignum by divisor, assigning the remainder to this and
+  // returning the quotient.
+  FMT_CONSTEXPR20 auto divmod_assign(const bigint& divisor) -> int {
+    FMT_ASSERT(this != &divisor, "");
+    if (compare(*this, divisor) < 0) return 0;
+    FMT_ASSERT(divisor.bigits_[divisor.bigits_.size() - 1u] != 0, "");
+    align(divisor);
+    int quotient = 0;
+    do {
+      subtract_aligned(divisor);
+      ++quotient;
+    } while (compare(*this, divisor) >= 0);
+    return quotient;
+  }
+};
+
+// format_dragon flags.
+enum dragon {
+  predecessor_closer = 1,
+  fixup = 2,  // Run fixup to correct exp10 which can be off by one.
+  fixed = 4,
+};
+
+// Formats a floating-point number using a variation of the Fixed-Precision
+// Positive Floating-Point Printout ((FPP)^2) algorithm by Steele & White:
+// https://fmt.dev/papers/p372-steele.pdf.
+FMT_CONSTEXPR20 inline void format_dragon(basic_fp<uint128_t> value,
+                                          unsigned flags, int num_digits,
+                                          buffer<char>& buf, int& exp10) {
+  bigint numerator;    // 2 * R in (FPP)^2.
+  bigint denominator;  // 2 * S in (FPP)^2.
+  // lower and upper are differences between value and corresponding boundaries.
+  bigint lower;             // (M^- in (FPP)^2).
+  bigint upper_store;       // upper's value if different from lower.
+  bigint* upper = nullptr;  // (M^+ in (FPP)^2).
+  // Shift numerator and denominator by an extra bit or two (if lower boundary
+  // is closer) to make lower and upper integers. This eliminates multiplication
+  // by 2 during later computations.
+  bool is_predecessor_closer = (flags & dragon::predecessor_closer) != 0;
+  int shift = is_predecessor_closer ? 2 : 1;
+  if (value.e >= 0) {
+    numerator = value.f;
+    numerator <<= value.e + shift;
+    lower = 1;
+    lower <<= value.e;
+    if (is_predecessor_closer) {
+      upper_store = 1;
+      upper_store <<= value.e + 1;
+      upper = &upper_store;
+    }
+    denominator.assign_pow10(exp10);
+    denominator <<= shift;
+  } else if (exp10 < 0) {
+    numerator.assign_pow10(-exp10);
+    lower.assign(numerator);
+    if (is_predecessor_closer) {
+      upper_store.assign(numerator);
+      upper_store <<= 1;
+      upper = &upper_store;
+    }
+    numerator *= value.f;
+    numerator <<= shift;
+    denominator = 1;
+    denominator <<= shift - value.e;
+  } else {
+    numerator = value.f;
+    numerator <<= shift;
+    denominator.assign_pow10(exp10);
+    denominator <<= shift - value.e;
+    lower = 1;
+    if (is_predecessor_closer) {
+      upper_store = 1ULL << 1;
+      upper = &upper_store;
+    }
+  }
+  int even = static_cast<int>((value.f & 1) == 0);
+  if (!upper) upper = &lower;
+  bool shortest = num_digits < 0;
+  if ((flags & dragon::fixup) != 0) {
+    if (add_compare(numerator, *upper, denominator) + even <= 0) {
+      --exp10;
+      numerator *= 10;
+      if (num_digits < 0) {
+        lower *= 10;
+        if (upper != &lower) *upper *= 10;
+      }
+    }
+    if ((flags & dragon::fixed) != 0) adjust_precision(num_digits, exp10 + 1);
+  }
+  // Invariant: value == (numerator / denominator) * pow(10, exp10).
+  if (shortest) {
+    // Generate the shortest representation.
+    num_digits = 0;
+    char* data = buf.data();
+    for (;;) {
+      int digit = numerator.divmod_assign(denominator);
+      bool low = compare(numerator, lower) - even < 0;  // numerator <[=] lower.
+      // numerator + upper >[=] pow10:
+      bool high = add_compare(numerator, *upper, denominator) + even > 0;
+      data[num_digits++] = static_cast<char>('0' + digit);
+      if (low || high) {
+        if (!low) {
+          ++data[num_digits - 1];
+        } else if (high) {
+          int result = add_compare(numerator, numerator, denominator);
+          // Round half to even.
+          if (result > 0 || (result == 0 && (digit % 2) != 0))
+            ++data[num_digits - 1];
+        }
+        buf.try_resize(to_unsigned(num_digits));
+        exp10 -= num_digits - 1;
+        return;
+      }
+      numerator *= 10;
+      lower *= 10;
+      if (upper != &lower) *upper *= 10;
+    }
+  }
+  // Generate the given number of digits.
+  exp10 -= num_digits - 1;
+  if (num_digits <= 0) {
+    auto digit = '0';
+    if (num_digits == 0) {
+      denominator *= 10;
+      digit = add_compare(numerator, numerator, denominator) > 0 ? '1' : '0';
+    }
+    buf.push_back(digit);
+    return;
+  }
+  buf.try_resize(to_unsigned(num_digits));
+  for (int i = 0; i < num_digits - 1; ++i) {
+    int digit = numerator.divmod_assign(denominator);
+    buf[i] = static_cast<char>('0' + digit);
+    numerator *= 10;
+  }
+  int digit = numerator.divmod_assign(denominator);
+  auto result = add_compare(numerator, numerator, denominator);
+  if (result > 0 || (result == 0 && (digit % 2) != 0)) {
+    if (digit == 9) {
+      const auto overflow = '0' + 10;
+      buf[num_digits - 1] = overflow;
+      // Propagate the carry.
+      for (int i = num_digits - 1; i > 0 && buf[i] == overflow; --i) {
+        buf[i] = '0';
+        ++buf[i - 1];
+      }
+      if (buf[0] == overflow) {
+        buf[0] = '1';
+        if ((flags & dragon::fixed) != 0)
+          buf.push_back('0');
+        else
+          ++exp10;
+      }
+      return;
+    }
+    ++digit;
+  }
+  buf[num_digits - 1] = static_cast<char>('0' + digit);
+}
+
+// Formats a floating-point number using the hexfloat format.
+template <typename Float, FMT_ENABLE_IF(!is_double_double<Float>::value)>
+FMT_CONSTEXPR20 void format_hexfloat(Float value, format_specs specs,
+                                     buffer<char>& buf) {
+  // float is passed as double to reduce the number of instantiations and to
+  // simplify implementation.
+  static_assert(!std::is_same<Float, float>::value, "");
+
+  using info = dragonbox::float_info<Float>;
+
+  // Assume Float is in the format [sign][exponent][significand].
+  using carrier_uint = typename info::carrier_uint;
+
+  constexpr auto num_float_significand_bits =
+      detail::num_significand_bits<Float>();
+
+  basic_fp<carrier_uint> f(value);
+  f.e += num_float_significand_bits;
+  if (!has_implicit_bit<Float>()) --f.e;
+
+  constexpr auto num_fraction_bits =
+      num_float_significand_bits + (has_implicit_bit<Float>() ? 1 : 0);
+  constexpr auto num_xdigits = (num_fraction_bits + 3) / 4;
+
+  constexpr auto leading_shift = ((num_xdigits - 1) * 4);
+  const auto leading_mask = carrier_uint(0xF) << leading_shift;
+  const auto leading_xdigit =
+      static_cast<uint32_t>((f.f & leading_mask) >> leading_shift);
+  if (leading_xdigit > 1) f.e -= (32 - countl_zero(leading_xdigit) - 1);
+
+  int print_xdigits = num_xdigits - 1;
+  if (specs.precision >= 0 && print_xdigits > specs.precision) {
+    const int shift = ((print_xdigits - specs.precision - 1) * 4);
+    const auto mask = carrier_uint(0xF) << shift;
+    const auto v = static_cast<uint32_t>((f.f & mask) >> shift);
+
+    if (v >= 8) {
+      const auto inc = carrier_uint(1) << (shift + 4);
+      f.f += inc;
+      f.f &= ~(inc - 1);
+    }
+
+    // Check long double overflow
+    if (!has_implicit_bit<Float>()) {
+      const auto implicit_bit = carrier_uint(1) << num_float_significand_bits;
+      if ((f.f & implicit_bit) == implicit_bit) {
+        f.f >>= 4;
+        f.e += 4;
+      }
+    }
+
+    print_xdigits = specs.precision;
+  }
+
+  char xdigits[num_bits<carrier_uint>() / 4];
+  detail::fill_n(xdigits, sizeof(xdigits), '0');
+  format_uint<4>(xdigits, f.f, num_xdigits, specs.upper);
+
+  // Remove zero tail
+  while (print_xdigits > 0 && xdigits[print_xdigits] == '0') --print_xdigits;
+
+  buf.push_back('0');
+  buf.push_back(specs.upper ? 'X' : 'x');
+  buf.push_back(xdigits[0]);
+  if (specs.alt || print_xdigits > 0 || print_xdigits < specs.precision)
+    buf.push_back('.');
+  buf.append(xdigits + 1, xdigits + 1 + print_xdigits);
+  for (; print_xdigits < specs.precision; ++print_xdigits) buf.push_back('0');
+
+  buf.push_back(specs.upper ? 'P' : 'p');
+
+  uint32_t abs_e;
+  if (f.e < 0) {
+    buf.push_back('-');
+    abs_e = static_cast<uint32_t>(-f.e);
+  } else {
+    buf.push_back('+');
+    abs_e = static_cast<uint32_t>(f.e);
+  }
+  format_decimal<char>(appender(buf), abs_e, detail::count_digits(abs_e));
+}
+
+template <typename Float, FMT_ENABLE_IF(is_double_double<Float>::value)>
+FMT_CONSTEXPR20 void format_hexfloat(Float value, format_specs specs,
+                                     buffer<char>& buf) {
+  format_hexfloat(static_cast<double>(value), specs, buf);
+}
+
+constexpr auto fractional_part_rounding_thresholds(int index) -> uint32_t {
+  // For checking rounding thresholds.
+  // The kth entry is chosen to be the smallest integer such that the
+  // upper 32-bits of 10^(k+1) times it is strictly bigger than 5 * 10^k.
+  // It is equal to ceil(2^31 + 2^32/10^(k + 1)).
+  // These are stored in a string literal because we cannot have static arrays
+  // in constexpr functions and non-static ones are poorly optimized.
+  return U"\x9999999a\x828f5c29\x80418938\x80068db9\x8000a7c6\x800010c7"
+         U"\x800001ae\x8000002b"[index];
+}
+
+template <typename Float>
+FMT_CONSTEXPR20 auto format_float(Float value, int precision, float_specs specs,
+                                  buffer<char>& buf) -> int {
+  // float is passed as double to reduce the number of instantiations.
+  static_assert(!std::is_same<Float, float>::value, "");
+  FMT_ASSERT(value >= 0, "value is negative");
+  auto converted_value = convert_float(value);
+
+  const bool fixed = specs.format == float_format::fixed;
+  if (value <= 0) {  // <= instead of == to silence a warning.
+    if (precision <= 0 || !fixed) {
+      buf.push_back('0');
+      return 0;
+    }
+    buf.try_resize(to_unsigned(precision));
+    fill_n(buf.data(), precision, '0');
+    return -precision;
+  }
+
+  int exp = 0;
+  bool use_dragon = true;
+  unsigned dragon_flags = 0;
+  if (!is_fast_float<Float>() || is_constant_evaluated()) {
+    const auto inv_log2_10 = 0.3010299956639812;  // 1 / log2(10)
+    using info = dragonbox::float_info<decltype(converted_value)>;
+    const auto f = basic_fp<typename info::carrier_uint>(converted_value);
+    // Compute exp, an approximate power of 10, such that
+    //   10^(exp - 1) <= value < 10^exp or 10^exp <= value < 10^(exp + 1).
+    // This is based on log10(value) == log2(value) / log2(10) and approximation
+    // of log2(value) by e + num_fraction_bits idea from double-conversion.
+    auto e = (f.e + count_digits<1>(f.f) - 1) * inv_log2_10 - 1e-10;
+    exp = static_cast<int>(e);
+    if (e > exp) ++exp;  // Compute ceil.
+    dragon_flags = dragon::fixup;
+  } else if (precision < 0) {
+    // Use Dragonbox for the shortest format.
+    if (specs.binary32) {
+      auto dec = dragonbox::to_decimal(static_cast<float>(value));
+      write<char>(appender(buf), dec.significand);
+      return dec.exponent;
+    }
+    auto dec = dragonbox::to_decimal(static_cast<double>(value));
+    write<char>(appender(buf), dec.significand);
+    return dec.exponent;
+  } else {
+    // Extract significand bits and exponent bits.
+    using info = dragonbox::float_info<double>;
+    auto br = bit_cast<uint64_t>(static_cast<double>(value));
+
+    const uint64_t significand_mask =
+        (static_cast<uint64_t>(1) << num_significand_bits<double>()) - 1;
+    uint64_t significand = (br & significand_mask);
+    int exponent = static_cast<int>((br & exponent_mask<double>()) >>
+                                    num_significand_bits<double>());
+
+    if (exponent != 0) {  // Check if normal.
+      exponent -= exponent_bias<double>() + num_significand_bits<double>();
+      significand |=
+          (static_cast<uint64_t>(1) << num_significand_bits<double>());
+      significand <<= 1;
+    } else {
+      // Normalize subnormal inputs.
+      FMT_ASSERT(significand != 0, "zeros should not appear here");
+      int shift = countl_zero(significand);
+      FMT_ASSERT(shift >= num_bits<uint64_t>() - num_significand_bits<double>(),
+                 "");
+      shift -= (num_bits<uint64_t>() - num_significand_bits<double>() - 2);
+      exponent = (std::numeric_limits<double>::min_exponent -
+                  num_significand_bits<double>()) -
+                 shift;
+      significand <<= shift;
+    }
+
+    // Compute the first several nonzero decimal significand digits.
+    // We call the number we get the first segment.
+    const int k = info::kappa - dragonbox::floor_log10_pow2(exponent);
+    exp = -k;
+    const int beta = exponent + dragonbox::floor_log2_pow10(k);
+    uint64_t first_segment;
+    bool has_more_segments;
+    int digits_in_the_first_segment;
+    {
+      const auto r = dragonbox::umul192_upper128(
+          significand << beta, dragonbox::get_cached_power(k));
+      first_segment = r.high();
+      has_more_segments = r.low() != 0;
+
+      // The first segment can have 18 ~ 19 digits.
+      if (first_segment >= 1000000000000000000ULL) {
+        digits_in_the_first_segment = 19;
+      } else {
+        // When it is of 18-digits, we align it to 19-digits by adding a bogus
+        // zero at the end.
+        digits_in_the_first_segment = 18;
+        first_segment *= 10;
+      }
+    }
+
+    // Compute the actual number of decimal digits to print.
+    if (fixed) adjust_precision(precision, exp + digits_in_the_first_segment);
+
+    // Use Dragon4 only when there might be not enough digits in the first
+    // segment.
+    if (digits_in_the_first_segment > precision) {
+      use_dragon = false;
+
+      if (precision <= 0) {
+        exp += digits_in_the_first_segment;
+
+        if (precision < 0) {
+          // Nothing to do, since all we have are just leading zeros.
+          buf.try_resize(0);
+        } else {
+          // We may need to round-up.
+          buf.try_resize(1);
+          if ((first_segment | static_cast<uint64_t>(has_more_segments)) >
+              5000000000000000000ULL) {
+            buf[0] = '1';
+          } else {
+            buf[0] = '0';
+          }
+        }
+      }  // precision <= 0
+      else {
+        exp += digits_in_the_first_segment - precision;
+
+        // When precision > 0, we divide the first segment into three
+        // subsegments, each with 9, 9, and 0 ~ 1 digits so that each fits
+        // in 32-bits which usually allows faster calculation than in
+        // 64-bits. Since some compiler (e.g. MSVC) doesn't know how to optimize
+        // division-by-constant for large 64-bit divisors, we do it here
+        // manually. The magic number 7922816251426433760 below is equal to
+        // ceil(2^(64+32) / 10^10).
+        const uint32_t first_subsegment = static_cast<uint32_t>(
+            dragonbox::umul128_upper64(first_segment, 7922816251426433760ULL) >>
+            32);
+        const uint64_t second_third_subsegments =
+            first_segment - first_subsegment * 10000000000ULL;
+
+        uint64_t prod;
+        uint32_t digits;
+        bool should_round_up;
+        int number_of_digits_to_print = precision > 9 ? 9 : precision;
+
+        // Print a 9-digits subsegment, either the first or the second.
+        auto print_subsegment = [&](uint32_t subsegment, char* buffer) {
+          int number_of_digits_printed = 0;
+
+          // If we want to print an odd number of digits from the subsegment,
+          if ((number_of_digits_to_print & 1) != 0) {
+            // Convert to 64-bit fixed-point fractional form with 1-digit
+            // integer part. The magic number 720575941 is a good enough
+            // approximation of 2^(32 + 24) / 10^8; see
+            // https://jk-jeon.github.io/posts/2022/12/fixed-precision-formatting/#fixed-length-case
+            // for details.
+            prod = ((subsegment * static_cast<uint64_t>(720575941)) >> 24) + 1;
+            digits = static_cast<uint32_t>(prod >> 32);
+            *buffer = static_cast<char>('0' + digits);
+            number_of_digits_printed++;
+          }
+          // If we want to print an even number of digits from the
+          // first_subsegment,
+          else {
+            // Convert to 64-bit fixed-point fractional form with 2-digits
+            // integer part. The magic number 450359963 is a good enough
+            // approximation of 2^(32 + 20) / 10^7; see
+            // https://jk-jeon.github.io/posts/2022/12/fixed-precision-formatting/#fixed-length-case
+            // for details.
+            prod = ((subsegment * static_cast<uint64_t>(450359963)) >> 20) + 1;
+            digits = static_cast<uint32_t>(prod >> 32);
+            copy2(buffer, digits2(digits));
+            number_of_digits_printed += 2;
+          }
+
+          // Print all digit pairs.
+          while (number_of_digits_printed < number_of_digits_to_print) {
+            prod = static_cast<uint32_t>(prod) * static_cast<uint64_t>(100);
+            digits = static_cast<uint32_t>(prod >> 32);
+            copy2(buffer + number_of_digits_printed, digits2(digits));
+            number_of_digits_printed += 2;
+          }
+        };
+
+        // Print first subsegment.
+        print_subsegment(first_subsegment, buf.data());
+
+        // Perform rounding if the first subsegment is the last subsegment to
+        // print.
+        if (precision <= 9) {
+          // Rounding inside the subsegment.
+          // We round-up if:
+          //  - either the fractional part is strictly larger than 1/2, or
+          //  - the fractional part is exactly 1/2 and the last digit is odd.
+          // We rely on the following observations:
+          //  - If fractional_part >= threshold, then the fractional part is
+          //    strictly larger than 1/2.
+          //  - If the MSB of fractional_part is set, then the fractional part
+          //    must be at least 1/2.
+          //  - When the MSB of fractional_part is set, either
+          //    second_third_subsegments being nonzero or has_more_segments
+          //    being true means there are further digits not printed, so the
+          //    fractional part is strictly larger than 1/2.
+          if (precision < 9) {
+            uint32_t fractional_part = static_cast<uint32_t>(prod);
+            should_round_up =
+                fractional_part >= fractional_part_rounding_thresholds(
+                                       8 - number_of_digits_to_print) ||
+                ((fractional_part >> 31) &
+                 ((digits & 1) | (second_third_subsegments != 0) |
+                  has_more_segments)) != 0;
+          }
+          // Rounding at the subsegment boundary.
+          // In this case, the fractional part is at least 1/2 if and only if
+          // second_third_subsegments >= 5000000000ULL, and is strictly larger
+          // than 1/2 if we further have either second_third_subsegments >
+          // 5000000000ULL or has_more_segments == true.
+          else {
+            should_round_up = second_third_subsegments > 5000000000ULL ||
+                              (second_third_subsegments == 5000000000ULL &&
+                               ((digits & 1) != 0 || has_more_segments));
+          }
+        }
+        // Otherwise, print the second subsegment.
+        else {
+          // Compilers are not aware of how to leverage the maximum value of
+          // second_third_subsegments to find out a better magic number which
+          // allows us to eliminate an additional shift. 1844674407370955162 =
+          // ceil(2^64/10) < ceil(2^64*(10^9/(10^10 - 1))).
+          const uint32_t second_subsegment =
+              static_cast<uint32_t>(dragonbox::umul128_upper64(
+                  second_third_subsegments, 1844674407370955162ULL));
+          const uint32_t third_subsegment =
+              static_cast<uint32_t>(second_third_subsegments) -
+              second_subsegment * 10;
+
+          number_of_digits_to_print = precision - 9;
+          print_subsegment(second_subsegment, buf.data() + 9);
+
+          // Rounding inside the subsegment.
+          if (precision < 18) {
+            // The condition third_subsegment != 0 implies that the segment was
+            // of 19 digits, so in this case the third segment should be
+            // consisting of a genuine digit from the input.
+            uint32_t fractional_part = static_cast<uint32_t>(prod);
+            should_round_up =
+                fractional_part >= fractional_part_rounding_thresholds(
+                                       8 - number_of_digits_to_print) ||
+                ((fractional_part >> 31) &
+                 ((digits & 1) | (third_subsegment != 0) |
+                  has_more_segments)) != 0;
+          }
+          // Rounding at the subsegment boundary.
+          else {
+            // In this case, the segment must be of 19 digits, thus
+            // the third subsegment should be consisting of a genuine digit from
+            // the input.
+            should_round_up = third_subsegment > 5 ||
+                              (third_subsegment == 5 &&
+                               ((digits & 1) != 0 || has_more_segments));
+          }
+        }
+
+        // Round-up if necessary.
+        if (should_round_up) {
+          ++buf[precision - 1];
+          for (int i = precision - 1; i > 0 && buf[i] > '9'; --i) {
+            buf[i] = '0';
+            ++buf[i - 1];
+          }
+          if (buf[0] > '9') {
+            buf[0] = '1';
+            if (fixed)
+              buf[precision++] = '0';
+            else
+              ++exp;
+          }
+        }
+        buf.try_resize(to_unsigned(precision));
+      }
+    }  // if (digits_in_the_first_segment > precision)
+    else {
+      // Adjust the exponent for its use in Dragon4.
+      exp += digits_in_the_first_segment - 1;
+    }
+  }
+  if (use_dragon) {
+    auto f = basic_fp<uint128_t>();
+    bool is_predecessor_closer = specs.binary32
+                                     ? f.assign(static_cast<float>(value))
+                                     : f.assign(converted_value);
+    if (is_predecessor_closer) dragon_flags |= dragon::predecessor_closer;
+    if (fixed) dragon_flags |= dragon::fixed;
+    // Limit precision to the maximum possible number of significant digits in
+    // an IEEE754 double because we don't need to generate zeros.
+    const int max_double_digits = 767;
+    if (precision > max_double_digits) precision = max_double_digits;
+    format_dragon(f, dragon_flags, precision, buf, exp);
+  }
+  if (!fixed && !specs.showpoint) {
+    // Remove trailing zeros.
+    auto num_digits = buf.size();
+    while (num_digits > 0 && buf[num_digits - 1] == '0') {
+      --num_digits;
+      ++exp;
+    }
+    buf.try_resize(num_digits);
+  }
+  return exp;
+}
+
+template <typename Char, typename OutputIt, typename T>
+FMT_CONSTEXPR20 auto write_float(OutputIt out, T value, format_specs specs,
+                                 locale_ref loc) -> OutputIt {
+  sign_t sign = specs.sign;
+  if (detail::signbit(value)) {  // value < 0 is false for NaN so use signbit.
+    sign = sign::minus;
+    value = -value;
+  } else if (sign == sign::minus) {
+    sign = sign::none;
+  }
+
+  if (!detail::isfinite(value))
+    return write_nonfinite<Char>(out, detail::isnan(value), specs, sign);
+
+  if (specs.align == align::numeric && sign) {
+    auto it = reserve(out, 1);
+    *it++ = detail::sign<Char>(sign);
+    out = base_iterator(out, it);
+    sign = sign::none;
+    if (specs.width != 0) --specs.width;
+  }
+
+  memory_buffer buffer;
+  if (specs.type == presentation_type::hexfloat) {
+    if (sign) buffer.push_back(detail::sign<char>(sign));
+    format_hexfloat(convert_float(value), specs, buffer);
+    return write_bytes<Char, align::right>(out, {buffer.data(), buffer.size()},
+                                           specs);
+  }
+
+  int precision = specs.precision >= 0 || specs.type == presentation_type::none
+                      ? specs.precision
+                      : 6;
+  if (specs.type == presentation_type::exp) {
+    if (precision == max_value<int>())
+      report_error("number is too big");
+    else
+      ++precision;
+  } else if (specs.type != presentation_type::fixed && precision == 0) {
+    precision = 1;
+  }
+  float_specs fspecs = parse_float_type_spec(specs);
+  fspecs.sign = sign;
+  if (const_check(std::is_same<T, float>())) fspecs.binary32 = true;
+  int exp = format_float(convert_float(value), precision, fspecs, buffer);
+  fspecs.precision = precision;
+  auto f = big_decimal_fp{buffer.data(), static_cast<int>(buffer.size()), exp};
+  return write_float<Char>(out, f, specs, fspecs, loc);
+}
+
+template <typename Char, typename OutputIt, typename T,
+          FMT_ENABLE_IF(is_floating_point<T>::value)>
+FMT_CONSTEXPR20 auto write(OutputIt out, T value, format_specs specs,
+                           locale_ref loc = {}) -> OutputIt {
+  if (const_check(!is_supported_floating_point(value))) return out;
+  return specs.localized && write_loc(out, value, specs, loc)
+             ? out
+             : write_float<Char>(out, value, specs, loc);
+}
+
+template <typename Char, typename OutputIt, typename T,
+          FMT_ENABLE_IF(is_fast_float<T>::value)>
+FMT_CONSTEXPR20 auto write(OutputIt out, T value) -> OutputIt {
+  if (is_constant_evaluated()) return write<Char>(out, value, format_specs());
+  if (const_check(!is_supported_floating_point(value))) return out;
+
+  auto sign = sign_t::none;
+  if (detail::signbit(value)) {
+    sign = sign::minus;
+    value = -value;
+  }
+
+  constexpr auto specs = format_specs();
+  using floaty = conditional_t<std::is_same<T, long double>::value, double, T>;
+  using floaty_uint = typename dragonbox::float_info<floaty>::carrier_uint;
+  floaty_uint mask = exponent_mask<floaty>();
+  if ((bit_cast<floaty_uint>(value) & mask) == mask)
+    return write_nonfinite<Char>(out, std::isnan(value), specs, sign);
+
+  auto fspecs = float_specs();
+  fspecs.sign = sign;
+  auto dec = dragonbox::to_decimal(static_cast<floaty>(value));
+  return write_float<Char>(out, dec, specs, fspecs, {});
+}
+
+template <typename Char, typename OutputIt, typename T,
+          FMT_ENABLE_IF(is_floating_point<T>::value &&
+                        !is_fast_float<T>::value)>
+inline auto write(OutputIt out, T value) -> OutputIt {
+  return write<Char>(out, value, format_specs());
+}
+
+template <typename Char, typename OutputIt>
+auto write(OutputIt out, monostate, format_specs = {}, locale_ref = {})
+    -> OutputIt {
+  FMT_ASSERT(false, "");
+  return out;
+}
+
+template <typename Char, typename OutputIt>
+FMT_CONSTEXPR auto write(OutputIt out, basic_string_view<Char> value)
+    -> OutputIt {
+  return copy_noinline<Char>(value.begin(), value.end(), out);
+}
+
+template <typename Char, typename OutputIt, typename T,
+          FMT_ENABLE_IF(has_to_string_view<T>::value)>
+constexpr auto write(OutputIt out, const T& value) -> OutputIt {
+  return write<Char>(out, to_string_view(value));
+}
+
+// FMT_ENABLE_IF() condition separated to workaround an MSVC bug.
+template <
+    typename Char, typename OutputIt, typename T,
+    bool check =
+        std::is_enum<T>::value && !std::is_same<T, Char>::value &&
+        mapped_type_constant<T, basic_format_context<OutputIt, Char>>::value !=
+            type::custom_type,
+    FMT_ENABLE_IF(check)>
+FMT_CONSTEXPR auto write(OutputIt out, T value) -> OutputIt {
+  return write<Char>(out, static_cast<underlying_t<T>>(value));
+}
+
+template <typename Char, typename OutputIt, typename T,
+          FMT_ENABLE_IF(std::is_same<T, bool>::value)>
+FMT_CONSTEXPR auto write(OutputIt out, T value, const format_specs& specs = {},
+                         locale_ref = {}) -> OutputIt {
+  return specs.type != presentation_type::none &&
+                 specs.type != presentation_type::string
+             ? write<Char>(out, value ? 1 : 0, specs, {})
+             : write_bytes<Char>(out, value ? "true" : "false", specs);
+}
+
+template <typename Char, typename OutputIt>
+FMT_CONSTEXPR auto write(OutputIt out, Char value) -> OutputIt {
+  auto it = reserve(out, 1);
+  *it++ = value;
+  return base_iterator(out, it);
+}
+
+template <typename Char, typename OutputIt>
+FMT_CONSTEXPR20 auto write(OutputIt out, const Char* value) -> OutputIt {
+  if (value) return write(out, basic_string_view<Char>(value));
+  report_error("string pointer is null");
+  return out;
+}
+
+template <typename Char, typename OutputIt, typename T,
+          FMT_ENABLE_IF(std::is_same<T, void>::value)>
+auto write(OutputIt out, const T* value, const format_specs& specs = {},
+           locale_ref = {}) -> OutputIt {
+  return write_ptr<Char>(out, bit_cast<uintptr_t>(value), &specs);
+}
+
+// A write overload that handles implicit conversions.
+template <typename Char, typename OutputIt, typename T,
+          typename Context = basic_format_context<OutputIt, Char>>
+FMT_CONSTEXPR auto write(OutputIt out, const T& value) -> enable_if_t<
+    std::is_class<T>::value && !has_to_string_view<T>::value &&
+        !is_floating_point<T>::value && !std::is_same<T, Char>::value &&
+        !std::is_same<T, remove_cvref_t<decltype(arg_mapper<Context>().map(
+                             value))>>::value,
+    OutputIt> {
+  return write<Char>(out, arg_mapper<Context>().map(value));
+}
+
+template <typename Char, typename OutputIt, typename T,
+          typename Context = basic_format_context<OutputIt, Char>>
+FMT_CONSTEXPR auto write(OutputIt out, const T& value)
+    -> enable_if_t<mapped_type_constant<T, Context>::value ==
+                           type::custom_type &&
+                       !std::is_fundamental<T>::value,
+                   OutputIt> {
+  auto formatter = typename Context::template formatter_type<T>();
+  auto parse_ctx = typename Context::parse_context_type({});
+  formatter.parse(parse_ctx);
+  auto ctx = Context(out, {}, {});
+  return formatter.format(value, ctx);
+}
+
+// An argument visitor that formats the argument and writes it via the output
+// iterator. It's a class and not a generic lambda for compatibility with C++11.
+template <typename Char> struct default_arg_formatter {
+  using iterator = basic_appender<Char>;
+  using context = buffered_context<Char>;
+
+  iterator out;
+  basic_format_args<context> args;
+  locale_ref loc;
+
+  template <typename T> auto operator()(T value) -> iterator {
+    return write<Char>(out, value);
+  }
+  auto operator()(typename basic_format_arg<context>::handle h) -> iterator {
+    basic_format_parse_context<Char> parse_ctx({});
+    context format_ctx(out, args, loc);
+    h.format(parse_ctx, format_ctx);
+    return format_ctx.out();
+  }
+};
+
+template <typename Char> struct arg_formatter {
+  using iterator = basic_appender<Char>;
+  using context = buffered_context<Char>;
+
+  iterator out;
+  const format_specs& specs;
+  locale_ref locale;
+
+  template <typename T>
+  FMT_CONSTEXPR FMT_INLINE auto operator()(T value) -> iterator {
+    return detail::write<Char>(out, value, specs, locale);
+  }
+  auto operator()(typename basic_format_arg<context>::handle) -> iterator {
+    // User-defined types are handled separately because they require access
+    // to the parse context.
+    return out;
+  }
+};
+
+struct width_checker {
+  template <typename T, FMT_ENABLE_IF(is_integer<T>::value)>
+  FMT_CONSTEXPR auto operator()(T value) -> unsigned long long {
+    if (is_negative(value)) report_error("negative width");
+    return static_cast<unsigned long long>(value);
+  }
+
+  template <typename T, FMT_ENABLE_IF(!is_integer<T>::value)>
+  FMT_CONSTEXPR auto operator()(T) -> unsigned long long {
+    report_error("width is not integer");
+    return 0;
+  }
+};
+
+struct precision_checker {
+  template <typename T, FMT_ENABLE_IF(is_integer<T>::value)>
+  FMT_CONSTEXPR auto operator()(T value) -> unsigned long long {
+    if (is_negative(value)) report_error("negative precision");
+    return static_cast<unsigned long long>(value);
+  }
+
+  template <typename T, FMT_ENABLE_IF(!is_integer<T>::value)>
+  FMT_CONSTEXPR auto operator()(T) -> unsigned long long {
+    report_error("precision is not integer");
+    return 0;
+  }
+};
+
+template <typename Handler, typename FormatArg>
+FMT_CONSTEXPR auto get_dynamic_spec(FormatArg arg) -> int {
+  unsigned long long value = arg.visit(Handler());
+  if (value > to_unsigned(max_value<int>())) report_error("number is too big");
+  return static_cast<int>(value);
+}
+
+template <typename Context, typename ID>
+FMT_CONSTEXPR auto get_arg(Context& ctx, ID id) -> decltype(ctx.arg(id)) {
+  auto arg = ctx.arg(id);
+  if (!arg) report_error("argument not found");
+  return arg;
+}
+
+template <typename Handler, typename Context>
+FMT_CONSTEXPR void handle_dynamic_spec(int& value,
+                                       arg_ref<typename Context::char_type> ref,
+                                       Context& ctx) {
+  switch (ref.kind) {
+  case arg_id_kind::none:
+    break;
+  case arg_id_kind::index:
+    value = detail::get_dynamic_spec<Handler>(get_arg(ctx, ref.val.index));
+    break;
+  case arg_id_kind::name:
+    value = detail::get_dynamic_spec<Handler>(get_arg(ctx, ref.val.name));
+    break;
+  }
+}
+
+#if FMT_USE_USER_DEFINED_LITERALS
+#  if FMT_USE_NONTYPE_TEMPLATE_ARGS
+template <typename T, typename Char, size_t N,
+          fmt::detail_exported::fixed_string<Char, N> Str>
+struct statically_named_arg : view {
+  static constexpr auto name = Str.data;
+
+  const T& value;
+  statically_named_arg(const T& v) : value(v) {}
+};
+
+template <typename T, typename Char, size_t N,
+          fmt::detail_exported::fixed_string<Char, N> Str>
+struct is_named_arg<statically_named_arg<T, Char, N, Str>> : std::true_type {};
+
+template <typename T, typename Char, size_t N,
+          fmt::detail_exported::fixed_string<Char, N> Str>
+struct is_statically_named_arg<statically_named_arg<T, Char, N, Str>>
+    : std::true_type {};
+
+template <typename Char, size_t N,
+          fmt::detail_exported::fixed_string<Char, N> Str>
+struct udl_arg {
+  template <typename T> auto operator=(T&& value) const {
+    return statically_named_arg<T, Char, N, Str>(std::forward<T>(value));
+  }
+};
+#  else
+template <typename Char> struct udl_arg {
+  const Char* str;
+
+  template <typename T> auto operator=(T&& value) const -> named_arg<Char, T> {
+    return {str, std::forward<T>(value)};
+  }
+};
+#  endif
+#endif  // FMT_USE_USER_DEFINED_LITERALS
+
+template <typename Locale, typename Char>
+auto vformat(const Locale& loc, basic_string_view<Char> fmt,
+             typename detail::vformat_args<Char>::type args)
+    -> std::basic_string<Char> {
+  auto buf = basic_memory_buffer<Char>();
+  detail::vformat_to(buf, fmt, args, detail::locale_ref(loc));
+  return {buf.data(), buf.size()};
+}
+
+using format_func = void (*)(detail::buffer<char>&, int, const char*);
+
+FMT_API void format_error_code(buffer<char>& out, int error_code,
+                               string_view message) noexcept;
+
+using fmt::report_error;
+FMT_API void report_error(format_func func, int error_code,
+                          const char* message) noexcept;
+}  // namespace detail
+
+FMT_BEGIN_EXPORT
+FMT_API auto vsystem_error(int error_code, string_view format_str,
+                           format_args args) -> std::system_error;
+
+/**
+ * Constructs `std::system_error` with a message formatted with
+ * `fmt::format(fmt, args...)`.
+ * `error_code` is a system error code as given by `errno`.
+ *
+ * **Example**:
+ *
+ *     // This throws std::system_error with the description
+ *     //   cannot open file 'madeup': No such file or directory
+ *     // or similar (system message may vary).
+ *     const char* filename = "madeup";
+ *     std::FILE* file = std::fopen(filename, "r");
+ *     if (!file)
+ *       throw fmt::system_error(errno, "cannot open file '{}'", filename);
+ */
+template <typename... T>
+auto system_error(int error_code, format_string<T...> fmt, T&&... args)
+    -> std::system_error {
+  return vsystem_error(error_code, fmt, fmt::make_format_args(args...));
+}
+
+/**
+ * Formats an error message for an error returned by an operating system or a
+ * language runtime, for example a file opening error, and writes it to `out`.
+ * The format is the same as the one used by `std::system_error(ec, message)`
+ * where `ec` is `std::error_code(error_code, std::generic_category())`.
+ * It is implementation-defined but normally looks like:
+ *
+ *     <message>: <system-message>
+ *
+ * where `<message>` is the passed message and `<system-message>` is the system
+ * message corresponding to the error code.
+ * `error_code` is a system error code as given by `errno`.
+ */
+FMT_API void format_system_error(detail::buffer<char>& out, int error_code,
+                                 const char* message) noexcept;
+
+// Reports a system error without throwing an exception.
+// Can be used to report errors from destructors.
+FMT_API void report_system_error(int error_code, const char* message) noexcept;
+
+/// A fast integer formatter.
+class format_int {
+ private:
+  // Buffer should be large enough to hold all digits (digits10 + 1),
+  // a sign and a null character.
+  enum { buffer_size = std::numeric_limits<unsigned long long>::digits10 + 3 };
+  mutable char buffer_[buffer_size];
+  char* str_;
+
+  template <typename UInt>
+  FMT_CONSTEXPR20 auto format_unsigned(UInt value) -> char* {
+    auto n = static_cast<detail::uint32_or_64_or_128_t<UInt>>(value);
+    return detail::format_decimal(buffer_, n, buffer_size - 1).begin;
+  }
+
+  template <typename Int>
+  FMT_CONSTEXPR20 auto format_signed(Int value) -> char* {
+    auto abs_value = static_cast<detail::uint32_or_64_or_128_t<Int>>(value);
+    bool negative = value < 0;
+    if (negative) abs_value = 0 - abs_value;
+    auto begin = format_unsigned(abs_value);
+    if (negative) *--begin = '-';
+    return begin;
+  }
+
+ public:
+  explicit FMT_CONSTEXPR20 format_int(int value) : str_(format_signed(value)) {}
+  explicit FMT_CONSTEXPR20 format_int(long value)
+      : str_(format_signed(value)) {}
+  explicit FMT_CONSTEXPR20 format_int(long long value)
+      : str_(format_signed(value)) {}
+  explicit FMT_CONSTEXPR20 format_int(unsigned value)
+      : str_(format_unsigned(value)) {}
+  explicit FMT_CONSTEXPR20 format_int(unsigned long value)
+      : str_(format_unsigned(value)) {}
+  explicit FMT_CONSTEXPR20 format_int(unsigned long long value)
+      : str_(format_unsigned(value)) {}
+
+  /// Returns the number of characters written to the output buffer.
+  FMT_CONSTEXPR20 auto size() const -> size_t {
+    return detail::to_unsigned(buffer_ - str_ + buffer_size - 1);
+  }
+
+  /// Returns a pointer to the output buffer content. No terminating null
+  /// character is appended.
+  FMT_CONSTEXPR20 auto data() const -> const char* { return str_; }
+
+  /// Returns a pointer to the output buffer content with terminating null
+  /// character appended.
+  FMT_CONSTEXPR20 auto c_str() const -> const char* {
+    buffer_[buffer_size - 1] = '\0';
+    return str_;
+  }
+
+  /// Returns the content of the output buffer as an `std::string`.
+  auto str() const -> std::string { return std::string(str_, size()); }
+};
+
+template <typename T, typename Char>
+struct formatter<T, Char, enable_if_t<detail::has_format_as<T>::value>>
+    : formatter<detail::format_as_t<T>, Char> {
+  template <typename FormatContext>
+  auto format(const T& value, FormatContext& ctx) const -> decltype(ctx.out()) {
+    auto&& val = format_as(value);  // Make an lvalue reference for format.
+    return formatter<detail::format_as_t<T>, Char>::format(val, ctx);
+  }
+};
+
+#define FMT_FORMAT_AS(Type, Base)                                              \
+  template <typename Char>                                                     \
+  struct formatter<Type, Char> : formatter<Base, Char> {                       \
+    template <typename FormatContext>                                          \
+    auto format(Type value, FormatContext& ctx) const -> decltype(ctx.out()) { \
+      return formatter<Base, Char>::format(value, ctx);                        \
+    }                                                                          \
+  }
+
+FMT_FORMAT_AS(signed char, int);
+FMT_FORMAT_AS(unsigned char, unsigned);
+FMT_FORMAT_AS(short, int);
+FMT_FORMAT_AS(unsigned short, unsigned);
+FMT_FORMAT_AS(long, detail::long_type);
+FMT_FORMAT_AS(unsigned long, detail::ulong_type);
+FMT_FORMAT_AS(Char*, const Char*);
+FMT_FORMAT_AS(std::nullptr_t, const void*);
+FMT_FORMAT_AS(detail::std_string_view<Char>, basic_string_view<Char>);
+FMT_FORMAT_AS(void*, const void*);
+
+template <typename Char, typename Traits, typename Allocator>
+class formatter<std::basic_string<Char, Traits, Allocator>, Char>
+    : public formatter<basic_string_view<Char>, Char> {};
+
+template <typename Char, size_t N>
+struct formatter<Char[N], Char> : formatter<basic_string_view<Char>, Char> {};
+
+/**
+ * Converts `p` to `const void*` for pointer formatting.
+ *
+ * **Example**:
+ *
+ *     auto s = fmt::format("{}", fmt::ptr(p));
+ */
+template <typename T> auto ptr(T p) -> const void* {
+  static_assert(std::is_pointer<T>::value, "");
+  return detail::bit_cast<const void*>(p);
+}
+
+/**
+ * Converts `e` to the underlying type.
+ *
+ * **Example**:
+ *
+ *     enum class color { red, green, blue };
+ *     auto s = fmt::format("{}", fmt::underlying(color::red));
+ */
+template <typename Enum>
+constexpr auto underlying(Enum e) noexcept -> underlying_t<Enum> {
+  return static_cast<underlying_t<Enum>>(e);
+}
+
+namespace enums {
+template <typename Enum, FMT_ENABLE_IF(std::is_enum<Enum>::value)>
+constexpr auto format_as(Enum e) noexcept -> underlying_t<Enum> {
+  return static_cast<underlying_t<Enum>>(e);
+}
+}  // namespace enums
+
+class bytes {
+ private:
+  string_view data_;
+  friend struct formatter<bytes>;
+
+ public:
+  explicit bytes(string_view data) : data_(data) {}
+};
+
+template <> struct formatter<bytes> {
+ private:
+  detail::dynamic_format_specs<> specs_;
+
+ public:
+  template <typename ParseContext>
+  FMT_CONSTEXPR auto parse(ParseContext& ctx) -> const char* {
+    return parse_format_specs(ctx.begin(), ctx.end(), specs_, ctx,
+                              detail::type::string_type);
+  }
+
+  template <typename FormatContext>
+  auto format(bytes b, FormatContext& ctx) const -> decltype(ctx.out()) {
+    auto specs = specs_;
+    detail::handle_dynamic_spec<detail::width_checker>(specs.width,
+                                                       specs.width_ref, ctx);
+    detail::handle_dynamic_spec<detail::precision_checker>(
+        specs.precision, specs.precision_ref, ctx);
+    return detail::write_bytes<char>(ctx.out(), b.data_, specs);
+  }
+};
+
+// group_digits_view is not derived from view because it copies the argument.
+template <typename T> struct group_digits_view {
+  T value;
+};
+
+/**
+ * Returns a view that formats an integer value using ',' as a
+ * locale-independent thousands separator.
+ *
+ * **Example**:
+ *
+ *     fmt::print("{}", fmt::group_digits(12345));
+ *     // Output: "12,345"
+ */
+template <typename T> auto group_digits(T value) -> group_digits_view<T> {
+  return {value};
+}
+
+template <typename T> struct formatter<group_digits_view<T>> : formatter<T> {
+ private:
+  detail::dynamic_format_specs<> specs_;
+
+ public:
+  template <typename ParseContext>
+  FMT_CONSTEXPR auto parse(ParseContext& ctx) -> const char* {
+    return parse_format_specs(ctx.begin(), ctx.end(), specs_, ctx,
+                              detail::type::int_type);
+  }
+
+  template <typename FormatContext>
+  auto format(group_digits_view<T> t, FormatContext& ctx) const
+      -> decltype(ctx.out()) {
+    auto specs = specs_;
+    detail::handle_dynamic_spec<detail::width_checker>(specs.width,
+                                                       specs.width_ref, ctx);
+    detail::handle_dynamic_spec<detail::precision_checker>(
+        specs.precision, specs.precision_ref, ctx);
+    auto arg = detail::make_write_int_arg(t.value, specs.sign);
+    return detail::write_int(
+        ctx.out(), static_cast<detail::uint64_or_128_t<T>>(arg.abs_value),
+        arg.prefix, specs, detail::digit_grouping<char>("\3", ","));
+  }
+};
+
+template <typename T, typename Char> struct nested_view {
+  const formatter<T, Char>* fmt;
+  const T* value;
+};
+
+template <typename T, typename Char>
+struct formatter<nested_view<T, Char>, Char> {
+  template <typename ParseContext>
+  FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
+    return ctx.begin();
+  }
+  template <typename FormatContext>
+  auto format(nested_view<T, Char> view, FormatContext& ctx) const
+      -> decltype(ctx.out()) {
+    return view.fmt->format(*view.value, ctx);
+  }
+};
+
+template <typename T, typename Char = char> struct nested_formatter {
+ private:
+  int width_;
+  detail::fill_t fill_;
+  align_t align_ : 4;
+  formatter<T, Char> formatter_;
+
+ public:
+  constexpr nested_formatter() : width_(0), align_(align_t::none) {}
+
+  FMT_CONSTEXPR auto parse(basic_format_parse_context<Char>& ctx)
+      -> decltype(ctx.begin()) {
+    auto specs = detail::dynamic_format_specs<Char>();
+    auto it = parse_format_specs(ctx.begin(), ctx.end(), specs, ctx,
+                                 detail::type::none_type);
+    width_ = specs.width;
+    fill_ = specs.fill;
+    align_ = specs.align;
+    ctx.advance_to(it);
+    return formatter_.parse(ctx);
+  }
+
+  template <typename FormatContext, typename F>
+  auto write_padded(FormatContext& ctx, F write) const -> decltype(ctx.out()) {
+    if (width_ == 0) return write(ctx.out());
+    auto buf = basic_memory_buffer<Char>();
+    write(basic_appender<Char>(buf));
+    auto specs = format_specs();
+    specs.width = width_;
+    specs.fill = fill_;
+    specs.align = align_;
+    return detail::write<Char>(
+        ctx.out(), basic_string_view<Char>(buf.data(), buf.size()), specs);
+  }
+
+  auto nested(const T& value) const -> nested_view<T, Char> {
+    return nested_view<T, Char>{&formatter_, &value};
+  }
+};
+
+/**
+ * Converts `value` to `std::string` using the default format for type `T`.
+ *
+ * **Example**:
+ *
+ *     std::string answer = fmt::to_string(42);
+ */
+template <typename T, FMT_ENABLE_IF(!std::is_integral<T>::value &&
+                                    !detail::has_format_as<T>::value)>
+inline auto to_string(const T& value) -> std::string {
+  auto buffer = memory_buffer();
+  detail::write<char>(appender(buffer), value);
+  return {buffer.data(), buffer.size()};
+}
+
+template <typename T, FMT_ENABLE_IF(std::is_integral<T>::value)>
+FMT_NODISCARD inline auto to_string(T value) -> std::string {
+  // The buffer should be large enough to store the number including the sign
+  // or "false" for bool.
+  constexpr int max_size = detail::digits10<T>() + 2;
+  char buffer[max_size > 5 ? static_cast<unsigned>(max_size) : 5];
+  char* begin = buffer;
+  return std::string(begin, detail::write<char>(begin, value));
+}
+
+template <typename Char, size_t SIZE>
+FMT_NODISCARD auto to_string(const basic_memory_buffer<Char, SIZE>& buf)
+    -> std::basic_string<Char> {
+  auto size = buf.size();
+  detail::assume(size < std::basic_string<Char>().max_size());
+  return std::basic_string<Char>(buf.data(), size);
+}
+
+template <typename T, FMT_ENABLE_IF(!std::is_integral<T>::value &&
+                                    detail::has_format_as<T>::value)>
+inline auto to_string(const T& value) -> std::string {
+  return to_string(format_as(value));
+}
+
+FMT_END_EXPORT
+
+namespace detail {
+
+template <typename Char>
+void vformat_to(buffer<Char>& buf, basic_string_view<Char> fmt,
+                typename vformat_args<Char>::type args, locale_ref loc) {
+  auto out = basic_appender<Char>(buf);
+  if (fmt.size() == 2 && equal2(fmt.data(), "{}")) {
+    auto arg = args.get(0);
+    if (!arg) report_error("argument not found");
+    arg.visit(default_arg_formatter<Char>{out, args, loc});
+    return;
+  }
+
+  struct format_handler {
+    basic_format_parse_context<Char> parse_context;
+    buffered_context<Char> context;
+
+    format_handler(basic_appender<Char> p_out, basic_string_view<Char> str,
+                   basic_format_args<buffered_context<Char>> p_args,
+                   locale_ref p_loc)
+        : parse_context(str), context(p_out, p_args, p_loc) {}
+
+    void on_text(const Char* begin, const Char* end) {
+      auto text = basic_string_view<Char>(begin, to_unsigned(end - begin));
+      context.advance_to(write<Char>(context.out(), text));
+    }
+
+    FMT_CONSTEXPR auto on_arg_id() -> int {
+      return parse_context.next_arg_id();
+    }
+    FMT_CONSTEXPR auto on_arg_id(int id) -> int {
+      parse_context.check_arg_id(id);
+      return id;
+    }
+    FMT_CONSTEXPR auto on_arg_id(basic_string_view<Char> id) -> int {
+      parse_context.check_arg_id(id);
+      int arg_id = context.arg_id(id);
+      if (arg_id < 0) report_error("argument not found");
+      return arg_id;
+    }
+
+    FMT_INLINE void on_replacement_field(int id, const Char*) {
+      auto arg = get_arg(context, id);
+      context.advance_to(arg.visit(default_arg_formatter<Char>{
+          context.out(), context.args(), context.locale()}));
+    }
+
+    auto on_format_specs(int id, const Char* begin, const Char* end)
+        -> const Char* {
+      auto arg = get_arg(context, id);
+      // Not using a visitor for custom types gives better codegen.
+      if (arg.format_custom(begin, parse_context, context))
+        return parse_context.begin();
+      auto specs = detail::dynamic_format_specs<Char>();
+      begin = parse_format_specs(begin, end, specs, parse_context, arg.type());
+      detail::handle_dynamic_spec<detail::width_checker>(
+          specs.width, specs.width_ref, context);
+      detail::handle_dynamic_spec<detail::precision_checker>(
+          specs.precision, specs.precision_ref, context);
+      if (begin == end || *begin != '}')
+        report_error("missing '}' in format string");
+      context.advance_to(arg.visit(
+          arg_formatter<Char>{context.out(), specs, context.locale()}));
+      return begin;
+    }
+
+    FMT_NORETURN void on_error(const char* message) { report_error(message); }
+  };
+  detail::parse_format_string<false>(fmt, format_handler(out, fmt, args, loc));
+}
+
+FMT_BEGIN_EXPORT
+
+#ifndef FMT_HEADER_ONLY
+extern template FMT_API void vformat_to(buffer<char>&, string_view,
+                                        typename vformat_args<>::type,
+                                        locale_ref);
+extern template FMT_API auto thousands_sep_impl<char>(locale_ref)
+    -> thousands_sep_result<char>;
+extern template FMT_API auto thousands_sep_impl<wchar_t>(locale_ref)
+    -> thousands_sep_result<wchar_t>;
+extern template FMT_API auto decimal_point_impl(locale_ref) -> char;
+extern template FMT_API auto decimal_point_impl(locale_ref) -> wchar_t;
+#endif  // FMT_HEADER_ONLY
+
+FMT_END_EXPORT
+
+template <typename T, typename Char, type TYPE>
+template <typename FormatContext>
+FMT_CONSTEXPR FMT_INLINE auto native_formatter<T, Char, TYPE>::format(
+    const T& val, FormatContext& ctx) const -> decltype(ctx.out()) {
+  if (specs_.width_ref.kind == arg_id_kind::none &&
+      specs_.precision_ref.kind == arg_id_kind::none) {
+    return write<Char>(ctx.out(), val, specs_, ctx.locale());
+  }
+  auto specs = specs_;
+  handle_dynamic_spec<width_checker>(specs.width, specs.width_ref, ctx);
+  handle_dynamic_spec<precision_checker>(specs.precision, specs.precision_ref,
+                                         ctx);
+  return write<Char>(ctx.out(), val, specs, ctx.locale());
+}
+
+}  // namespace detail
+
+FMT_BEGIN_EXPORT
+
+template <typename Char>
+struct formatter<detail::float128, Char>
+    : detail::native_formatter<detail::float128, Char,
+                               detail::type::float_type> {};
+
+#if FMT_USE_USER_DEFINED_LITERALS
+inline namespace literals {
+/**
+ * User-defined literal equivalent of `fmt::arg`.
+ *
+ * **Example**:
+ *
+ *     using namespace fmt::literals;
+ *     fmt::print("The answer is {answer}.", "answer"_a=42);
+ */
+#  if FMT_USE_NONTYPE_TEMPLATE_ARGS
+template <detail_exported::fixed_string Str> constexpr auto operator""_a() {
+  using char_t = remove_cvref_t<decltype(Str.data[0])>;
+  return detail::udl_arg<char_t, sizeof(Str.data) / sizeof(char_t), Str>();
+}
+#  else
+constexpr auto operator""_a(const char* s, size_t) -> detail::udl_arg<char> {
+  return {s};
+}
+#  endif
+}  // namespace literals
+#endif  // FMT_USE_USER_DEFINED_LITERALS
+
+FMT_API auto vformat(string_view fmt, format_args args) -> std::string;
+
+/**
+ * Formats `args` according to specifications in `fmt` and returns the result
+ * as a string.
+ *
+ * **Example**:
+ *
+ *     #include <fmt/format.h>
+ *     std::string message = fmt::format("The answer is {}.", 42);
+ */
+template <typename... T>
+FMT_NODISCARD FMT_INLINE auto format(format_string<T...> fmt, T&&... args)
+    -> std::string {
+  return vformat(fmt, fmt::make_format_args(args...));
+}
+
+template <typename Locale, FMT_ENABLE_IF(detail::is_locale<Locale>::value)>
+inline auto vformat(const Locale& loc, string_view fmt, format_args args)
+    -> std::string {
+  return detail::vformat(loc, fmt, args);
+}
+
+template <typename Locale, typename... T,
+          FMT_ENABLE_IF(detail::is_locale<Locale>::value)>
+inline auto format(const Locale& loc, format_string<T...> fmt, T&&... args)
+    -> std::string {
+  return fmt::vformat(loc, string_view(fmt), fmt::make_format_args(args...));
+}
+
+template <typename OutputIt, typename Locale,
+          FMT_ENABLE_IF(detail::is_output_iterator<OutputIt, char>::value&&
+                            detail::is_locale<Locale>::value)>
+auto vformat_to(OutputIt out, const Locale& loc, string_view fmt,
+                format_args args) -> OutputIt {
+  using detail::get_buffer;
+  auto&& buf = get_buffer<char>(out);
+  detail::vformat_to(buf, fmt, args, detail::locale_ref(loc));
+  return detail::get_iterator(buf, out);
+}
+
+template <typename OutputIt, typename Locale, typename... T,
+          FMT_ENABLE_IF(detail::is_output_iterator<OutputIt, char>::value&&
+                            detail::is_locale<Locale>::value)>
+FMT_INLINE auto format_to(OutputIt out, const Locale& loc,
+                          format_string<T...> fmt, T&&... args) -> OutputIt {
+  return vformat_to(out, loc, fmt, fmt::make_format_args(args...));
+}
+
+template <typename Locale, typename... T,
+          FMT_ENABLE_IF(detail::is_locale<Locale>::value)>
+FMT_NODISCARD FMT_INLINE auto formatted_size(const Locale& loc,
+                                             format_string<T...> fmt,
+                                             T&&... args) -> size_t {
+  auto buf = detail::counting_buffer<>();
+  detail::vformat_to<char>(buf, fmt, fmt::make_format_args(args...),
+                           detail::locale_ref(loc));
+  return buf.count();
+}
+
+FMT_END_EXPORT
+
+FMT_END_NAMESPACE
+
+#ifdef FMT_HEADER_ONLY
+#  define FMT_FUNC inline
+#  include "format-inl.h"
+#else
+#  define FMT_FUNC
+#endif
+
+// Restore _LIBCPP_REMOVE_TRANSITIVE_INCLUDES.
+#ifdef FMT_REMOVE_TRANSITIVE_INCLUDES
+#  undef _LIBCPP_REMOVE_TRANSITIVE_INCLUDES
+#endif
+
+#endif  // FMT_FORMAT_H_
diff --git a/ext/spdlog/include/spdlog/fmt/bundled/locale.h b/ext/spdlog/include/spdlog/fmt/bundled/locale.h
new file mode 100644
index 0000000..7571b52
--- /dev/null
+++ b/ext/spdlog/include/spdlog/fmt/bundled/locale.h
@@ -0,0 +1,2 @@
+#include "xchar.h"
+#warning fmt/locale.h is deprecated, include fmt/format.h or fmt/xchar.h instead
diff --git a/ext/spdlog/include/spdlog/fmt/bundled/os.h b/ext/spdlog/include/spdlog/fmt/bundled/os.h
new file mode 100644
index 0000000..5c85ea0
--- /dev/null
+++ b/ext/spdlog/include/spdlog/fmt/bundled/os.h
@@ -0,0 +1,439 @@
+// Formatting library for C++ - optional OS-specific functionality
+//
+// Copyright (c) 2012 - present, Victor Zverovich
+// All rights reserved.
+//
+// For the license information refer to format.h.
+
+#ifndef FMT_OS_H_
+#define FMT_OS_H_
+
+#include "format.h"
+
+#ifndef FMT_MODULE
+#  include <cerrno>
+#  include <cstddef>
+#  include <cstdio>
+#  include <system_error>  // std::system_error
+
+#  if FMT_HAS_INCLUDE(<xlocale.h>)
+#    include <xlocale.h>  // LC_NUMERIC_MASK on macOS
+#  endif
+#endif  // FMT_MODULE
+
+#ifndef FMT_USE_FCNTL
+// UWP doesn't provide _pipe.
+#  if FMT_HAS_INCLUDE("winapifamily.h")
+#    include <winapifamily.h>
+#  endif
+#  if (FMT_HAS_INCLUDE(<fcntl.h>) || defined(__APPLE__) || \
+       defined(__linux__)) &&                              \
+      (!defined(WINAPI_FAMILY) ||                          \
+       (WINAPI_FAMILY == WINAPI_FAMILY_DESKTOP_APP))
+#    include <fcntl.h>  // for O_RDONLY
+#    define FMT_USE_FCNTL 1
+#  else
+#    define FMT_USE_FCNTL 0
+#  endif
+#endif
+
+#ifndef FMT_POSIX
+#  if defined(_WIN32) && !defined(__MINGW32__)
+// Fix warnings about deprecated symbols.
+#    define FMT_POSIX(call) _##call
+#  else
+#    define FMT_POSIX(call) call
+#  endif
+#endif
+
+// Calls to system functions are wrapped in FMT_SYSTEM for testability.
+#ifdef FMT_SYSTEM
+#  define FMT_HAS_SYSTEM
+#  define FMT_POSIX_CALL(call) FMT_SYSTEM(call)
+#else
+#  define FMT_SYSTEM(call) ::call
+#  ifdef _WIN32
+// Fix warnings about deprecated symbols.
+#    define FMT_POSIX_CALL(call) ::_##call
+#  else
+#    define FMT_POSIX_CALL(call) ::call
+#  endif
+#endif
+
+// Retries the expression while it evaluates to error_result and errno
+// equals to EINTR.
+#ifndef _WIN32
+#  define FMT_RETRY_VAL(result, expression, error_result) \
+    do {                                                  \
+      (result) = (expression);                            \
+    } while ((result) == (error_result) && errno == EINTR)
+#else
+#  define FMT_RETRY_VAL(result, expression, error_result) result = (expression)
+#endif
+
+#define FMT_RETRY(result, expression) FMT_RETRY_VAL(result, expression, -1)
+
+FMT_BEGIN_NAMESPACE
+FMT_BEGIN_EXPORT
+
+/**
+ * A reference to a null-terminated string. It can be constructed from a C
+ * string or `std::string`.
+ *
+ * You can use one of the following type aliases for common character types:
+ *
+ * +---------------+-----------------------------+
+ * | Type          | Definition                  |
+ * +===============+=============================+
+ * | cstring_view  | basic_cstring_view<char>    |
+ * +---------------+-----------------------------+
+ * | wcstring_view | basic_cstring_view<wchar_t> |
+ * +---------------+-----------------------------+
+ *
+ * This class is most useful as a parameter type for functions that wrap C APIs.
+ */
+template <typename Char> class basic_cstring_view {
+ private:
+  const Char* data_;
+
+ public:
+  /// Constructs a string reference object from a C string.
+  basic_cstring_view(const Char* s) : data_(s) {}
+
+  /// Constructs a string reference from an `std::string` object.
+  basic_cstring_view(const std::basic_string<Char>& s) : data_(s.c_str()) {}
+
+  /// Returns the pointer to a C string.
+  auto c_str() const -> const Char* { return data_; }
+};
+
+using cstring_view = basic_cstring_view<char>;
+using wcstring_view = basic_cstring_view<wchar_t>;
+
+#ifdef _WIN32
+FMT_API const std::error_category& system_category() noexcept;
+
+namespace detail {
+FMT_API void format_windows_error(buffer<char>& out, int error_code,
+                                  const char* message) noexcept;
+}
+
+FMT_API std::system_error vwindows_error(int error_code, string_view format_str,
+                                         format_args args);
+
+/**
+ * Constructs a `std::system_error` object with the description of the form
+ *
+ *     <message>: <system-message>
+ *
+ * where `<message>` is the formatted message and `<system-message>` is the
+ * system message corresponding to the error code.
+ * `error_code` is a Windows error code as given by `GetLastError`.
+ * If `error_code` is not a valid error code such as -1, the system message
+ * will look like "error -1".
+ *
+ * **Example**:
+ *
+ *     // This throws a system_error with the description
+ *     //   cannot open file 'madeup': The system cannot find the file
+ * specified.
+ *     // or similar (system message may vary).
+ *     const char *filename = "madeup";
+ *     LPOFSTRUCT of = LPOFSTRUCT();
+ *     HFILE file = OpenFile(filename, &of, OF_READ);
+ *     if (file == HFILE_ERROR) {
+ *       throw fmt::windows_error(GetLastError(),
+ *                                "cannot open file '{}'", filename);
+ *     }
+ */
+template <typename... Args>
+std::system_error windows_error(int error_code, string_view message,
+                                const Args&... args) {
+  return vwindows_error(error_code, message, fmt::make_format_args(args...));
+}
+
+// Reports a Windows error without throwing an exception.
+// Can be used to report errors from destructors.
+FMT_API void report_windows_error(int error_code, const char* message) noexcept;
+#else
+inline auto system_category() noexcept -> const std::error_category& {
+  return std::system_category();
+}
+#endif  // _WIN32
+
+// std::system is not available on some platforms such as iOS (#2248).
+#ifdef __OSX__
+template <typename S, typename... Args, typename Char = char_t<S>>
+void say(const S& format_str, Args&&... args) {
+  std::system(format("say \"{}\"", format(format_str, args...)).c_str());
+}
+#endif
+
+// A buffered file.
+class buffered_file {
+ private:
+  FILE* file_;
+
+  friend class file;
+
+  explicit buffered_file(FILE* f) : file_(f) {}
+
+ public:
+  buffered_file(const buffered_file&) = delete;
+  void operator=(const buffered_file&) = delete;
+
+  // Constructs a buffered_file object which doesn't represent any file.
+  buffered_file() noexcept : file_(nullptr) {}
+
+  // Destroys the object closing the file it represents if any.
+  FMT_API ~buffered_file() noexcept;
+
+ public:
+  buffered_file(buffered_file&& other) noexcept : file_(other.file_) {
+    other.file_ = nullptr;
+  }
+
+  auto operator=(buffered_file&& other) -> buffered_file& {
+    close();
+    file_ = other.file_;
+    other.file_ = nullptr;
+    return *this;
+  }
+
+  // Opens a file.
+  FMT_API buffered_file(cstring_view filename, cstring_view mode);
+
+  // Closes the file.
+  FMT_API void close();
+
+  // Returns the pointer to a FILE object representing this file.
+  auto get() const noexcept -> FILE* { return file_; }
+
+  FMT_API auto descriptor() const -> int;
+
+  template <typename... T>
+  inline void print(string_view fmt, const T&... args) {
+    const auto& vargs = fmt::make_format_args(args...);
+    detail::is_locking<T...>() ? fmt::vprint_buffered(file_, fmt, vargs)
+                               : fmt::vprint(file_, fmt, vargs);
+  }
+};
+
+#if FMT_USE_FCNTL
+
+// A file. Closed file is represented by a file object with descriptor -1.
+// Methods that are not declared with noexcept may throw
+// fmt::system_error in case of failure. Note that some errors such as
+// closing the file multiple times will cause a crash on Windows rather
+// than an exception. You can get standard behavior by overriding the
+// invalid parameter handler with _set_invalid_parameter_handler.
+class FMT_API file {
+ private:
+  int fd_;  // File descriptor.
+
+  // Constructs a file object with a given descriptor.
+  explicit file(int fd) : fd_(fd) {}
+
+  friend struct pipe;
+
+ public:
+  // Possible values for the oflag argument to the constructor.
+  enum {
+    RDONLY = FMT_POSIX(O_RDONLY),  // Open for reading only.
+    WRONLY = FMT_POSIX(O_WRONLY),  // Open for writing only.
+    RDWR = FMT_POSIX(O_RDWR),      // Open for reading and writing.
+    CREATE = FMT_POSIX(O_CREAT),   // Create if the file doesn't exist.
+    APPEND = FMT_POSIX(O_APPEND),  // Open in append mode.
+    TRUNC = FMT_POSIX(O_TRUNC)     // Truncate the content of the file.
+  };
+
+  // Constructs a file object which doesn't represent any file.
+  file() noexcept : fd_(-1) {}
+
+  // Opens a file and constructs a file object representing this file.
+  file(cstring_view path, int oflag);
+
+ public:
+  file(const file&) = delete;
+  void operator=(const file&) = delete;
+
+  file(file&& other) noexcept : fd_(other.fd_) { other.fd_ = -1; }
+
+  // Move assignment is not noexcept because close may throw.
+  auto operator=(file&& other) -> file& {
+    close();
+    fd_ = other.fd_;
+    other.fd_ = -1;
+    return *this;
+  }
+
+  // Destroys the object closing the file it represents if any.
+  ~file() noexcept;
+
+  // Returns the file descriptor.
+  auto descriptor() const noexcept -> int { return fd_; }
+
+  // Closes the file.
+  void close();
+
+  // Returns the file size. The size has signed type for consistency with
+  // stat::st_size.
+  auto size() const -> long long;
+
+  // Attempts to read count bytes from the file into the specified buffer.
+  auto read(void* buffer, size_t count) -> size_t;
+
+  // Attempts to write count bytes from the specified buffer to the file.
+  auto write(const void* buffer, size_t count) -> size_t;
+
+  // Duplicates a file descriptor with the dup function and returns
+  // the duplicate as a file object.
+  static auto dup(int fd) -> file;
+
+  // Makes fd be the copy of this file descriptor, closing fd first if
+  // necessary.
+  void dup2(int fd);
+
+  // Makes fd be the copy of this file descriptor, closing fd first if
+  // necessary.
+  void dup2(int fd, std::error_code& ec) noexcept;
+
+  // Creates a buffered_file object associated with this file and detaches
+  // this file object from the file.
+  auto fdopen(const char* mode) -> buffered_file;
+
+#  if defined(_WIN32) && !defined(__MINGW32__)
+  // Opens a file and constructs a file object representing this file by
+  // wcstring_view filename. Windows only.
+  static file open_windows_file(wcstring_view path, int oflag);
+#  endif
+};
+
+struct FMT_API pipe {
+  file read_end;
+  file write_end;
+
+  // Creates a pipe setting up read_end and write_end file objects for reading
+  // and writing respectively.
+  pipe();
+};
+
+// Returns the memory page size.
+auto getpagesize() -> long;
+
+namespace detail {
+
+struct buffer_size {
+  buffer_size() = default;
+  size_t value = 0;
+  auto operator=(size_t val) const -> buffer_size {
+    auto bs = buffer_size();
+    bs.value = val;
+    return bs;
+  }
+};
+
+struct ostream_params {
+  int oflag = file::WRONLY | file::CREATE | file::TRUNC;
+  size_t buffer_size = BUFSIZ > 32768 ? BUFSIZ : 32768;
+
+  ostream_params() {}
+
+  template <typename... T>
+  ostream_params(T... params, int new_oflag) : ostream_params(params...) {
+    oflag = new_oflag;
+  }
+
+  template <typename... T>
+  ostream_params(T... params, detail::buffer_size bs)
+      : ostream_params(params...) {
+    this->buffer_size = bs.value;
+  }
+
+// Intel has a bug that results in failure to deduce a constructor
+// for empty parameter packs.
+#  if defined(__INTEL_COMPILER) && __INTEL_COMPILER < 2000
+  ostream_params(int new_oflag) : oflag(new_oflag) {}
+  ostream_params(detail::buffer_size bs) : buffer_size(bs.value) {}
+#  endif
+};
+
+class file_buffer final : public buffer<char> {
+ private:
+  file file_;
+
+  FMT_API static void grow(buffer<char>& buf, size_t);
+
+ public:
+  FMT_API file_buffer(cstring_view path, const ostream_params& params);
+  FMT_API file_buffer(file_buffer&& other) noexcept;
+  FMT_API ~file_buffer();
+
+  void flush() {
+    if (size() == 0) return;
+    file_.write(data(), size() * sizeof(data()[0]));
+    clear();
+  }
+
+  void close() {
+    flush();
+    file_.close();
+  }
+};
+
+}  // namespace detail
+
+constexpr auto buffer_size = detail::buffer_size();
+
+/// A fast output stream for writing from a single thread. Writing from
+/// multiple threads without external synchronization may result in a data race.
+class FMT_API ostream {
+ private:
+  FMT_MSC_WARNING(suppress : 4251)
+  detail::file_buffer buffer_;
+
+  ostream(cstring_view path, const detail::ostream_params& params)
+      : buffer_(path, params) {}
+
+ public:
+  ostream(ostream&& other) : buffer_(std::move(other.buffer_)) {}
+
+  ~ostream();
+
+  void flush() { buffer_.flush(); }
+
+  template <typename... T>
+  friend auto output_file(cstring_view path, T... params) -> ostream;
+
+  void close() { buffer_.close(); }
+
+  /// Formats `args` according to specifications in `fmt` and writes the
+  /// output to the file.
+  template <typename... T> void print(format_string<T...> fmt, T&&... args) {
+    vformat_to(appender(buffer_), fmt, fmt::make_format_args(args...));
+  }
+};
+
+/**
+ * Opens a file for writing. Supported parameters passed in `params`:
+ *
+ * - `<integer>`: Flags passed to [open](
+ *   https://pubs.opengroup.org/onlinepubs/007904875/functions/open.html)
+ *   (`file::WRONLY | file::CREATE | file::TRUNC` by default)
+ * - `buffer_size=<integer>`: Output buffer size
+ *
+ * **Example**:
+ *
+ *     auto out = fmt::output_file("guide.txt");
+ *     out.print("Don't {}", "Panic");
+ */
+template <typename... T>
+inline auto output_file(cstring_view path, T... params) -> ostream {
+  return {path, detail::ostream_params(params...)};
+}
+#endif  // FMT_USE_FCNTL
+
+FMT_END_EXPORT
+FMT_END_NAMESPACE
+
+#endif  // FMT_OS_H_
diff --git a/ext/spdlog/include/spdlog/fmt/bundled/ostream.h b/ext/spdlog/include/spdlog/fmt/bundled/ostream.h
new file mode 100644
index 0000000..98faef6
--- /dev/null
+++ b/ext/spdlog/include/spdlog/fmt/bundled/ostream.h
@@ -0,0 +1,211 @@
+// Formatting library for C++ - std::ostream support
+//
+// Copyright (c) 2012 - present, Victor Zverovich
+// All rights reserved.
+//
+// For the license information refer to format.h.
+
+#ifndef FMT_OSTREAM_H_
+#define FMT_OSTREAM_H_
+
+#ifndef FMT_MODULE
+#  include <fstream>  // std::filebuf
+#endif
+
+#ifdef _WIN32
+#  ifdef __GLIBCXX__
+#    include <ext/stdio_filebuf.h>
+#    include <ext/stdio_sync_filebuf.h>
+#  endif
+#  include <io.h>
+#endif
+
+#include "chrono.h"  // formatbuf
+
+FMT_BEGIN_NAMESPACE
+namespace detail {
+
+// Generate a unique explicit instantion in every translation unit using a tag
+// type in an anonymous namespace.
+namespace {
+struct file_access_tag {};
+}  // namespace
+template <typename Tag, typename BufType, FILE* BufType::*FileMemberPtr>
+class file_access {
+  friend auto get_file(BufType& obj) -> FILE* { return obj.*FileMemberPtr; }
+};
+
+#if FMT_MSC_VERSION
+template class file_access<file_access_tag, std::filebuf,
+                           &std::filebuf::_Myfile>;
+auto get_file(std::filebuf&) -> FILE*;
+#endif
+
+inline auto write_ostream_unicode(std::ostream& os, fmt::string_view data)
+    -> bool {
+  FILE* f = nullptr;
+#if FMT_MSC_VERSION && FMT_USE_RTTI
+  if (auto* buf = dynamic_cast<std::filebuf*>(os.rdbuf()))
+    f = get_file(*buf);
+  else
+    return false;
+#elif defined(_WIN32) && defined(__GLIBCXX__) && FMT_USE_RTTI
+  auto* rdbuf = os.rdbuf();
+  if (auto* sfbuf = dynamic_cast<__gnu_cxx::stdio_sync_filebuf<char>*>(rdbuf))
+    f = sfbuf->file();
+  else if (auto* fbuf = dynamic_cast<__gnu_cxx::stdio_filebuf<char>*>(rdbuf))
+    f = fbuf->file();
+  else
+    return false;
+#else
+  ignore_unused(os, data, f);
+#endif
+#ifdef _WIN32
+  if (f) {
+    int fd = _fileno(f);
+    if (_isatty(fd)) {
+      os.flush();
+      return write_console(fd, data);
+    }
+  }
+#endif
+  return false;
+}
+inline auto write_ostream_unicode(std::wostream&,
+                                  fmt::basic_string_view<wchar_t>) -> bool {
+  return false;
+}
+
+// Write the content of buf to os.
+// It is a separate function rather than a part of vprint to simplify testing.
+template <typename Char>
+void write_buffer(std::basic_ostream<Char>& os, buffer<Char>& buf) {
+  const Char* buf_data = buf.data();
+  using unsigned_streamsize = std::make_unsigned<std::streamsize>::type;
+  unsigned_streamsize size = buf.size();
+  unsigned_streamsize max_size = to_unsigned(max_value<std::streamsize>());
+  do {
+    unsigned_streamsize n = size <= max_size ? size : max_size;
+    os.write(buf_data, static_cast<std::streamsize>(n));
+    buf_data += n;
+    size -= n;
+  } while (size != 0);
+}
+
+template <typename Char, typename T>
+void format_value(buffer<Char>& buf, const T& value) {
+  auto&& format_buf = formatbuf<std::basic_streambuf<Char>>(buf);
+  auto&& output = std::basic_ostream<Char>(&format_buf);
+#if !defined(FMT_STATIC_THOUSANDS_SEPARATOR)
+  output.imbue(std::locale::classic());  // The default is always unlocalized.
+#endif
+  output << value;
+  output.exceptions(std::ios_base::failbit | std::ios_base::badbit);
+}
+
+template <typename T> struct streamed_view {
+  const T& value;
+};
+
+}  // namespace detail
+
+// Formats an object of type T that has an overloaded ostream operator<<.
+template <typename Char>
+struct basic_ostream_formatter : formatter<basic_string_view<Char>, Char> {
+  void set_debug_format() = delete;
+
+  template <typename T, typename Context>
+  auto format(const T& value, Context& ctx) const -> decltype(ctx.out()) {
+    auto buffer = basic_memory_buffer<Char>();
+    detail::format_value(buffer, value);
+    return formatter<basic_string_view<Char>, Char>::format(
+        {buffer.data(), buffer.size()}, ctx);
+  }
+};
+
+using ostream_formatter = basic_ostream_formatter<char>;
+
+template <typename T, typename Char>
+struct formatter<detail::streamed_view<T>, Char>
+    : basic_ostream_formatter<Char> {
+  template <typename Context>
+  auto format(detail::streamed_view<T> view, Context& ctx) const
+      -> decltype(ctx.out()) {
+    return basic_ostream_formatter<Char>::format(view.value, ctx);
+  }
+};
+
+/**
+ * Returns a view that formats `value` via an ostream `operator<<`.
+ *
+ * **Example**:
+ *
+ *     fmt::print("Current thread id: {}\n",
+ *                fmt::streamed(std::this_thread::get_id()));
+ */
+template <typename T>
+constexpr auto streamed(const T& value) -> detail::streamed_view<T> {
+  return {value};
+}
+
+namespace detail {
+
+inline void vprint_directly(std::ostream& os, string_view format_str,
+                            format_args args) {
+  auto buffer = memory_buffer();
+  detail::vformat_to(buffer, format_str, args);
+  detail::write_buffer(os, buffer);
+}
+
+}  // namespace detail
+
+FMT_EXPORT template <typename Char>
+void vprint(std::basic_ostream<Char>& os,
+            basic_string_view<type_identity_t<Char>> format_str,
+            typename detail::vformat_args<Char>::type args) {
+  auto buffer = basic_memory_buffer<Char>();
+  detail::vformat_to(buffer, format_str, args);
+  if (detail::write_ostream_unicode(os, {buffer.data(), buffer.size()})) return;
+  detail::write_buffer(os, buffer);
+}
+
+/**
+ * Prints formatted data to the stream `os`.
+ *
+ * **Example**:
+ *
+ *     fmt::print(cerr, "Don't {}!", "panic");
+ */
+FMT_EXPORT template <typename... T>
+void print(std::ostream& os, format_string<T...> fmt, T&&... args) {
+  const auto& vargs = fmt::make_format_args(args...);
+  if (detail::use_utf8())
+    vprint(os, fmt, vargs);
+  else
+    detail::vprint_directly(os, fmt, vargs);
+}
+
+FMT_EXPORT
+template <typename... Args>
+void print(std::wostream& os,
+           basic_format_string<wchar_t, type_identity_t<Args>...> fmt,
+           Args&&... args) {
+  vprint(os, fmt, fmt::make_format_args<buffered_context<wchar_t>>(args...));
+}
+
+FMT_EXPORT template <typename... T>
+void println(std::ostream& os, format_string<T...> fmt, T&&... args) {
+  fmt::print(os, "{}\n", fmt::format(fmt, std::forward<T>(args)...));
+}
+
+FMT_EXPORT
+template <typename... Args>
+void println(std::wostream& os,
+             basic_format_string<wchar_t, type_identity_t<Args>...> fmt,
+             Args&&... args) {
+  print(os, L"{}\n", fmt::format(fmt, std::forward<Args>(args)...));
+}
+
+FMT_END_NAMESPACE
+
+#endif  // FMT_OSTREAM_H_
diff --git a/ext/spdlog/include/spdlog/fmt/bundled/printf.h b/ext/spdlog/include/spdlog/fmt/bundled/printf.h
new file mode 100644
index 0000000..072cc6b
--- /dev/null
+++ b/ext/spdlog/include/spdlog/fmt/bundled/printf.h
@@ -0,0 +1,656 @@
+// Formatting library for C++ - legacy printf implementation
+//
+// Copyright (c) 2012 - 2016, Victor Zverovich
+// All rights reserved.
+//
+// For the license information refer to format.h.
+
+#ifndef FMT_PRINTF_H_
+#define FMT_PRINTF_H_
+
+#ifndef FMT_MODULE
+#  include <algorithm>  // std::max
+#  include <limits>     // std::numeric_limits
+#endif
+
+#include "format.h"
+
+FMT_BEGIN_NAMESPACE
+FMT_BEGIN_EXPORT
+
+template <typename T> struct printf_formatter {
+  printf_formatter() = delete;
+};
+
+template <typename Char> class basic_printf_context {
+ private:
+  basic_appender<Char> out_;
+  basic_format_args<basic_printf_context> args_;
+
+  static_assert(std::is_same<Char, char>::value ||
+                    std::is_same<Char, wchar_t>::value,
+                "Unsupported code unit type.");
+
+ public:
+  using char_type = Char;
+  using parse_context_type = basic_format_parse_context<Char>;
+  template <typename T> using formatter_type = printf_formatter<T>;
+
+  /// Constructs a `printf_context` object. References to the arguments are
+  /// stored in the context object so make sure they have appropriate lifetimes.
+  basic_printf_context(basic_appender<Char> out,
+                       basic_format_args<basic_printf_context> args)
+      : out_(out), args_(args) {}
+
+  auto out() -> basic_appender<Char> { return out_; }
+  void advance_to(basic_appender<Char>) {}
+
+  auto locale() -> detail::locale_ref { return {}; }
+
+  auto arg(int id) const -> basic_format_arg<basic_printf_context> {
+    return args_.get(id);
+  }
+};
+
+namespace detail {
+
+// Checks if a value fits in int - used to avoid warnings about comparing
+// signed and unsigned integers.
+template <bool IsSigned> struct int_checker {
+  template <typename T> static auto fits_in_int(T value) -> bool {
+    unsigned max = to_unsigned(max_value<int>());
+    return value <= max;
+  }
+  static auto fits_in_int(bool) -> bool { return true; }
+};
+
+template <> struct int_checker<true> {
+  template <typename T> static auto fits_in_int(T value) -> bool {
+    return value >= (std::numeric_limits<int>::min)() &&
+           value <= max_value<int>();
+  }
+  static auto fits_in_int(int) -> bool { return true; }
+};
+
+struct printf_precision_handler {
+  template <typename T, FMT_ENABLE_IF(std::is_integral<T>::value)>
+  auto operator()(T value) -> int {
+    if (!int_checker<std::numeric_limits<T>::is_signed>::fits_in_int(value))
+      report_error("number is too big");
+    return (std::max)(static_cast<int>(value), 0);
+  }
+
+  template <typename T, FMT_ENABLE_IF(!std::is_integral<T>::value)>
+  auto operator()(T) -> int {
+    report_error("precision is not integer");
+    return 0;
+  }
+};
+
+// An argument visitor that returns true iff arg is a zero integer.
+struct is_zero_int {
+  template <typename T, FMT_ENABLE_IF(std::is_integral<T>::value)>
+  auto operator()(T value) -> bool {
+    return value == 0;
+  }
+
+  template <typename T, FMT_ENABLE_IF(!std::is_integral<T>::value)>
+  auto operator()(T) -> bool {
+    return false;
+  }
+};
+
+template <typename T> struct make_unsigned_or_bool : std::make_unsigned<T> {};
+
+template <> struct make_unsigned_or_bool<bool> {
+  using type = bool;
+};
+
+template <typename T, typename Context> class arg_converter {
+ private:
+  using char_type = typename Context::char_type;
+
+  basic_format_arg<Context>& arg_;
+  char_type type_;
+
+ public:
+  arg_converter(basic_format_arg<Context>& arg, char_type type)
+      : arg_(arg), type_(type) {}
+
+  void operator()(bool value) {
+    if (type_ != 's') operator()<bool>(value);
+  }
+
+  template <typename U, FMT_ENABLE_IF(std::is_integral<U>::value)>
+  void operator()(U value) {
+    bool is_signed = type_ == 'd' || type_ == 'i';
+    using target_type = conditional_t<std::is_same<T, void>::value, U, T>;
+    if (const_check(sizeof(target_type) <= sizeof(int))) {
+      // Extra casts are used to silence warnings.
+      if (is_signed) {
+        auto n = static_cast<int>(static_cast<target_type>(value));
+        arg_ = detail::make_arg<Context>(n);
+      } else {
+        using unsigned_type = typename make_unsigned_or_bool<target_type>::type;
+        auto n = static_cast<unsigned>(static_cast<unsigned_type>(value));
+        arg_ = detail::make_arg<Context>(n);
+      }
+    } else {
+      if (is_signed) {
+        // glibc's printf doesn't sign extend arguments of smaller types:
+        //   std::printf("%lld", -42);  // prints "4294967254"
+        // but we don't have to do the same because it's a UB.
+        auto n = static_cast<long long>(value);
+        arg_ = detail::make_arg<Context>(n);
+      } else {
+        auto n = static_cast<typename make_unsigned_or_bool<U>::type>(value);
+        arg_ = detail::make_arg<Context>(n);
+      }
+    }
+  }
+
+  template <typename U, FMT_ENABLE_IF(!std::is_integral<U>::value)>
+  void operator()(U) {}  // No conversion needed for non-integral types.
+};
+
+// Converts an integer argument to T for printf, if T is an integral type.
+// If T is void, the argument is converted to corresponding signed or unsigned
+// type depending on the type specifier: 'd' and 'i' - signed, other -
+// unsigned).
+template <typename T, typename Context, typename Char>
+void convert_arg(basic_format_arg<Context>& arg, Char type) {
+  arg.visit(arg_converter<T, Context>(arg, type));
+}
+
+// Converts an integer argument to char for printf.
+template <typename Context> class char_converter {
+ private:
+  basic_format_arg<Context>& arg_;
+
+ public:
+  explicit char_converter(basic_format_arg<Context>& arg) : arg_(arg) {}
+
+  template <typename T, FMT_ENABLE_IF(std::is_integral<T>::value)>
+  void operator()(T value) {
+    auto c = static_cast<typename Context::char_type>(value);
+    arg_ = detail::make_arg<Context>(c);
+  }
+
+  template <typename T, FMT_ENABLE_IF(!std::is_integral<T>::value)>
+  void operator()(T) {}  // No conversion needed for non-integral types.
+};
+
+// An argument visitor that return a pointer to a C string if argument is a
+// string or null otherwise.
+template <typename Char> struct get_cstring {
+  template <typename T> auto operator()(T) -> const Char* { return nullptr; }
+  auto operator()(const Char* s) -> const Char* { return s; }
+};
+
+// Checks if an argument is a valid printf width specifier and sets
+// left alignment if it is negative.
+class printf_width_handler {
+ private:
+  format_specs& specs_;
+
+ public:
+  explicit printf_width_handler(format_specs& specs) : specs_(specs) {}
+
+  template <typename T, FMT_ENABLE_IF(std::is_integral<T>::value)>
+  auto operator()(T value) -> unsigned {
+    auto width = static_cast<uint32_or_64_or_128_t<T>>(value);
+    if (detail::is_negative(value)) {
+      specs_.align = align::left;
+      width = 0 - width;
+    }
+    unsigned int_max = to_unsigned(max_value<int>());
+    if (width > int_max) report_error("number is too big");
+    return static_cast<unsigned>(width);
+  }
+
+  template <typename T, FMT_ENABLE_IF(!std::is_integral<T>::value)>
+  auto operator()(T) -> unsigned {
+    report_error("width is not integer");
+    return 0;
+  }
+};
+
+// Workaround for a bug with the XL compiler when initializing
+// printf_arg_formatter's base class.
+template <typename Char>
+auto make_arg_formatter(basic_appender<Char> iter, format_specs& s)
+    -> arg_formatter<Char> {
+  return {iter, s, locale_ref()};
+}
+
+// The `printf` argument formatter.
+template <typename Char>
+class printf_arg_formatter : public arg_formatter<Char> {
+ private:
+  using base = arg_formatter<Char>;
+  using context_type = basic_printf_context<Char>;
+
+  context_type& context_;
+
+  void write_null_pointer(bool is_string = false) {
+    auto s = this->specs;
+    s.type = presentation_type::none;
+    write_bytes<Char>(this->out, is_string ? "(null)" : "(nil)", s);
+  }
+
+ public:
+  printf_arg_formatter(basic_appender<Char> iter, format_specs& s,
+                       context_type& ctx)
+      : base(make_arg_formatter(iter, s)), context_(ctx) {}
+
+  void operator()(monostate value) { base::operator()(value); }
+
+  template <typename T, FMT_ENABLE_IF(detail::is_integral<T>::value)>
+  void operator()(T value) {
+    // MSVC2013 fails to compile separate overloads for bool and Char so use
+    // std::is_same instead.
+    if (!std::is_same<T, Char>::value) {
+      base::operator()(value);
+      return;
+    }
+    format_specs s = this->specs;
+    if (s.type != presentation_type::none && s.type != presentation_type::chr) {
+      return (*this)(static_cast<int>(value));
+    }
+    s.sign = sign::none;
+    s.alt = false;
+    s.fill = ' ';  // Ignore '0' flag for char types.
+    // align::numeric needs to be overwritten here since the '0' flag is
+    // ignored for non-numeric types
+    if (s.align == align::none || s.align == align::numeric)
+      s.align = align::right;
+    write<Char>(this->out, static_cast<Char>(value), s);
+  }
+
+  template <typename T, FMT_ENABLE_IF(std::is_floating_point<T>::value)>
+  void operator()(T value) {
+    base::operator()(value);
+  }
+
+  void operator()(const char* value) {
+    if (value)
+      base::operator()(value);
+    else
+      write_null_pointer(this->specs.type != presentation_type::pointer);
+  }
+
+  void operator()(const wchar_t* value) {
+    if (value)
+      base::operator()(value);
+    else
+      write_null_pointer(this->specs.type != presentation_type::pointer);
+  }
+
+  void operator()(basic_string_view<Char> value) { base::operator()(value); }
+
+  void operator()(const void* value) {
+    if (value)
+      base::operator()(value);
+    else
+      write_null_pointer();
+  }
+
+  void operator()(typename basic_format_arg<context_type>::handle handle) {
+    auto parse_ctx = basic_format_parse_context<Char>({});
+    handle.format(parse_ctx, context_);
+  }
+};
+
+template <typename Char>
+void parse_flags(format_specs& specs, const Char*& it, const Char* end) {
+  for (; it != end; ++it) {
+    switch (*it) {
+    case '-':
+      specs.align = align::left;
+      break;
+    case '+':
+      specs.sign = sign::plus;
+      break;
+    case '0':
+      specs.fill = '0';
+      break;
+    case ' ':
+      if (specs.sign != sign::plus) specs.sign = sign::space;
+      break;
+    case '#':
+      specs.alt = true;
+      break;
+    default:
+      return;
+    }
+  }
+}
+
+template <typename Char, typename GetArg>
+auto parse_header(const Char*& it, const Char* end, format_specs& specs,
+                  GetArg get_arg) -> int {
+  int arg_index = -1;
+  Char c = *it;
+  if (c >= '0' && c <= '9') {
+    // Parse an argument index (if followed by '$') or a width possibly
+    // preceded with '0' flag(s).
+    int value = parse_nonnegative_int(it, end, -1);
+    if (it != end && *it == '$') {  // value is an argument index
+      ++it;
+      arg_index = value != -1 ? value : max_value<int>();
+    } else {
+      if (c == '0') specs.fill = '0';
+      if (value != 0) {
+        // Nonzero value means that we parsed width and don't need to
+        // parse it or flags again, so return now.
+        if (value == -1) report_error("number is too big");
+        specs.width = value;
+        return arg_index;
+      }
+    }
+  }
+  parse_flags(specs, it, end);
+  // Parse width.
+  if (it != end) {
+    if (*it >= '0' && *it <= '9') {
+      specs.width = parse_nonnegative_int(it, end, -1);
+      if (specs.width == -1) report_error("number is too big");
+    } else if (*it == '*') {
+      ++it;
+      specs.width = static_cast<int>(
+          get_arg(-1).visit(detail::printf_width_handler(specs)));
+    }
+  }
+  return arg_index;
+}
+
+inline auto parse_printf_presentation_type(char c, type t, bool& upper)
+    -> presentation_type {
+  using pt = presentation_type;
+  constexpr auto integral_set = sint_set | uint_set | bool_set | char_set;
+  switch (c) {
+  case 'd':
+    return in(t, integral_set) ? pt::dec : pt::none;
+  case 'o':
+    return in(t, integral_set) ? pt::oct : pt::none;
+  case 'X':
+    upper = true;
+    FMT_FALLTHROUGH;
+  case 'x':
+    return in(t, integral_set) ? pt::hex : pt::none;
+  case 'E':
+    upper = true;
+    FMT_FALLTHROUGH;
+  case 'e':
+    return in(t, float_set) ? pt::exp : pt::none;
+  case 'F':
+    upper = true;
+    FMT_FALLTHROUGH;
+  case 'f':
+    return in(t, float_set) ? pt::fixed : pt::none;
+  case 'G':
+    upper = true;
+    FMT_FALLTHROUGH;
+  case 'g':
+    return in(t, float_set) ? pt::general : pt::none;
+  case 'A':
+    upper = true;
+    FMT_FALLTHROUGH;
+  case 'a':
+    return in(t, float_set) ? pt::hexfloat : pt::none;
+  case 'c':
+    return in(t, integral_set) ? pt::chr : pt::none;
+  case 's':
+    return in(t, string_set | cstring_set) ? pt::string : pt::none;
+  case 'p':
+    return in(t, pointer_set | cstring_set) ? pt::pointer : pt::none;
+  default:
+    return pt::none;
+  }
+}
+
+template <typename Char, typename Context>
+void vprintf(buffer<Char>& buf, basic_string_view<Char> format,
+             basic_format_args<Context> args) {
+  using iterator = basic_appender<Char>;
+  auto out = iterator(buf);
+  auto context = basic_printf_context<Char>(out, args);
+  auto parse_ctx = basic_format_parse_context<Char>(format);
+
+  // Returns the argument with specified index or, if arg_index is -1, the next
+  // argument.
+  auto get_arg = [&](int arg_index) {
+    if (arg_index < 0)
+      arg_index = parse_ctx.next_arg_id();
+    else
+      parse_ctx.check_arg_id(--arg_index);
+    return detail::get_arg(context, arg_index);
+  };
+
+  const Char* start = parse_ctx.begin();
+  const Char* end = parse_ctx.end();
+  auto it = start;
+  while (it != end) {
+    if (!find<false, Char>(it, end, '%', it)) {
+      it = end;  // find leaves it == nullptr if it doesn't find '%'.
+      break;
+    }
+    Char c = *it++;
+    if (it != end && *it == c) {
+      write(out, basic_string_view<Char>(start, to_unsigned(it - start)));
+      start = ++it;
+      continue;
+    }
+    write(out, basic_string_view<Char>(start, to_unsigned(it - 1 - start)));
+
+    auto specs = format_specs();
+    specs.align = align::right;
+
+    // Parse argument index, flags and width.
+    int arg_index = parse_header(it, end, specs, get_arg);
+    if (arg_index == 0) report_error("argument not found");
+
+    // Parse precision.
+    if (it != end && *it == '.') {
+      ++it;
+      c = it != end ? *it : 0;
+      if ('0' <= c && c <= '9') {
+        specs.precision = parse_nonnegative_int(it, end, 0);
+      } else if (c == '*') {
+        ++it;
+        specs.precision =
+            static_cast<int>(get_arg(-1).visit(printf_precision_handler()));
+      } else {
+        specs.precision = 0;
+      }
+    }
+
+    auto arg = get_arg(arg_index);
+    // For d, i, o, u, x, and X conversion specifiers, if a precision is
+    // specified, the '0' flag is ignored
+    if (specs.precision >= 0 && arg.is_integral()) {
+      // Ignore '0' for non-numeric types or if '-' present.
+      specs.fill = ' ';
+    }
+    if (specs.precision >= 0 && arg.type() == type::cstring_type) {
+      auto str = arg.visit(get_cstring<Char>());
+      auto str_end = str + specs.precision;
+      auto nul = std::find(str, str_end, Char());
+      auto sv = basic_string_view<Char>(
+          str, to_unsigned(nul != str_end ? nul - str : specs.precision));
+      arg = make_arg<basic_printf_context<Char>>(sv);
+    }
+    if (specs.alt && arg.visit(is_zero_int())) specs.alt = false;
+    if (specs.fill.template get<Char>() == '0') {
+      if (arg.is_arithmetic() && specs.align != align::left)
+        specs.align = align::numeric;
+      else
+        specs.fill = ' ';  // Ignore '0' flag for non-numeric types or if '-'
+                           // flag is also present.
+    }
+
+    // Parse length and convert the argument to the required type.
+    c = it != end ? *it++ : 0;
+    Char t = it != end ? *it : 0;
+    switch (c) {
+    case 'h':
+      if (t == 'h') {
+        ++it;
+        t = it != end ? *it : 0;
+        convert_arg<signed char>(arg, t);
+      } else {
+        convert_arg<short>(arg, t);
+      }
+      break;
+    case 'l':
+      if (t == 'l') {
+        ++it;
+        t = it != end ? *it : 0;
+        convert_arg<long long>(arg, t);
+      } else {
+        convert_arg<long>(arg, t);
+      }
+      break;
+    case 'j':
+      convert_arg<intmax_t>(arg, t);
+      break;
+    case 'z':
+      convert_arg<size_t>(arg, t);
+      break;
+    case 't':
+      convert_arg<std::ptrdiff_t>(arg, t);
+      break;
+    case 'L':
+      // printf produces garbage when 'L' is omitted for long double, no
+      // need to do the same.
+      break;
+    default:
+      --it;
+      convert_arg<void>(arg, c);
+    }
+
+    // Parse type.
+    if (it == end) report_error("invalid format string");
+    char type = static_cast<char>(*it++);
+    if (arg.is_integral()) {
+      // Normalize type.
+      switch (type) {
+      case 'i':
+      case 'u':
+        type = 'd';
+        break;
+      case 'c':
+        arg.visit(char_converter<basic_printf_context<Char>>(arg));
+        break;
+      }
+    }
+    bool upper = false;
+    specs.type = parse_printf_presentation_type(type, arg.type(), upper);
+    if (specs.type == presentation_type::none)
+      report_error("invalid format specifier");
+    specs.upper = upper;
+
+    start = it;
+
+    // Format argument.
+    arg.visit(printf_arg_formatter<Char>(out, specs, context));
+  }
+  write(out, basic_string_view<Char>(start, to_unsigned(it - start)));
+}
+}  // namespace detail
+
+using printf_context = basic_printf_context<char>;
+using wprintf_context = basic_printf_context<wchar_t>;
+
+using printf_args = basic_format_args<printf_context>;
+using wprintf_args = basic_format_args<wprintf_context>;
+
+/// Constructs an `format_arg_store` object that contains references to
+/// arguments and can be implicitly converted to `printf_args`.
+template <typename Char = char, typename... T>
+inline auto make_printf_args(T&... args)
+    -> decltype(fmt::make_format_args<basic_printf_context<Char>>(args...)) {
+  return fmt::make_format_args<basic_printf_context<Char>>(args...);
+}
+
+template <typename Char> struct vprintf_args {
+  using type = basic_format_args<basic_printf_context<Char>>;
+};
+
+template <typename Char>
+inline auto vsprintf(basic_string_view<Char> fmt,
+                     typename vprintf_args<Char>::type args)
+    -> std::basic_string<Char> {
+  auto buf = basic_memory_buffer<Char>();
+  detail::vprintf(buf, fmt, args);
+  return to_string(buf);
+}
+
+/**
+ * Formats `args` according to specifications in `fmt` and returns the result
+ * as as string.
+ *
+ * **Example**:
+ *
+ *     std::string message = fmt::sprintf("The answer is %d", 42);
+ */
+template <typename S, typename... T, typename Char = char_t<S>>
+inline auto sprintf(const S& fmt, const T&... args) -> std::basic_string<Char> {
+  return vsprintf(detail::to_string_view(fmt),
+                  fmt::make_format_args<basic_printf_context<Char>>(args...));
+}
+
+template <typename Char>
+inline auto vfprintf(std::FILE* f, basic_string_view<Char> fmt,
+                     typename vprintf_args<Char>::type args) -> int {
+  auto buf = basic_memory_buffer<Char>();
+  detail::vprintf(buf, fmt, args);
+  size_t size = buf.size();
+  return std::fwrite(buf.data(), sizeof(Char), size, f) < size
+             ? -1
+             : static_cast<int>(size);
+}
+
+/**
+ * Formats `args` according to specifications in `fmt` and writes the output
+ * to `f`.
+ *
+ * **Example**:
+ *
+ *     fmt::fprintf(stderr, "Don't %s!", "panic");
+ */
+template <typename S, typename... T, typename Char = char_t<S>>
+inline auto fprintf(std::FILE* f, const S& fmt, const T&... args) -> int {
+  return vfprintf(f, detail::to_string_view(fmt),
+                  make_printf_args<Char>(args...));
+}
+
+template <typename Char>
+FMT_DEPRECATED inline auto vprintf(basic_string_view<Char> fmt,
+                                   typename vprintf_args<Char>::type args)
+    -> int {
+  return vfprintf(stdout, fmt, args);
+}
+
+/**
+ * Formats `args` according to specifications in `fmt` and writes the output
+ * to `stdout`.
+ *
+ * **Example**:
+ *
+ *   fmt::printf("Elapsed time: %.2f seconds", 1.23);
+ */
+template <typename... T>
+inline auto printf(string_view fmt, const T&... args) -> int {
+  return vfprintf(stdout, fmt, make_printf_args(args...));
+}
+template <typename... T>
+FMT_DEPRECATED inline auto printf(basic_string_view<wchar_t> fmt,
+                                  const T&... args) -> int {
+  return vfprintf(stdout, fmt, make_printf_args<wchar_t>(args...));
+}
+
+FMT_END_EXPORT
+FMT_END_NAMESPACE
+
+#endif  // FMT_PRINTF_H_
diff --git a/ext/spdlog/include/spdlog/fmt/bundled/ranges.h b/ext/spdlog/include/spdlog/fmt/bundled/ranges.h
new file mode 100644
index 0000000..0d3dfbd
--- /dev/null
+++ b/ext/spdlog/include/spdlog/fmt/bundled/ranges.h
@@ -0,0 +1,882 @@
+// Formatting library for C++ - range and tuple support
+//
+// Copyright (c) 2012 - present, Victor Zverovich and {fmt} contributors
+// All rights reserved.
+//
+// For the license information refer to format.h.
+
+#ifndef FMT_RANGES_H_
+#define FMT_RANGES_H_
+
+#ifndef FMT_MODULE
+#  include <initializer_list>
+#  include <iterator>
+#  include <string>
+#  include <tuple>
+#  include <type_traits>
+#  include <utility>
+#endif
+
+#include "format.h"
+
+FMT_BEGIN_NAMESPACE
+
+FMT_EXPORT
+enum class range_format { disabled, map, set, sequence, string, debug_string };
+
+namespace detail {
+
+template <typename T> class is_map {
+  template <typename U> static auto check(U*) -> typename U::mapped_type;
+  template <typename> static void check(...);
+
+ public:
+  static constexpr const bool value =
+      !std::is_void<decltype(check<T>(nullptr))>::value;
+};
+
+template <typename T> class is_set {
+  template <typename U> static auto check(U*) -> typename U::key_type;
+  template <typename> static void check(...);
+
+ public:
+  static constexpr const bool value =
+      !std::is_void<decltype(check<T>(nullptr))>::value && !is_map<T>::value;
+};
+
+template <typename... Ts> struct conditional_helper {};
+
+template <typename T, typename _ = void> struct is_range_ : std::false_type {};
+
+#if !FMT_MSC_VERSION || FMT_MSC_VERSION > 1800
+
+#  define FMT_DECLTYPE_RETURN(val)  \
+    ->decltype(val) { return val; } \
+    static_assert(                  \
+        true, "")  // This makes it so that a semicolon is required after the
+                   // macro, which helps clang-format handle the formatting.
+
+// C array overload
+template <typename T, std::size_t N>
+auto range_begin(const T (&arr)[N]) -> const T* {
+  return arr;
+}
+template <typename T, std::size_t N>
+auto range_end(const T (&arr)[N]) -> const T* {
+  return arr + N;
+}
+
+template <typename T, typename Enable = void>
+struct has_member_fn_begin_end_t : std::false_type {};
+
+template <typename T>
+struct has_member_fn_begin_end_t<T, void_t<decltype(*std::declval<T>().begin()),
+                                           decltype(std::declval<T>().end())>>
+    : std::true_type {};
+
+// Member function overloads.
+template <typename T>
+auto range_begin(T&& rng) FMT_DECLTYPE_RETURN(static_cast<T&&>(rng).begin());
+template <typename T>
+auto range_end(T&& rng) FMT_DECLTYPE_RETURN(static_cast<T&&>(rng).end());
+
+// ADL overloads. Only participate in overload resolution if member functions
+// are not found.
+template <typename T>
+auto range_begin(T&& rng)
+    -> enable_if_t<!has_member_fn_begin_end_t<T&&>::value,
+                   decltype(begin(static_cast<T&&>(rng)))> {
+  return begin(static_cast<T&&>(rng));
+}
+template <typename T>
+auto range_end(T&& rng) -> enable_if_t<!has_member_fn_begin_end_t<T&&>::value,
+                                       decltype(end(static_cast<T&&>(rng)))> {
+  return end(static_cast<T&&>(rng));
+}
+
+template <typename T, typename Enable = void>
+struct has_const_begin_end : std::false_type {};
+template <typename T, typename Enable = void>
+struct has_mutable_begin_end : std::false_type {};
+
+template <typename T>
+struct has_const_begin_end<
+    T, void_t<decltype(*detail::range_begin(
+                  std::declval<const remove_cvref_t<T>&>())),
+              decltype(detail::range_end(
+                  std::declval<const remove_cvref_t<T>&>()))>>
+    : std::true_type {};
+
+template <typename T>
+struct has_mutable_begin_end<
+    T, void_t<decltype(*detail::range_begin(std::declval<T&>())),
+              decltype(detail::range_end(std::declval<T&>())),
+              // the extra int here is because older versions of MSVC don't
+              // SFINAE properly unless there are distinct types
+              int>> : std::true_type {};
+
+template <typename T>
+struct is_range_<T, void>
+    : std::integral_constant<bool, (has_const_begin_end<T>::value ||
+                                    has_mutable_begin_end<T>::value)> {};
+#  undef FMT_DECLTYPE_RETURN
+#endif
+
+// tuple_size and tuple_element check.
+template <typename T> class is_tuple_like_ {
+  template <typename U>
+  static auto check(U* p) -> decltype(std::tuple_size<U>::value, int());
+  template <typename> static void check(...);
+
+ public:
+  static constexpr const bool value =
+      !std::is_void<decltype(check<T>(nullptr))>::value;
+};
+
+// Check for integer_sequence
+#if defined(__cpp_lib_integer_sequence) || FMT_MSC_VERSION >= 1900
+template <typename T, T... N>
+using integer_sequence = std::integer_sequence<T, N...>;
+template <size_t... N> using index_sequence = std::index_sequence<N...>;
+template <size_t N> using make_index_sequence = std::make_index_sequence<N>;
+#else
+template <typename T, T... N> struct integer_sequence {
+  using value_type = T;
+
+  static FMT_CONSTEXPR auto size() -> size_t { return sizeof...(N); }
+};
+
+template <size_t... N> using index_sequence = integer_sequence<size_t, N...>;
+
+template <typename T, size_t N, T... Ns>
+struct make_integer_sequence : make_integer_sequence<T, N - 1, N - 1, Ns...> {};
+template <typename T, T... Ns>
+struct make_integer_sequence<T, 0, Ns...> : integer_sequence<T, Ns...> {};
+
+template <size_t N>
+using make_index_sequence = make_integer_sequence<size_t, N>;
+#endif
+
+template <typename T>
+using tuple_index_sequence = make_index_sequence<std::tuple_size<T>::value>;
+
+template <typename T, typename C, bool = is_tuple_like_<T>::value>
+class is_tuple_formattable_ {
+ public:
+  static constexpr const bool value = false;
+};
+template <typename T, typename C> class is_tuple_formattable_<T, C, true> {
+  template <size_t... Is>
+  static auto all_true(index_sequence<Is...>,
+                       integer_sequence<bool, (Is >= 0)...>) -> std::true_type;
+  static auto all_true(...) -> std::false_type;
+
+  template <size_t... Is>
+  static auto check(index_sequence<Is...>) -> decltype(all_true(
+      index_sequence<Is...>{},
+      integer_sequence<bool,
+                       (is_formattable<typename std::tuple_element<Is, T>::type,
+                                       C>::value)...>{}));
+
+ public:
+  static constexpr const bool value =
+      decltype(check(tuple_index_sequence<T>{}))::value;
+};
+
+template <typename Tuple, typename F, size_t... Is>
+FMT_CONSTEXPR void for_each(index_sequence<Is...>, Tuple&& t, F&& f) {
+  using std::get;
+  // Using a free function get<Is>(Tuple) now.
+  const int unused[] = {0, ((void)f(get<Is>(t)), 0)...};
+  ignore_unused(unused);
+}
+
+template <typename Tuple, typename F>
+FMT_CONSTEXPR void for_each(Tuple&& t, F&& f) {
+  for_each(tuple_index_sequence<remove_cvref_t<Tuple>>(),
+           std::forward<Tuple>(t), std::forward<F>(f));
+}
+
+template <typename Tuple1, typename Tuple2, typename F, size_t... Is>
+void for_each2(index_sequence<Is...>, Tuple1&& t1, Tuple2&& t2, F&& f) {
+  using std::get;
+  const int unused[] = {0, ((void)f(get<Is>(t1), get<Is>(t2)), 0)...};
+  ignore_unused(unused);
+}
+
+template <typename Tuple1, typename Tuple2, typename F>
+void for_each2(Tuple1&& t1, Tuple2&& t2, F&& f) {
+  for_each2(tuple_index_sequence<remove_cvref_t<Tuple1>>(),
+            std::forward<Tuple1>(t1), std::forward<Tuple2>(t2),
+            std::forward<F>(f));
+}
+
+namespace tuple {
+// Workaround a bug in MSVC 2019 (v140).
+template <typename Char, typename... T>
+using result_t = std::tuple<formatter<remove_cvref_t<T>, Char>...>;
+
+using std::get;
+template <typename Tuple, typename Char, std::size_t... Is>
+auto get_formatters(index_sequence<Is...>)
+    -> result_t<Char, decltype(get<Is>(std::declval<Tuple>()))...>;
+}  // namespace tuple
+
+#if FMT_MSC_VERSION && FMT_MSC_VERSION < 1920
+// Older MSVC doesn't get the reference type correctly for arrays.
+template <typename R> struct range_reference_type_impl {
+  using type = decltype(*detail::range_begin(std::declval<R&>()));
+};
+
+template <typename T, std::size_t N> struct range_reference_type_impl<T[N]> {
+  using type = T&;
+};
+
+template <typename T>
+using range_reference_type = typename range_reference_type_impl<T>::type;
+#else
+template <typename Range>
+using range_reference_type =
+    decltype(*detail::range_begin(std::declval<Range&>()));
+#endif
+
+// We don't use the Range's value_type for anything, but we do need the Range's
+// reference type, with cv-ref stripped.
+template <typename Range>
+using uncvref_type = remove_cvref_t<range_reference_type<Range>>;
+
+template <typename Formatter>
+FMT_CONSTEXPR auto maybe_set_debug_format(Formatter& f, bool set)
+    -> decltype(f.set_debug_format(set)) {
+  f.set_debug_format(set);
+}
+template <typename Formatter>
+FMT_CONSTEXPR void maybe_set_debug_format(Formatter&, ...) {}
+
+template <typename T>
+struct range_format_kind_
+    : std::integral_constant<range_format,
+                             std::is_same<uncvref_type<T>, T>::value
+                                 ? range_format::disabled
+                             : is_map<T>::value ? range_format::map
+                             : is_set<T>::value ? range_format::set
+                                                : range_format::sequence> {};
+
+template <range_format K>
+using range_format_constant = std::integral_constant<range_format, K>;
+
+// These are not generic lambdas for compatibility with C++11.
+template <typename ParseContext> struct parse_empty_specs {
+  template <typename Formatter> FMT_CONSTEXPR void operator()(Formatter& f) {
+    f.parse(ctx);
+    detail::maybe_set_debug_format(f, true);
+  }
+  ParseContext& ctx;
+};
+template <typename FormatContext> struct format_tuple_element {
+  using char_type = typename FormatContext::char_type;
+
+  template <typename T>
+  void operator()(const formatter<T, char_type>& f, const T& v) {
+    if (i > 0) ctx.advance_to(detail::copy<char_type>(separator, ctx.out()));
+    ctx.advance_to(f.format(v, ctx));
+    ++i;
+  }
+
+  int i;
+  FormatContext& ctx;
+  basic_string_view<char_type> separator;
+};
+
+}  // namespace detail
+
+template <typename T> struct is_tuple_like {
+  static constexpr const bool value =
+      detail::is_tuple_like_<T>::value && !detail::is_range_<T>::value;
+};
+
+template <typename T, typename C> struct is_tuple_formattable {
+  static constexpr const bool value =
+      detail::is_tuple_formattable_<T, C>::value;
+};
+
+template <typename Tuple, typename Char>
+struct formatter<Tuple, Char,
+                 enable_if_t<fmt::is_tuple_like<Tuple>::value &&
+                             fmt::is_tuple_formattable<Tuple, Char>::value>> {
+ private:
+  decltype(detail::tuple::get_formatters<Tuple, Char>(
+      detail::tuple_index_sequence<Tuple>())) formatters_;
+
+  basic_string_view<Char> separator_ = detail::string_literal<Char, ',', ' '>{};
+  basic_string_view<Char> opening_bracket_ =
+      detail::string_literal<Char, '('>{};
+  basic_string_view<Char> closing_bracket_ =
+      detail::string_literal<Char, ')'>{};
+
+ public:
+  FMT_CONSTEXPR formatter() {}
+
+  FMT_CONSTEXPR void set_separator(basic_string_view<Char> sep) {
+    separator_ = sep;
+  }
+
+  FMT_CONSTEXPR void set_brackets(basic_string_view<Char> open,
+                                  basic_string_view<Char> close) {
+    opening_bracket_ = open;
+    closing_bracket_ = close;
+  }
+
+  template <typename ParseContext>
+  FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
+    auto it = ctx.begin();
+    if (it != ctx.end() && *it != '}') report_error("invalid format specifier");
+    detail::for_each(formatters_, detail::parse_empty_specs<ParseContext>{ctx});
+    return it;
+  }
+
+  template <typename FormatContext>
+  auto format(const Tuple& value, FormatContext& ctx) const
+      -> decltype(ctx.out()) {
+    ctx.advance_to(detail::copy<Char>(opening_bracket_, ctx.out()));
+    detail::for_each2(
+        formatters_, value,
+        detail::format_tuple_element<FormatContext>{0, ctx, separator_});
+    return detail::copy<Char>(closing_bracket_, ctx.out());
+  }
+};
+
+template <typename T, typename Char> struct is_range {
+  static constexpr const bool value =
+      detail::is_range_<T>::value && !detail::has_to_string_view<T>::value;
+};
+
+namespace detail {
+template <typename Context> struct range_mapper {
+  using mapper = arg_mapper<Context>;
+
+  template <typename T,
+            FMT_ENABLE_IF(has_formatter<remove_cvref_t<T>, Context>::value)>
+  static auto map(T&& value) -> T&& {
+    return static_cast<T&&>(value);
+  }
+  template <typename T,
+            FMT_ENABLE_IF(!has_formatter<remove_cvref_t<T>, Context>::value)>
+  static auto map(T&& value)
+      -> decltype(mapper().map(static_cast<T&&>(value))) {
+    return mapper().map(static_cast<T&&>(value));
+  }
+};
+
+template <typename Char, typename Element>
+using range_formatter_type =
+    formatter<remove_cvref_t<decltype(range_mapper<buffered_context<Char>>{}
+                                          .map(std::declval<Element>()))>,
+              Char>;
+
+template <typename R>
+using maybe_const_range =
+    conditional_t<has_const_begin_end<R>::value, const R, R>;
+
+// Workaround a bug in MSVC 2015 and earlier.
+#if !FMT_MSC_VERSION || FMT_MSC_VERSION >= 1910
+template <typename R, typename Char>
+struct is_formattable_delayed
+    : is_formattable<uncvref_type<maybe_const_range<R>>, Char> {};
+#endif
+}  // namespace detail
+
+template <typename...> struct conjunction : std::true_type {};
+template <typename P> struct conjunction<P> : P {};
+template <typename P1, typename... Pn>
+struct conjunction<P1, Pn...>
+    : conditional_t<bool(P1::value), conjunction<Pn...>, P1> {};
+
+template <typename T, typename Char, typename Enable = void>
+struct range_formatter;
+
+template <typename T, typename Char>
+struct range_formatter<
+    T, Char,
+    enable_if_t<conjunction<std::is_same<T, remove_cvref_t<T>>,
+                            is_formattable<T, Char>>::value>> {
+ private:
+  detail::range_formatter_type<Char, T> underlying_;
+  basic_string_view<Char> separator_ = detail::string_literal<Char, ',', ' '>{};
+  basic_string_view<Char> opening_bracket_ =
+      detail::string_literal<Char, '['>{};
+  basic_string_view<Char> closing_bracket_ =
+      detail::string_literal<Char, ']'>{};
+  bool is_debug = false;
+
+  template <typename Output, typename It, typename Sentinel, typename U = T,
+            FMT_ENABLE_IF(std::is_same<U, Char>::value)>
+  auto write_debug_string(Output& out, It it, Sentinel end) const -> Output {
+    auto buf = basic_memory_buffer<Char>();
+    for (; it != end; ++it) buf.push_back(*it);
+    auto specs = format_specs();
+    specs.type = presentation_type::debug;
+    return detail::write<Char>(
+        out, basic_string_view<Char>(buf.data(), buf.size()), specs);
+  }
+
+  template <typename Output, typename It, typename Sentinel, typename U = T,
+            FMT_ENABLE_IF(!std::is_same<U, Char>::value)>
+  auto write_debug_string(Output& out, It, Sentinel) const -> Output {
+    return out;
+  }
+
+ public:
+  FMT_CONSTEXPR range_formatter() {}
+
+  FMT_CONSTEXPR auto underlying() -> detail::range_formatter_type<Char, T>& {
+    return underlying_;
+  }
+
+  FMT_CONSTEXPR void set_separator(basic_string_view<Char> sep) {
+    separator_ = sep;
+  }
+
+  FMT_CONSTEXPR void set_brackets(basic_string_view<Char> open,
+                                  basic_string_view<Char> close) {
+    opening_bracket_ = open;
+    closing_bracket_ = close;
+  }
+
+  template <typename ParseContext>
+  FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
+    auto it = ctx.begin();
+    auto end = ctx.end();
+    detail::maybe_set_debug_format(underlying_, true);
+    if (it == end) return underlying_.parse(ctx);
+
+    switch (detail::to_ascii(*it)) {
+    case 'n':
+      set_brackets({}, {});
+      ++it;
+      break;
+    case '?':
+      is_debug = true;
+      set_brackets({}, {});
+      ++it;
+      if (it == end || *it != 's') report_error("invalid format specifier");
+      FMT_FALLTHROUGH;
+    case 's':
+      if (!std::is_same<T, Char>::value)
+        report_error("invalid format specifier");
+      if (!is_debug) {
+        set_brackets(detail::string_literal<Char, '"'>{},
+                     detail::string_literal<Char, '"'>{});
+        set_separator({});
+        detail::maybe_set_debug_format(underlying_, false);
+      }
+      ++it;
+      return it;
+    }
+
+    if (it != end && *it != '}') {
+      if (*it != ':') report_error("invalid format specifier");
+      detail::maybe_set_debug_format(underlying_, false);
+      ++it;
+    }
+
+    ctx.advance_to(it);
+    return underlying_.parse(ctx);
+  }
+
+  template <typename R, typename FormatContext>
+  auto format(R&& range, FormatContext& ctx) const -> decltype(ctx.out()) {
+    auto mapper = detail::range_mapper<buffered_context<Char>>();
+    auto out = ctx.out();
+    auto it = detail::range_begin(range);
+    auto end = detail::range_end(range);
+    if (is_debug) return write_debug_string(out, std::move(it), end);
+
+    out = detail::copy<Char>(opening_bracket_, out);
+    int i = 0;
+    for (; it != end; ++it) {
+      if (i > 0) out = detail::copy<Char>(separator_, out);
+      ctx.advance_to(out);
+      auto&& item = *it;  // Need an lvalue
+      out = underlying_.format(mapper.map(item), ctx);
+      ++i;
+    }
+    out = detail::copy<Char>(closing_bracket_, out);
+    return out;
+  }
+};
+
+FMT_EXPORT
+template <typename T, typename Char, typename Enable = void>
+struct range_format_kind
+    : conditional_t<
+          is_range<T, Char>::value, detail::range_format_kind_<T>,
+          std::integral_constant<range_format, range_format::disabled>> {};
+
+template <typename R, typename Char>
+struct formatter<
+    R, Char,
+    enable_if_t<conjunction<
+        bool_constant<
+            range_format_kind<R, Char>::value != range_format::disabled &&
+            range_format_kind<R, Char>::value != range_format::map &&
+            range_format_kind<R, Char>::value != range_format::string &&
+            range_format_kind<R, Char>::value != range_format::debug_string>
+// Workaround a bug in MSVC 2015 and earlier.
+#if !FMT_MSC_VERSION || FMT_MSC_VERSION >= 1910
+        ,
+        detail::is_formattable_delayed<R, Char>
+#endif
+        >::value>> {
+ private:
+  using range_type = detail::maybe_const_range<R>;
+  range_formatter<detail::uncvref_type<range_type>, Char> range_formatter_;
+
+ public:
+  using nonlocking = void;
+
+  FMT_CONSTEXPR formatter() {
+    if (detail::const_check(range_format_kind<R, Char>::value !=
+                            range_format::set))
+      return;
+    range_formatter_.set_brackets(detail::string_literal<Char, '{'>{},
+                                  detail::string_literal<Char, '}'>{});
+  }
+
+  template <typename ParseContext>
+  FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
+    return range_formatter_.parse(ctx);
+  }
+
+  template <typename FormatContext>
+  auto format(range_type& range, FormatContext& ctx) const
+      -> decltype(ctx.out()) {
+    return range_formatter_.format(range, ctx);
+  }
+};
+
+// A map formatter.
+template <typename R, typename Char>
+struct formatter<
+    R, Char,
+    enable_if_t<range_format_kind<R, Char>::value == range_format::map>> {
+ private:
+  using map_type = detail::maybe_const_range<R>;
+  using element_type = detail::uncvref_type<map_type>;
+
+  decltype(detail::tuple::get_formatters<element_type, Char>(
+      detail::tuple_index_sequence<element_type>())) formatters_;
+  bool no_delimiters_ = false;
+
+ public:
+  FMT_CONSTEXPR formatter() {}
+
+  template <typename ParseContext>
+  FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
+    auto it = ctx.begin();
+    auto end = ctx.end();
+    if (it != end) {
+      if (detail::to_ascii(*it) == 'n') {
+        no_delimiters_ = true;
+        ++it;
+      }
+      if (it != end && *it != '}') {
+        if (*it != ':') report_error("invalid format specifier");
+        ++it;
+      }
+      ctx.advance_to(it);
+    }
+    detail::for_each(formatters_, detail::parse_empty_specs<ParseContext>{ctx});
+    return it;
+  }
+
+  template <typename FormatContext>
+  auto format(map_type& map, FormatContext& ctx) const -> decltype(ctx.out()) {
+    auto out = ctx.out();
+    basic_string_view<Char> open = detail::string_literal<Char, '{'>{};
+    if (!no_delimiters_) out = detail::copy<Char>(open, out);
+    int i = 0;
+    auto mapper = detail::range_mapper<buffered_context<Char>>();
+    basic_string_view<Char> sep = detail::string_literal<Char, ',', ' '>{};
+    for (auto&& value : map) {
+      if (i > 0) out = detail::copy<Char>(sep, out);
+      ctx.advance_to(out);
+      detail::for_each2(formatters_, mapper.map(value),
+                        detail::format_tuple_element<FormatContext>{
+                            0, ctx, detail::string_literal<Char, ':', ' '>{}});
+      ++i;
+    }
+    basic_string_view<Char> close = detail::string_literal<Char, '}'>{};
+    if (!no_delimiters_) out = detail::copy<Char>(close, out);
+    return out;
+  }
+};
+
+// A (debug_)string formatter.
+template <typename R, typename Char>
+struct formatter<
+    R, Char,
+    enable_if_t<range_format_kind<R, Char>::value == range_format::string ||
+                range_format_kind<R, Char>::value ==
+                    range_format::debug_string>> {
+ private:
+  using range_type = detail::maybe_const_range<R>;
+  using string_type =
+      conditional_t<std::is_constructible<
+                        detail::std_string_view<Char>,
+                        decltype(detail::range_begin(std::declval<R>())),
+                        decltype(detail::range_end(std::declval<R>()))>::value,
+                    detail::std_string_view<Char>, std::basic_string<Char>>;
+
+  formatter<string_type, Char> underlying_;
+
+ public:
+  template <typename ParseContext>
+  FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
+    return underlying_.parse(ctx);
+  }
+
+  template <typename FormatContext>
+  auto format(range_type& range, FormatContext& ctx) const
+      -> decltype(ctx.out()) {
+    auto out = ctx.out();
+    if (detail::const_check(range_format_kind<R, Char>::value ==
+                            range_format::debug_string))
+      *out++ = '"';
+    out = underlying_.format(
+        string_type{detail::range_begin(range), detail::range_end(range)}, ctx);
+    if (detail::const_check(range_format_kind<R, Char>::value ==
+                            range_format::debug_string))
+      *out++ = '"';
+    return out;
+  }
+};
+
+template <typename It, typename Sentinel, typename Char = char>
+struct join_view : detail::view {
+  It begin;
+  Sentinel end;
+  basic_string_view<Char> sep;
+
+  join_view(It b, Sentinel e, basic_string_view<Char> s)
+      : begin(std::move(b)), end(e), sep(s) {}
+};
+
+template <typename It, typename Sentinel, typename Char>
+struct formatter<join_view<It, Sentinel, Char>, Char> {
+ private:
+  using value_type =
+#ifdef __cpp_lib_ranges
+      std::iter_value_t<It>;
+#else
+      typename std::iterator_traits<It>::value_type;
+#endif
+  formatter<remove_cvref_t<value_type>, Char> value_formatter_;
+
+  using view_ref = conditional_t<std::is_copy_constructible<It>::value,
+                                 const join_view<It, Sentinel, Char>&,
+                                 join_view<It, Sentinel, Char>&&>;
+
+ public:
+  using nonlocking = void;
+
+  template <typename ParseContext>
+  FMT_CONSTEXPR auto parse(ParseContext& ctx) -> const Char* {
+    return value_formatter_.parse(ctx);
+  }
+
+  template <typename FormatContext>
+  auto format(view_ref& value, FormatContext& ctx) const
+      -> decltype(ctx.out()) {
+    auto it = std::forward<view_ref>(value).begin;
+    auto out = ctx.out();
+    if (it == value.end) return out;
+    out = value_formatter_.format(*it, ctx);
+    ++it;
+    while (it != value.end) {
+      out = detail::copy<Char>(value.sep.begin(), value.sep.end(), out);
+      ctx.advance_to(out);
+      out = value_formatter_.format(*it, ctx);
+      ++it;
+    }
+    return out;
+  }
+};
+
+/// Returns a view that formats the iterator range `[begin, end)` with elements
+/// separated by `sep`.
+template <typename It, typename Sentinel>
+auto join(It begin, Sentinel end, string_view sep) -> join_view<It, Sentinel> {
+  return {std::move(begin), end, sep};
+}
+
+/**
+ * Returns a view that formats `range` with elements separated by `sep`.
+ *
+ * **Example**:
+ *
+ *     auto v = std::vector<int>{1, 2, 3};
+ *     fmt::print("{}", fmt::join(v, ", "));
+ *     // Output: 1, 2, 3
+ *
+ * `fmt::join` applies passed format specifiers to the range elements:
+ *
+ *     fmt::print("{:02}", fmt::join(v, ", "));
+ *     // Output: 01, 02, 03
+ */
+template <typename Range>
+auto join(Range&& r, string_view sep)
+    -> join_view<decltype(detail::range_begin(r)),
+                 decltype(detail::range_end(r))> {
+  return {detail::range_begin(r), detail::range_end(r), sep};
+}
+
+template <typename Char, typename... T> struct tuple_join_view : detail::view {
+  const std::tuple<T...>& tuple;
+  basic_string_view<Char> sep;
+
+  tuple_join_view(const std::tuple<T...>& t, basic_string_view<Char> s)
+      : tuple(t), sep{s} {}
+};
+
+// Define FMT_TUPLE_JOIN_SPECIFIERS to enable experimental format specifiers
+// support in tuple_join. It is disabled by default because of issues with
+// the dynamic width and precision.
+#ifndef FMT_TUPLE_JOIN_SPECIFIERS
+#  define FMT_TUPLE_JOIN_SPECIFIERS 0
+#endif
+
+template <typename Char, typename... T>
+struct formatter<tuple_join_view<Char, T...>, Char> {
+  template <typename ParseContext>
+  FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
+    return do_parse(ctx, std::integral_constant<size_t, sizeof...(T)>());
+  }
+
+  template <typename FormatContext>
+  auto format(const tuple_join_view<Char, T...>& value,
+              FormatContext& ctx) const -> typename FormatContext::iterator {
+    return do_format(value, ctx,
+                     std::integral_constant<size_t, sizeof...(T)>());
+  }
+
+ private:
+  std::tuple<formatter<typename std::decay<T>::type, Char>...> formatters_;
+
+  template <typename ParseContext>
+  FMT_CONSTEXPR auto do_parse(ParseContext& ctx,
+                              std::integral_constant<size_t, 0>)
+      -> decltype(ctx.begin()) {
+    return ctx.begin();
+  }
+
+  template <typename ParseContext, size_t N>
+  FMT_CONSTEXPR auto do_parse(ParseContext& ctx,
+                              std::integral_constant<size_t, N>)
+      -> decltype(ctx.begin()) {
+    auto end = ctx.begin();
+#if FMT_TUPLE_JOIN_SPECIFIERS
+    end = std::get<sizeof...(T) - N>(formatters_).parse(ctx);
+    if (N > 1) {
+      auto end1 = do_parse(ctx, std::integral_constant<size_t, N - 1>());
+      if (end != end1)
+        report_error("incompatible format specs for tuple elements");
+    }
+#endif
+    return end;
+  }
+
+  template <typename FormatContext>
+  auto do_format(const tuple_join_view<Char, T...>&, FormatContext& ctx,
+                 std::integral_constant<size_t, 0>) const ->
+      typename FormatContext::iterator {
+    return ctx.out();
+  }
+
+  template <typename FormatContext, size_t N>
+  auto do_format(const tuple_join_view<Char, T...>& value, FormatContext& ctx,
+                 std::integral_constant<size_t, N>) const ->
+      typename FormatContext::iterator {
+    auto out = std::get<sizeof...(T) - N>(formatters_)
+                   .format(std::get<sizeof...(T) - N>(value.tuple), ctx);
+    if (N <= 1) return out;
+    out = detail::copy<Char>(value.sep, out);
+    ctx.advance_to(out);
+    return do_format(value, ctx, std::integral_constant<size_t, N - 1>());
+  }
+};
+
+namespace detail {
+// Check if T has an interface like a container adaptor (e.g. std::stack,
+// std::queue, std::priority_queue).
+template <typename T> class is_container_adaptor_like {
+  template <typename U> static auto check(U* p) -> typename U::container_type;
+  template <typename> static void check(...);
+
+ public:
+  static constexpr const bool value =
+      !std::is_void<decltype(check<T>(nullptr))>::value;
+};
+
+template <typename Container> struct all {
+  const Container& c;
+  auto begin() const -> typename Container::const_iterator { return c.begin(); }
+  auto end() const -> typename Container::const_iterator { return c.end(); }
+};
+}  // namespace detail
+
+template <typename T, typename Char>
+struct formatter<
+    T, Char,
+    enable_if_t<conjunction<detail::is_container_adaptor_like<T>,
+                            bool_constant<range_format_kind<T, Char>::value ==
+                                          range_format::disabled>>::value>>
+    : formatter<detail::all<typename T::container_type>, Char> {
+  using all = detail::all<typename T::container_type>;
+  template <typename FormatContext>
+  auto format(const T& t, FormatContext& ctx) const -> decltype(ctx.out()) {
+    struct getter : T {
+      static auto get(const T& t) -> all {
+        return {t.*(&getter::c)};  // Access c through the derived class.
+      }
+    };
+    return formatter<all>::format(getter::get(t), ctx);
+  }
+};
+
+FMT_BEGIN_EXPORT
+
+/**
+ * Returns an object that formats `std::tuple` with elements separated by `sep`.
+ *
+ * **Example**:
+ *
+ *     auto t = std::tuple<int, char>{1, 'a'};
+ *     fmt::print("{}", fmt::join(t, ", "));
+ *     // Output: 1, a
+ */
+template <typename... T>
+FMT_CONSTEXPR auto join(const std::tuple<T...>& tuple, string_view sep)
+    -> tuple_join_view<char, T...> {
+  return {tuple, sep};
+}
+
+/**
+ * Returns an object that formats `std::initializer_list` with elements
+ * separated by `sep`.
+ *
+ * **Example**:
+ *
+ *     fmt::print("{}", fmt::join({1, 2, 3}, ", "));
+ *     // Output: "1, 2, 3"
+ */
+template <typename T>
+auto join(std::initializer_list<T> list, string_view sep)
+    -> join_view<const T*, const T*> {
+  return join(std::begin(list), std::end(list), sep);
+}
+
+FMT_END_EXPORT
+FMT_END_NAMESPACE
+
+#endif  // FMT_RANGES_H_
diff --git a/ext/spdlog/include/spdlog/fmt/bundled/std.h b/ext/spdlog/include/spdlog/fmt/bundled/std.h
new file mode 100644
index 0000000..fb43940
--- /dev/null
+++ b/ext/spdlog/include/spdlog/fmt/bundled/std.h
@@ -0,0 +1,699 @@
+// Formatting library for C++ - formatters for standard library types
+//
+// Copyright (c) 2012 - present, Victor Zverovich
+// All rights reserved.
+//
+// For the license information refer to format.h.
+
+#ifndef FMT_STD_H_
+#define FMT_STD_H_
+
+#include "format.h"
+#include "ostream.h"
+
+#ifndef FMT_MODULE
+#  include <atomic>
+#  include <bitset>
+#  include <complex>
+#  include <cstdlib>
+#  include <exception>
+#  include <memory>
+#  include <thread>
+#  include <type_traits>
+#  include <typeinfo>
+#  include <utility>
+#  include <vector>
+
+// Check FMT_CPLUSPLUS to suppress a bogus warning in MSVC.
+#  if FMT_CPLUSPLUS >= 201703L
+#    if FMT_HAS_INCLUDE(<filesystem>)
+#      include <filesystem>
+#    endif
+#    if FMT_HAS_INCLUDE(<variant>)
+#      include <variant>
+#    endif
+#    if FMT_HAS_INCLUDE(<optional>)
+#      include <optional>
+#    endif
+#  endif
+// Use > instead of >= in the version check because <source_location> may be
+// available after C++17 but before C++20 is marked as implemented.
+#  if FMT_CPLUSPLUS > 201703L && FMT_HAS_INCLUDE(<source_location>)
+#    include <source_location>
+#  endif
+#  if FMT_CPLUSPLUS > 202002L && FMT_HAS_INCLUDE(<expected>)
+#    include <expected>
+#  endif
+#endif  // FMT_MODULE
+
+#if FMT_HAS_INCLUDE(<version>)
+#  include <version>
+#endif
+
+// GCC 4 does not support FMT_HAS_INCLUDE.
+#if FMT_HAS_INCLUDE(<cxxabi.h>) || defined(__GLIBCXX__)
+#  include <cxxabi.h>
+// Android NDK with gabi++ library on some architectures does not implement
+// abi::__cxa_demangle().
+#  ifndef __GABIXX_CXXABI_H__
+#    define FMT_HAS_ABI_CXA_DEMANGLE
+#  endif
+#endif
+
+// For older Xcode versions, __cpp_lib_xxx flags are inaccurately defined.
+#ifndef FMT_CPP_LIB_FILESYSTEM
+#  ifdef __cpp_lib_filesystem
+#    define FMT_CPP_LIB_FILESYSTEM __cpp_lib_filesystem
+#  else
+#    define FMT_CPP_LIB_FILESYSTEM 0
+#  endif
+#endif
+
+#ifndef FMT_CPP_LIB_VARIANT
+#  ifdef __cpp_lib_variant
+#    define FMT_CPP_LIB_VARIANT __cpp_lib_variant
+#  else
+#    define FMT_CPP_LIB_VARIANT 0
+#  endif
+#endif
+
+#if FMT_CPP_LIB_FILESYSTEM
+FMT_BEGIN_NAMESPACE
+
+namespace detail {
+
+template <typename Char, typename PathChar>
+auto get_path_string(const std::filesystem::path& p,
+                     const std::basic_string<PathChar>& native) {
+  if constexpr (std::is_same_v<Char, char> && std::is_same_v<PathChar, wchar_t>)
+    return to_utf8<wchar_t>(native, to_utf8_error_policy::replace);
+  else
+    return p.string<Char>();
+}
+
+template <typename Char, typename PathChar>
+void write_escaped_path(basic_memory_buffer<Char>& quoted,
+                        const std::filesystem::path& p,
+                        const std::basic_string<PathChar>& native) {
+  if constexpr (std::is_same_v<Char, char> &&
+                std::is_same_v<PathChar, wchar_t>) {
+    auto buf = basic_memory_buffer<wchar_t>();
+    write_escaped_string<wchar_t>(std::back_inserter(buf), native);
+    bool valid = to_utf8<wchar_t>::convert(quoted, {buf.data(), buf.size()});
+    FMT_ASSERT(valid, "invalid utf16");
+  } else if constexpr (std::is_same_v<Char, PathChar>) {
+    write_escaped_string<std::filesystem::path::value_type>(
+        std::back_inserter(quoted), native);
+  } else {
+    write_escaped_string<Char>(std::back_inserter(quoted), p.string<Char>());
+  }
+}
+
+}  // namespace detail
+
+FMT_EXPORT
+template <typename Char> struct formatter<std::filesystem::path, Char> {
+ private:
+  format_specs specs_;
+  detail::arg_ref<Char> width_ref_;
+  bool debug_ = false;
+  char path_type_ = 0;
+
+ public:
+  FMT_CONSTEXPR void set_debug_format(bool set = true) { debug_ = set; }
+
+  template <typename ParseContext> FMT_CONSTEXPR auto parse(ParseContext& ctx) {
+    auto it = ctx.begin(), end = ctx.end();
+    if (it == end) return it;
+
+    it = detail::parse_align(it, end, specs_);
+    if (it == end) return it;
+
+    it = detail::parse_dynamic_spec(it, end, specs_.width, width_ref_, ctx);
+    if (it != end && *it == '?') {
+      debug_ = true;
+      ++it;
+    }
+    if (it != end && (*it == 'g')) path_type_ = detail::to_ascii(*it++);
+    return it;
+  }
+
+  template <typename FormatContext>
+  auto format(const std::filesystem::path& p, FormatContext& ctx) const {
+    auto specs = specs_;
+    auto path_string =
+        !path_type_ ? p.native()
+                    : p.generic_string<std::filesystem::path::value_type>();
+
+    detail::handle_dynamic_spec<detail::width_checker>(specs.width, width_ref_,
+                                                       ctx);
+    if (!debug_) {
+      auto s = detail::get_path_string<Char>(p, path_string);
+      return detail::write(ctx.out(), basic_string_view<Char>(s), specs);
+    }
+    auto quoted = basic_memory_buffer<Char>();
+    detail::write_escaped_path(quoted, p, path_string);
+    return detail::write(ctx.out(),
+                         basic_string_view<Char>(quoted.data(), quoted.size()),
+                         specs);
+  }
+};
+
+class path : public std::filesystem::path {
+ public:
+  auto display_string() const -> std::string {
+    const std::filesystem::path& base = *this;
+    return fmt::format(FMT_STRING("{}"), base);
+  }
+  auto system_string() const -> std::string { return string(); }
+
+  auto generic_display_string() const -> std::string {
+    const std::filesystem::path& base = *this;
+    return fmt::format(FMT_STRING("{:g}"), base);
+  }
+  auto generic_system_string() const -> std::string { return generic_string(); }
+};
+
+FMT_END_NAMESPACE
+#endif  // FMT_CPP_LIB_FILESYSTEM
+
+FMT_BEGIN_NAMESPACE
+FMT_EXPORT
+template <std::size_t N, typename Char>
+struct formatter<std::bitset<N>, Char> : nested_formatter<string_view> {
+ private:
+  // Functor because C++11 doesn't support generic lambdas.
+  struct writer {
+    const std::bitset<N>& bs;
+
+    template <typename OutputIt>
+    FMT_CONSTEXPR auto operator()(OutputIt out) -> OutputIt {
+      for (auto pos = N; pos > 0; --pos) {
+        out = detail::write<Char>(out, bs[pos - 1] ? Char('1') : Char('0'));
+      }
+
+      return out;
+    }
+  };
+
+ public:
+  template <typename FormatContext>
+  auto format(const std::bitset<N>& bs, FormatContext& ctx) const
+      -> decltype(ctx.out()) {
+    return write_padded(ctx, writer{bs});
+  }
+};
+
+FMT_EXPORT
+template <typename Char>
+struct formatter<std::thread::id, Char> : basic_ostream_formatter<Char> {};
+FMT_END_NAMESPACE
+
+#ifdef __cpp_lib_optional
+FMT_BEGIN_NAMESPACE
+FMT_EXPORT
+template <typename T, typename Char>
+struct formatter<std::optional<T>, Char,
+                 std::enable_if_t<is_formattable<T, Char>::value>> {
+ private:
+  formatter<T, Char> underlying_;
+  static constexpr basic_string_view<Char> optional =
+      detail::string_literal<Char, 'o', 'p', 't', 'i', 'o', 'n', 'a', 'l',
+                             '('>{};
+  static constexpr basic_string_view<Char> none =
+      detail::string_literal<Char, 'n', 'o', 'n', 'e'>{};
+
+  template <class U>
+  FMT_CONSTEXPR static auto maybe_set_debug_format(U& u, bool set)
+      -> decltype(u.set_debug_format(set)) {
+    u.set_debug_format(set);
+  }
+
+  template <class U>
+  FMT_CONSTEXPR static void maybe_set_debug_format(U&, ...) {}
+
+ public:
+  template <typename ParseContext> FMT_CONSTEXPR auto parse(ParseContext& ctx) {
+    maybe_set_debug_format(underlying_, true);
+    return underlying_.parse(ctx);
+  }
+
+  template <typename FormatContext>
+  auto format(const std::optional<T>& opt, FormatContext& ctx) const
+      -> decltype(ctx.out()) {
+    if (!opt) return detail::write<Char>(ctx.out(), none);
+
+    auto out = ctx.out();
+    out = detail::write<Char>(out, optional);
+    ctx.advance_to(out);
+    out = underlying_.format(*opt, ctx);
+    return detail::write(out, ')');
+  }
+};
+FMT_END_NAMESPACE
+#endif  // __cpp_lib_optional
+
+#if defined(__cpp_lib_expected) || FMT_CPP_LIB_VARIANT
+
+FMT_BEGIN_NAMESPACE
+namespace detail {
+
+template <typename Char, typename OutputIt, typename T>
+auto write_escaped_alternative(OutputIt out, const T& v) -> OutputIt {
+  if constexpr (has_to_string_view<T>::value)
+    return write_escaped_string<Char>(out, detail::to_string_view(v));
+  if constexpr (std::is_same_v<T, Char>) return write_escaped_char(out, v);
+  return write<Char>(out, v);
+}
+
+}  // namespace detail
+
+FMT_END_NAMESPACE
+#endif
+
+#ifdef __cpp_lib_expected
+FMT_BEGIN_NAMESPACE
+
+FMT_EXPORT
+template <typename T, typename E, typename Char>
+struct formatter<std::expected<T, E>, Char,
+                 std::enable_if_t<is_formattable<T, Char>::value &&
+                                  is_formattable<E, Char>::value>> {
+  template <typename ParseContext>
+  FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
+    return ctx.begin();
+  }
+
+  template <typename FormatContext>
+  auto format(const std::expected<T, E>& value, FormatContext& ctx) const
+      -> decltype(ctx.out()) {
+    auto out = ctx.out();
+
+    if (value.has_value()) {
+      out = detail::write<Char>(out, "expected(");
+      out = detail::write_escaped_alternative<Char>(out, *value);
+    } else {
+      out = detail::write<Char>(out, "unexpected(");
+      out = detail::write_escaped_alternative<Char>(out, value.error());
+    }
+    *out++ = ')';
+    return out;
+  }
+};
+FMT_END_NAMESPACE
+#endif  // __cpp_lib_expected
+
+#ifdef __cpp_lib_source_location
+FMT_BEGIN_NAMESPACE
+FMT_EXPORT
+template <> struct formatter<std::source_location> {
+  template <typename ParseContext> FMT_CONSTEXPR auto parse(ParseContext& ctx) {
+    return ctx.begin();
+  }
+
+  template <typename FormatContext>
+  auto format(const std::source_location& loc, FormatContext& ctx) const
+      -> decltype(ctx.out()) {
+    auto out = ctx.out();
+    out = detail::write(out, loc.file_name());
+    out = detail::write(out, ':');
+    out = detail::write<char>(out, loc.line());
+    out = detail::write(out, ':');
+    out = detail::write<char>(out, loc.column());
+    out = detail::write(out, ": ");
+    out = detail::write(out, loc.function_name());
+    return out;
+  }
+};
+FMT_END_NAMESPACE
+#endif
+
+#if FMT_CPP_LIB_VARIANT
+FMT_BEGIN_NAMESPACE
+namespace detail {
+
+template <typename T>
+using variant_index_sequence =
+    std::make_index_sequence<std::variant_size<T>::value>;
+
+template <typename> struct is_variant_like_ : std::false_type {};
+template <typename... Types>
+struct is_variant_like_<std::variant<Types...>> : std::true_type {};
+
+// formattable element check.
+template <typename T, typename C> class is_variant_formattable_ {
+  template <std::size_t... Is>
+  static std::conjunction<
+      is_formattable<std::variant_alternative_t<Is, T>, C>...>
+      check(std::index_sequence<Is...>);
+
+ public:
+  static constexpr const bool value =
+      decltype(check(variant_index_sequence<T>{}))::value;
+};
+
+}  // namespace detail
+
+template <typename T> struct is_variant_like {
+  static constexpr const bool value = detail::is_variant_like_<T>::value;
+};
+
+template <typename T, typename C> struct is_variant_formattable {
+  static constexpr const bool value =
+      detail::is_variant_formattable_<T, C>::value;
+};
+
+FMT_EXPORT
+template <typename Char> struct formatter<std::monostate, Char> {
+  template <typename ParseContext>
+  FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
+    return ctx.begin();
+  }
+
+  template <typename FormatContext>
+  auto format(const std::monostate&, FormatContext& ctx) const
+      -> decltype(ctx.out()) {
+    return detail::write<Char>(ctx.out(), "monostate");
+  }
+};
+
+FMT_EXPORT
+template <typename Variant, typename Char>
+struct formatter<
+    Variant, Char,
+    std::enable_if_t<std::conjunction_v<
+        is_variant_like<Variant>, is_variant_formattable<Variant, Char>>>> {
+  template <typename ParseContext>
+  FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
+    return ctx.begin();
+  }
+
+  template <typename FormatContext>
+  auto format(const Variant& value, FormatContext& ctx) const
+      -> decltype(ctx.out()) {
+    auto out = ctx.out();
+
+    out = detail::write<Char>(out, "variant(");
+    FMT_TRY {
+      std::visit(
+          [&](const auto& v) {
+            out = detail::write_escaped_alternative<Char>(out, v);
+          },
+          value);
+    }
+    FMT_CATCH(const std::bad_variant_access&) {
+      detail::write<Char>(out, "valueless by exception");
+    }
+    *out++ = ')';
+    return out;
+  }
+};
+FMT_END_NAMESPACE
+#endif  // FMT_CPP_LIB_VARIANT
+
+FMT_BEGIN_NAMESPACE
+FMT_EXPORT
+template <typename Char> struct formatter<std::error_code, Char> {
+  template <typename ParseContext>
+  FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
+    return ctx.begin();
+  }
+
+  template <typename FormatContext>
+  FMT_CONSTEXPR auto format(const std::error_code& ec, FormatContext& ctx) const
+      -> decltype(ctx.out()) {
+    auto out = ctx.out();
+    out = detail::write_bytes<Char>(out, ec.category().name(), format_specs());
+    out = detail::write<Char>(out, Char(':'));
+    out = detail::write<Char>(out, ec.value());
+    return out;
+  }
+};
+
+#if FMT_USE_RTTI
+namespace detail {
+
+template <typename Char, typename OutputIt>
+auto write_demangled_name(OutputIt out, const std::type_info& ti) -> OutputIt {
+#  ifdef FMT_HAS_ABI_CXA_DEMANGLE
+  int status = 0;
+  std::size_t size = 0;
+  std::unique_ptr<char, void (*)(void*)> demangled_name_ptr(
+      abi::__cxa_demangle(ti.name(), nullptr, &size, &status), &std::free);
+
+  string_view demangled_name_view;
+  if (demangled_name_ptr) {
+    demangled_name_view = demangled_name_ptr.get();
+
+    // Normalization of stdlib inline namespace names.
+    // libc++ inline namespaces.
+    //  std::__1::*       -> std::*
+    //  std::__1::__fs::* -> std::*
+    // libstdc++ inline namespaces.
+    //  std::__cxx11::*             -> std::*
+    //  std::filesystem::__cxx11::* -> std::filesystem::*
+    if (demangled_name_view.starts_with("std::")) {
+      char* begin = demangled_name_ptr.get();
+      char* to = begin + 5;  // std::
+      for (char *from = to, *end = begin + demangled_name_view.size();
+           from < end;) {
+        // This is safe, because demangled_name is NUL-terminated.
+        if (from[0] == '_' && from[1] == '_') {
+          char* next = from + 1;
+          while (next < end && *next != ':') next++;
+          if (next[0] == ':' && next[1] == ':') {
+            from = next + 2;
+            continue;
+          }
+        }
+        *to++ = *from++;
+      }
+      demangled_name_view = {begin, detail::to_unsigned(to - begin)};
+    }
+  } else {
+    demangled_name_view = string_view(ti.name());
+  }
+  return detail::write_bytes<Char>(out, demangled_name_view);
+#  elif FMT_MSC_VERSION
+  const string_view demangled_name(ti.name());
+  for (std::size_t i = 0; i < demangled_name.size(); ++i) {
+    auto sub = demangled_name;
+    sub.remove_prefix(i);
+    if (sub.starts_with("enum ")) {
+      i += 4;
+      continue;
+    }
+    if (sub.starts_with("class ") || sub.starts_with("union ")) {
+      i += 5;
+      continue;
+    }
+    if (sub.starts_with("struct ")) {
+      i += 6;
+      continue;
+    }
+    if (*sub.begin() != ' ') *out++ = *sub.begin();
+  }
+  return out;
+#  else
+  return detail::write_bytes<Char>(out, string_view(ti.name()));
+#  endif
+}
+
+}  // namespace detail
+
+FMT_EXPORT
+template <typename Char>
+struct formatter<std::type_info, Char  // DEPRECATED! Mixing code unit types.
+                 > {
+ public:
+  FMT_CONSTEXPR auto parse(basic_format_parse_context<Char>& ctx)
+      -> decltype(ctx.begin()) {
+    return ctx.begin();
+  }
+
+  template <typename Context>
+  auto format(const std::type_info& ti, Context& ctx) const
+      -> decltype(ctx.out()) {
+    return detail::write_demangled_name<Char>(ctx.out(), ti);
+  }
+};
+#endif
+
+FMT_EXPORT
+template <typename T, typename Char>
+struct formatter<
+    T, Char,  // DEPRECATED! Mixing code unit types.
+    typename std::enable_if<std::is_base_of<std::exception, T>::value>::type> {
+ private:
+  bool with_typename_ = false;
+
+ public:
+  FMT_CONSTEXPR auto parse(basic_format_parse_context<Char>& ctx)
+      -> decltype(ctx.begin()) {
+    auto it = ctx.begin();
+    auto end = ctx.end();
+    if (it == end || *it == '}') return it;
+    if (*it == 't') {
+      ++it;
+      with_typename_ = FMT_USE_RTTI != 0;
+    }
+    return it;
+  }
+
+  template <typename Context>
+  auto format(const std::exception& ex, Context& ctx) const
+      -> decltype(ctx.out()) {
+    auto out = ctx.out();
+#if FMT_USE_RTTI
+    if (with_typename_) {
+      out = detail::write_demangled_name<Char>(out, typeid(ex));
+      *out++ = ':';
+      *out++ = ' ';
+    }
+#endif
+    return detail::write_bytes<Char>(out, string_view(ex.what()));
+  }
+};
+
+namespace detail {
+
+template <typename T, typename Enable = void>
+struct has_flip : std::false_type {};
+
+template <typename T>
+struct has_flip<T, void_t<decltype(std::declval<T>().flip())>>
+    : std::true_type {};
+
+template <typename T> struct is_bit_reference_like {
+  static constexpr const bool value =
+      std::is_convertible<T, bool>::value &&
+      std::is_nothrow_assignable<T, bool>::value && has_flip<T>::value;
+};
+
+#ifdef _LIBCPP_VERSION
+
+// Workaround for libc++ incompatibility with C++ standard.
+// According to the Standard, `bitset::operator[] const` returns bool.
+template <typename C>
+struct is_bit_reference_like<std::__bit_const_reference<C>> {
+  static constexpr const bool value = true;
+};
+
+#endif
+
+}  // namespace detail
+
+// We can't use std::vector<bool, Allocator>::reference and
+// std::bitset<N>::reference because the compiler can't deduce Allocator and N
+// in partial specialization.
+FMT_EXPORT
+template <typename BitRef, typename Char>
+struct formatter<BitRef, Char,
+                 enable_if_t<detail::is_bit_reference_like<BitRef>::value>>
+    : formatter<bool, Char> {
+  template <typename FormatContext>
+  FMT_CONSTEXPR auto format(const BitRef& v, FormatContext& ctx) const
+      -> decltype(ctx.out()) {
+    return formatter<bool, Char>::format(v, ctx);
+  }
+};
+
+template <typename T, typename Deleter>
+auto ptr(const std::unique_ptr<T, Deleter>& p) -> const void* {
+  return p.get();
+}
+template <typename T> auto ptr(const std::shared_ptr<T>& p) -> const void* {
+  return p.get();
+}
+
+FMT_EXPORT
+template <typename T, typename Char>
+struct formatter<std::atomic<T>, Char,
+                 enable_if_t<is_formattable<T, Char>::value>>
+    : formatter<T, Char> {
+  template <typename FormatContext>
+  auto format(const std::atomic<T>& v, FormatContext& ctx) const
+      -> decltype(ctx.out()) {
+    return formatter<T, Char>::format(v.load(), ctx);
+  }
+};
+
+#ifdef __cpp_lib_atomic_flag_test
+FMT_EXPORT
+template <typename Char>
+struct formatter<std::atomic_flag, Char> : formatter<bool, Char> {
+  template <typename FormatContext>
+  auto format(const std::atomic_flag& v, FormatContext& ctx) const
+      -> decltype(ctx.out()) {
+    return formatter<bool, Char>::format(v.test(), ctx);
+  }
+};
+#endif  // __cpp_lib_atomic_flag_test
+
+FMT_EXPORT
+template <typename T, typename Char> struct formatter<std::complex<T>, Char> {
+ private:
+  detail::dynamic_format_specs<Char> specs_;
+
+  template <typename FormatContext, typename OutputIt>
+  FMT_CONSTEXPR auto do_format(const std::complex<T>& c,
+                               detail::dynamic_format_specs<Char>& specs,
+                               FormatContext& ctx, OutputIt out) const
+      -> OutputIt {
+    if (c.real() != 0) {
+      *out++ = Char('(');
+      out = detail::write<Char>(out, c.real(), specs, ctx.locale());
+      specs.sign = sign::plus;
+      out = detail::write<Char>(out, c.imag(), specs, ctx.locale());
+      if (!detail::isfinite(c.imag())) *out++ = Char(' ');
+      *out++ = Char('i');
+      *out++ = Char(')');
+      return out;
+    }
+    out = detail::write<Char>(out, c.imag(), specs, ctx.locale());
+    if (!detail::isfinite(c.imag())) *out++ = Char(' ');
+    *out++ = Char('i');
+    return out;
+  }
+
+ public:
+  FMT_CONSTEXPR auto parse(basic_format_parse_context<Char>& ctx)
+      -> decltype(ctx.begin()) {
+    if (ctx.begin() == ctx.end() || *ctx.begin() == '}') return ctx.begin();
+    return parse_format_specs(ctx.begin(), ctx.end(), specs_, ctx,
+                              detail::type_constant<T, Char>::value);
+  }
+
+  template <typename FormatContext>
+  auto format(const std::complex<T>& c, FormatContext& ctx) const
+      -> decltype(ctx.out()) {
+    auto specs = specs_;
+    if (specs.width_ref.kind != detail::arg_id_kind::none ||
+        specs.precision_ref.kind != detail::arg_id_kind::none) {
+      detail::handle_dynamic_spec<detail::width_checker>(specs.width,
+                                                         specs.width_ref, ctx);
+      detail::handle_dynamic_spec<detail::precision_checker>(
+          specs.precision, specs.precision_ref, ctx);
+    }
+
+    if (specs.width == 0) return do_format(c, specs, ctx, ctx.out());
+    auto buf = basic_memory_buffer<Char>();
+
+    auto outer_specs = format_specs();
+    outer_specs.width = specs.width;
+    outer_specs.fill = specs.fill;
+    outer_specs.align = specs.align;
+
+    specs.width = 0;
+    specs.fill = {};
+    specs.align = align::none;
+
+    do_format(c, specs, ctx, basic_appender<Char>(buf));
+    return detail::write<Char>(ctx.out(),
+                               basic_string_view<Char>(buf.data(), buf.size()),
+                               outer_specs);
+  }
+};
+
+FMT_END_NAMESPACE
+#endif  // FMT_STD_H_
diff --git a/ext/spdlog/include/spdlog/fmt/bundled/xchar.h b/ext/spdlog/include/spdlog/fmt/bundled/xchar.h
new file mode 100644
index 0000000..b1f39ed
--- /dev/null
+++ b/ext/spdlog/include/spdlog/fmt/bundled/xchar.h
@@ -0,0 +1,322 @@
+// Formatting library for C++ - optional wchar_t and exotic character support
+//
+// Copyright (c) 2012 - present, Victor Zverovich
+// All rights reserved.
+//
+// For the license information refer to format.h.
+
+#ifndef FMT_XCHAR_H_
+#define FMT_XCHAR_H_
+
+#include "color.h"
+#include "format.h"
+#include "ranges.h"
+
+#ifndef FMT_MODULE
+#  include <cwchar>
+#  if !defined(FMT_STATIC_THOUSANDS_SEPARATOR)
+#    include <locale>
+#  endif
+#endif
+
+FMT_BEGIN_NAMESPACE
+namespace detail {
+
+template <typename T>
+using is_exotic_char = bool_constant<!std::is_same<T, char>::value>;
+
+template <typename S, typename = void> struct format_string_char {};
+
+template <typename S>
+struct format_string_char<
+    S, void_t<decltype(sizeof(detail::to_string_view(std::declval<S>())))>> {
+  using type = char_t<S>;
+};
+
+template <typename S>
+struct format_string_char<S, enable_if_t<is_compile_string<S>::value>> {
+  using type = typename S::char_type;
+};
+
+template <typename S>
+using format_string_char_t = typename format_string_char<S>::type;
+
+inline auto write_loc(basic_appender<wchar_t> out, loc_value value,
+                      const format_specs& specs, locale_ref loc) -> bool {
+#ifndef FMT_STATIC_THOUSANDS_SEPARATOR
+  auto& numpunct =
+      std::use_facet<std::numpunct<wchar_t>>(loc.get<std::locale>());
+  auto separator = std::wstring();
+  auto grouping = numpunct.grouping();
+  if (!grouping.empty()) separator = std::wstring(1, numpunct.thousands_sep());
+  return value.visit(loc_writer<wchar_t>{out, specs, separator, grouping, {}});
+#endif
+  return false;
+}
+}  // namespace detail
+
+FMT_BEGIN_EXPORT
+
+using wstring_view = basic_string_view<wchar_t>;
+using wformat_parse_context = basic_format_parse_context<wchar_t>;
+using wformat_context = buffered_context<wchar_t>;
+using wformat_args = basic_format_args<wformat_context>;
+using wmemory_buffer = basic_memory_buffer<wchar_t>;
+
+#if FMT_GCC_VERSION && FMT_GCC_VERSION < 409
+// Workaround broken conversion on older gcc.
+template <typename... Args> using wformat_string = wstring_view;
+inline auto runtime(wstring_view s) -> wstring_view { return s; }
+#else
+template <typename... Args>
+using wformat_string = basic_format_string<wchar_t, type_identity_t<Args>...>;
+inline auto runtime(wstring_view s) -> runtime_format_string<wchar_t> {
+  return {{s}};
+}
+#endif
+
+template <> struct is_char<wchar_t> : std::true_type {};
+template <> struct is_char<char16_t> : std::true_type {};
+template <> struct is_char<char32_t> : std::true_type {};
+
+#ifdef __cpp_char8_t
+template <>
+struct is_char<char8_t> : bool_constant<detail::is_utf8_enabled()> {};
+#endif
+
+template <typename... T>
+constexpr auto make_wformat_args(T&... args)
+    -> decltype(fmt::make_format_args<wformat_context>(args...)) {
+  return fmt::make_format_args<wformat_context>(args...);
+}
+
+inline namespace literals {
+#if FMT_USE_USER_DEFINED_LITERALS && !FMT_USE_NONTYPE_TEMPLATE_ARGS
+constexpr auto operator""_a(const wchar_t* s, size_t)
+    -> detail::udl_arg<wchar_t> {
+  return {s};
+}
+#endif
+}  // namespace literals
+
+template <typename It, typename Sentinel>
+auto join(It begin, Sentinel end, wstring_view sep)
+    -> join_view<It, Sentinel, wchar_t> {
+  return {begin, end, sep};
+}
+
+template <typename Range>
+auto join(Range&& range, wstring_view sep)
+    -> join_view<detail::iterator_t<Range>, detail::sentinel_t<Range>,
+                 wchar_t> {
+  return join(std::begin(range), std::end(range), sep);
+}
+
+template <typename T>
+auto join(std::initializer_list<T> list, wstring_view sep)
+    -> join_view<const T*, const T*, wchar_t> {
+  return join(std::begin(list), std::end(list), sep);
+}
+
+template <typename... T>
+auto join(const std::tuple<T...>& tuple, basic_string_view<wchar_t> sep)
+    -> tuple_join_view<wchar_t, T...> {
+  return {tuple, sep};
+}
+
+template <typename Char, FMT_ENABLE_IF(!std::is_same<Char, char>::value)>
+auto vformat(basic_string_view<Char> format_str,
+             typename detail::vformat_args<Char>::type args)
+    -> std::basic_string<Char> {
+  auto buf = basic_memory_buffer<Char>();
+  detail::vformat_to(buf, format_str, args);
+  return to_string(buf);
+}
+
+template <typename... T>
+auto format(wformat_string<T...> fmt, T&&... args) -> std::wstring {
+  return vformat(fmt::wstring_view(fmt), fmt::make_wformat_args(args...));
+}
+
+template <typename OutputIt, typename... T>
+auto format_to(OutputIt out, wformat_string<T...> fmt, T&&... args)
+    -> OutputIt {
+  return vformat_to(out, fmt::wstring_view(fmt),
+                    fmt::make_wformat_args(args...));
+}
+
+// Pass char_t as a default template parameter instead of using
+// std::basic_string<char_t<S>> to reduce the symbol size.
+template <typename S, typename... T,
+          typename Char = detail::format_string_char_t<S>,
+          FMT_ENABLE_IF(!std::is_same<Char, char>::value &&
+                        !std::is_same<Char, wchar_t>::value)>
+auto format(const S& format_str, T&&... args) -> std::basic_string<Char> {
+  return vformat(detail::to_string_view(format_str),
+                 fmt::make_format_args<buffered_context<Char>>(args...));
+}
+
+template <typename Locale, typename S,
+          typename Char = detail::format_string_char_t<S>,
+          FMT_ENABLE_IF(detail::is_locale<Locale>::value&&
+                            detail::is_exotic_char<Char>::value)>
+inline auto vformat(const Locale& loc, const S& format_str,
+                    typename detail::vformat_args<Char>::type args)
+    -> std::basic_string<Char> {
+  return detail::vformat(loc, detail::to_string_view(format_str), args);
+}
+
+template <typename Locale, typename S, typename... T,
+          typename Char = detail::format_string_char_t<S>,
+          FMT_ENABLE_IF(detail::is_locale<Locale>::value&&
+                            detail::is_exotic_char<Char>::value)>
+inline auto format(const Locale& loc, const S& format_str, T&&... args)
+    -> std::basic_string<Char> {
+  return detail::vformat(
+      loc, detail::to_string_view(format_str),
+      fmt::make_format_args<buffered_context<Char>>(args...));
+}
+
+template <typename OutputIt, typename S,
+          typename Char = detail::format_string_char_t<S>,
+          FMT_ENABLE_IF(detail::is_output_iterator<OutputIt, Char>::value&&
+                            detail::is_exotic_char<Char>::value)>
+auto vformat_to(OutputIt out, const S& format_str,
+                typename detail::vformat_args<Char>::type args) -> OutputIt {
+  auto&& buf = detail::get_buffer<Char>(out);
+  detail::vformat_to(buf, detail::to_string_view(format_str), args);
+  return detail::get_iterator(buf, out);
+}
+
+template <typename OutputIt, typename S, typename... T,
+          typename Char = detail::format_string_char_t<S>,
+          FMT_ENABLE_IF(detail::is_output_iterator<OutputIt, Char>::value &&
+                        !std::is_same<Char, char>::value &&
+                        !std::is_same<Char, wchar_t>::value)>
+inline auto format_to(OutputIt out, const S& fmt, T&&... args) -> OutputIt {
+  return vformat_to(out, detail::to_string_view(fmt),
+                    fmt::make_format_args<buffered_context<Char>>(args...));
+}
+
+template <typename Locale, typename S, typename OutputIt, typename... Args,
+          typename Char = detail::format_string_char_t<S>,
+          FMT_ENABLE_IF(detail::is_output_iterator<OutputIt, Char>::value&&
+                            detail::is_locale<Locale>::value&&
+                                detail::is_exotic_char<Char>::value)>
+inline auto vformat_to(OutputIt out, const Locale& loc, const S& format_str,
+                       typename detail::vformat_args<Char>::type args)
+    -> OutputIt {
+  auto&& buf = detail::get_buffer<Char>(out);
+  vformat_to(buf, detail::to_string_view(format_str), args,
+             detail::locale_ref(loc));
+  return detail::get_iterator(buf, out);
+}
+
+template <typename OutputIt, typename Locale, typename S, typename... T,
+          typename Char = detail::format_string_char_t<S>,
+          bool enable = detail::is_output_iterator<OutputIt, Char>::value &&
+                        detail::is_locale<Locale>::value &&
+                        detail::is_exotic_char<Char>::value>
+inline auto format_to(OutputIt out, const Locale& loc, const S& format_str,
+                      T&&... args) ->
+    typename std::enable_if<enable, OutputIt>::type {
+  return vformat_to(out, loc, detail::to_string_view(format_str),
+                    fmt::make_format_args<buffered_context<Char>>(args...));
+}
+
+template <typename OutputIt, typename Char, typename... Args,
+          FMT_ENABLE_IF(detail::is_output_iterator<OutputIt, Char>::value&&
+                            detail::is_exotic_char<Char>::value)>
+inline auto vformat_to_n(OutputIt out, size_t n,
+                         basic_string_view<Char> format_str,
+                         typename detail::vformat_args<Char>::type args)
+    -> format_to_n_result<OutputIt> {
+  using traits = detail::fixed_buffer_traits;
+  auto buf = detail::iterator_buffer<OutputIt, Char, traits>(out, n);
+  detail::vformat_to(buf, format_str, args);
+  return {buf.out(), buf.count()};
+}
+
+template <typename OutputIt, typename S, typename... T,
+          typename Char = detail::format_string_char_t<S>,
+          FMT_ENABLE_IF(detail::is_output_iterator<OutputIt, Char>::value&&
+                            detail::is_exotic_char<Char>::value)>
+inline auto format_to_n(OutputIt out, size_t n, const S& fmt, T&&... args)
+    -> format_to_n_result<OutputIt> {
+  return vformat_to_n(out, n, fmt::basic_string_view<Char>(fmt),
+                      fmt::make_format_args<buffered_context<Char>>(args...));
+}
+
+template <typename S, typename... T,
+          typename Char = detail::format_string_char_t<S>,
+          FMT_ENABLE_IF(detail::is_exotic_char<Char>::value)>
+inline auto formatted_size(const S& fmt, T&&... args) -> size_t {
+  auto buf = detail::counting_buffer<Char>();
+  detail::vformat_to(buf, detail::to_string_view(fmt),
+                     fmt::make_format_args<buffered_context<Char>>(args...));
+  return buf.count();
+}
+
+inline void vprint(std::FILE* f, wstring_view fmt, wformat_args args) {
+  auto buf = wmemory_buffer();
+  detail::vformat_to(buf, fmt, args);
+  buf.push_back(L'\0');
+  if (std::fputws(buf.data(), f) == -1)
+    FMT_THROW(system_error(errno, FMT_STRING("cannot write to file")));
+}
+
+inline void vprint(wstring_view fmt, wformat_args args) {
+  vprint(stdout, fmt, args);
+}
+
+template <typename... T>
+void print(std::FILE* f, wformat_string<T...> fmt, T&&... args) {
+  return vprint(f, wstring_view(fmt), fmt::make_wformat_args(args...));
+}
+
+template <typename... T> void print(wformat_string<T...> fmt, T&&... args) {
+  return vprint(wstring_view(fmt), fmt::make_wformat_args(args...));
+}
+
+template <typename... T>
+void println(std::FILE* f, wformat_string<T...> fmt, T&&... args) {
+  return print(f, L"{}\n", fmt::format(fmt, std::forward<T>(args)...));
+}
+
+template <typename... T> void println(wformat_string<T...> fmt, T&&... args) {
+  return print(L"{}\n", fmt::format(fmt, std::forward<T>(args)...));
+}
+
+inline auto vformat(const text_style& ts, wstring_view fmt, wformat_args args)
+    -> std::wstring {
+  auto buf = wmemory_buffer();
+  detail::vformat_to(buf, ts, fmt, args);
+  return fmt::to_string(buf);
+}
+
+template <typename... T>
+inline auto format(const text_style& ts, wformat_string<T...> fmt, T&&... args)
+    -> std::wstring {
+  return fmt::vformat(ts, fmt, fmt::make_wformat_args(args...));
+}
+
+template <typename... T>
+FMT_DEPRECATED void print(std::FILE* f, const text_style& ts,
+                          wformat_string<T...> fmt, const T&... args) {
+  vprint(f, ts, fmt, fmt::make_wformat_args(args...));
+}
+
+template <typename... T>
+FMT_DEPRECATED void print(const text_style& ts, wformat_string<T...> fmt,
+                          const T&... args) {
+  return print(stdout, ts, fmt, args...);
+}
+
+/// Converts `value` to `std::wstring` using the default format for type `T`.
+template <typename T> inline auto to_wstring(const T& value) -> std::wstring {
+  return format(FMT_STRING(L"{}"), value);
+}
+FMT_END_EXPORT
+FMT_END_NAMESPACE
+
+#endif  // FMT_XCHAR_H_
diff --git a/ext/spdlog/include/spdlog/fmt/chrono.h b/ext/spdlog/include/spdlog/fmt/chrono.h
new file mode 100644
index 0000000..a72a5bd
--- /dev/null
+++ b/ext/spdlog/include/spdlog/fmt/chrono.h
@@ -0,0 +1,23 @@
+//
+// Copyright(c) 2016 Gabi Melman.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+//
+
+#pragma once
+//
+// include bundled or external copy of fmtlib's chrono support
+//
+#include <spdlog/tweakme.h>
+
+#if !defined(SPDLOG_USE_STD_FORMAT)
+    #if !defined(SPDLOG_FMT_EXTERNAL)
+        #ifdef SPDLOG_HEADER_ONLY
+            #ifndef FMT_HEADER_ONLY
+                #define FMT_HEADER_ONLY
+            #endif
+        #endif
+        #include <spdlog/fmt/bundled/chrono.h>
+    #else
+        #include <fmt/chrono.h>
+    #endif
+#endif
diff --git a/ext/spdlog/include/spdlog/fmt/compile.h b/ext/spdlog/include/spdlog/fmt/compile.h
new file mode 100644
index 0000000..3c9c25d
--- /dev/null
+++ b/ext/spdlog/include/spdlog/fmt/compile.h
@@ -0,0 +1,23 @@
+//
+// Copyright(c) 2016 Gabi Melman.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+//
+
+#pragma once
+//
+// include bundled or external copy of fmtlib's compile-time support
+//
+#include <spdlog/tweakme.h>
+
+#if !defined(SPDLOG_USE_STD_FORMAT)
+    #if !defined(SPDLOG_FMT_EXTERNAL)
+        #ifdef SPDLOG_HEADER_ONLY
+            #ifndef FMT_HEADER_ONLY
+                #define FMT_HEADER_ONLY
+            #endif
+        #endif
+        #include <spdlog/fmt/bundled/compile.h>
+    #else
+        #include <fmt/compile.h>
+    #endif
+#endif
diff --git a/ext/spdlog/include/spdlog/fmt/fmt.h b/ext/spdlog/include/spdlog/fmt/fmt.h
new file mode 100644
index 0000000..7fa6b09
--- /dev/null
+++ b/ext/spdlog/include/spdlog/fmt/fmt.h
@@ -0,0 +1,30 @@
+//
+// Copyright(c) 2016-2018 Gabi Melman.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+//
+
+#pragma once
+
+//
+// Include a bundled header-only copy of fmtlib or an external one.
+// By default spdlog include its own copy.
+//
+#include <spdlog/tweakme.h>
+
+#if defined(SPDLOG_USE_STD_FORMAT)  // SPDLOG_USE_STD_FORMAT is defined - use std::format
+    #include <format>
+#elif !defined(SPDLOG_FMT_EXTERNAL)
+    #if !defined(SPDLOG_COMPILED_LIB) && !defined(FMT_HEADER_ONLY)
+        #define FMT_HEADER_ONLY
+    #endif
+    #ifndef FMT_USE_WINDOWS_H
+        #define FMT_USE_WINDOWS_H 0
+    #endif
+
+    #include <spdlog/fmt/bundled/core.h>
+    #include <spdlog/fmt/bundled/format.h>
+
+#else  // SPDLOG_FMT_EXTERNAL is defined - use external fmtlib
+    #include <fmt/core.h>
+    #include <fmt/format.h>
+#endif
diff --git a/ext/spdlog/include/spdlog/fmt/ostr.h b/ext/spdlog/include/spdlog/fmt/ostr.h
new file mode 100644
index 0000000..2b90105
--- /dev/null
+++ b/ext/spdlog/include/spdlog/fmt/ostr.h
@@ -0,0 +1,23 @@
+//
+// Copyright(c) 2016 Gabi Melman.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+//
+
+#pragma once
+//
+// include bundled or external copy of fmtlib's ostream support
+//
+#include <spdlog/tweakme.h>
+
+#if !defined(SPDLOG_USE_STD_FORMAT)
+    #if !defined(SPDLOG_FMT_EXTERNAL)
+        #ifdef SPDLOG_HEADER_ONLY
+            #ifndef FMT_HEADER_ONLY
+                #define FMT_HEADER_ONLY
+            #endif
+        #endif
+        #include <spdlog/fmt/bundled/ostream.h>
+    #else
+        #include <fmt/ostream.h>
+    #endif
+#endif
diff --git a/ext/spdlog/include/spdlog/fmt/ranges.h b/ext/spdlog/include/spdlog/fmt/ranges.h
new file mode 100644
index 0000000..5bb91e9
--- /dev/null
+++ b/ext/spdlog/include/spdlog/fmt/ranges.h
@@ -0,0 +1,23 @@
+//
+// Copyright(c) 2016 Gabi Melman.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+//
+
+#pragma once
+//
+// include bundled or external copy of fmtlib's ranges support
+//
+#include <spdlog/tweakme.h>
+
+#if !defined(SPDLOG_USE_STD_FORMAT)
+    #if !defined(SPDLOG_FMT_EXTERNAL)
+        #ifdef SPDLOG_HEADER_ONLY
+            #ifndef FMT_HEADER_ONLY
+                #define FMT_HEADER_ONLY
+            #endif
+        #endif
+        #include <spdlog/fmt/bundled/ranges.h>
+    #else
+        #include <fmt/ranges.h>
+    #endif
+#endif
diff --git a/ext/spdlog/include/spdlog/fmt/std.h b/ext/spdlog/include/spdlog/fmt/std.h
new file mode 100644
index 0000000..dabe6f6
--- /dev/null
+++ b/ext/spdlog/include/spdlog/fmt/std.h
@@ -0,0 +1,24 @@
+//
+// Copyright(c) 2016 Gabi Melman.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+//
+
+#pragma once
+//
+// include bundled or external copy of fmtlib's std support (for formatting e.g.
+// std::filesystem::path, std::thread::id, std::monostate, std::variant, ...)
+//
+#include <spdlog/tweakme.h>
+
+#if !defined(SPDLOG_USE_STD_FORMAT)
+    #if !defined(SPDLOG_FMT_EXTERNAL)
+        #ifdef SPDLOG_HEADER_ONLY
+            #ifndef FMT_HEADER_ONLY
+                #define FMT_HEADER_ONLY
+            #endif
+        #endif
+        #include <spdlog/fmt/bundled/std.h>
+    #else
+        #include <fmt/std.h>
+    #endif
+#endif
diff --git a/ext/spdlog/include/spdlog/fmt/xchar.h b/ext/spdlog/include/spdlog/fmt/xchar.h
new file mode 100644
index 0000000..2525f05
--- /dev/null
+++ b/ext/spdlog/include/spdlog/fmt/xchar.h
@@ -0,0 +1,23 @@
+//
+// Copyright(c) 2016 Gabi Melman.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+//
+
+#pragma once
+//
+// include bundled or external copy of fmtlib's xchar support
+//
+#include <spdlog/tweakme.h>
+
+#if !defined(SPDLOG_USE_STD_FORMAT)
+    #if !defined(SPDLOG_FMT_EXTERNAL)
+        #ifdef SPDLOG_HEADER_ONLY
+            #ifndef FMT_HEADER_ONLY
+                #define FMT_HEADER_ONLY
+            #endif
+        #endif
+        #include <spdlog/fmt/bundled/xchar.h>
+    #else
+        #include <fmt/xchar.h>
+    #endif
+#endif
diff --git a/ext/spdlog/include/spdlog/formatter.h b/ext/spdlog/include/spdlog/formatter.h
new file mode 100644
index 0000000..4d482f8
--- /dev/null
+++ b/ext/spdlog/include/spdlog/formatter.h
@@ -0,0 +1,17 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#include <spdlog/details/log_msg.h>
+#include <spdlog/fmt/fmt.h>
+
+namespace spdlog {
+
+class formatter {
+public:
+    virtual ~formatter() = default;
+    virtual void format(const details::log_msg &msg, memory_buf_t &dest) = 0;
+    virtual std::unique_ptr<formatter> clone() const = 0;
+};
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/fwd.h b/ext/spdlog/include/spdlog/fwd.h
new file mode 100644
index 0000000..647b16b
--- /dev/null
+++ b/ext/spdlog/include/spdlog/fwd.h
@@ -0,0 +1,18 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+namespace spdlog {
+class logger;
+class formatter;
+
+namespace sinks {
+class sink;
+}
+
+namespace level {
+enum level_enum : int;
+}
+
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/logger-inl.h b/ext/spdlog/include/spdlog/logger-inl.h
new file mode 100644
index 0000000..5218fe4
--- /dev/null
+++ b/ext/spdlog/include/spdlog/logger-inl.h
@@ -0,0 +1,198 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#ifndef SPDLOG_HEADER_ONLY
+    #include <spdlog/logger.h>
+#endif
+
+#include <spdlog/details/backtracer.h>
+#include <spdlog/pattern_formatter.h>
+#include <spdlog/sinks/sink.h>
+
+#include <cstdio>
+
+namespace spdlog {
+
+// public methods
+SPDLOG_INLINE logger::logger(const logger &other)
+    : name_(other.name_),
+      sinks_(other.sinks_),
+      level_(other.level_.load(std::memory_order_relaxed)),
+      flush_level_(other.flush_level_.load(std::memory_order_relaxed)),
+      custom_err_handler_(other.custom_err_handler_),
+      tracer_(other.tracer_) {}
+
+SPDLOG_INLINE logger::logger(logger &&other) SPDLOG_NOEXCEPT
+    : name_(std::move(other.name_)),
+      sinks_(std::move(other.sinks_)),
+      level_(other.level_.load(std::memory_order_relaxed)),
+      flush_level_(other.flush_level_.load(std::memory_order_relaxed)),
+      custom_err_handler_(std::move(other.custom_err_handler_)),
+      tracer_(std::move(other.tracer_))
+
+{}
+
+SPDLOG_INLINE logger &logger::operator=(logger other) SPDLOG_NOEXCEPT {
+    this->swap(other);
+    return *this;
+}
+
+SPDLOG_INLINE void logger::swap(spdlog::logger &other) SPDLOG_NOEXCEPT {
+    name_.swap(other.name_);
+    sinks_.swap(other.sinks_);
+
+    // swap level_
+    auto other_level = other.level_.load();
+    auto my_level = level_.exchange(other_level);
+    other.level_.store(my_level);
+
+    // swap flush level_
+    other_level = other.flush_level_.load();
+    my_level = flush_level_.exchange(other_level);
+    other.flush_level_.store(my_level);
+
+    custom_err_handler_.swap(other.custom_err_handler_);
+    std::swap(tracer_, other.tracer_);
+}
+
+SPDLOG_INLINE void swap(logger &a, logger &b) { a.swap(b); }
+
+SPDLOG_INLINE void logger::set_level(level::level_enum log_level) { level_.store(log_level); }
+
+SPDLOG_INLINE level::level_enum logger::level() const {
+    return static_cast<level::level_enum>(level_.load(std::memory_order_relaxed));
+}
+
+SPDLOG_INLINE const std::string &logger::name() const { return name_; }
+
+// set formatting for the sinks in this logger.
+// each sink will get a separate instance of the formatter object.
+SPDLOG_INLINE void logger::set_formatter(std::unique_ptr<formatter> f) {
+    for (auto it = sinks_.begin(); it != sinks_.end(); ++it) {
+        if (std::next(it) == sinks_.end()) {
+            // last element - we can be move it.
+            (*it)->set_formatter(std::move(f));
+            break;  // to prevent clang-tidy warning
+        } else {
+            (*it)->set_formatter(f->clone());
+        }
+    }
+}
+
+SPDLOG_INLINE void logger::set_pattern(std::string pattern, pattern_time_type time_type) {
+    auto new_formatter = details::make_unique<pattern_formatter>(std::move(pattern), time_type);
+    set_formatter(std::move(new_formatter));
+}
+
+// create new backtrace sink and move to it all our child sinks
+SPDLOG_INLINE void logger::enable_backtrace(size_t n_messages) { tracer_.enable(n_messages); }
+
+// restore orig sinks and level and delete the backtrace sink
+SPDLOG_INLINE void logger::disable_backtrace() { tracer_.disable(); }
+
+SPDLOG_INLINE void logger::dump_backtrace() { dump_backtrace_(); }
+
+// flush functions
+SPDLOG_INLINE void logger::flush() { flush_(); }
+
+SPDLOG_INLINE void logger::flush_on(level::level_enum log_level) { flush_level_.store(log_level); }
+
+SPDLOG_INLINE level::level_enum logger::flush_level() const {
+    return static_cast<level::level_enum>(flush_level_.load(std::memory_order_relaxed));
+}
+
+// sinks
+SPDLOG_INLINE const std::vector<sink_ptr> &logger::sinks() const { return sinks_; }
+
+SPDLOG_INLINE std::vector<sink_ptr> &logger::sinks() { return sinks_; }
+
+// error handler
+SPDLOG_INLINE void logger::set_error_handler(err_handler handler) {
+    custom_err_handler_ = std::move(handler);
+}
+
+// create new logger with same sinks and configuration.
+SPDLOG_INLINE std::shared_ptr<logger> logger::clone(std::string logger_name) {
+    auto cloned = std::make_shared<logger>(*this);
+    cloned->name_ = std::move(logger_name);
+    return cloned;
+}
+
+// protected methods
+SPDLOG_INLINE void logger::log_it_(const spdlog::details::log_msg &log_msg,
+                                   bool log_enabled,
+                                   bool traceback_enabled) {
+    if (log_enabled) {
+        sink_it_(log_msg);
+    }
+    if (traceback_enabled) {
+        tracer_.push_back(log_msg);
+    }
+}
+
+SPDLOG_INLINE void logger::sink_it_(const details::log_msg &msg) {
+    for (auto &sink : sinks_) {
+        if (sink->should_log(msg.level)) {
+            SPDLOG_TRY { sink->log(msg); }
+            SPDLOG_LOGGER_CATCH(msg.source)
+        }
+    }
+
+    if (should_flush_(msg)) {
+        flush_();
+    }
+}
+
+SPDLOG_INLINE void logger::flush_() {
+    for (auto &sink : sinks_) {
+        SPDLOG_TRY { sink->flush(); }
+        SPDLOG_LOGGER_CATCH(source_loc())
+    }
+}
+
+SPDLOG_INLINE void logger::dump_backtrace_() {
+    using details::log_msg;
+    if (tracer_.enabled() && !tracer_.empty()) {
+        sink_it_(
+            log_msg{name(), level::info, "****************** Backtrace Start ******************"});
+        tracer_.foreach_pop([this](const log_msg &msg) { this->sink_it_(msg); });
+        sink_it_(
+            log_msg{name(), level::info, "****************** Backtrace End ********************"});
+    }
+}
+
+SPDLOG_INLINE bool logger::should_flush_(const details::log_msg &msg) {
+    auto flush_level = flush_level_.load(std::memory_order_relaxed);
+    return (msg.level >= flush_level) && (msg.level != level::off);
+}
+
+SPDLOG_INLINE void logger::err_handler_(const std::string &msg) {
+    if (custom_err_handler_) {
+        custom_err_handler_(msg);
+    } else {
+        using std::chrono::system_clock;
+        static std::mutex mutex;
+        static std::chrono::system_clock::time_point last_report_time;
+        static size_t err_counter = 0;
+        std::lock_guard<std::mutex> lk{mutex};
+        auto now = system_clock::now();
+        err_counter++;
+        if (now - last_report_time < std::chrono::seconds(1)) {
+            return;
+        }
+        last_report_time = now;
+        auto tm_time = details::os::localtime(system_clock::to_time_t(now));
+        char date_buf[64];
+        std::strftime(date_buf, sizeof(date_buf), "%Y-%m-%d %H:%M:%S", &tm_time);
+#if defined(USING_R) && defined(R_R_H)  // if in R environment
+        REprintf("[*** LOG ERROR #%04zu ***] [%s] [%s] %s\n", err_counter, date_buf, name().c_str(),
+                 msg.c_str());
+#else
+        std::fprintf(stderr, "[*** LOG ERROR #%04zu ***] [%s] [%s] %s\n", err_counter, date_buf,
+                     name().c_str(), msg.c_str());
+#endif
+    }
+}
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/logger.h b/ext/spdlog/include/spdlog/logger.h
new file mode 100644
index 0000000..f49bdc0
--- /dev/null
+++ b/ext/spdlog/include/spdlog/logger.h
@@ -0,0 +1,379 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+// Thread safe logger (except for set_error_handler())
+// Has name, log level, vector of std::shared sink pointers and formatter
+// Upon each log write the logger:
+// 1. Checks if its log level is enough to log the message and if yes:
+// 2. Call the underlying sinks to do the job.
+// 3. Each sink use its own private copy of a formatter to format the message
+// and send to its destination.
+//
+// The use of private formatter per sink provides the opportunity to cache some
+// formatted data, and support for different format per sink.
+
+#include <spdlog/common.h>
+#include <spdlog/details/backtracer.h>
+#include <spdlog/details/log_msg.h>
+
+#ifdef SPDLOG_WCHAR_TO_UTF8_SUPPORT
+    #ifndef _WIN32
+        #error SPDLOG_WCHAR_TO_UTF8_SUPPORT only supported on windows
+    #endif
+    #include <spdlog/details/os.h>
+#endif
+
+#include <vector>
+
+#ifndef SPDLOG_NO_EXCEPTIONS
+    #define SPDLOG_LOGGER_CATCH(location)                                                 \
+        catch (const std::exception &ex) {                                                \
+            if (location.filename) {                                                      \
+                err_handler_(fmt_lib::format(SPDLOG_FMT_STRING("{} [{}({})]"), ex.what(), \
+                                             location.filename, location.line));          \
+            } else {                                                                      \
+                err_handler_(ex.what());                                                  \
+            }                                                                             \
+        }                                                                                 \
+        catch (...) {                                                                     \
+            err_handler_("Rethrowing unknown exception in logger");                       \
+            throw;                                                                        \
+        }
+#else
+    #define SPDLOG_LOGGER_CATCH(location)
+#endif
+
+namespace spdlog {
+
+class SPDLOG_API logger {
+public:
+    // Empty logger
+    explicit logger(std::string name)
+        : name_(std::move(name)),
+          sinks_() {}
+
+    // Logger with range on sinks
+    template <typename It>
+    logger(std::string name, It begin, It end)
+        : name_(std::move(name)),
+          sinks_(begin, end) {}
+
+    // Logger with single sink
+    logger(std::string name, sink_ptr single_sink)
+        : logger(std::move(name), {std::move(single_sink)}) {}
+
+    // Logger with sinks init list
+    logger(std::string name, sinks_init_list sinks)
+        : logger(std::move(name), sinks.begin(), sinks.end()) {}
+
+    virtual ~logger() = default;
+
+    logger(const logger &other);
+    logger(logger &&other) SPDLOG_NOEXCEPT;
+    logger &operator=(logger other) SPDLOG_NOEXCEPT;
+    void swap(spdlog::logger &other) SPDLOG_NOEXCEPT;
+
+    template <typename... Args>
+    void log(source_loc loc, level::level_enum lvl, format_string_t<Args...> fmt, Args &&...args) {
+        log_(loc, lvl, details::to_string_view(fmt), std::forward<Args>(args)...);
+    }
+
+    template <typename... Args>
+    void log(level::level_enum lvl, format_string_t<Args...> fmt, Args &&...args) {
+        log(source_loc{}, lvl, fmt, std::forward<Args>(args)...);
+    }
+
+    template <typename T>
+    void log(level::level_enum lvl, const T &msg) {
+        log(source_loc{}, lvl, msg);
+    }
+
+    // T cannot be statically converted to format string (including string_view/wstring_view)
+    template <class T,
+              typename std::enable_if<!is_convertible_to_any_format_string<const T &>::value,
+                                      int>::type = 0>
+    void log(source_loc loc, level::level_enum lvl, const T &msg) {
+        log(loc, lvl, "{}", msg);
+    }
+
+    void log(log_clock::time_point log_time,
+             source_loc loc,
+             level::level_enum lvl,
+             string_view_t msg) {
+        bool log_enabled = should_log(lvl);
+        bool traceback_enabled = tracer_.enabled();
+        if (!log_enabled && !traceback_enabled) {
+            return;
+        }
+
+        details::log_msg log_msg(log_time, loc, name_, lvl, msg);
+        log_it_(log_msg, log_enabled, traceback_enabled);
+    }
+
+    void log(source_loc loc, level::level_enum lvl, string_view_t msg) {
+        bool log_enabled = should_log(lvl);
+        bool traceback_enabled = tracer_.enabled();
+        if (!log_enabled && !traceback_enabled) {
+            return;
+        }
+
+        details::log_msg log_msg(loc, name_, lvl, msg);
+        log_it_(log_msg, log_enabled, traceback_enabled);
+    }
+
+    void log(level::level_enum lvl, string_view_t msg) { log(source_loc{}, lvl, msg); }
+
+    template <typename... Args>
+    void trace(format_string_t<Args...> fmt, Args &&...args) {
+        log(level::trace, fmt, std::forward<Args>(args)...);
+    }
+
+    template <typename... Args>
+    void debug(format_string_t<Args...> fmt, Args &&...args) {
+        log(level::debug, fmt, std::forward<Args>(args)...);
+    }
+
+    template <typename... Args>
+    void info(format_string_t<Args...> fmt, Args &&...args) {
+        log(level::info, fmt, std::forward<Args>(args)...);
+    }
+
+    template <typename... Args>
+    void warn(format_string_t<Args...> fmt, Args &&...args) {
+        log(level::warn, fmt, std::forward<Args>(args)...);
+    }
+
+    template <typename... Args>
+    void error(format_string_t<Args...> fmt, Args &&...args) {
+        log(level::err, fmt, std::forward<Args>(args)...);
+    }
+
+    template <typename... Args>
+    void critical(format_string_t<Args...> fmt, Args &&...args) {
+        log(level::critical, fmt, std::forward<Args>(args)...);
+    }
+
+#ifdef SPDLOG_WCHAR_TO_UTF8_SUPPORT
+    template <typename... Args>
+    void log(source_loc loc, level::level_enum lvl, wformat_string_t<Args...> fmt, Args &&...args) {
+        log_(loc, lvl, details::to_string_view(fmt), std::forward<Args>(args)...);
+    }
+
+    template <typename... Args>
+    void log(level::level_enum lvl, wformat_string_t<Args...> fmt, Args &&...args) {
+        log(source_loc{}, lvl, fmt, std::forward<Args>(args)...);
+    }
+
+    void log(log_clock::time_point log_time,
+             source_loc loc,
+             level::level_enum lvl,
+             wstring_view_t msg) {
+        bool log_enabled = should_log(lvl);
+        bool traceback_enabled = tracer_.enabled();
+        if (!log_enabled && !traceback_enabled) {
+            return;
+        }
+
+        memory_buf_t buf;
+        details::os::wstr_to_utf8buf(wstring_view_t(msg.data(), msg.size()), buf);
+        details::log_msg log_msg(log_time, loc, name_, lvl, string_view_t(buf.data(), buf.size()));
+        log_it_(log_msg, log_enabled, traceback_enabled);
+    }
+
+    void log(source_loc loc, level::level_enum lvl, wstring_view_t msg) {
+        bool log_enabled = should_log(lvl);
+        bool traceback_enabled = tracer_.enabled();
+        if (!log_enabled && !traceback_enabled) {
+            return;
+        }
+
+        memory_buf_t buf;
+        details::os::wstr_to_utf8buf(wstring_view_t(msg.data(), msg.size()), buf);
+        details::log_msg log_msg(loc, name_, lvl, string_view_t(buf.data(), buf.size()));
+        log_it_(log_msg, log_enabled, traceback_enabled);
+    }
+
+    void log(level::level_enum lvl, wstring_view_t msg) { log(source_loc{}, lvl, msg); }
+
+    template <typename... Args>
+    void trace(wformat_string_t<Args...> fmt, Args &&...args) {
+        log(level::trace, fmt, std::forward<Args>(args)...);
+    }
+
+    template <typename... Args>
+    void debug(wformat_string_t<Args...> fmt, Args &&...args) {
+        log(level::debug, fmt, std::forward<Args>(args)...);
+    }
+
+    template <typename... Args>
+    void info(wformat_string_t<Args...> fmt, Args &&...args) {
+        log(level::info, fmt, std::forward<Args>(args)...);
+    }
+
+    template <typename... Args>
+    void warn(wformat_string_t<Args...> fmt, Args &&...args) {
+        log(level::warn, fmt, std::forward<Args>(args)...);
+    }
+
+    template <typename... Args>
+    void error(wformat_string_t<Args...> fmt, Args &&...args) {
+        log(level::err, fmt, std::forward<Args>(args)...);
+    }
+
+    template <typename... Args>
+    void critical(wformat_string_t<Args...> fmt, Args &&...args) {
+        log(level::critical, fmt, std::forward<Args>(args)...);
+    }
+#endif
+
+    template <typename T>
+    void trace(const T &msg) {
+        log(level::trace, msg);
+    }
+
+    template <typename T>
+    void debug(const T &msg) {
+        log(level::debug, msg);
+    }
+
+    template <typename T>
+    void info(const T &msg) {
+        log(level::info, msg);
+    }
+
+    template <typename T>
+    void warn(const T &msg) {
+        log(level::warn, msg);
+    }
+
+    template <typename T>
+    void error(const T &msg) {
+        log(level::err, msg);
+    }
+
+    template <typename T>
+    void critical(const T &msg) {
+        log(level::critical, msg);
+    }
+
+    // return true logging is enabled for the given level.
+    bool should_log(level::level_enum msg_level) const {
+        return msg_level >= level_.load(std::memory_order_relaxed);
+    }
+
+    // return true if backtrace logging is enabled.
+    bool should_backtrace() const { return tracer_.enabled(); }
+
+    void set_level(level::level_enum log_level);
+
+    level::level_enum level() const;
+
+    const std::string &name() const;
+
+    // set formatting for the sinks in this logger.
+    // each sink will get a separate instance of the formatter object.
+    void set_formatter(std::unique_ptr<formatter> f);
+
+    // set formatting for the sinks in this logger.
+    // equivalent to
+    //     set_formatter(make_unique<pattern_formatter>(pattern, time_type))
+    // Note: each sink will get a new instance of a formatter object, replacing the old one.
+    void set_pattern(std::string pattern, pattern_time_type time_type = pattern_time_type::local);
+
+    // backtrace support.
+    // efficiently store all debug/trace messages in a circular buffer until needed for debugging.
+    void enable_backtrace(size_t n_messages);
+    void disable_backtrace();
+    void dump_backtrace();
+
+    // flush functions
+    void flush();
+    void flush_on(level::level_enum log_level);
+    level::level_enum flush_level() const;
+
+    // sinks
+    const std::vector<sink_ptr> &sinks() const;
+
+    std::vector<sink_ptr> &sinks();
+
+    // error handler
+    void set_error_handler(err_handler);
+
+    // create new logger with same sinks and configuration.
+    virtual std::shared_ptr<logger> clone(std::string logger_name);
+
+protected:
+    std::string name_;
+    std::vector<sink_ptr> sinks_;
+    spdlog::level_t level_{level::info};
+    spdlog::level_t flush_level_{level::off};
+    err_handler custom_err_handler_{nullptr};
+    details::backtracer tracer_;
+
+    // common implementation for after templated public api has been resolved
+    template <typename... Args>
+    void log_(source_loc loc, level::level_enum lvl, string_view_t fmt, Args &&...args) {
+        bool log_enabled = should_log(lvl);
+        bool traceback_enabled = tracer_.enabled();
+        if (!log_enabled && !traceback_enabled) {
+            return;
+        }
+        SPDLOG_TRY {
+            memory_buf_t buf;
+#ifdef SPDLOG_USE_STD_FORMAT
+            fmt_lib::vformat_to(std::back_inserter(buf), fmt, fmt_lib::make_format_args(args...));
+#else
+            fmt::vformat_to(fmt::appender(buf), fmt, fmt::make_format_args(args...));
+#endif
+
+            details::log_msg log_msg(loc, name_, lvl, string_view_t(buf.data(), buf.size()));
+            log_it_(log_msg, log_enabled, traceback_enabled);
+        }
+        SPDLOG_LOGGER_CATCH(loc)
+    }
+
+#ifdef SPDLOG_WCHAR_TO_UTF8_SUPPORT
+    template <typename... Args>
+    void log_(source_loc loc, level::level_enum lvl, wstring_view_t fmt, Args &&...args) {
+        bool log_enabled = should_log(lvl);
+        bool traceback_enabled = tracer_.enabled();
+        if (!log_enabled && !traceback_enabled) {
+            return;
+        }
+        SPDLOG_TRY {
+            // format to wmemory_buffer and convert to utf8
+            wmemory_buf_t wbuf;
+            fmt_lib::vformat_to(std::back_inserter(wbuf), fmt,
+                                fmt_lib::make_format_args<fmt_lib::wformat_context>(args...));
+
+            memory_buf_t buf;
+            details::os::wstr_to_utf8buf(wstring_view_t(wbuf.data(), wbuf.size()), buf);
+            details::log_msg log_msg(loc, name_, lvl, string_view_t(buf.data(), buf.size()));
+            log_it_(log_msg, log_enabled, traceback_enabled);
+        }
+        SPDLOG_LOGGER_CATCH(loc)
+    }
+#endif  // SPDLOG_WCHAR_TO_UTF8_SUPPORT
+
+    // log the given message (if the given log level is high enough),
+    // and save backtrace (if backtrace is enabled).
+    void log_it_(const details::log_msg &log_msg, bool log_enabled, bool traceback_enabled);
+    virtual void sink_it_(const details::log_msg &msg);
+    virtual void flush_();
+    void dump_backtrace_();
+    bool should_flush_(const details::log_msg &msg);
+
+    // handle errors during logging.
+    // default handler prints the error to stderr at max rate of 1 message/sec.
+    void err_handler_(const std::string &msg);
+};
+
+void swap(logger &a, logger &b);
+
+}  // namespace spdlog
+
+#ifdef SPDLOG_HEADER_ONLY
+    #include "logger-inl.h"
+#endif
diff --git a/ext/spdlog/include/spdlog/mdc.h b/ext/spdlog/include/spdlog/mdc.h
new file mode 100644
index 0000000..80b6f25
--- /dev/null
+++ b/ext/spdlog/include/spdlog/mdc.h
@@ -0,0 +1,50 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#if defined(SPDLOG_NO_TLS)
+    #error "This header requires thread local storage support, but SPDLOG_NO_TLS is defined."
+#endif
+
+#include <map>
+#include <string>
+
+#include <spdlog/common.h>
+
+// MDC is a simple map of key->string values stored in thread local storage whose content will be printed by the loggers.
+// Note: Not supported in async mode (thread local storage - so the async thread pool have different copy).
+//
+// Usage example:
+// spdlog::mdc::put("mdc_key_1", "mdc_value_1");
+// spdlog::info("Hello, {}", "World!");  // => [2024-04-26 02:08:05.040] [info] [mdc_key_1:mdc_value_1] Hello, World!
+
+namespace spdlog {
+class SPDLOG_API mdc {
+public:
+    using mdc_map_t = std::map<std::string, std::string>;
+
+    static void put(const std::string &key, const std::string &value) {
+        get_context()[key] = value;
+    }
+
+    static std::string get(const std::string &key) {
+        auto &context = get_context();
+        auto it = context.find(key);
+        if (it != context.end()) {
+            return it->second;
+        }
+        return "";
+    }
+
+    static void remove(const std::string &key) { get_context().erase(key); }
+
+    static void clear() { get_context().clear(); }
+
+    static mdc_map_t &get_context() {
+        static thread_local mdc_map_t context;
+        return context;
+    }
+};
+
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/pattern_formatter-inl.h b/ext/spdlog/include/spdlog/pattern_formatter-inl.h
new file mode 100644
index 0000000..b53d805
--- /dev/null
+++ b/ext/spdlog/include/spdlog/pattern_formatter-inl.h
@@ -0,0 +1,1338 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#ifndef SPDLOG_HEADER_ONLY
+    #include <spdlog/pattern_formatter.h>
+#endif
+
+#include <spdlog/details/fmt_helper.h>
+#include <spdlog/details/log_msg.h>
+#include <spdlog/details/os.h>
+
+#ifndef SPDLOG_NO_TLS
+    #include <spdlog/mdc.h>
+#endif
+
+#include <spdlog/fmt/fmt.h>
+#include <spdlog/formatter.h>
+
+#include <algorithm>
+#include <array>
+#include <cctype>
+#include <chrono>
+#include <cstring>
+#include <ctime>
+#include <iterator>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <thread>
+#include <utility>
+#include <vector>
+
+namespace spdlog {
+namespace details {
+
+///////////////////////////////////////////////////////////////////////
+// name & level pattern appender
+///////////////////////////////////////////////////////////////////////
+
+class scoped_padder {
+public:
+    scoped_padder(size_t wrapped_size, const padding_info &padinfo, memory_buf_t &dest)
+        : padinfo_(padinfo),
+          dest_(dest) {
+        remaining_pad_ = static_cast<long>(padinfo.width_) - static_cast<long>(wrapped_size);
+        if (remaining_pad_ <= 0) {
+            return;
+        }
+
+        if (padinfo_.side_ == padding_info::pad_side::left) {
+            pad_it(remaining_pad_);
+            remaining_pad_ = 0;
+        } else if (padinfo_.side_ == padding_info::pad_side::center) {
+            auto half_pad = remaining_pad_ / 2;
+            auto reminder = remaining_pad_ & 1;
+            pad_it(half_pad);
+            remaining_pad_ = half_pad + reminder;  // for the right side
+        }
+    }
+
+    template <typename T>
+    static unsigned int count_digits(T n) {
+        return fmt_helper::count_digits(n);
+    }
+
+    ~scoped_padder() {
+        if (remaining_pad_ >= 0) {
+            pad_it(remaining_pad_);
+        } else if (padinfo_.truncate_) {
+            long new_size = static_cast<long>(dest_.size()) + remaining_pad_;
+            dest_.resize(static_cast<size_t>(new_size));
+        }
+    }
+
+private:
+    void pad_it(long count) {
+        fmt_helper::append_string_view(string_view_t(spaces_.data(), static_cast<size_t>(count)),
+                                       dest_);
+    }
+
+    const padding_info &padinfo_;
+    memory_buf_t &dest_;
+    long remaining_pad_;
+    string_view_t spaces_{"                                                                ", 64};
+};
+
+struct null_scoped_padder {
+    null_scoped_padder(size_t /*wrapped_size*/,
+                       const padding_info & /*padinfo*/,
+                       memory_buf_t & /*dest*/) {}
+
+    template <typename T>
+    static unsigned int count_digits(T /* number */) {
+        return 0;
+    }
+};
+
+template <typename ScopedPadder>
+class name_formatter final : public flag_formatter {
+public:
+    explicit name_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+    void format(const details::log_msg &msg, const std::tm &, memory_buf_t &dest) override {
+        ScopedPadder p(msg.logger_name.size(), padinfo_, dest);
+        fmt_helper::append_string_view(msg.logger_name, dest);
+    }
+};
+
+// log level appender
+template <typename ScopedPadder>
+class level_formatter final : public flag_formatter {
+public:
+    explicit level_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+    void format(const details::log_msg &msg, const std::tm &, memory_buf_t &dest) override {
+        const string_view_t &level_name = level::to_string_view(msg.level);
+        ScopedPadder p(level_name.size(), padinfo_, dest);
+        fmt_helper::append_string_view(level_name, dest);
+    }
+};
+
+// short log level appender
+template <typename ScopedPadder>
+class short_level_formatter final : public flag_formatter {
+public:
+    explicit short_level_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+    void format(const details::log_msg &msg, const std::tm &, memory_buf_t &dest) override {
+        string_view_t level_name{level::to_short_c_str(msg.level)};
+        ScopedPadder p(level_name.size(), padinfo_, dest);
+        fmt_helper::append_string_view(level_name, dest);
+    }
+};
+
+///////////////////////////////////////////////////////////////////////
+// Date time pattern appenders
+///////////////////////////////////////////////////////////////////////
+
+static const char *ampm(const tm &t) { return t.tm_hour >= 12 ? "PM" : "AM"; }
+
+static int to12h(const tm &t) { return t.tm_hour > 12 ? t.tm_hour - 12 : t.tm_hour; }
+
+// Abbreviated weekday name
+static std::array<const char *, 7> days{{"Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"}};
+
+template <typename ScopedPadder>
+class a_formatter final : public flag_formatter {
+public:
+    explicit a_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+    void format(const details::log_msg &, const std::tm &tm_time, memory_buf_t &dest) override {
+        string_view_t field_value{days[static_cast<size_t>(tm_time.tm_wday)]};
+        ScopedPadder p(field_value.size(), padinfo_, dest);
+        fmt_helper::append_string_view(field_value, dest);
+    }
+};
+
+// Full weekday name
+static std::array<const char *, 7> full_days{
+    {"Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"}};
+
+template <typename ScopedPadder>
+class A_formatter : public flag_formatter {
+public:
+    explicit A_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+    void format(const details::log_msg &, const std::tm &tm_time, memory_buf_t &dest) override {
+        string_view_t field_value{full_days[static_cast<size_t>(tm_time.tm_wday)]};
+        ScopedPadder p(field_value.size(), padinfo_, dest);
+        fmt_helper::append_string_view(field_value, dest);
+    }
+};
+
+// Abbreviated month
+static const std::array<const char *, 12> months{
+    {"Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"}};
+
+template <typename ScopedPadder>
+class b_formatter final : public flag_formatter {
+public:
+    explicit b_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+    void format(const details::log_msg &, const std::tm &tm_time, memory_buf_t &dest) override {
+        string_view_t field_value{months[static_cast<size_t>(tm_time.tm_mon)]};
+        ScopedPadder p(field_value.size(), padinfo_, dest);
+        fmt_helper::append_string_view(field_value, dest);
+    }
+};
+
+// Full month name
+static const std::array<const char *, 12> full_months{{"January", "February", "March", "April",
+                                                       "May", "June", "July", "August", "September",
+                                                       "October", "November", "December"}};
+
+template <typename ScopedPadder>
+class B_formatter final : public flag_formatter {
+public:
+    explicit B_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+    void format(const details::log_msg &, const std::tm &tm_time, memory_buf_t &dest) override {
+        string_view_t field_value{full_months[static_cast<size_t>(tm_time.tm_mon)]};
+        ScopedPadder p(field_value.size(), padinfo_, dest);
+        fmt_helper::append_string_view(field_value, dest);
+    }
+};
+
+// Date and time representation (Thu Aug 23 15:35:46 2014)
+template <typename ScopedPadder>
+class c_formatter final : public flag_formatter {
+public:
+    explicit c_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+    void format(const details::log_msg &, const std::tm &tm_time, memory_buf_t &dest) override {
+        const size_t field_size = 24;
+        ScopedPadder p(field_size, padinfo_, dest);
+
+        fmt_helper::append_string_view(days[static_cast<size_t>(tm_time.tm_wday)], dest);
+        dest.push_back(' ');
+        fmt_helper::append_string_view(months[static_cast<size_t>(tm_time.tm_mon)], dest);
+        dest.push_back(' ');
+        fmt_helper::append_int(tm_time.tm_mday, dest);
+        dest.push_back(' ');
+        // time
+
+        fmt_helper::pad2(tm_time.tm_hour, dest);
+        dest.push_back(':');
+        fmt_helper::pad2(tm_time.tm_min, dest);
+        dest.push_back(':');
+        fmt_helper::pad2(tm_time.tm_sec, dest);
+        dest.push_back(' ');
+        fmt_helper::append_int(tm_time.tm_year + 1900, dest);
+    }
+};
+
+// year - 2 digit
+template <typename ScopedPadder>
+class C_formatter final : public flag_formatter {
+public:
+    explicit C_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+    void format(const details::log_msg &, const std::tm &tm_time, memory_buf_t &dest) override {
+        const size_t field_size = 2;
+        ScopedPadder p(field_size, padinfo_, dest);
+        fmt_helper::pad2(tm_time.tm_year % 100, dest);
+    }
+};
+
+// Short MM/DD/YY date, equivalent to %m/%d/%y 08/23/01
+template <typename ScopedPadder>
+class D_formatter final : public flag_formatter {
+public:
+    explicit D_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+    void format(const details::log_msg &, const std::tm &tm_time, memory_buf_t &dest) override {
+        const size_t field_size = 10;
+        ScopedPadder p(field_size, padinfo_, dest);
+
+        fmt_helper::pad2(tm_time.tm_mon + 1, dest);
+        dest.push_back('/');
+        fmt_helper::pad2(tm_time.tm_mday, dest);
+        dest.push_back('/');
+        fmt_helper::pad2(tm_time.tm_year % 100, dest);
+    }
+};
+
+// year - 4 digit
+template <typename ScopedPadder>
+class Y_formatter final : public flag_formatter {
+public:
+    explicit Y_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+    void format(const details::log_msg &, const std::tm &tm_time, memory_buf_t &dest) override {
+        const size_t field_size = 4;
+        ScopedPadder p(field_size, padinfo_, dest);
+        fmt_helper::append_int(tm_time.tm_year + 1900, dest);
+    }
+};
+
+// month 1-12
+template <typename ScopedPadder>
+class m_formatter final : public flag_formatter {
+public:
+    explicit m_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+    void format(const details::log_msg &, const std::tm &tm_time, memory_buf_t &dest) override {
+        const size_t field_size = 2;
+        ScopedPadder p(field_size, padinfo_, dest);
+        fmt_helper::pad2(tm_time.tm_mon + 1, dest);
+    }
+};
+
+// day of month 1-31
+template <typename ScopedPadder>
+class d_formatter final : public flag_formatter {
+public:
+    explicit d_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+    void format(const details::log_msg &, const std::tm &tm_time, memory_buf_t &dest) override {
+        const size_t field_size = 2;
+        ScopedPadder p(field_size, padinfo_, dest);
+        fmt_helper::pad2(tm_time.tm_mday, dest);
+    }
+};
+
+// hours in 24 format 0-23
+template <typename ScopedPadder>
+class H_formatter final : public flag_formatter {
+public:
+    explicit H_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+    void format(const details::log_msg &, const std::tm &tm_time, memory_buf_t &dest) override {
+        const size_t field_size = 2;
+        ScopedPadder p(field_size, padinfo_, dest);
+        fmt_helper::pad2(tm_time.tm_hour, dest);
+    }
+};
+
+// hours in 12 format 1-12
+template <typename ScopedPadder>
+class I_formatter final : public flag_formatter {
+public:
+    explicit I_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+    void format(const details::log_msg &, const std::tm &tm_time, memory_buf_t &dest) override {
+        const size_t field_size = 2;
+        ScopedPadder p(field_size, padinfo_, dest);
+        fmt_helper::pad2(to12h(tm_time), dest);
+    }
+};
+
+// minutes 0-59
+template <typename ScopedPadder>
+class M_formatter final : public flag_formatter {
+public:
+    explicit M_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+    void format(const details::log_msg &, const std::tm &tm_time, memory_buf_t &dest) override {
+        const size_t field_size = 2;
+        ScopedPadder p(field_size, padinfo_, dest);
+        fmt_helper::pad2(tm_time.tm_min, dest);
+    }
+};
+
+// seconds 0-59
+template <typename ScopedPadder>
+class S_formatter final : public flag_formatter {
+public:
+    explicit S_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+    void format(const details::log_msg &, const std::tm &tm_time, memory_buf_t &dest) override {
+        const size_t field_size = 2;
+        ScopedPadder p(field_size, padinfo_, dest);
+        fmt_helper::pad2(tm_time.tm_sec, dest);
+    }
+};
+
+// milliseconds
+template <typename ScopedPadder>
+class e_formatter final : public flag_formatter {
+public:
+    explicit e_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+    void format(const details::log_msg &msg, const std::tm &, memory_buf_t &dest) override {
+        auto millis = fmt_helper::time_fraction<std::chrono::milliseconds>(msg.time);
+        const size_t field_size = 3;
+        ScopedPadder p(field_size, padinfo_, dest);
+        fmt_helper::pad3(static_cast<uint32_t>(millis.count()), dest);
+    }
+};
+
+// microseconds
+template <typename ScopedPadder>
+class f_formatter final : public flag_formatter {
+public:
+    explicit f_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+    void format(const details::log_msg &msg, const std::tm &, memory_buf_t &dest) override {
+        auto micros = fmt_helper::time_fraction<std::chrono::microseconds>(msg.time);
+
+        const size_t field_size = 6;
+        ScopedPadder p(field_size, padinfo_, dest);
+        fmt_helper::pad6(static_cast<size_t>(micros.count()), dest);
+    }
+};
+
+// nanoseconds
+template <typename ScopedPadder>
+class F_formatter final : public flag_formatter {
+public:
+    explicit F_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+    void format(const details::log_msg &msg, const std::tm &, memory_buf_t &dest) override {
+        auto ns = fmt_helper::time_fraction<std::chrono::nanoseconds>(msg.time);
+        const size_t field_size = 9;
+        ScopedPadder p(field_size, padinfo_, dest);
+        fmt_helper::pad9(static_cast<size_t>(ns.count()), dest);
+    }
+};
+
+// seconds since epoch
+template <typename ScopedPadder>
+class E_formatter final : public flag_formatter {
+public:
+    explicit E_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+    void format(const details::log_msg &msg, const std::tm &, memory_buf_t &dest) override {
+        const size_t field_size = 10;
+        ScopedPadder p(field_size, padinfo_, dest);
+        auto duration = msg.time.time_since_epoch();
+        auto seconds = std::chrono::duration_cast<std::chrono::seconds>(duration).count();
+        fmt_helper::append_int(seconds, dest);
+    }
+};
+
+// AM/PM
+template <typename ScopedPadder>
+class p_formatter final : public flag_formatter {
+public:
+    explicit p_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+    void format(const details::log_msg &, const std::tm &tm_time, memory_buf_t &dest) override {
+        const size_t field_size = 2;
+        ScopedPadder p(field_size, padinfo_, dest);
+        fmt_helper::append_string_view(ampm(tm_time), dest);
+    }
+};
+
+// 12 hour clock 02:55:02 pm
+template <typename ScopedPadder>
+class r_formatter final : public flag_formatter {
+public:
+    explicit r_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+    void format(const details::log_msg &, const std::tm &tm_time, memory_buf_t &dest) override {
+        const size_t field_size = 11;
+        ScopedPadder p(field_size, padinfo_, dest);
+
+        fmt_helper::pad2(to12h(tm_time), dest);
+        dest.push_back(':');
+        fmt_helper::pad2(tm_time.tm_min, dest);
+        dest.push_back(':');
+        fmt_helper::pad2(tm_time.tm_sec, dest);
+        dest.push_back(' ');
+        fmt_helper::append_string_view(ampm(tm_time), dest);
+    }
+};
+
+// 24-hour HH:MM time, equivalent to %H:%M
+template <typename ScopedPadder>
+class R_formatter final : public flag_formatter {
+public:
+    explicit R_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+    void format(const details::log_msg &, const std::tm &tm_time, memory_buf_t &dest) override {
+        const size_t field_size = 5;
+        ScopedPadder p(field_size, padinfo_, dest);
+
+        fmt_helper::pad2(tm_time.tm_hour, dest);
+        dest.push_back(':');
+        fmt_helper::pad2(tm_time.tm_min, dest);
+    }
+};
+
+// ISO 8601 time format (HH:MM:SS), equivalent to %H:%M:%S
+template <typename ScopedPadder>
+class T_formatter final : public flag_formatter {
+public:
+    explicit T_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+    void format(const details::log_msg &, const std::tm &tm_time, memory_buf_t &dest) override {
+        const size_t field_size = 8;
+        ScopedPadder p(field_size, padinfo_, dest);
+
+        fmt_helper::pad2(tm_time.tm_hour, dest);
+        dest.push_back(':');
+        fmt_helper::pad2(tm_time.tm_min, dest);
+        dest.push_back(':');
+        fmt_helper::pad2(tm_time.tm_sec, dest);
+    }
+};
+
+// ISO 8601 offset from UTC in timezone (+-HH:MM)
+template <typename ScopedPadder>
+class z_formatter final : public flag_formatter {
+public:
+    explicit z_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+    z_formatter() = default;
+    z_formatter(const z_formatter &) = delete;
+    z_formatter &operator=(const z_formatter &) = delete;
+
+    void format(const details::log_msg &msg, const std::tm &tm_time, memory_buf_t &dest) override {
+        const size_t field_size = 6;
+        ScopedPadder p(field_size, padinfo_, dest);
+
+        auto total_minutes = get_cached_offset(msg, tm_time);
+        bool is_negative = total_minutes < 0;
+        if (is_negative) {
+            total_minutes = -total_minutes;
+            dest.push_back('-');
+        } else {
+            dest.push_back('+');
+        }
+
+        fmt_helper::pad2(total_minutes / 60, dest);  // hours
+        dest.push_back(':');
+        fmt_helper::pad2(total_minutes % 60, dest);  // minutes
+    }
+
+private:
+    log_clock::time_point last_update_{std::chrono::seconds(0)};
+    int offset_minutes_{0};
+
+    int get_cached_offset(const log_msg &msg, const std::tm &tm_time) {
+        // refresh every 10 seconds
+        if (msg.time - last_update_ >= std::chrono::seconds(10)) {
+            offset_minutes_ = os::utc_minutes_offset(tm_time);
+            last_update_ = msg.time;
+        }
+        return offset_minutes_;
+    }
+};
+
+// Thread id
+template <typename ScopedPadder>
+class t_formatter final : public flag_formatter {
+public:
+    explicit t_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+    void format(const details::log_msg &msg, const std::tm &, memory_buf_t &dest) override {
+        const auto field_size = ScopedPadder::count_digits(msg.thread_id);
+        ScopedPadder p(field_size, padinfo_, dest);
+        fmt_helper::append_int(msg.thread_id, dest);
+    }
+};
+
+// Current pid
+template <typename ScopedPadder>
+class pid_formatter final : public flag_formatter {
+public:
+    explicit pid_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+    void format(const details::log_msg &, const std::tm &, memory_buf_t &dest) override {
+        const auto pid = static_cast<uint32_t>(details::os::pid());
+        auto field_size = ScopedPadder::count_digits(pid);
+        ScopedPadder p(field_size, padinfo_, dest);
+        fmt_helper::append_int(pid, dest);
+    }
+};
+
+template <typename ScopedPadder>
+class v_formatter final : public flag_formatter {
+public:
+    explicit v_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+    void format(const details::log_msg &msg, const std::tm &, memory_buf_t &dest) override {
+        ScopedPadder p(msg.payload.size(), padinfo_, dest);
+        fmt_helper::append_string_view(msg.payload, dest);
+    }
+};
+
+class ch_formatter final : public flag_formatter {
+public:
+    explicit ch_formatter(char ch)
+        : ch_(ch) {}
+
+    void format(const details::log_msg &, const std::tm &, memory_buf_t &dest) override {
+        dest.push_back(ch_);
+    }
+
+private:
+    char ch_;
+};
+
+// aggregate user chars to display as is
+class aggregate_formatter final : public flag_formatter {
+public:
+    aggregate_formatter() = default;
+
+    void add_ch(char ch) { str_ += ch; }
+    void format(const details::log_msg &, const std::tm &, memory_buf_t &dest) override {
+        fmt_helper::append_string_view(str_, dest);
+    }
+
+private:
+    std::string str_;
+};
+
+// mark the color range. expect it to be in the form of "%^colored text%$"
+class color_start_formatter final : public flag_formatter {
+public:
+    explicit color_start_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+    void format(const details::log_msg &msg, const std::tm &, memory_buf_t &dest) override {
+        msg.color_range_start = dest.size();
+    }
+};
+
+class color_stop_formatter final : public flag_formatter {
+public:
+    explicit color_stop_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+    void format(const details::log_msg &msg, const std::tm &, memory_buf_t &dest) override {
+        msg.color_range_end = dest.size();
+    }
+};
+
+// print source location
+template <typename ScopedPadder>
+class source_location_formatter final : public flag_formatter {
+public:
+    explicit source_location_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+    void format(const details::log_msg &msg, const std::tm &, memory_buf_t &dest) override {
+        if (msg.source.empty()) {
+            ScopedPadder p(0, padinfo_, dest);
+            return;
+        }
+
+        size_t text_size;
+        if (padinfo_.enabled()) {
+            // calc text size for padding based on "filename:line"
+            text_size = std::char_traits<char>::length(msg.source.filename) +
+                        ScopedPadder::count_digits(msg.source.line) + 1;
+        } else {
+            text_size = 0;
+        }
+
+        ScopedPadder p(text_size, padinfo_, dest);
+        fmt_helper::append_string_view(msg.source.filename, dest);
+        dest.push_back(':');
+        fmt_helper::append_int(msg.source.line, dest);
+    }
+};
+
+// print source filename
+template <typename ScopedPadder>
+class source_filename_formatter final : public flag_formatter {
+public:
+    explicit source_filename_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+    void format(const details::log_msg &msg, const std::tm &, memory_buf_t &dest) override {
+        if (msg.source.empty()) {
+            ScopedPadder p(0, padinfo_, dest);
+            return;
+        }
+        size_t text_size =
+            padinfo_.enabled() ? std::char_traits<char>::length(msg.source.filename) : 0;
+        ScopedPadder p(text_size, padinfo_, dest);
+        fmt_helper::append_string_view(msg.source.filename, dest);
+    }
+};
+
+template <typename ScopedPadder>
+class short_filename_formatter final : public flag_formatter {
+public:
+    explicit short_filename_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+#ifdef _MSC_VER
+    #pragma warning(push)
+    #pragma warning(disable : 4127)  // consider using 'if constexpr' instead
+#endif                               // _MSC_VER
+    static const char *basename(const char *filename) {
+        // if the size is 2 (1 character + null terminator) we can use the more efficient strrchr
+        // the branch will be elided by optimizations
+        if (sizeof(os::folder_seps) == 2) {
+            const char *rv = std::strrchr(filename, os::folder_seps[0]);
+            return rv != nullptr ? rv + 1 : filename;
+        } else {
+            const std::reverse_iterator<const char *> begin(filename + std::strlen(filename));
+            const std::reverse_iterator<const char *> end(filename);
+
+            const auto it = std::find_first_of(begin, end, std::begin(os::folder_seps),
+                                               std::end(os::folder_seps) - 1);
+            return it != end ? it.base() : filename;
+        }
+    }
+#ifdef _MSC_VER
+    #pragma warning(pop)
+#endif  // _MSC_VER
+
+    void format(const details::log_msg &msg, const std::tm &, memory_buf_t &dest) override {
+        if (msg.source.empty()) {
+            ScopedPadder p(0, padinfo_, dest);
+            return;
+        }
+        auto filename = basename(msg.source.filename);
+        size_t text_size = padinfo_.enabled() ? std::char_traits<char>::length(filename) : 0;
+        ScopedPadder p(text_size, padinfo_, dest);
+        fmt_helper::append_string_view(filename, dest);
+    }
+};
+
+template <typename ScopedPadder>
+class source_linenum_formatter final : public flag_formatter {
+public:
+    explicit source_linenum_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+    void format(const details::log_msg &msg, const std::tm &, memory_buf_t &dest) override {
+        if (msg.source.empty()) {
+            ScopedPadder p(0, padinfo_, dest);
+            return;
+        }
+
+        auto field_size = ScopedPadder::count_digits(msg.source.line);
+        ScopedPadder p(field_size, padinfo_, dest);
+        fmt_helper::append_int(msg.source.line, dest);
+    }
+};
+
+// print source funcname
+template <typename ScopedPadder>
+class source_funcname_formatter final : public flag_formatter {
+public:
+    explicit source_funcname_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+    void format(const details::log_msg &msg, const std::tm &, memory_buf_t &dest) override {
+        if (msg.source.empty()) {
+            ScopedPadder p(0, padinfo_, dest);
+            return;
+        }
+        size_t text_size =
+            padinfo_.enabled() ? std::char_traits<char>::length(msg.source.funcname) : 0;
+        ScopedPadder p(text_size, padinfo_, dest);
+        fmt_helper::append_string_view(msg.source.funcname, dest);
+    }
+};
+
+// print elapsed time since last message
+template <typename ScopedPadder, typename Units>
+class elapsed_formatter final : public flag_formatter {
+public:
+    using DurationUnits = Units;
+
+    explicit elapsed_formatter(padding_info padinfo)
+        : flag_formatter(padinfo),
+          last_message_time_(log_clock::now()) {}
+
+    void format(const details::log_msg &msg, const std::tm &, memory_buf_t &dest) override {
+        auto delta = (std::max)(msg.time - last_message_time_, log_clock::duration::zero());
+        auto delta_units = std::chrono::duration_cast<DurationUnits>(delta);
+        last_message_time_ = msg.time;
+        auto delta_count = static_cast<size_t>(delta_units.count());
+        auto n_digits = static_cast<size_t>(ScopedPadder::count_digits(delta_count));
+        ScopedPadder p(n_digits, padinfo_, dest);
+        fmt_helper::append_int(delta_count, dest);
+    }
+
+private:
+    log_clock::time_point last_message_time_;
+};
+
+// Class for formatting Mapped Diagnostic Context (MDC) in log messages.
+// Example: [logger-name] [info] [mdc_key_1:mdc_value_1 mdc_key_2:mdc_value_2] some message
+#ifndef SPDLOG_NO_TLS
+template <typename ScopedPadder>
+class mdc_formatter : public flag_formatter {
+public:
+    explicit mdc_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+    void format(const details::log_msg &, const std::tm &, memory_buf_t &dest) override {
+        auto &mdc_map = mdc::get_context();
+        if (mdc_map.empty()) {
+            ScopedPadder p(0, padinfo_, dest);
+            return;
+        } else {
+            format_mdc(mdc_map, dest);
+        }
+    }
+
+    void format_mdc(const mdc::mdc_map_t &mdc_map, memory_buf_t &dest) {
+        auto last_element = --mdc_map.end();
+        for (auto it = mdc_map.begin(); it != mdc_map.end(); ++it) {
+            auto &pair = *it;
+            const auto &key = pair.first;
+            const auto &value = pair.second;
+            size_t content_size = key.size() + value.size() + 1;  // 1 for ':'
+
+            if (it != last_element) {
+                content_size++;  // 1 for ' '
+            }
+
+            ScopedPadder p(content_size, padinfo_, dest);
+            fmt_helper::append_string_view(key, dest);
+            fmt_helper::append_string_view(":", dest);
+            fmt_helper::append_string_view(value, dest);
+            if (it != last_element) {
+                fmt_helper::append_string_view(" ", dest);
+            }
+        }
+    }
+};
+#endif
+
+// Full info formatter
+// pattern: [%Y-%m-%d %H:%M:%S.%e] [%n] [%l] [%s:%#] %v
+class full_formatter final : public flag_formatter {
+public:
+    explicit full_formatter(padding_info padinfo)
+        : flag_formatter(padinfo) {}
+
+    void format(const details::log_msg &msg, const std::tm &tm_time, memory_buf_t &dest) override {
+        using std::chrono::duration_cast;
+        using std::chrono::milliseconds;
+        using std::chrono::seconds;
+
+        // cache the date/time part for the next second.
+        auto duration = msg.time.time_since_epoch();
+        auto secs = duration_cast<seconds>(duration);
+
+        if (cache_timestamp_ != secs || cached_datetime_.size() == 0) {
+            cached_datetime_.clear();
+            cached_datetime_.push_back('[');
+            fmt_helper::append_int(tm_time.tm_year + 1900, cached_datetime_);
+            cached_datetime_.push_back('-');
+
+            fmt_helper::pad2(tm_time.tm_mon + 1, cached_datetime_);
+            cached_datetime_.push_back('-');
+
+            fmt_helper::pad2(tm_time.tm_mday, cached_datetime_);
+            cached_datetime_.push_back(' ');
+
+            fmt_helper::pad2(tm_time.tm_hour, cached_datetime_);
+            cached_datetime_.push_back(':');
+
+            fmt_helper::pad2(tm_time.tm_min, cached_datetime_);
+            cached_datetime_.push_back(':');
+
+            fmt_helper::pad2(tm_time.tm_sec, cached_datetime_);
+            cached_datetime_.push_back('.');
+
+            cache_timestamp_ = secs;
+        }
+        dest.append(cached_datetime_.begin(), cached_datetime_.end());
+
+        auto millis = fmt_helper::time_fraction<milliseconds>(msg.time);
+        fmt_helper::pad3(static_cast<uint32_t>(millis.count()), dest);
+        dest.push_back(']');
+        dest.push_back(' ');
+
+        // append logger name if exists
+        if (msg.logger_name.size() > 0) {
+            dest.push_back('[');
+            fmt_helper::append_string_view(msg.logger_name, dest);
+            dest.push_back(']');
+            dest.push_back(' ');
+        }
+
+        dest.push_back('[');
+        // wrap the level name with color
+        msg.color_range_start = dest.size();
+        // fmt_helper::append_string_view(level::to_c_str(msg.level), dest);
+        fmt_helper::append_string_view(level::to_string_view(msg.level), dest);
+        msg.color_range_end = dest.size();
+        dest.push_back(']');
+        dest.push_back(' ');
+
+        // add source location if present
+        if (!msg.source.empty()) {
+            dest.push_back('[');
+            const char *filename =
+                details::short_filename_formatter<details::null_scoped_padder>::basename(
+                    msg.source.filename);
+            fmt_helper::append_string_view(filename, dest);
+            dest.push_back(':');
+            fmt_helper::append_int(msg.source.line, dest);
+            dest.push_back(']');
+            dest.push_back(' ');
+        }
+
+#ifndef SPDLOG_NO_TLS
+        // add mdc if present
+        auto &mdc_map = mdc::get_context();
+        if (!mdc_map.empty()) {
+            dest.push_back('[');
+            mdc_formatter_.format_mdc(mdc_map, dest);
+            dest.push_back(']');
+            dest.push_back(' ');
+        }
+#endif
+        // fmt_helper::append_string_view(msg.msg(), dest);
+        fmt_helper::append_string_view(msg.payload, dest);
+    }
+
+private:
+    std::chrono::seconds cache_timestamp_{0};
+    memory_buf_t cached_datetime_;
+
+#ifndef SPDLOG_NO_TLS
+    mdc_formatter<null_scoped_padder> mdc_formatter_{padding_info{}};
+#endif
+
+};
+
+}  // namespace details
+
+SPDLOG_INLINE pattern_formatter::pattern_formatter(std::string pattern,
+                                                   pattern_time_type time_type,
+                                                   std::string eol,
+                                                   custom_flags custom_user_flags)
+    : pattern_(std::move(pattern)),
+      eol_(std::move(eol)),
+      pattern_time_type_(time_type),
+      need_localtime_(false),
+      last_log_secs_(0),
+      custom_handlers_(std::move(custom_user_flags)) {
+    std::memset(&cached_tm_, 0, sizeof(cached_tm_));
+    compile_pattern_(pattern_);
+}
+
+// use by default full formatter for if pattern is not given
+SPDLOG_INLINE pattern_formatter::pattern_formatter(pattern_time_type time_type, std::string eol)
+    : pattern_("%+"),
+      eol_(std::move(eol)),
+      pattern_time_type_(time_type),
+      need_localtime_(true),
+      last_log_secs_(0) {
+    std::memset(&cached_tm_, 0, sizeof(cached_tm_));
+    formatters_.push_back(details::make_unique<details::full_formatter>(details::padding_info{}));
+}
+
+SPDLOG_INLINE std::unique_ptr<formatter> pattern_formatter::clone() const {
+    custom_flags cloned_custom_formatters;
+    for (auto &it : custom_handlers_) {
+        cloned_custom_formatters[it.first] = it.second->clone();
+    }
+    auto cloned = details::make_unique<pattern_formatter>(pattern_, pattern_time_type_, eol_,
+                                                          std::move(cloned_custom_formatters));
+    cloned->need_localtime(need_localtime_);
+#if defined(__GNUC__) && __GNUC__ < 5
+    return std::move(cloned);
+#else
+    return cloned;
+#endif
+}
+
+SPDLOG_INLINE void pattern_formatter::format(const details::log_msg &msg, memory_buf_t &dest) {
+    if (need_localtime_) {
+        const auto secs =
+            std::chrono::duration_cast<std::chrono::seconds>(msg.time.time_since_epoch());
+        if (secs != last_log_secs_) {
+            cached_tm_ = get_time_(msg);
+            last_log_secs_ = secs;
+        }
+    }
+
+    for (auto &f : formatters_) {
+        f->format(msg, cached_tm_, dest);
+    }
+    // write eol
+    details::fmt_helper::append_string_view(eol_, dest);
+}
+
+SPDLOG_INLINE void pattern_formatter::set_pattern(std::string pattern) {
+    pattern_ = std::move(pattern);
+    need_localtime_ = false;
+    compile_pattern_(pattern_);
+}
+
+SPDLOG_INLINE void pattern_formatter::need_localtime(bool need) { need_localtime_ = need; }
+
+SPDLOG_INLINE std::tm pattern_formatter::get_time_(const details::log_msg &msg) {
+    if (pattern_time_type_ == pattern_time_type::local) {
+        return details::os::localtime(log_clock::to_time_t(msg.time));
+    }
+    return details::os::gmtime(log_clock::to_time_t(msg.time));
+}
+
+template <typename Padder>
+SPDLOG_INLINE void pattern_formatter::handle_flag_(char flag, details::padding_info padding) {
+    // process custom flags
+    auto it = custom_handlers_.find(flag);
+    if (it != custom_handlers_.end()) {
+        auto custom_handler = it->second->clone();
+        custom_handler->set_padding_info(padding);
+        formatters_.push_back(std::move(custom_handler));
+        return;
+    }
+
+    // process built-in flags
+    switch (flag) {
+        case ('+'):  // default formatter
+            formatters_.push_back(details::make_unique<details::full_formatter>(padding));
+            need_localtime_ = true;
+            break;
+
+        case 'n':  // logger name
+            formatters_.push_back(details::make_unique<details::name_formatter<Padder>>(padding));
+            break;
+
+        case 'l':  // level
+            formatters_.push_back(details::make_unique<details::level_formatter<Padder>>(padding));
+            break;
+
+        case 'L':  // short level
+            formatters_.push_back(
+                details::make_unique<details::short_level_formatter<Padder>>(padding));
+            break;
+
+        case ('t'):  // thread id
+            formatters_.push_back(details::make_unique<details::t_formatter<Padder>>(padding));
+            break;
+
+        case ('v'):  // the message text
+            formatters_.push_back(details::make_unique<details::v_formatter<Padder>>(padding));
+            break;
+
+        case ('a'):  // weekday
+            formatters_.push_back(details::make_unique<details::a_formatter<Padder>>(padding));
+            need_localtime_ = true;
+            break;
+
+        case ('A'):  // short weekday
+            formatters_.push_back(details::make_unique<details::A_formatter<Padder>>(padding));
+            need_localtime_ = true;
+            break;
+
+        case ('b'):
+        case ('h'):  // month
+            formatters_.push_back(details::make_unique<details::b_formatter<Padder>>(padding));
+            need_localtime_ = true;
+            break;
+
+        case ('B'):  // short month
+            formatters_.push_back(details::make_unique<details::B_formatter<Padder>>(padding));
+            need_localtime_ = true;
+            break;
+
+        case ('c'):  // datetime
+            formatters_.push_back(details::make_unique<details::c_formatter<Padder>>(padding));
+            need_localtime_ = true;
+            break;
+
+        case ('C'):  // year 2 digits
+            formatters_.push_back(details::make_unique<details::C_formatter<Padder>>(padding));
+            need_localtime_ = true;
+            break;
+
+        case ('Y'):  // year 4 digits
+            formatters_.push_back(details::make_unique<details::Y_formatter<Padder>>(padding));
+            need_localtime_ = true;
+            break;
+
+        case ('D'):
+        case ('x'):  // datetime MM/DD/YY
+            formatters_.push_back(details::make_unique<details::D_formatter<Padder>>(padding));
+            need_localtime_ = true;
+            break;
+
+        case ('m'):  // month 1-12
+            formatters_.push_back(details::make_unique<details::m_formatter<Padder>>(padding));
+            need_localtime_ = true;
+            break;
+
+        case ('d'):  // day of month 1-31
+            formatters_.push_back(details::make_unique<details::d_formatter<Padder>>(padding));
+            need_localtime_ = true;
+            break;
+
+        case ('H'):  // hours 24
+            formatters_.push_back(details::make_unique<details::H_formatter<Padder>>(padding));
+            need_localtime_ = true;
+            break;
+
+        case ('I'):  // hours 12
+            formatters_.push_back(details::make_unique<details::I_formatter<Padder>>(padding));
+            need_localtime_ = true;
+            break;
+
+        case ('M'):  // minutes
+            formatters_.push_back(details::make_unique<details::M_formatter<Padder>>(padding));
+            need_localtime_ = true;
+            break;
+
+        case ('S'):  // seconds
+            formatters_.push_back(details::make_unique<details::S_formatter<Padder>>(padding));
+            need_localtime_ = true;
+            break;
+
+        case ('e'):  // milliseconds
+            formatters_.push_back(details::make_unique<details::e_formatter<Padder>>(padding));
+            break;
+
+        case ('f'):  // microseconds
+            formatters_.push_back(details::make_unique<details::f_formatter<Padder>>(padding));
+            break;
+
+        case ('F'):  // nanoseconds
+            formatters_.push_back(details::make_unique<details::F_formatter<Padder>>(padding));
+            break;
+
+        case ('E'):  // seconds since epoch
+            formatters_.push_back(details::make_unique<details::E_formatter<Padder>>(padding));
+            break;
+
+        case ('p'):  // am/pm
+            formatters_.push_back(details::make_unique<details::p_formatter<Padder>>(padding));
+            need_localtime_ = true;
+            break;
+
+        case ('r'):  // 12 hour clock 02:55:02 pm
+            formatters_.push_back(details::make_unique<details::r_formatter<Padder>>(padding));
+            need_localtime_ = true;
+            break;
+
+        case ('R'):  // 24-hour HH:MM time
+            formatters_.push_back(details::make_unique<details::R_formatter<Padder>>(padding));
+            need_localtime_ = true;
+            break;
+
+        case ('T'):
+        case ('X'):  // ISO 8601 time format (HH:MM:SS)
+            formatters_.push_back(details::make_unique<details::T_formatter<Padder>>(padding));
+            need_localtime_ = true;
+            break;
+
+        case ('z'):  // timezone
+            formatters_.push_back(details::make_unique<details::z_formatter<Padder>>(padding));
+            need_localtime_ = true;
+            break;
+
+        case ('P'):  // pid
+            formatters_.push_back(details::make_unique<details::pid_formatter<Padder>>(padding));
+            break;
+
+        case ('^'):  // color range start
+            formatters_.push_back(details::make_unique<details::color_start_formatter>(padding));
+            break;
+
+        case ('$'):  // color range end
+            formatters_.push_back(details::make_unique<details::color_stop_formatter>(padding));
+            break;
+
+        case ('@'):  // source location (filename:filenumber)
+            formatters_.push_back(
+                details::make_unique<details::source_location_formatter<Padder>>(padding));
+            break;
+
+        case ('s'):  // short source filename - without directory name
+            formatters_.push_back(
+                details::make_unique<details::short_filename_formatter<Padder>>(padding));
+            break;
+
+        case ('g'):  // full source filename
+            formatters_.push_back(
+                details::make_unique<details::source_filename_formatter<Padder>>(padding));
+            break;
+
+        case ('#'):  // source line number
+            formatters_.push_back(
+                details::make_unique<details::source_linenum_formatter<Padder>>(padding));
+            break;
+
+        case ('!'):  // source funcname
+            formatters_.push_back(
+                details::make_unique<details::source_funcname_formatter<Padder>>(padding));
+            break;
+
+        case ('%'):  // % char
+            formatters_.push_back(details::make_unique<details::ch_formatter>('%'));
+            break;
+
+        case ('u'):  // elapsed time since last log message in nanos
+            formatters_.push_back(
+                details::make_unique<details::elapsed_formatter<Padder, std::chrono::nanoseconds>>(
+                    padding));
+            break;
+
+        case ('i'):  // elapsed time since last log message in micros
+            formatters_.push_back(
+                details::make_unique<details::elapsed_formatter<Padder, std::chrono::microseconds>>(
+                    padding));
+            break;
+
+        case ('o'):  // elapsed time since last log message in millis
+            formatters_.push_back(
+                details::make_unique<details::elapsed_formatter<Padder, std::chrono::milliseconds>>(
+                    padding));
+            break;
+
+        case ('O'):  // elapsed time since last log message in seconds
+            formatters_.push_back(
+                details::make_unique<details::elapsed_formatter<Padder, std::chrono::seconds>>(
+                    padding));
+            break;
+
+#ifndef SPDLOG_NO_TLS  // mdc formatter requires TLS support
+        case ('&'):
+            formatters_.push_back(details::make_unique<details::mdc_formatter<Padder>>(padding));
+            break;
+#endif
+
+        default:  // Unknown flag appears as is
+            auto unknown_flag = details::make_unique<details::aggregate_formatter>();
+
+            if (!padding.truncate_) {
+                unknown_flag->add_ch('%');
+                unknown_flag->add_ch(flag);
+                formatters_.push_back((std::move(unknown_flag)));
+            }
+            // fix issue #1617 (prev char was '!' and should have been treated as funcname flag
+            // instead of truncating flag) spdlog::set_pattern("[%10!] %v") => "[      main] some
+            // message" spdlog::set_pattern("[%3!!] %v") => "[mai] some message"
+            else {
+                padding.truncate_ = false;
+                formatters_.push_back(
+                    details::make_unique<details::source_funcname_formatter<Padder>>(padding));
+                unknown_flag->add_ch(flag);
+                formatters_.push_back((std::move(unknown_flag)));
+            }
+
+            break;
+    }
+}
+
+// Extract given pad spec (e.g. %8X, %=8X, %-8!X, %8!X, %=8!X, %-8!X, %+8!X)
+// Advance the given it pass the end of the padding spec found (if any)
+// Return padding.
+SPDLOG_INLINE details::padding_info pattern_formatter::handle_padspec_(
+    std::string::const_iterator &it, std::string::const_iterator end) {
+    using details::padding_info;
+    using details::scoped_padder;
+    const size_t max_width = 64;
+    if (it == end) {
+        return padding_info{};
+    }
+
+    padding_info::pad_side side;
+    switch (*it) {
+        case '-':
+            side = padding_info::pad_side::right;
+            ++it;
+            break;
+        case '=':
+            side = padding_info::pad_side::center;
+            ++it;
+            break;
+        default:
+            side = details::padding_info::pad_side::left;
+            break;
+    }
+
+    if (it == end || !std::isdigit(static_cast<unsigned char>(*it))) {
+        return padding_info{};  // no padding if no digit found here
+    }
+
+    auto width = static_cast<size_t>(*it) - '0';
+    for (++it; it != end && std::isdigit(static_cast<unsigned char>(*it)); ++it) {
+        auto digit = static_cast<size_t>(*it) - '0';
+        width = width * 10 + digit;
+    }
+
+    // search for the optional truncate marker '!'
+    bool truncate;
+    if (it != end && *it == '!') {
+        truncate = true;
+        ++it;
+    } else {
+        truncate = false;
+    }
+    return details::padding_info{std::min<size_t>(width, max_width), side, truncate};
+}
+
+SPDLOG_INLINE void pattern_formatter::compile_pattern_(const std::string &pattern) {
+    auto end = pattern.end();
+    std::unique_ptr<details::aggregate_formatter> user_chars;
+    formatters_.clear();
+    for (auto it = pattern.begin(); it != end; ++it) {
+        if (*it == '%') {
+            if (user_chars)  // append user chars found so far
+            {
+                formatters_.push_back(std::move(user_chars));
+            }
+
+            auto padding = handle_padspec_(++it, end);
+
+            if (it != end) {
+                if (padding.enabled()) {
+                    handle_flag_<details::scoped_padder>(*it, padding);
+                } else {
+                    handle_flag_<details::null_scoped_padder>(*it, padding);
+                }
+            } else {
+                break;
+            }
+        } else  // chars not following the % sign should be displayed as is
+        {
+            if (!user_chars) {
+                user_chars = details::make_unique<details::aggregate_formatter>();
+            }
+            user_chars->add_ch(*it);
+        }
+    }
+    if (user_chars)  // append raw chars found so far
+    {
+        formatters_.push_back(std::move(user_chars));
+    }
+}
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/pattern_formatter.h b/ext/spdlog/include/spdlog/pattern_formatter.h
new file mode 100644
index 0000000..ececd67
--- /dev/null
+++ b/ext/spdlog/include/spdlog/pattern_formatter.h
@@ -0,0 +1,118 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#include <spdlog/common.h>
+#include <spdlog/details/log_msg.h>
+#include <spdlog/details/os.h>
+#include <spdlog/formatter.h>
+
+#include <chrono>
+#include <ctime>
+#include <memory>
+
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+namespace spdlog {
+namespace details {
+
+// padding information.
+struct padding_info {
+    enum class pad_side { left, right, center };
+
+    padding_info() = default;
+    padding_info(size_t width, padding_info::pad_side side, bool truncate)
+        : width_(width),
+          side_(side),
+          truncate_(truncate),
+          enabled_(true) {}
+
+    bool enabled() const { return enabled_; }
+    size_t width_ = 0;
+    pad_side side_ = pad_side::left;
+    bool truncate_ = false;
+    bool enabled_ = false;
+};
+
+class SPDLOG_API flag_formatter {
+public:
+    explicit flag_formatter(padding_info padinfo)
+        : padinfo_(padinfo) {}
+    flag_formatter() = default;
+    virtual ~flag_formatter() = default;
+    virtual void format(const details::log_msg &msg,
+                        const std::tm &tm_time,
+                        memory_buf_t &dest) = 0;
+
+protected:
+    padding_info padinfo_;
+};
+
+}  // namespace details
+
+class SPDLOG_API custom_flag_formatter : public details::flag_formatter {
+public:
+    virtual std::unique_ptr<custom_flag_formatter> clone() const = 0;
+
+    void set_padding_info(const details::padding_info &padding) {
+        flag_formatter::padinfo_ = padding;
+    }
+};
+
+class SPDLOG_API pattern_formatter final : public formatter {
+public:
+    using custom_flags = std::unordered_map<char, std::unique_ptr<custom_flag_formatter>>;
+
+    explicit pattern_formatter(std::string pattern,
+                               pattern_time_type time_type = pattern_time_type::local,
+                               std::string eol = spdlog::details::os::default_eol,
+                               custom_flags custom_user_flags = custom_flags());
+
+    // use default pattern is not given
+    explicit pattern_formatter(pattern_time_type time_type = pattern_time_type::local,
+                               std::string eol = spdlog::details::os::default_eol);
+
+    pattern_formatter(const pattern_formatter &other) = delete;
+    pattern_formatter &operator=(const pattern_formatter &other) = delete;
+
+    std::unique_ptr<formatter> clone() const override;
+    void format(const details::log_msg &msg, memory_buf_t &dest) override;
+
+    template <typename T, typename... Args>
+    pattern_formatter &add_flag(char flag, Args &&...args) {
+        custom_handlers_[flag] = details::make_unique<T>(std::forward<Args>(args)...);
+        return *this;
+    }
+    void set_pattern(std::string pattern);
+    void need_localtime(bool need = true);
+
+private:
+    std::string pattern_;
+    std::string eol_;
+    pattern_time_type pattern_time_type_;
+    bool need_localtime_;
+    std::tm cached_tm_;
+    std::chrono::seconds last_log_secs_;
+    std::vector<std::unique_ptr<details::flag_formatter>> formatters_;
+    custom_flags custom_handlers_;
+
+    std::tm get_time_(const details::log_msg &msg);
+    template <typename Padder>
+    void handle_flag_(char flag, details::padding_info padding);
+
+    // Extract given pad spec (e.g. %8X)
+    // Advance the given it pass the end of the padding spec found (if any)
+    // Return padding.
+    static details::padding_info handle_padspec_(std::string::const_iterator &it,
+                                                 std::string::const_iterator end);
+
+    void compile_pattern_(const std::string &pattern);
+};
+}  // namespace spdlog
+
+#ifdef SPDLOG_HEADER_ONLY
+    #include "pattern_formatter-inl.h"
+#endif
diff --git a/ext/spdlog/include/spdlog/sinks/android_sink.h b/ext/spdlog/include/spdlog/sinks/android_sink.h
new file mode 100644
index 0000000..4435a56
--- /dev/null
+++ b/ext/spdlog/include/spdlog/sinks/android_sink.h
@@ -0,0 +1,137 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#ifdef __ANDROID__
+
+    #include <spdlog/details/fmt_helper.h>
+    #include <spdlog/details/null_mutex.h>
+    #include <spdlog/details/os.h>
+    #include <spdlog/details/synchronous_factory.h>
+    #include <spdlog/sinks/base_sink.h>
+
+    #include <android/log.h>
+    #include <chrono>
+    #include <mutex>
+    #include <string>
+    #include <thread>
+    #include <type_traits>
+
+    #if !defined(SPDLOG_ANDROID_RETRIES)
+        #define SPDLOG_ANDROID_RETRIES 2
+    #endif
+
+namespace spdlog {
+namespace sinks {
+
+/*
+ * Android sink
+ * (logging using __android_log_write or __android_log_buf_write depending on the specified
+ * BufferID)
+ */
+template <typename Mutex, int BufferID = log_id::LOG_ID_MAIN>
+class android_sink final : public base_sink<Mutex> {
+public:
+    explicit android_sink(std::string tag = "spdlog", bool use_raw_msg = false)
+        : tag_(std::move(tag)),
+          use_raw_msg_(use_raw_msg) {}
+
+protected:
+    void sink_it_(const details::log_msg &msg) override {
+        const android_LogPriority priority = convert_to_android_(msg.level);
+        memory_buf_t formatted;
+        if (use_raw_msg_) {
+            details::fmt_helper::append_string_view(msg.payload, formatted);
+        } else {
+            base_sink<Mutex>::formatter_->format(msg, formatted);
+        }
+        formatted.push_back('\0');
+        const char *msg_output = formatted.data();
+
+        // See system/core/liblog/logger_write.c for explanation of return value
+        int ret = android_log(priority, tag_.c_str(), msg_output);
+        if (ret == -EPERM) {
+            return;  // !__android_log_is_loggable
+        }
+        int retry_count = 0;
+        while ((ret == -11 /*EAGAIN*/) && (retry_count < SPDLOG_ANDROID_RETRIES)) {
+            details::os::sleep_for_millis(5);
+            ret = android_log(priority, tag_.c_str(), msg_output);
+            retry_count++;
+        }
+
+        if (ret < 0) {
+            throw_spdlog_ex("logging to Android failed", ret);
+        }
+    }
+
+    void flush_() override {}
+
+private:
+    // There might be liblog versions used, that do not support __android_log_buf_write. So we only
+    // compile and link against
+    // __android_log_buf_write, if user explicitly provides a non-default log buffer. Otherwise,
+    // when using the default log buffer, always log via __android_log_write.
+    template <int ID = BufferID>
+    typename std::enable_if<ID == static_cast<int>(log_id::LOG_ID_MAIN), int>::type android_log(
+        int prio, const char *tag, const char *text) {
+        return __android_log_write(prio, tag, text);
+    }
+
+    template <int ID = BufferID>
+    typename std::enable_if<ID != static_cast<int>(log_id::LOG_ID_MAIN), int>::type android_log(
+        int prio, const char *tag, const char *text) {
+        return __android_log_buf_write(ID, prio, tag, text);
+    }
+
+    static android_LogPriority convert_to_android_(spdlog::level::level_enum level) {
+        switch (level) {
+            case spdlog::level::trace:
+                return ANDROID_LOG_VERBOSE;
+            case spdlog::level::debug:
+                return ANDROID_LOG_DEBUG;
+            case spdlog::level::info:
+                return ANDROID_LOG_INFO;
+            case spdlog::level::warn:
+                return ANDROID_LOG_WARN;
+            case spdlog::level::err:
+                return ANDROID_LOG_ERROR;
+            case spdlog::level::critical:
+                return ANDROID_LOG_FATAL;
+            default:
+                return ANDROID_LOG_DEFAULT;
+        }
+    }
+
+    std::string tag_;
+    bool use_raw_msg_;
+};
+
+using android_sink_mt = android_sink<std::mutex>;
+using android_sink_st = android_sink<details::null_mutex>;
+
+template <int BufferId = log_id::LOG_ID_MAIN>
+using android_sink_buf_mt = android_sink<std::mutex, BufferId>;
+template <int BufferId = log_id::LOG_ID_MAIN>
+using android_sink_buf_st = android_sink<details::null_mutex, BufferId>;
+
+}  // namespace sinks
+
+// Create and register android syslog logger
+
+template <typename Factory = spdlog::synchronous_factory>
+inline std::shared_ptr<logger> android_logger_mt(const std::string &logger_name,
+                                                 const std::string &tag = "spdlog") {
+    return Factory::template create<sinks::android_sink_mt>(logger_name, tag);
+}
+
+template <typename Factory = spdlog::synchronous_factory>
+inline std::shared_ptr<logger> android_logger_st(const std::string &logger_name,
+                                                 const std::string &tag = "spdlog") {
+    return Factory::template create<sinks::android_sink_st>(logger_name, tag);
+}
+
+}  // namespace spdlog
+
+#endif  // __ANDROID__
diff --git a/ext/spdlog/include/spdlog/sinks/ansicolor_sink-inl.h b/ext/spdlog/include/spdlog/sinks/ansicolor_sink-inl.h
new file mode 100644
index 0000000..8fd3078
--- /dev/null
+++ b/ext/spdlog/include/spdlog/sinks/ansicolor_sink-inl.h
@@ -0,0 +1,135 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#ifndef SPDLOG_HEADER_ONLY
+    #include <spdlog/sinks/ansicolor_sink.h>
+#endif
+
+#include <spdlog/details/os.h>
+#include <spdlog/pattern_formatter.h>
+
+namespace spdlog {
+namespace sinks {
+
+template <typename ConsoleMutex>
+SPDLOG_INLINE ansicolor_sink<ConsoleMutex>::ansicolor_sink(FILE *target_file, color_mode mode)
+    : target_file_(target_file),
+      mutex_(ConsoleMutex::mutex()),
+      formatter_(details::make_unique<spdlog::pattern_formatter>())
+
+{
+    set_color_mode(mode);
+    colors_.at(level::trace) = to_string_(white);
+    colors_.at(level::debug) = to_string_(cyan);
+    colors_.at(level::info) = to_string_(green);
+    colors_.at(level::warn) = to_string_(yellow_bold);
+    colors_.at(level::err) = to_string_(red_bold);
+    colors_.at(level::critical) = to_string_(bold_on_red);
+    colors_.at(level::off) = to_string_(reset);
+}
+
+template <typename ConsoleMutex>
+SPDLOG_INLINE void ansicolor_sink<ConsoleMutex>::set_color(level::level_enum color_level,
+                                                           string_view_t color) {
+    std::lock_guard<mutex_t> lock(mutex_);
+    colors_.at(static_cast<size_t>(color_level)) = to_string_(color);
+}
+
+template <typename ConsoleMutex>
+SPDLOG_INLINE void ansicolor_sink<ConsoleMutex>::log(const details::log_msg &msg) {
+    // Wrap the originally formatted message in color codes.
+    // If color is not supported in the terminal, log as is instead.
+    std::lock_guard<mutex_t> lock(mutex_);
+    msg.color_range_start = 0;
+    msg.color_range_end = 0;
+    memory_buf_t formatted;
+    formatter_->format(msg, formatted);
+    if (should_do_colors_ && msg.color_range_end > msg.color_range_start) {
+        // before color range
+        print_range_(formatted, 0, msg.color_range_start);
+        // in color range
+        print_ccode_(colors_.at(static_cast<size_t>(msg.level)));
+        print_range_(formatted, msg.color_range_start, msg.color_range_end);
+        print_ccode_(reset);
+        // after color range
+        print_range_(formatted, msg.color_range_end, formatted.size());
+    } else  // no color
+    {
+        print_range_(formatted, 0, formatted.size());
+    }
+    fflush(target_file_);
+}
+
+template <typename ConsoleMutex>
+SPDLOG_INLINE void ansicolor_sink<ConsoleMutex>::flush() {
+    std::lock_guard<mutex_t> lock(mutex_);
+    fflush(target_file_);
+}
+
+template <typename ConsoleMutex>
+SPDLOG_INLINE void ansicolor_sink<ConsoleMutex>::set_pattern(const std::string &pattern) {
+    std::lock_guard<mutex_t> lock(mutex_);
+    formatter_ = std::unique_ptr<spdlog::formatter>(new pattern_formatter(pattern));
+}
+
+template <typename ConsoleMutex>
+SPDLOG_INLINE void ansicolor_sink<ConsoleMutex>::set_formatter(
+    std::unique_ptr<spdlog::formatter> sink_formatter) {
+    std::lock_guard<mutex_t> lock(mutex_);
+    formatter_ = std::move(sink_formatter);
+}
+
+template <typename ConsoleMutex>
+SPDLOG_INLINE bool ansicolor_sink<ConsoleMutex>::should_color() {
+    return should_do_colors_;
+}
+
+template <typename ConsoleMutex>
+SPDLOG_INLINE void ansicolor_sink<ConsoleMutex>::set_color_mode(color_mode mode) {
+    switch (mode) {
+        case color_mode::always:
+            should_do_colors_ = true;
+            return;
+        case color_mode::automatic:
+            should_do_colors_ =
+                details::os::in_terminal(target_file_) && details::os::is_color_terminal();
+            return;
+        case color_mode::never:
+            should_do_colors_ = false;
+            return;
+        default:
+            should_do_colors_ = false;
+    }
+}
+
+template <typename ConsoleMutex>
+SPDLOG_INLINE void ansicolor_sink<ConsoleMutex>::print_ccode_(const string_view_t &color_code) {
+    details::os::fwrite_bytes(color_code.data(), color_code.size(), target_file_);
+}
+
+template <typename ConsoleMutex>
+SPDLOG_INLINE void ansicolor_sink<ConsoleMutex>::print_range_(const memory_buf_t &formatted,
+                                                              size_t start,
+                                                              size_t end) {
+    details::os::fwrite_bytes(formatted.data() + start, end - start, target_file_);
+}
+
+template <typename ConsoleMutex>
+SPDLOG_INLINE std::string ansicolor_sink<ConsoleMutex>::to_string_(const string_view_t &sv) {
+    return std::string(sv.data(), sv.size());
+}
+
+// ansicolor_stdout_sink
+template <typename ConsoleMutex>
+SPDLOG_INLINE ansicolor_stdout_sink<ConsoleMutex>::ansicolor_stdout_sink(color_mode mode)
+    : ansicolor_sink<ConsoleMutex>(stdout, mode) {}
+
+// ansicolor_stderr_sink
+template <typename ConsoleMutex>
+SPDLOG_INLINE ansicolor_stderr_sink<ConsoleMutex>::ansicolor_stderr_sink(color_mode mode)
+    : ansicolor_sink<ConsoleMutex>(stderr, mode) {}
+
+}  // namespace sinks
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/sinks/ansicolor_sink.h b/ext/spdlog/include/spdlog/sinks/ansicolor_sink.h
new file mode 100644
index 0000000..d0dadd7
--- /dev/null
+++ b/ext/spdlog/include/spdlog/sinks/ansicolor_sink.h
@@ -0,0 +1,115 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#include <array>
+#include <memory>
+#include <mutex>
+#include <spdlog/details/console_globals.h>
+#include <spdlog/details/null_mutex.h>
+#include <spdlog/sinks/sink.h>
+#include <string>
+
+namespace spdlog {
+namespace sinks {
+
+/**
+ * This sink prefixes the output with an ANSI escape sequence color code
+ * depending on the severity
+ * of the message.
+ * If no color terminal detected, omit the escape codes.
+ */
+
+template <typename ConsoleMutex>
+class ansicolor_sink : public sink {
+public:
+    using mutex_t = typename ConsoleMutex::mutex_t;
+    ansicolor_sink(FILE *target_file, color_mode mode);
+    ~ansicolor_sink() override = default;
+
+    ansicolor_sink(const ansicolor_sink &other) = delete;
+    ansicolor_sink(ansicolor_sink &&other) = delete;
+
+    ansicolor_sink &operator=(const ansicolor_sink &other) = delete;
+    ansicolor_sink &operator=(ansicolor_sink &&other) = delete;
+
+    void set_color(level::level_enum color_level, string_view_t color);
+    void set_color_mode(color_mode mode);
+    bool should_color();
+
+    void log(const details::log_msg &msg) override;
+    void flush() override;
+    void set_pattern(const std::string &pattern) final override;
+    void set_formatter(std::unique_ptr<spdlog::formatter> sink_formatter) override;
+
+    // Formatting codes
+    const string_view_t reset = "\033[m";
+    const string_view_t bold = "\033[1m";
+    const string_view_t dark = "\033[2m";
+    const string_view_t underline = "\033[4m";
+    const string_view_t blink = "\033[5m";
+    const string_view_t reverse = "\033[7m";
+    const string_view_t concealed = "\033[8m";
+    const string_view_t clear_line = "\033[K";
+
+    // Foreground colors
+    const string_view_t black = "\033[30m";
+    const string_view_t red = "\033[31m";
+    const string_view_t green = "\033[32m";
+    const string_view_t yellow = "\033[33m";
+    const string_view_t blue = "\033[34m";
+    const string_view_t magenta = "\033[35m";
+    const string_view_t cyan = "\033[36m";
+    const string_view_t white = "\033[37m";
+
+    /// Background colors
+    const string_view_t on_black = "\033[40m";
+    const string_view_t on_red = "\033[41m";
+    const string_view_t on_green = "\033[42m";
+    const string_view_t on_yellow = "\033[43m";
+    const string_view_t on_blue = "\033[44m";
+    const string_view_t on_magenta = "\033[45m";
+    const string_view_t on_cyan = "\033[46m";
+    const string_view_t on_white = "\033[47m";
+
+    /// Bold colors
+    const string_view_t yellow_bold = "\033[33m\033[1m";
+    const string_view_t red_bold = "\033[31m\033[1m";
+    const string_view_t bold_on_red = "\033[1m\033[41m";
+
+private:
+    FILE *target_file_;
+    mutex_t &mutex_;
+    bool should_do_colors_;
+    std::unique_ptr<spdlog::formatter> formatter_;
+    std::array<std::string, level::n_levels> colors_;
+    void print_ccode_(const string_view_t &color_code);
+    void print_range_(const memory_buf_t &formatted, size_t start, size_t end);
+    static std::string to_string_(const string_view_t &sv);
+};
+
+template <typename ConsoleMutex>
+class ansicolor_stdout_sink : public ansicolor_sink<ConsoleMutex> {
+public:
+    explicit ansicolor_stdout_sink(color_mode mode = color_mode::automatic);
+};
+
+template <typename ConsoleMutex>
+class ansicolor_stderr_sink : public ansicolor_sink<ConsoleMutex> {
+public:
+    explicit ansicolor_stderr_sink(color_mode mode = color_mode::automatic);
+};
+
+using ansicolor_stdout_sink_mt = ansicolor_stdout_sink<details::console_mutex>;
+using ansicolor_stdout_sink_st = ansicolor_stdout_sink<details::console_nullmutex>;
+
+using ansicolor_stderr_sink_mt = ansicolor_stderr_sink<details::console_mutex>;
+using ansicolor_stderr_sink_st = ansicolor_stderr_sink<details::console_nullmutex>;
+
+}  // namespace sinks
+}  // namespace spdlog
+
+#ifdef SPDLOG_HEADER_ONLY
+    #include "ansicolor_sink-inl.h"
+#endif
diff --git a/ext/spdlog/include/spdlog/sinks/base_sink-inl.h b/ext/spdlog/include/spdlog/sinks/base_sink-inl.h
new file mode 100644
index 0000000..ada161b
--- /dev/null
+++ b/ext/spdlog/include/spdlog/sinks/base_sink-inl.h
@@ -0,0 +1,59 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#ifndef SPDLOG_HEADER_ONLY
+    #include <spdlog/sinks/base_sink.h>
+#endif
+
+#include <spdlog/common.h>
+#include <spdlog/pattern_formatter.h>
+
+#include <memory>
+#include <mutex>
+
+template <typename Mutex>
+SPDLOG_INLINE spdlog::sinks::base_sink<Mutex>::base_sink()
+    : formatter_{details::make_unique<spdlog::pattern_formatter>()} {}
+
+template <typename Mutex>
+SPDLOG_INLINE spdlog::sinks::base_sink<Mutex>::base_sink(
+    std::unique_ptr<spdlog::formatter> formatter)
+    : formatter_{std::move(formatter)} {}
+
+template <typename Mutex>
+void SPDLOG_INLINE spdlog::sinks::base_sink<Mutex>::log(const details::log_msg &msg) {
+    std::lock_guard<Mutex> lock(mutex_);
+    sink_it_(msg);
+}
+
+template <typename Mutex>
+void SPDLOG_INLINE spdlog::sinks::base_sink<Mutex>::flush() {
+    std::lock_guard<Mutex> lock(mutex_);
+    flush_();
+}
+
+template <typename Mutex>
+void SPDLOG_INLINE spdlog::sinks::base_sink<Mutex>::set_pattern(const std::string &pattern) {
+    std::lock_guard<Mutex> lock(mutex_);
+    set_pattern_(pattern);
+}
+
+template <typename Mutex>
+void SPDLOG_INLINE
+spdlog::sinks::base_sink<Mutex>::set_formatter(std::unique_ptr<spdlog::formatter> sink_formatter) {
+    std::lock_guard<Mutex> lock(mutex_);
+    set_formatter_(std::move(sink_formatter));
+}
+
+template <typename Mutex>
+void SPDLOG_INLINE spdlog::sinks::base_sink<Mutex>::set_pattern_(const std::string &pattern) {
+    set_formatter_(details::make_unique<spdlog::pattern_formatter>(pattern));
+}
+
+template <typename Mutex>
+void SPDLOG_INLINE
+spdlog::sinks::base_sink<Mutex>::set_formatter_(std::unique_ptr<spdlog::formatter> sink_formatter) {
+    formatter_ = std::move(sink_formatter);
+}
diff --git a/ext/spdlog/include/spdlog/sinks/base_sink.h b/ext/spdlog/include/spdlog/sinks/base_sink.h
new file mode 100644
index 0000000..1b4bb06
--- /dev/null
+++ b/ext/spdlog/include/spdlog/sinks/base_sink.h
@@ -0,0 +1,51 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+//
+// base sink templated over a mutex (either dummy or real)
+// concrete implementation should override the sink_it_() and flush_()  methods.
+// locking is taken care of in this class - no locking needed by the
+// implementers..
+//
+
+#include <spdlog/common.h>
+#include <spdlog/details/log_msg.h>
+#include <spdlog/sinks/sink.h>
+
+namespace spdlog {
+namespace sinks {
+template <typename Mutex>
+class SPDLOG_API base_sink : public sink {
+public:
+    base_sink();
+    explicit base_sink(std::unique_ptr<spdlog::formatter> formatter);
+    ~base_sink() override = default;
+
+    base_sink(const base_sink &) = delete;
+    base_sink(base_sink &&) = delete;
+
+    base_sink &operator=(const base_sink &) = delete;
+    base_sink &operator=(base_sink &&) = delete;
+
+    void log(const details::log_msg &msg) final override;
+    void flush() final override;
+    void set_pattern(const std::string &pattern) final override;
+    void set_formatter(std::unique_ptr<spdlog::formatter> sink_formatter) final override;
+
+protected:
+    // sink formatter
+    std::unique_ptr<spdlog::formatter> formatter_;
+    Mutex mutex_;
+
+    virtual void sink_it_(const details::log_msg &msg) = 0;
+    virtual void flush_() = 0;
+    virtual void set_pattern_(const std::string &pattern);
+    virtual void set_formatter_(std::unique_ptr<spdlog::formatter> sink_formatter);
+};
+}  // namespace sinks
+}  // namespace spdlog
+
+#ifdef SPDLOG_HEADER_ONLY
+    #include "base_sink-inl.h"
+#endif
diff --git a/ext/spdlog/include/spdlog/sinks/basic_file_sink-inl.h b/ext/spdlog/include/spdlog/sinks/basic_file_sink-inl.h
new file mode 100644
index 0000000..ce0ddad
--- /dev/null
+++ b/ext/spdlog/include/spdlog/sinks/basic_file_sink-inl.h
@@ -0,0 +1,48 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#ifndef SPDLOG_HEADER_ONLY
+    #include <spdlog/sinks/basic_file_sink.h>
+#endif
+
+#include <spdlog/common.h>
+#include <spdlog/details/os.h>
+
+namespace spdlog {
+namespace sinks {
+
+template <typename Mutex>
+SPDLOG_INLINE basic_file_sink<Mutex>::basic_file_sink(const filename_t &filename,
+                                                      bool truncate,
+                                                      const file_event_handlers &event_handlers)
+    : file_helper_{event_handlers} {
+    file_helper_.open(filename, truncate);
+}
+
+template <typename Mutex>
+SPDLOG_INLINE const filename_t &basic_file_sink<Mutex>::filename() const {
+    return file_helper_.filename();
+}
+
+template <typename Mutex>
+SPDLOG_INLINE void basic_file_sink<Mutex>::truncate() {
+    std::lock_guard<Mutex> lock(base_sink<Mutex>::mutex_);
+    file_helper_.reopen(true);
+}
+
+template <typename Mutex>
+SPDLOG_INLINE void basic_file_sink<Mutex>::sink_it_(const details::log_msg &msg) {
+    memory_buf_t formatted;
+    base_sink<Mutex>::formatter_->format(msg, formatted);
+    file_helper_.write(formatted);
+}
+
+template <typename Mutex>
+SPDLOG_INLINE void basic_file_sink<Mutex>::flush_() {
+    file_helper_.flush();
+}
+
+}  // namespace sinks
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/sinks/basic_file_sink.h b/ext/spdlog/include/spdlog/sinks/basic_file_sink.h
new file mode 100644
index 0000000..48c0767
--- /dev/null
+++ b/ext/spdlog/include/spdlog/sinks/basic_file_sink.h
@@ -0,0 +1,66 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#include <spdlog/details/file_helper.h>
+#include <spdlog/details/null_mutex.h>
+#include <spdlog/details/synchronous_factory.h>
+#include <spdlog/sinks/base_sink.h>
+
+#include <mutex>
+#include <string>
+
+namespace spdlog {
+namespace sinks {
+/*
+ * Trivial file sink with single file as target
+ */
+template <typename Mutex>
+class basic_file_sink final : public base_sink<Mutex> {
+public:
+    explicit basic_file_sink(const filename_t &filename,
+                             bool truncate = false,
+                             const file_event_handlers &event_handlers = {});
+    const filename_t &filename() const;
+    void truncate();
+
+protected:
+    void sink_it_(const details::log_msg &msg) override;
+    void flush_() override;
+
+private:
+    details::file_helper file_helper_;
+};
+
+using basic_file_sink_mt = basic_file_sink<std::mutex>;
+using basic_file_sink_st = basic_file_sink<details::null_mutex>;
+
+}  // namespace sinks
+
+//
+// factory functions
+//
+template <typename Factory = spdlog::synchronous_factory>
+inline std::shared_ptr<logger> basic_logger_mt(const std::string &logger_name,
+                                               const filename_t &filename,
+                                               bool truncate = false,
+                                               const file_event_handlers &event_handlers = {}) {
+    return Factory::template create<sinks::basic_file_sink_mt>(logger_name, filename, truncate,
+                                                               event_handlers);
+}
+
+template <typename Factory = spdlog::synchronous_factory>
+inline std::shared_ptr<logger> basic_logger_st(const std::string &logger_name,
+                                               const filename_t &filename,
+                                               bool truncate = false,
+                                               const file_event_handlers &event_handlers = {}) {
+    return Factory::template create<sinks::basic_file_sink_st>(logger_name, filename, truncate,
+                                                               event_handlers);
+}
+
+}  // namespace spdlog
+
+#ifdef SPDLOG_HEADER_ONLY
+    #include "basic_file_sink-inl.h"
+#endif
diff --git a/ext/spdlog/include/spdlog/sinks/callback_sink.h b/ext/spdlog/include/spdlog/sinks/callback_sink.h
new file mode 100644
index 0000000..5f8b6bc
--- /dev/null
+++ b/ext/spdlog/include/spdlog/sinks/callback_sink.h
@@ -0,0 +1,56 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#include <spdlog/details/null_mutex.h>
+#include <spdlog/details/synchronous_factory.h>
+#include <spdlog/sinks/base_sink.h>
+
+#include <mutex>
+#include <string>
+
+namespace spdlog {
+
+// callbacks type
+typedef std::function<void(const details::log_msg &msg)> custom_log_callback;
+
+namespace sinks {
+/*
+ * Trivial callback sink, gets a callback function and calls it on each log
+ */
+template <typename Mutex>
+class callback_sink final : public base_sink<Mutex> {
+public:
+    explicit callback_sink(const custom_log_callback &callback)
+        : callback_{callback} {}
+
+protected:
+    void sink_it_(const details::log_msg &msg) override { callback_(msg); }
+    void flush_() override{}
+
+private:
+    custom_log_callback callback_;
+};
+
+using callback_sink_mt = callback_sink<std::mutex>;
+using callback_sink_st = callback_sink<details::null_mutex>;
+
+}  // namespace sinks
+
+//
+// factory functions
+//
+template <typename Factory = spdlog::synchronous_factory>
+inline std::shared_ptr<logger> callback_logger_mt(const std::string &logger_name,
+                                                  const custom_log_callback &callback) {
+    return Factory::template create<sinks::callback_sink_mt>(logger_name, callback);
+}
+
+template <typename Factory = spdlog::synchronous_factory>
+inline std::shared_ptr<logger> callback_logger_st(const std::string &logger_name,
+                                                  const custom_log_callback &callback) {
+    return Factory::template create<sinks::callback_sink_st>(logger_name, callback);
+}
+
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/sinks/daily_file_sink.h b/ext/spdlog/include/spdlog/sinks/daily_file_sink.h
new file mode 100644
index 0000000..615c9f7
--- /dev/null
+++ b/ext/spdlog/include/spdlog/sinks/daily_file_sink.h
@@ -0,0 +1,254 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#include <spdlog/common.h>
+#include <spdlog/details/circular_q.h>
+#include <spdlog/details/file_helper.h>
+#include <spdlog/details/null_mutex.h>
+#include <spdlog/details/os.h>
+#include <spdlog/details/synchronous_factory.h>
+#include <spdlog/fmt/chrono.h>
+#include <spdlog/fmt/fmt.h>
+#include <spdlog/sinks/base_sink.h>
+
+#include <chrono>
+#include <cstdio>
+#include <iomanip>
+#include <mutex>
+#include <sstream>
+#include <string>
+
+namespace spdlog {
+namespace sinks {
+
+/*
+ * Generator of daily log file names in format basename.YYYY-MM-DD.ext
+ */
+struct daily_filename_calculator {
+    // Create filename for the form basename.YYYY-MM-DD
+    static filename_t calc_filename(const filename_t &filename, const tm &now_tm) {
+        filename_t basename, ext;
+        std::tie(basename, ext) = details::file_helper::split_by_extension(filename);
+        return fmt_lib::format(SPDLOG_FMT_STRING(SPDLOG_FILENAME_T("{}_{:04d}-{:02d}-{:02d}{}")),
+                               basename, now_tm.tm_year + 1900, now_tm.tm_mon + 1, now_tm.tm_mday,
+                               ext);
+    }
+};
+
+/*
+ * Generator of daily log file names with strftime format.
+ * Usages:
+ *    auto sink =
+ * std::make_shared<spdlog::sinks::daily_file_format_sink_mt>("myapp-%Y-%m-%d:%H:%M:%S.log", hour,
+ * minute);" auto logger = spdlog::daily_logger_format_mt("loggername, "myapp-%Y-%m-%d:%X.log",
+ * hour,  minute)"
+ *
+ */
+struct daily_filename_format_calculator {
+    static filename_t calc_filename(const filename_t &file_path, const tm &now_tm) {
+#if defined(_WIN32) && defined(SPDLOG_WCHAR_FILENAMES)
+        std::wstringstream stream;
+#else
+        std::stringstream stream;
+#endif
+        stream << std::put_time(&now_tm, file_path.c_str());
+        return stream.str();
+    }
+};
+
+/*
+ * Rotating file sink based on date.
+ * If truncate != false , the created file will be truncated.
+ * If max_files > 0, retain only the last max_files and delete previous.
+ * Note that old log files from previous executions will not be deleted by this class,
+ * rotation and deletion is only applied while the program is running.
+ */
+template <typename Mutex, typename FileNameCalc = daily_filename_calculator>
+class daily_file_sink final : public base_sink<Mutex> {
+public:
+    // create daily file sink which rotates on given time
+    daily_file_sink(filename_t base_filename,
+                    int rotation_hour,
+                    int rotation_minute,
+                    bool truncate = false,
+                    uint16_t max_files = 0,
+                    const file_event_handlers &event_handlers = {})
+        : base_filename_(std::move(base_filename)),
+          rotation_h_(rotation_hour),
+          rotation_m_(rotation_minute),
+          file_helper_{event_handlers},
+          truncate_(truncate),
+          max_files_(max_files),
+          filenames_q_() {
+        if (rotation_hour < 0 || rotation_hour > 23 || rotation_minute < 0 ||
+            rotation_minute > 59) {
+            throw_spdlog_ex("daily_file_sink: Invalid rotation time in ctor");
+        }
+
+        auto now = log_clock::now();
+        auto filename = FileNameCalc::calc_filename(base_filename_, now_tm(now));
+        file_helper_.open(filename, truncate_);
+        rotation_tp_ = next_rotation_tp_();
+
+        if (max_files_ > 0) {
+            init_filenames_q_();
+        }
+    }
+
+    filename_t filename() {
+        std::lock_guard<Mutex> lock(base_sink<Mutex>::mutex_);
+        return file_helper_.filename();
+    }
+
+protected:
+    void sink_it_(const details::log_msg &msg) override {
+        auto time = msg.time;
+        bool should_rotate = time >= rotation_tp_;
+        if (should_rotate) {
+            auto filename = FileNameCalc::calc_filename(base_filename_, now_tm(time));
+            file_helper_.open(filename, truncate_);
+            rotation_tp_ = next_rotation_tp_();
+        }
+        memory_buf_t formatted;
+        base_sink<Mutex>::formatter_->format(msg, formatted);
+        file_helper_.write(formatted);
+
+        // Do the cleaning only at the end because it might throw on failure.
+        if (should_rotate && max_files_ > 0) {
+            delete_old_();
+        }
+    }
+
+    void flush_() override { file_helper_.flush(); }
+
+private:
+    void init_filenames_q_() {
+        using details::os::path_exists;
+
+        filenames_q_ = details::circular_q<filename_t>(static_cast<size_t>(max_files_));
+        std::vector<filename_t> filenames;
+        auto now = log_clock::now();
+        while (filenames.size() < max_files_) {
+            auto filename = FileNameCalc::calc_filename(base_filename_, now_tm(now));
+            if (!path_exists(filename)) {
+                break;
+            }
+            filenames.emplace_back(filename);
+            now -= std::chrono::hours(24);
+        }
+        for (auto iter = filenames.rbegin(); iter != filenames.rend(); ++iter) {
+            filenames_q_.push_back(std::move(*iter));
+        }
+    }
+
+    tm now_tm(log_clock::time_point tp) {
+        time_t tnow = log_clock::to_time_t(tp);
+        return spdlog::details::os::localtime(tnow);
+    }
+
+    log_clock::time_point next_rotation_tp_() {
+        auto now = log_clock::now();
+        tm date = now_tm(now);
+        date.tm_hour = rotation_h_;
+        date.tm_min = rotation_m_;
+        date.tm_sec = 0;
+        auto rotation_time = log_clock::from_time_t(std::mktime(&date));
+        if (rotation_time > now) {
+            return rotation_time;
+        }
+        return {rotation_time + std::chrono::hours(24)};
+    }
+
+    // Delete the file N rotations ago.
+    // Throw spdlog_ex on failure to delete the old file.
+    void delete_old_() {
+        using details::os::filename_to_str;
+        using details::os::remove_if_exists;
+
+        filename_t current_file = file_helper_.filename();
+        if (filenames_q_.full()) {
+            auto old_filename = std::move(filenames_q_.front());
+            filenames_q_.pop_front();
+            bool ok = remove_if_exists(old_filename) == 0;
+            if (!ok) {
+                filenames_q_.push_back(std::move(current_file));
+                throw_spdlog_ex("Failed removing daily file " + filename_to_str(old_filename),
+                                errno);
+            }
+        }
+        filenames_q_.push_back(std::move(current_file));
+    }
+
+    filename_t base_filename_;
+    int rotation_h_;
+    int rotation_m_;
+    log_clock::time_point rotation_tp_;
+    details::file_helper file_helper_;
+    bool truncate_;
+    uint16_t max_files_;
+    details::circular_q<filename_t> filenames_q_;
+};
+
+using daily_file_sink_mt = daily_file_sink<std::mutex>;
+using daily_file_sink_st = daily_file_sink<details::null_mutex>;
+using daily_file_format_sink_mt = daily_file_sink<std::mutex, daily_filename_format_calculator>;
+using daily_file_format_sink_st =
+    daily_file_sink<details::null_mutex, daily_filename_format_calculator>;
+
+}  // namespace sinks
+
+//
+// factory functions
+//
+template <typename Factory = spdlog::synchronous_factory>
+inline std::shared_ptr<logger> daily_logger_mt(const std::string &logger_name,
+                                               const filename_t &filename,
+                                               int hour = 0,
+                                               int minute = 0,
+                                               bool truncate = false,
+                                               uint16_t max_files = 0,
+                                               const file_event_handlers &event_handlers = {}) {
+    return Factory::template create<sinks::daily_file_sink_mt>(logger_name, filename, hour, minute,
+                                                               truncate, max_files, event_handlers);
+}
+
+template <typename Factory = spdlog::synchronous_factory>
+inline std::shared_ptr<logger> daily_logger_format_mt(
+    const std::string &logger_name,
+    const filename_t &filename,
+    int hour = 0,
+    int minute = 0,
+    bool truncate = false,
+    uint16_t max_files = 0,
+    const file_event_handlers &event_handlers = {}) {
+    return Factory::template create<sinks::daily_file_format_sink_mt>(
+        logger_name, filename, hour, minute, truncate, max_files, event_handlers);
+}
+
+template <typename Factory = spdlog::synchronous_factory>
+inline std::shared_ptr<logger> daily_logger_st(const std::string &logger_name,
+                                               const filename_t &filename,
+                                               int hour = 0,
+                                               int minute = 0,
+                                               bool truncate = false,
+                                               uint16_t max_files = 0,
+                                               const file_event_handlers &event_handlers = {}) {
+    return Factory::template create<sinks::daily_file_sink_st>(logger_name, filename, hour, minute,
+                                                               truncate, max_files, event_handlers);
+}
+
+template <typename Factory = spdlog::synchronous_factory>
+inline std::shared_ptr<logger> daily_logger_format_st(
+    const std::string &logger_name,
+    const filename_t &filename,
+    int hour = 0,
+    int minute = 0,
+    bool truncate = false,
+    uint16_t max_files = 0,
+    const file_event_handlers &event_handlers = {}) {
+    return Factory::template create<sinks::daily_file_format_sink_st>(
+        logger_name, filename, hour, minute, truncate, max_files, event_handlers);
+}
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/sinks/dist_sink.h b/ext/spdlog/include/spdlog/sinks/dist_sink.h
new file mode 100644
index 0000000..69c4971
--- /dev/null
+++ b/ext/spdlog/include/spdlog/sinks/dist_sink.h
@@ -0,0 +1,81 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#include "base_sink.h"
+#include <spdlog/details/log_msg.h>
+#include <spdlog/details/null_mutex.h>
+#include <spdlog/pattern_formatter.h>
+
+#include <algorithm>
+#include <memory>
+#include <mutex>
+#include <vector>
+
+// Distribution sink (mux). Stores a vector of sinks which get called when log
+// is called
+
+namespace spdlog {
+namespace sinks {
+
+template <typename Mutex>
+class dist_sink : public base_sink<Mutex> {
+public:
+    dist_sink() = default;
+    explicit dist_sink(std::vector<std::shared_ptr<sink>> sinks)
+        : sinks_(sinks) {}
+
+    dist_sink(const dist_sink &) = delete;
+    dist_sink &operator=(const dist_sink &) = delete;
+
+    void add_sink(std::shared_ptr<sink> sub_sink) {
+        std::lock_guard<Mutex> lock(base_sink<Mutex>::mutex_);
+        sinks_.push_back(sub_sink);
+    }
+
+    void remove_sink(std::shared_ptr<sink> sub_sink) {
+        std::lock_guard<Mutex> lock(base_sink<Mutex>::mutex_);
+        sinks_.erase(std::remove(sinks_.begin(), sinks_.end(), sub_sink), sinks_.end());
+    }
+
+    void set_sinks(std::vector<std::shared_ptr<sink>> sinks) {
+        std::lock_guard<Mutex> lock(base_sink<Mutex>::mutex_);
+        sinks_ = std::move(sinks);
+    }
+
+    std::vector<std::shared_ptr<sink>> &sinks() { return sinks_; }
+
+protected:
+    void sink_it_(const details::log_msg &msg) override {
+        for (auto &sub_sink : sinks_) {
+            if (sub_sink->should_log(msg.level)) {
+                sub_sink->log(msg);
+            }
+        }
+    }
+
+    void flush_() override {
+        for (auto &sub_sink : sinks_) {
+            sub_sink->flush();
+        }
+    }
+
+    void set_pattern_(const std::string &pattern) override {
+        set_formatter_(details::make_unique<spdlog::pattern_formatter>(pattern));
+    }
+
+    void set_formatter_(std::unique_ptr<spdlog::formatter> sink_formatter) override {
+        base_sink<Mutex>::formatter_ = std::move(sink_formatter);
+        for (auto &sub_sink : sinks_) {
+            sub_sink->set_formatter(base_sink<Mutex>::formatter_->clone());
+        }
+    }
+    std::vector<std::shared_ptr<sink>> sinks_;
+};
+
+using dist_sink_mt = dist_sink<std::mutex>;
+using dist_sink_st = dist_sink<details::null_mutex>;
+
+}  // namespace sinks
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/sinks/dup_filter_sink.h b/ext/spdlog/include/spdlog/sinks/dup_filter_sink.h
new file mode 100644
index 0000000..1498142
--- /dev/null
+++ b/ext/spdlog/include/spdlog/sinks/dup_filter_sink.h
@@ -0,0 +1,92 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#include "dist_sink.h"
+#include <spdlog/details/log_msg.h>
+#include <spdlog/details/null_mutex.h>
+
+#include <chrono>
+#include <cstdio>
+#include <mutex>
+#include <string>
+
+// Duplicate message removal sink.
+// Skip the message if previous one is identical and less than "max_skip_duration" have passed
+//
+// Example:
+//
+//     #include <spdlog/sinks/dup_filter_sink.h>
+//
+//     int main() {
+//         auto dup_filter = std::make_shared<dup_filter_sink_st>(std::chrono::seconds(5),
+//         level::info); dup_filter->add_sink(std::make_shared<stdout_color_sink_mt>());
+//         spdlog::logger l("logger", dup_filter);
+//         l.info("Hello");
+//         l.info("Hello");
+//         l.info("Hello");
+//         l.info("Different Hello");
+//     }
+//
+// Will produce:
+//       [2019-06-25 17:50:56.511] [logger] [info] Hello
+//       [2019-06-25 17:50:56.512] [logger] [info] Skipped 3 duplicate messages..
+//       [2019-06-25 17:50:56.512] [logger] [info] Different Hello
+
+namespace spdlog {
+namespace sinks {
+template <typename Mutex>
+class dup_filter_sink : public dist_sink<Mutex> {
+public:
+    template <class Rep, class Period>
+    explicit dup_filter_sink(std::chrono::duration<Rep, Period> max_skip_duration,
+                             level::level_enum notification_level = level::info)
+        : max_skip_duration_{max_skip_duration},
+          log_level_{notification_level} {}
+
+protected:
+    std::chrono::microseconds max_skip_duration_;
+    log_clock::time_point last_msg_time_;
+    std::string last_msg_payload_;
+    size_t skip_counter_ = 0;
+    level::level_enum log_level_;
+
+    void sink_it_(const details::log_msg &msg) override {
+        bool filtered = filter_(msg);
+        if (!filtered) {
+            skip_counter_ += 1;
+            return;
+        }
+
+        // log the "skipped.." message
+        if (skip_counter_ > 0) {
+            char buf[64];
+            auto msg_size = ::snprintf(buf, sizeof(buf), "Skipped %u duplicate messages..",
+                                       static_cast<unsigned>(skip_counter_));
+            if (msg_size > 0 && static_cast<size_t>(msg_size) < sizeof(buf)) {
+                details::log_msg skipped_msg{msg.source, msg.logger_name, log_level_,
+                                             string_view_t{buf, static_cast<size_t>(msg_size)}};
+                dist_sink<Mutex>::sink_it_(skipped_msg);
+            }
+        }
+
+        // log current message
+        dist_sink<Mutex>::sink_it_(msg);
+        last_msg_time_ = msg.time;
+        skip_counter_ = 0;
+        last_msg_payload_.assign(msg.payload.data(), msg.payload.data() + msg.payload.size());
+    }
+
+    // return whether the log msg should be displayed (true) or skipped (false)
+    bool filter_(const details::log_msg &msg) {
+        auto filter_duration = msg.time - last_msg_time_;
+        return (filter_duration > max_skip_duration_) || (msg.payload != last_msg_payload_);
+    }
+};
+
+using dup_filter_sink_mt = dup_filter_sink<std::mutex>;
+using dup_filter_sink_st = dup_filter_sink<details::null_mutex>;
+
+}  // namespace sinks
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/sinks/hourly_file_sink.h b/ext/spdlog/include/spdlog/sinks/hourly_file_sink.h
new file mode 100644
index 0000000..3e61872
--- /dev/null
+++ b/ext/spdlog/include/spdlog/sinks/hourly_file_sink.h
@@ -0,0 +1,193 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#include <spdlog/common.h>
+#include <spdlog/details/circular_q.h>
+#include <spdlog/details/file_helper.h>
+#include <spdlog/details/null_mutex.h>
+#include <spdlog/details/os.h>
+#include <spdlog/details/synchronous_factory.h>
+#include <spdlog/fmt/fmt.h>
+#include <spdlog/sinks/base_sink.h>
+
+#include <chrono>
+#include <cstdio>
+#include <ctime>
+#include <mutex>
+#include <string>
+
+namespace spdlog {
+namespace sinks {
+
+/*
+ * Generator of Hourly log file names in format basename.YYYY-MM-DD-HH.ext
+ */
+struct hourly_filename_calculator {
+    // Create filename for the form basename.YYYY-MM-DD-H
+    static filename_t calc_filename(const filename_t &filename, const tm &now_tm) {
+        filename_t basename, ext;
+        std::tie(basename, ext) = details::file_helper::split_by_extension(filename);
+        return fmt_lib::format(SPDLOG_FILENAME_T("{}_{:04d}-{:02d}-{:02d}_{:02d}{}"), basename,
+                               now_tm.tm_year + 1900, now_tm.tm_mon + 1, now_tm.tm_mday,
+                               now_tm.tm_hour, ext);
+    }
+};
+
+/*
+ * Rotating file sink based on time.
+ * If truncate != false , the created file will be truncated.
+ * If max_files > 0, retain only the last max_files and delete previous.
+ * Note that old log files from previous executions will not be deleted by this class,
+ * rotation and deletion is only applied while the program is running.
+ */
+template <typename Mutex, typename FileNameCalc = hourly_filename_calculator>
+class hourly_file_sink final : public base_sink<Mutex> {
+public:
+    // create hourly file sink which rotates on given time
+    hourly_file_sink(filename_t base_filename,
+                     bool truncate = false,
+                     uint16_t max_files = 0,
+                     const file_event_handlers &event_handlers = {})
+        : base_filename_(std::move(base_filename)),
+          file_helper_{event_handlers},
+          truncate_(truncate),
+          max_files_(max_files),
+          filenames_q_() {
+        auto now = log_clock::now();
+        auto filename = FileNameCalc::calc_filename(base_filename_, now_tm(now));
+        file_helper_.open(filename, truncate_);
+        remove_init_file_ = file_helper_.size() == 0;
+        rotation_tp_ = next_rotation_tp_();
+
+        if (max_files_ > 0) {
+            init_filenames_q_();
+        }
+    }
+
+    filename_t filename() {
+        std::lock_guard<Mutex> lock(base_sink<Mutex>::mutex_);
+        return file_helper_.filename();
+    }
+
+protected:
+    void sink_it_(const details::log_msg &msg) override {
+        auto time = msg.time;
+        bool should_rotate = time >= rotation_tp_;
+        if (should_rotate) {
+            if (remove_init_file_) {
+                file_helper_.close();
+                details::os::remove(file_helper_.filename());
+            }
+            auto filename = FileNameCalc::calc_filename(base_filename_, now_tm(time));
+            file_helper_.open(filename, truncate_);
+            rotation_tp_ = next_rotation_tp_();
+        }
+        remove_init_file_ = false;
+        memory_buf_t formatted;
+        base_sink<Mutex>::formatter_->format(msg, formatted);
+        file_helper_.write(formatted);
+
+        // Do the cleaning only at the end because it might throw on failure.
+        if (should_rotate && max_files_ > 0) {
+            delete_old_();
+        }
+    }
+
+    void flush_() override { file_helper_.flush(); }
+
+private:
+    void init_filenames_q_() {
+        using details::os::path_exists;
+
+        filenames_q_ = details::circular_q<filename_t>(static_cast<size_t>(max_files_));
+        std::vector<filename_t> filenames;
+        auto now = log_clock::now();
+        while (filenames.size() < max_files_) {
+            auto filename = FileNameCalc::calc_filename(base_filename_, now_tm(now));
+            if (!path_exists(filename)) {
+                break;
+            }
+            filenames.emplace_back(filename);
+            now -= std::chrono::hours(1);
+        }
+        for (auto iter = filenames.rbegin(); iter != filenames.rend(); ++iter) {
+            filenames_q_.push_back(std::move(*iter));
+        }
+    }
+
+    tm now_tm(log_clock::time_point tp) {
+        time_t tnow = log_clock::to_time_t(tp);
+        return spdlog::details::os::localtime(tnow);
+    }
+
+    log_clock::time_point next_rotation_tp_() {
+        auto now = log_clock::now();
+        tm date = now_tm(now);
+        date.tm_min = 0;
+        date.tm_sec = 0;
+        auto rotation_time = log_clock::from_time_t(std::mktime(&date));
+        if (rotation_time > now) {
+            return rotation_time;
+        }
+        return {rotation_time + std::chrono::hours(1)};
+    }
+
+    // Delete the file N rotations ago.
+    // Throw spdlog_ex on failure to delete the old file.
+    void delete_old_() {
+        using details::os::filename_to_str;
+        using details::os::remove_if_exists;
+
+        filename_t current_file = file_helper_.filename();
+        if (filenames_q_.full()) {
+            auto old_filename = std::move(filenames_q_.front());
+            filenames_q_.pop_front();
+            bool ok = remove_if_exists(old_filename) == 0;
+            if (!ok) {
+                filenames_q_.push_back(std::move(current_file));
+                SPDLOG_THROW(spdlog_ex(
+                    "Failed removing hourly file " + filename_to_str(old_filename), errno));
+            }
+        }
+        filenames_q_.push_back(std::move(current_file));
+    }
+
+    filename_t base_filename_;
+    log_clock::time_point rotation_tp_;
+    details::file_helper file_helper_;
+    bool truncate_;
+    uint16_t max_files_;
+    details::circular_q<filename_t> filenames_q_;
+    bool remove_init_file_;
+};
+
+using hourly_file_sink_mt = hourly_file_sink<std::mutex>;
+using hourly_file_sink_st = hourly_file_sink<details::null_mutex>;
+
+}  // namespace sinks
+
+//
+// factory functions
+//
+template <typename Factory = spdlog::synchronous_factory>
+inline std::shared_ptr<logger> hourly_logger_mt(const std::string &logger_name,
+                                                const filename_t &filename,
+                                                bool truncate = false,
+                                                uint16_t max_files = 0,
+                                                const file_event_handlers &event_handlers = {}) {
+    return Factory::template create<sinks::hourly_file_sink_mt>(logger_name, filename, truncate,
+                                                                max_files, event_handlers);
+}
+
+template <typename Factory = spdlog::synchronous_factory>
+inline std::shared_ptr<logger> hourly_logger_st(const std::string &logger_name,
+                                                const filename_t &filename,
+                                                bool truncate = false,
+                                                uint16_t max_files = 0,
+                                                const file_event_handlers &event_handlers = {}) {
+    return Factory::template create<sinks::hourly_file_sink_st>(logger_name, filename, truncate,
+                                                                max_files, event_handlers);
+}
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/sinks/kafka_sink.h b/ext/spdlog/include/spdlog/sinks/kafka_sink.h
new file mode 100644
index 0000000..91e9878
--- /dev/null
+++ b/ext/spdlog/include/spdlog/sinks/kafka_sink.h
@@ -0,0 +1,119 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+//
+// Custom sink for kafka
+// Building and using requires librdkafka library.
+// For building librdkafka library check the url below
+// https://github.com/confluentinc/librdkafka
+//
+
+#include "spdlog/async.h"
+#include "spdlog/details/log_msg.h"
+#include "spdlog/details/null_mutex.h"
+#include "spdlog/details/synchronous_factory.h"
+#include "spdlog/sinks/base_sink.h"
+#include <mutex>
+#include <spdlog/common.h>
+
+// kafka header
+#include <librdkafka/rdkafkacpp.h>
+
+namespace spdlog {
+namespace sinks {
+
+struct kafka_sink_config {
+    std::string server_addr;
+    std::string produce_topic;
+    int32_t flush_timeout_ms = 1000;
+
+    kafka_sink_config(std::string addr, std::string topic, int flush_timeout_ms = 1000)
+        : server_addr{std::move(addr)},
+          produce_topic{std::move(topic)},
+          flush_timeout_ms(flush_timeout_ms) {}
+};
+
+template <typename Mutex>
+class kafka_sink : public base_sink<Mutex> {
+public:
+    kafka_sink(kafka_sink_config config)
+        : config_{std::move(config)} {
+        try {
+            std::string errstr;
+            conf_.reset(RdKafka::Conf::create(RdKafka::Conf::CONF_GLOBAL));
+            RdKafka::Conf::ConfResult confRes =
+                conf_->set("bootstrap.servers", config_.server_addr, errstr);
+            if (confRes != RdKafka::Conf::CONF_OK) {
+                throw_spdlog_ex(
+                    fmt_lib::format("conf set bootstrap.servers failed err:{}", errstr));
+            }
+
+            tconf_.reset(RdKafka::Conf::create(RdKafka::Conf::CONF_TOPIC));
+            if (tconf_ == nullptr) {
+                throw_spdlog_ex(fmt_lib::format("create topic config failed"));
+            }
+
+            producer_.reset(RdKafka::Producer::create(conf_.get(), errstr));
+            if (producer_ == nullptr) {
+                throw_spdlog_ex(fmt_lib::format("create producer failed err:{}", errstr));
+            }
+            topic_.reset(RdKafka::Topic::create(producer_.get(), config_.produce_topic,
+                                                tconf_.get(), errstr));
+            if (topic_ == nullptr) {
+                throw_spdlog_ex(fmt_lib::format("create topic failed err:{}", errstr));
+            }
+        } catch (const std::exception &e) {
+            throw_spdlog_ex(fmt_lib::format("error create kafka instance: {}", e.what()));
+        }
+    }
+
+    ~kafka_sink() { producer_->flush(config_.flush_timeout_ms); }
+
+protected:
+    void sink_it_(const details::log_msg &msg) override {
+        producer_->produce(topic_.get(), 0, RdKafka::Producer::RK_MSG_COPY,
+                           (void *)msg.payload.data(), msg.payload.size(), NULL, NULL);
+    }
+
+    void flush_() override { producer_->flush(config_.flush_timeout_ms); }
+
+private:
+    kafka_sink_config config_;
+    std::unique_ptr<RdKafka::Producer> producer_ = nullptr;
+    std::unique_ptr<RdKafka::Conf> conf_ = nullptr;
+    std::unique_ptr<RdKafka::Conf> tconf_ = nullptr;
+    std::unique_ptr<RdKafka::Topic> topic_ = nullptr;
+};
+
+using kafka_sink_mt = kafka_sink<std::mutex>;
+using kafka_sink_st = kafka_sink<spdlog::details::null_mutex>;
+
+}  // namespace sinks
+
+template <typename Factory = spdlog::synchronous_factory>
+inline std::shared_ptr<logger> kafka_logger_mt(const std::string &logger_name,
+                                               spdlog::sinks::kafka_sink_config config) {
+    return Factory::template create<sinks::kafka_sink_mt>(logger_name, config);
+}
+
+template <typename Factory = spdlog::synchronous_factory>
+inline std::shared_ptr<logger> kafka_logger_st(const std::string &logger_name,
+                                               spdlog::sinks::kafka_sink_config config) {
+    return Factory::template create<sinks::kafka_sink_st>(logger_name, config);
+}
+
+template <typename Factory = spdlog::async_factory>
+inline std::shared_ptr<spdlog::logger> kafka_logger_async_mt(
+    std::string logger_name, spdlog::sinks::kafka_sink_config config) {
+    return Factory::template create<sinks::kafka_sink_mt>(logger_name, config);
+}
+
+template <typename Factory = spdlog::async_factory>
+inline std::shared_ptr<spdlog::logger> kafka_logger_async_st(
+    std::string logger_name, spdlog::sinks::kafka_sink_config config) {
+    return Factory::template create<sinks::kafka_sink_st>(logger_name, config);
+}
+
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/sinks/mongo_sink.h b/ext/spdlog/include/spdlog/sinks/mongo_sink.h
new file mode 100644
index 0000000..c5b38ab
--- /dev/null
+++ b/ext/spdlog/include/spdlog/sinks/mongo_sink.h
@@ -0,0 +1,108 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+//
+// Custom sink for mongodb
+// Building and using requires mongocxx library.
+// For building mongocxx library check the url below
+// http://mongocxx.org/mongocxx-v3/installation/
+//
+
+#include "spdlog/common.h"
+#include "spdlog/details/log_msg.h"
+#include "spdlog/sinks/base_sink.h"
+#include <spdlog/details/synchronous_factory.h>
+
+#include <bsoncxx/builder/stream/document.hpp>
+#include <bsoncxx/types.hpp>
+#include <bsoncxx/view_or_value.hpp>
+
+#include <mongocxx/client.hpp>
+#include <mongocxx/instance.hpp>
+#include <mongocxx/uri.hpp>
+
+namespace spdlog {
+namespace sinks {
+template <typename Mutex>
+class mongo_sink : public base_sink<Mutex> {
+public:
+    mongo_sink(const std::string &db_name,
+               const std::string &collection_name,
+               const std::string &uri = "mongodb://localhost:27017") try
+        : mongo_sink(std::make_shared<mongocxx::instance>(), db_name, collection_name, uri) {
+    } catch (const std::exception &e) {
+        throw_spdlog_ex(fmt_lib::format("Error opening database: {}", e.what()));
+    }
+
+    mongo_sink(std::shared_ptr<mongocxx::instance> instance,
+               const std::string &db_name,
+               const std::string &collection_name,
+               const std::string &uri = "mongodb://localhost:27017")
+        : instance_(std::move(instance)),
+          db_name_(db_name),
+          coll_name_(collection_name) {
+        try {
+            client_ = spdlog::details::make_unique<mongocxx::client>(mongocxx::uri{uri});
+        } catch (const std::exception &e) {
+            throw_spdlog_ex(fmt_lib::format("Error opening database: {}", e.what()));
+        }
+    }
+
+    ~mongo_sink() { flush_(); }
+
+protected:
+    void sink_it_(const details::log_msg &msg) override {
+        using bsoncxx::builder::stream::document;
+        using bsoncxx::builder::stream::finalize;
+
+        if (client_ != nullptr) {
+            auto doc = document{} << "timestamp" << bsoncxx::types::b_date(msg.time) << "level"
+                                  << level::to_string_view(msg.level).data() << "level_num"
+                                  << msg.level << "message"
+                                  << std::string(msg.payload.begin(), msg.payload.end())
+                                  << "logger_name"
+                                  << std::string(msg.logger_name.begin(), msg.logger_name.end())
+                                  << "thread_id" << static_cast<int>(msg.thread_id) << finalize;
+            client_->database(db_name_).collection(coll_name_).insert_one(doc.view());
+        }
+    }
+
+    void flush_() override {}
+
+private:
+    std::shared_ptr<mongocxx::instance> instance_;
+    std::string db_name_;
+    std::string coll_name_;
+    std::unique_ptr<mongocxx::client> client_ = nullptr;
+};
+
+#include "spdlog/details/null_mutex.h"
+#include <mutex>
+using mongo_sink_mt = mongo_sink<std::mutex>;
+using mongo_sink_st = mongo_sink<spdlog::details::null_mutex>;
+
+}  // namespace sinks
+
+template <typename Factory = spdlog::synchronous_factory>
+inline std::shared_ptr<logger> mongo_logger_mt(
+    const std::string &logger_name,
+    const std::string &db_name,
+    const std::string &collection_name,
+    const std::string &uri = "mongodb://localhost:27017") {
+    return Factory::template create<sinks::mongo_sink_mt>(logger_name, db_name, collection_name,
+                                                          uri);
+}
+
+template <typename Factory = spdlog::synchronous_factory>
+inline std::shared_ptr<logger> mongo_logger_st(
+    const std::string &logger_name,
+    const std::string &db_name,
+    const std::string &collection_name,
+    const std::string &uri = "mongodb://localhost:27017") {
+    return Factory::template create<sinks::mongo_sink_st>(logger_name, db_name, collection_name,
+                                                          uri);
+}
+
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/sinks/msvc_sink.h b/ext/spdlog/include/spdlog/sinks/msvc_sink.h
new file mode 100644
index 0000000..c28d6eb
--- /dev/null
+++ b/ext/spdlog/include/spdlog/sinks/msvc_sink.h
@@ -0,0 +1,68 @@
+// Copyright(c) 2016 Alexander Dalshov & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#if defined(_WIN32)
+
+    #include <spdlog/details/null_mutex.h>
+    #if defined(SPDLOG_WCHAR_TO_UTF8_SUPPORT)
+        #include <spdlog/details/os.h>
+    #endif
+    #include <spdlog/sinks/base_sink.h>
+
+    #include <mutex>
+    #include <string>
+
+    // Avoid including windows.h (https://stackoverflow.com/a/30741042)
+    #if defined(SPDLOG_WCHAR_TO_UTF8_SUPPORT)
+extern "C" __declspec(dllimport) void __stdcall OutputDebugStringW(const wchar_t *lpOutputString);
+    #else
+extern "C" __declspec(dllimport) void __stdcall OutputDebugStringA(const char *lpOutputString);
+    #endif
+extern "C" __declspec(dllimport) int __stdcall IsDebuggerPresent();
+
+namespace spdlog {
+namespace sinks {
+/*
+ * MSVC sink (logging using OutputDebugStringA)
+ */
+template <typename Mutex>
+class msvc_sink : public base_sink<Mutex> {
+public:
+    msvc_sink() = default;
+    msvc_sink(bool check_debugger_present)
+        : check_debugger_present_{check_debugger_present} {}
+
+protected:
+    void sink_it_(const details::log_msg &msg) override {
+        if (check_debugger_present_ && !IsDebuggerPresent()) {
+            return;
+        }
+        memory_buf_t formatted;
+        base_sink<Mutex>::formatter_->format(msg, formatted);
+        formatted.push_back('\0');  // add a null terminator for OutputDebugString
+    #if defined(SPDLOG_WCHAR_TO_UTF8_SUPPORT)
+        wmemory_buf_t wformatted;
+        details::os::utf8_to_wstrbuf(string_view_t(formatted.data(), formatted.size()), wformatted);
+        OutputDebugStringW(wformatted.data());
+    #else
+        OutputDebugStringA(formatted.data());
+    #endif
+    }
+
+    void flush_() override {}
+
+    bool check_debugger_present_ = true;
+};
+
+using msvc_sink_mt = msvc_sink<std::mutex>;
+using msvc_sink_st = msvc_sink<details::null_mutex>;
+
+using windebug_sink_mt = msvc_sink_mt;
+using windebug_sink_st = msvc_sink_st;
+
+}  // namespace sinks
+}  // namespace spdlog
+
+#endif
diff --git a/ext/spdlog/include/spdlog/sinks/null_sink.h b/ext/spdlog/include/spdlog/sinks/null_sink.h
new file mode 100644
index 0000000..74530b5
--- /dev/null
+++ b/ext/spdlog/include/spdlog/sinks/null_sink.h
@@ -0,0 +1,41 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#include <spdlog/details/null_mutex.h>
+#include <spdlog/details/synchronous_factory.h>
+#include <spdlog/sinks/base_sink.h>
+
+#include <mutex>
+
+namespace spdlog {
+namespace sinks {
+
+template <typename Mutex>
+class null_sink final : public base_sink<Mutex> {
+protected:
+    void sink_it_(const details::log_msg &) override {}
+    void flush_() override {}
+};
+
+using null_sink_mt = null_sink<details::null_mutex>;
+using null_sink_st = null_sink<details::null_mutex>;
+
+}  // namespace sinks
+
+template <typename Factory = spdlog::synchronous_factory>
+inline std::shared_ptr<logger> null_logger_mt(const std::string &logger_name) {
+    auto null_logger = Factory::template create<sinks::null_sink_mt>(logger_name);
+    null_logger->set_level(level::off);
+    return null_logger;
+}
+
+template <typename Factory = spdlog::synchronous_factory>
+inline std::shared_ptr<logger> null_logger_st(const std::string &logger_name) {
+    auto null_logger = Factory::template create<sinks::null_sink_st>(logger_name);
+    null_logger->set_level(level::off);
+    return null_logger;
+}
+
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/sinks/ostream_sink.h b/ext/spdlog/include/spdlog/sinks/ostream_sink.h
new file mode 100644
index 0000000..6af9dd0
--- /dev/null
+++ b/ext/spdlog/include/spdlog/sinks/ostream_sink.h
@@ -0,0 +1,43 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#include <spdlog/details/null_mutex.h>
+#include <spdlog/sinks/base_sink.h>
+
+#include <mutex>
+#include <ostream>
+
+namespace spdlog {
+namespace sinks {
+template <typename Mutex>
+class ostream_sink final : public base_sink<Mutex> {
+public:
+    explicit ostream_sink(std::ostream &os, bool force_flush = false)
+        : ostream_(os),
+          force_flush_(force_flush) {}
+    ostream_sink(const ostream_sink &) = delete;
+    ostream_sink &operator=(const ostream_sink &) = delete;
+
+protected:
+    void sink_it_(const details::log_msg &msg) override {
+        memory_buf_t formatted;
+        base_sink<Mutex>::formatter_->format(msg, formatted);
+        ostream_.write(formatted.data(), static_cast<std::streamsize>(formatted.size()));
+        if (force_flush_) {
+            ostream_.flush();
+        }
+    }
+
+    void flush_() override { ostream_.flush(); }
+
+    std::ostream &ostream_;
+    bool force_flush_;
+};
+
+using ostream_sink_mt = ostream_sink<std::mutex>;
+using ostream_sink_st = ostream_sink<details::null_mutex>;
+
+}  // namespace sinks
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/sinks/qt_sinks.h b/ext/spdlog/include/spdlog/sinks/qt_sinks.h
new file mode 100644
index 0000000..d319e84
--- /dev/null
+++ b/ext/spdlog/include/spdlog/sinks/qt_sinks.h
@@ -0,0 +1,304 @@
+// Copyright(c) 2015-present, Gabi Melman, mguludag and spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+//
+// Custom sink for QPlainTextEdit or QTextEdit and its children (QTextBrowser...
+// etc) Building and using requires Qt library.
+//
+// Warning: the qt_sink won't be notified if the target widget is destroyed.
+// If the widget's lifetime can be shorter than the logger's one, you should provide some permanent
+// QObject, and then use a standard signal/slot.
+//
+
+#include "spdlog/common.h"
+#include "spdlog/details/log_msg.h"
+#include "spdlog/details/synchronous_factory.h"
+#include "spdlog/sinks/base_sink.h"
+#include <array>
+
+#include <QPlainTextEdit>
+#include <QTextEdit>
+
+//
+// qt_sink class
+//
+namespace spdlog {
+namespace sinks {
+template <typename Mutex>
+class qt_sink : public base_sink<Mutex> {
+public:
+    qt_sink(QObject *qt_object, std::string meta_method)
+        : qt_object_(qt_object),
+          meta_method_(std::move(meta_method)) {
+        if (!qt_object_) {
+            throw_spdlog_ex("qt_sink: qt_object is null");
+        }
+    }
+
+    ~qt_sink() { flush_(); }
+
+protected:
+    void sink_it_(const details::log_msg &msg) override {
+        memory_buf_t formatted;
+        base_sink<Mutex>::formatter_->format(msg, formatted);
+        const string_view_t str = string_view_t(formatted.data(), formatted.size());
+        QMetaObject::invokeMethod(
+            qt_object_, meta_method_.c_str(), Qt::AutoConnection,
+            Q_ARG(QString, QString::fromUtf8(str.data(), static_cast<int>(str.size())).trimmed()));
+    }
+
+    void flush_() override {}
+
+private:
+    QObject *qt_object_ = nullptr;
+    std::string meta_method_;
+};
+
+// Qt color sink to QTextEdit.
+// Color location is determined by the sink log pattern like in the rest of spdlog sinks.
+// Colors can be modified if needed using sink->set_color(level, qtTextCharFormat).
+// max_lines is the maximum number of lines that the sink will hold before removing the oldest
+// lines. By default, only ascii (latin1) is supported by this sink. Set is_utf8 to true if utf8
+// support is needed.
+template <typename Mutex>
+class qt_color_sink : public base_sink<Mutex> {
+public:
+    qt_color_sink(QTextEdit *qt_text_edit,
+                  int max_lines,
+                  bool dark_colors = false,
+                  bool is_utf8 = false)
+        : qt_text_edit_(qt_text_edit),
+          max_lines_(max_lines),
+          is_utf8_(is_utf8) {
+        if (!qt_text_edit_) {
+            throw_spdlog_ex("qt_color_text_sink: text_edit is null");
+        }
+
+        default_color_ = qt_text_edit_->currentCharFormat();
+        // set colors
+        QTextCharFormat format;
+        // trace
+        format.setForeground(dark_colors ? Qt::darkGray : Qt::gray);
+        colors_.at(level::trace) = format;
+        // debug
+        format.setForeground(dark_colors ? Qt::darkCyan : Qt::cyan);
+        colors_.at(level::debug) = format;
+        // info
+        format.setForeground(dark_colors ? Qt::darkGreen : Qt::green);
+        colors_.at(level::info) = format;
+        // warn
+        format.setForeground(dark_colors ? Qt::darkYellow : Qt::yellow);
+        colors_.at(level::warn) = format;
+        // err
+        format.setForeground(Qt::red);
+        colors_.at(level::err) = format;
+        // critical
+        format.setForeground(Qt::white);
+        format.setBackground(Qt::red);
+        colors_.at(level::critical) = format;
+    }
+
+    ~qt_color_sink() { flush_(); }
+
+    void set_default_color(QTextCharFormat format) {
+        // std::lock_guard<Mutex> lock(base_sink<Mutex>::mutex_);
+        default_color_ = format;
+    }
+
+    void set_level_color(level::level_enum color_level, QTextCharFormat format) {
+        // std::lock_guard<Mutex> lock(base_sink<Mutex>::mutex_);
+        colors_.at(static_cast<size_t>(color_level)) = format;
+    }
+
+    QTextCharFormat &get_level_color(level::level_enum color_level) {
+        std::lock_guard<Mutex> lock(base_sink<Mutex>::mutex_);
+        return colors_.at(static_cast<size_t>(color_level));
+    }
+
+    QTextCharFormat &get_default_color() {
+        std::lock_guard<Mutex> lock(base_sink<Mutex>::mutex_);
+        return default_color_;
+    }
+
+protected:
+    struct invoke_params {
+        invoke_params(int max_lines,
+                      QTextEdit *q_text_edit,
+                      QString payload,
+                      QTextCharFormat default_color,
+                      QTextCharFormat level_color,
+                      int color_range_start,
+                      int color_range_end)
+            : max_lines(max_lines),
+              q_text_edit(q_text_edit),
+              payload(std::move(payload)),
+              default_color(default_color),
+              level_color(level_color),
+              color_range_start(color_range_start),
+              color_range_end(color_range_end) {}
+        int max_lines;
+        QTextEdit *q_text_edit;
+        QString payload;
+        QTextCharFormat default_color;
+        QTextCharFormat level_color;
+        int color_range_start;
+        int color_range_end;
+    };
+
+    void sink_it_(const details::log_msg &msg) override {
+        memory_buf_t formatted;
+        base_sink<Mutex>::formatter_->format(msg, formatted);
+
+        const string_view_t str = string_view_t(formatted.data(), formatted.size());
+        // apply the color to the color range in the formatted message.
+        QString payload;
+        int color_range_start = static_cast<int>(msg.color_range_start);
+        int color_range_end = static_cast<int>(msg.color_range_end);
+        if (is_utf8_) {
+            payload = QString::fromUtf8(str.data(), static_cast<int>(str.size()));
+            // convert color ranges from byte index to character index.
+            if (msg.color_range_start < msg.color_range_end) {
+                color_range_start = QString::fromUtf8(str.data(), msg.color_range_start).size();
+                color_range_end = QString::fromUtf8(str.data(), msg.color_range_end).size();
+            }
+        } else {
+            payload = QString::fromLatin1(str.data(), static_cast<int>(str.size()));
+        }
+
+        invoke_params params{max_lines_,             // max lines
+                             qt_text_edit_,          // text edit to append to
+                             std::move(payload),     // text to append
+                             default_color_,         // default color
+                             colors_.at(msg.level),  // color to apply
+                             color_range_start,      // color range start
+                             color_range_end};       // color range end
+
+        QMetaObject::invokeMethod(
+            qt_text_edit_, [params]() { invoke_method_(params); }, Qt::AutoConnection);
+    }
+
+    void flush_() override {}
+
+    // Add colored text to the text edit widget. This method is invoked in the GUI thread.
+    // It is a static method to ensure that it is handled correctly even if the sink is destroyed
+    // prematurely before it is invoked.
+
+    static void invoke_method_(invoke_params params) {
+        auto *document = params.q_text_edit->document();
+        QTextCursor cursor(document);
+
+        // remove first blocks if number of blocks exceeds max_lines
+        while (document->blockCount() > params.max_lines) {
+            cursor.select(QTextCursor::BlockUnderCursor);
+            cursor.removeSelectedText();
+            cursor.deleteChar();  // delete the newline after the block
+        }
+
+        cursor.movePosition(QTextCursor::End);
+        cursor.setCharFormat(params.default_color);
+
+        // if color range not specified or not not valid, just append the text with default color
+        if (params.color_range_end <= params.color_range_start) {
+            cursor.insertText(params.payload);
+            return;
+        }
+
+        // insert the text before the color range
+        cursor.insertText(params.payload.left(params.color_range_start));
+
+        // insert the colorized text
+        cursor.setCharFormat(params.level_color);
+        cursor.insertText(params.payload.mid(params.color_range_start,
+                                             params.color_range_end - params.color_range_start));
+
+        // insert the text after the color range with default format
+        cursor.setCharFormat(params.default_color);
+        cursor.insertText(params.payload.mid(params.color_range_end));
+    }
+
+    QTextEdit *qt_text_edit_;
+    int max_lines_;
+    bool is_utf8_;
+    QTextCharFormat default_color_;
+    std::array<QTextCharFormat, level::n_levels> colors_;
+};
+
+#include "spdlog/details/null_mutex.h"
+#include <mutex>
+
+using qt_sink_mt = qt_sink<std::mutex>;
+using qt_sink_st = qt_sink<details::null_mutex>;
+using qt_color_sink_mt = qt_color_sink<std::mutex>;
+using qt_color_sink_st = qt_color_sink<details::null_mutex>;
+}  // namespace sinks
+
+//
+// Factory functions
+//
+
+// log to QTextEdit
+template <typename Factory = spdlog::synchronous_factory>
+inline std::shared_ptr<logger> qt_logger_mt(const std::string &logger_name,
+                                            QTextEdit *qt_object,
+                                            const std::string &meta_method = "append") {
+    return Factory::template create<sinks::qt_sink_mt>(logger_name, qt_object, meta_method);
+}
+
+template <typename Factory = spdlog::synchronous_factory>
+inline std::shared_ptr<logger> qt_logger_st(const std::string &logger_name,
+                                            QTextEdit *qt_object,
+                                            const std::string &meta_method = "append") {
+    return Factory::template create<sinks::qt_sink_st>(logger_name, qt_object, meta_method);
+}
+
+// log to QPlainTextEdit
+template <typename Factory = spdlog::synchronous_factory>
+inline std::shared_ptr<logger> qt_logger_mt(const std::string &logger_name,
+                                            QPlainTextEdit *qt_object,
+                                            const std::string &meta_method = "appendPlainText") {
+    return Factory::template create<sinks::qt_sink_mt>(logger_name, qt_object, meta_method);
+}
+
+template <typename Factory = spdlog::synchronous_factory>
+inline std::shared_ptr<logger> qt_logger_st(const std::string &logger_name,
+                                            QPlainTextEdit *qt_object,
+                                            const std::string &meta_method = "appendPlainText") {
+    return Factory::template create<sinks::qt_sink_st>(logger_name, qt_object, meta_method);
+}
+// log to QObject
+template <typename Factory = spdlog::synchronous_factory>
+inline std::shared_ptr<logger> qt_logger_mt(const std::string &logger_name,
+                                            QObject *qt_object,
+                                            const std::string &meta_method) {
+    return Factory::template create<sinks::qt_sink_mt>(logger_name, qt_object, meta_method);
+}
+
+template <typename Factory = spdlog::synchronous_factory>
+inline std::shared_ptr<logger> qt_logger_st(const std::string &logger_name,
+                                            QObject *qt_object,
+                                            const std::string &meta_method) {
+    return Factory::template create<sinks::qt_sink_st>(logger_name, qt_object, meta_method);
+}
+
+// log to QTextEdit with colorized output
+template <typename Factory = spdlog::synchronous_factory>
+inline std::shared_ptr<logger> qt_color_logger_mt(const std::string &logger_name,
+                                                  QTextEdit *qt_text_edit,
+                                                  int max_lines,
+                                                  bool is_utf8 = false) {
+    return Factory::template create<sinks::qt_color_sink_mt>(logger_name, qt_text_edit, max_lines,
+                                                             false, is_utf8);
+}
+
+template <typename Factory = spdlog::synchronous_factory>
+inline std::shared_ptr<logger> qt_color_logger_st(const std::string &logger_name,
+                                                  QTextEdit *qt_text_edit,
+                                                  int max_lines,
+                                                  bool is_utf8 = false) {
+    return Factory::template create<sinks::qt_color_sink_st>(logger_name, qt_text_edit, max_lines,
+                                                             false, is_utf8);
+}
+
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/sinks/ringbuffer_sink.h b/ext/spdlog/include/spdlog/sinks/ringbuffer_sink.h
new file mode 100644
index 0000000..6156c6a
--- /dev/null
+++ b/ext/spdlog/include/spdlog/sinks/ringbuffer_sink.h
@@ -0,0 +1,67 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#include "spdlog/details/circular_q.h"
+#include "spdlog/details/log_msg_buffer.h"
+#include "spdlog/details/null_mutex.h"
+#include "spdlog/sinks/base_sink.h"
+
+#include <mutex>
+#include <string>
+#include <vector>
+
+namespace spdlog {
+namespace sinks {
+/*
+ * Ring buffer sink
+ */
+template <typename Mutex>
+class ringbuffer_sink final : public base_sink<Mutex> {
+public:
+    explicit ringbuffer_sink(size_t n_items)
+        : q_{n_items} {}
+
+    std::vector<details::log_msg_buffer> last_raw(size_t lim = 0) {
+        std::lock_guard<Mutex> lock(base_sink<Mutex>::mutex_);
+        auto items_available = q_.size();
+        auto n_items = lim > 0 ? (std::min)(lim, items_available) : items_available;
+        std::vector<details::log_msg_buffer> ret;
+        ret.reserve(n_items);
+        for (size_t i = (items_available - n_items); i < items_available; i++) {
+            ret.push_back(q_.at(i));
+        }
+        return ret;
+    }
+
+    std::vector<std::string> last_formatted(size_t lim = 0) {
+        std::lock_guard<Mutex> lock(base_sink<Mutex>::mutex_);
+        auto items_available = q_.size();
+        auto n_items = lim > 0 ? (std::min)(lim, items_available) : items_available;
+        std::vector<std::string> ret;
+        ret.reserve(n_items);
+        for (size_t i = (items_available - n_items); i < items_available; i++) {
+            memory_buf_t formatted;
+            base_sink<Mutex>::formatter_->format(q_.at(i), formatted);
+            ret.push_back(SPDLOG_BUF_TO_STRING(formatted));
+        }
+        return ret;
+    }
+
+protected:
+    void sink_it_(const details::log_msg &msg) override {
+        q_.push_back(details::log_msg_buffer{msg});
+    }
+    void flush_() override {}
+
+private:
+    details::circular_q<details::log_msg_buffer> q_;
+};
+
+using ringbuffer_sink_mt = ringbuffer_sink<std::mutex>;
+using ringbuffer_sink_st = ringbuffer_sink<details::null_mutex>;
+
+}  // namespace sinks
+
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/sinks/rotating_file_sink-inl.h b/ext/spdlog/include/spdlog/sinks/rotating_file_sink-inl.h
new file mode 100644
index 0000000..420bafb
--- /dev/null
+++ b/ext/spdlog/include/spdlog/sinks/rotating_file_sink-inl.h
@@ -0,0 +1,150 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#ifndef SPDLOG_HEADER_ONLY
+    #include <spdlog/sinks/rotating_file_sink.h>
+#endif
+
+#include <spdlog/common.h>
+
+#include <spdlog/details/file_helper.h>
+#include <spdlog/details/null_mutex.h>
+#include <spdlog/fmt/fmt.h>
+
+#include <cerrno>
+#include <chrono>
+#include <ctime>
+#include <mutex>
+#include <string>
+#include <tuple>
+
+namespace spdlog {
+namespace sinks {
+
+template <typename Mutex>
+SPDLOG_INLINE rotating_file_sink<Mutex>::rotating_file_sink(
+    filename_t base_filename,
+    std::size_t max_size,
+    std::size_t max_files,
+    bool rotate_on_open,
+    const file_event_handlers &event_handlers)
+    : base_filename_(std::move(base_filename)),
+      max_size_(max_size),
+      max_files_(max_files),
+      file_helper_{event_handlers} {
+    if (max_size == 0) {
+        throw_spdlog_ex("rotating sink constructor: max_size arg cannot be zero");
+    }
+
+    if (max_files > 200000) {
+        throw_spdlog_ex("rotating sink constructor: max_files arg cannot exceed 200000");
+    }
+    file_helper_.open(calc_filename(base_filename_, 0));
+    current_size_ = file_helper_.size();  // expensive. called only once
+    if (rotate_on_open && current_size_ > 0) {
+        rotate_();
+        current_size_ = 0;
+    }
+}
+
+// calc filename according to index and file extension if exists.
+// e.g. calc_filename("logs/mylog.txt, 3) => "logs/mylog.3.txt".
+template <typename Mutex>
+SPDLOG_INLINE filename_t rotating_file_sink<Mutex>::calc_filename(const filename_t &filename,
+                                                                  std::size_t index) {
+    if (index == 0u) {
+        return filename;
+    }
+
+    filename_t basename, ext;
+    std::tie(basename, ext) = details::file_helper::split_by_extension(filename);
+    return fmt_lib::format(SPDLOG_FMT_STRING(SPDLOG_FILENAME_T("{}.{}{}")), basename, index, ext);
+}
+
+template <typename Mutex>
+SPDLOG_INLINE filename_t rotating_file_sink<Mutex>::filename() {
+    std::lock_guard<Mutex> lock(base_sink<Mutex>::mutex_);
+    return file_helper_.filename();
+}
+
+template <typename Mutex>
+SPDLOG_INLINE void rotating_file_sink<Mutex>::rotate_now() {
+    std::lock_guard<Mutex> lock(base_sink<Mutex>::mutex_);
+    rotate_();
+}
+
+template <typename Mutex>
+SPDLOG_INLINE void rotating_file_sink<Mutex>::sink_it_(const details::log_msg &msg) {
+    memory_buf_t formatted;
+    base_sink<Mutex>::formatter_->format(msg, formatted);
+    auto new_size = current_size_ + formatted.size();
+
+    // rotate if the new estimated file size exceeds max size.
+    // rotate only if the real size > 0 to better deal with full disk (see issue #2261).
+    // we only check the real size when new_size > max_size_ because it is relatively expensive.
+    if (new_size > max_size_) {
+        file_helper_.flush();
+        if (file_helper_.size() > 0) {
+            rotate_();
+            new_size = formatted.size();
+        }
+    }
+    file_helper_.write(formatted);
+    current_size_ = new_size;
+}
+
+template <typename Mutex>
+SPDLOG_INLINE void rotating_file_sink<Mutex>::flush_() {
+    file_helper_.flush();
+}
+
+// Rotate files:
+// log.txt -> log.1.txt
+// log.1.txt -> log.2.txt
+// log.2.txt -> log.3.txt
+// log.3.txt -> delete
+template <typename Mutex>
+SPDLOG_INLINE void rotating_file_sink<Mutex>::rotate_() {
+    using details::os::filename_to_str;
+    using details::os::path_exists;
+
+    file_helper_.close();
+    for (auto i = max_files_; i > 0; --i) {
+        filename_t src = calc_filename(base_filename_, i - 1);
+        if (!path_exists(src)) {
+            continue;
+        }
+        filename_t target = calc_filename(base_filename_, i);
+
+        if (!rename_file_(src, target)) {
+            // if failed try again after a small delay.
+            // this is a workaround to a windows issue, where very high rotation
+            // rates can cause the rename to fail with permission denied (because of antivirus?).
+            details::os::sleep_for_millis(100);
+            if (!rename_file_(src, target)) {
+                file_helper_.reopen(
+                    true);  // truncate the log file anyway to prevent it to grow beyond its limit!
+                current_size_ = 0;
+                throw_spdlog_ex("rotating_file_sink: failed renaming " + filename_to_str(src) +
+                                    " to " + filename_to_str(target),
+                                errno);
+            }
+        }
+    }
+    file_helper_.reopen(true);
+}
+
+// delete the target if exists, and rename the src file  to target
+// return true on success, false otherwise.
+template <typename Mutex>
+SPDLOG_INLINE bool rotating_file_sink<Mutex>::rename_file_(const filename_t &src_filename,
+                                                           const filename_t &target_filename) {
+    // try to delete the target file in case it already exists.
+    (void)details::os::remove(target_filename);
+    return details::os::rename(src_filename, target_filename) == 0;
+}
+
+}  // namespace sinks
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/sinks/rotating_file_sink.h b/ext/spdlog/include/spdlog/sinks/rotating_file_sink.h
new file mode 100644
index 0000000..42bd376
--- /dev/null
+++ b/ext/spdlog/include/spdlog/sinks/rotating_file_sink.h
@@ -0,0 +1,90 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#include <spdlog/details/file_helper.h>
+#include <spdlog/details/null_mutex.h>
+#include <spdlog/details/synchronous_factory.h>
+#include <spdlog/sinks/base_sink.h>
+
+#include <chrono>
+#include <mutex>
+#include <string>
+
+namespace spdlog {
+namespace sinks {
+
+//
+// Rotating file sink based on size
+//
+template <typename Mutex>
+class rotating_file_sink final : public base_sink<Mutex> {
+public:
+    rotating_file_sink(filename_t base_filename,
+                       std::size_t max_size,
+                       std::size_t max_files,
+                       bool rotate_on_open = false,
+                       const file_event_handlers &event_handlers = {});
+    static filename_t calc_filename(const filename_t &filename, std::size_t index);
+    filename_t filename();
+    void rotate_now();
+
+protected:
+    void sink_it_(const details::log_msg &msg) override;
+    void flush_() override;
+
+private:
+    // Rotate files:
+    // log.txt -> log.1.txt
+    // log.1.txt -> log.2.txt
+    // log.2.txt -> log.3.txt
+    // log.3.txt -> delete
+    void rotate_();
+
+    // delete the target if exists, and rename the src file  to target
+    // return true on success, false otherwise.
+    bool rename_file_(const filename_t &src_filename, const filename_t &target_filename);
+
+    filename_t base_filename_;
+    std::size_t max_size_;
+    std::size_t max_files_;
+    std::size_t current_size_;
+    details::file_helper file_helper_;
+};
+
+using rotating_file_sink_mt = rotating_file_sink<std::mutex>;
+using rotating_file_sink_st = rotating_file_sink<details::null_mutex>;
+
+}  // namespace sinks
+
+//
+// factory functions
+//
+
+template <typename Factory = spdlog::synchronous_factory>
+inline std::shared_ptr<logger> rotating_logger_mt(const std::string &logger_name,
+                                                  const filename_t &filename,
+                                                  size_t max_file_size,
+                                                  size_t max_files,
+                                                  bool rotate_on_open = false,
+                                                  const file_event_handlers &event_handlers = {}) {
+    return Factory::template create<sinks::rotating_file_sink_mt>(
+        logger_name, filename, max_file_size, max_files, rotate_on_open, event_handlers);
+}
+
+template <typename Factory = spdlog::synchronous_factory>
+inline std::shared_ptr<logger> rotating_logger_st(const std::string &logger_name,
+                                                  const filename_t &filename,
+                                                  size_t max_file_size,
+                                                  size_t max_files,
+                                                  bool rotate_on_open = false,
+                                                  const file_event_handlers &event_handlers = {}) {
+    return Factory::template create<sinks::rotating_file_sink_st>(
+        logger_name, filename, max_file_size, max_files, rotate_on_open, event_handlers);
+}
+}  // namespace spdlog
+
+#ifdef SPDLOG_HEADER_ONLY
+    #include "rotating_file_sink-inl.h"
+#endif
diff --git a/ext/spdlog/include/spdlog/sinks/sink-inl.h b/ext/spdlog/include/spdlog/sinks/sink-inl.h
new file mode 100644
index 0000000..e4b2714
--- /dev/null
+++ b/ext/spdlog/include/spdlog/sinks/sink-inl.h
@@ -0,0 +1,22 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#ifndef SPDLOG_HEADER_ONLY
+    #include <spdlog/sinks/sink.h>
+#endif
+
+#include <spdlog/common.h>
+
+SPDLOG_INLINE bool spdlog::sinks::sink::should_log(spdlog::level::level_enum msg_level) const {
+    return msg_level >= level_.load(std::memory_order_relaxed);
+}
+
+SPDLOG_INLINE void spdlog::sinks::sink::set_level(level::level_enum log_level) {
+    level_.store(log_level, std::memory_order_relaxed);
+}
+
+SPDLOG_INLINE spdlog::level::level_enum spdlog::sinks::sink::level() const {
+    return static_cast<spdlog::level::level_enum>(level_.load(std::memory_order_relaxed));
+}
diff --git a/ext/spdlog/include/spdlog/sinks/sink.h b/ext/spdlog/include/spdlog/sinks/sink.h
new file mode 100644
index 0000000..5850685
--- /dev/null
+++ b/ext/spdlog/include/spdlog/sinks/sink.h
@@ -0,0 +1,34 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#include <spdlog/details/log_msg.h>
+#include <spdlog/formatter.h>
+
+namespace spdlog {
+
+namespace sinks {
+class SPDLOG_API sink {
+public:
+    virtual ~sink() = default;
+    virtual void log(const details::log_msg &msg) = 0;
+    virtual void flush() = 0;
+    virtual void set_pattern(const std::string &pattern) = 0;
+    virtual void set_formatter(std::unique_ptr<spdlog::formatter> sink_formatter) = 0;
+
+    void set_level(level::level_enum log_level);
+    level::level_enum level() const;
+    bool should_log(level::level_enum msg_level) const;
+
+protected:
+    // sink log level - default is all
+    level_t level_{level::trace};
+};
+
+}  // namespace sinks
+}  // namespace spdlog
+
+#ifdef SPDLOG_HEADER_ONLY
+    #include "sink-inl.h"
+#endif
diff --git a/ext/spdlog/include/spdlog/sinks/stdout_color_sinks-inl.h b/ext/spdlog/include/spdlog/sinks/stdout_color_sinks-inl.h
new file mode 100644
index 0000000..166e386
--- /dev/null
+++ b/ext/spdlog/include/spdlog/sinks/stdout_color_sinks-inl.h
@@ -0,0 +1,38 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#ifndef SPDLOG_HEADER_ONLY
+    #include <spdlog/sinks/stdout_color_sinks.h>
+#endif
+
+#include <spdlog/common.h>
+#include <spdlog/logger.h>
+
+namespace spdlog {
+
+template <typename Factory>
+SPDLOG_INLINE std::shared_ptr<logger> stdout_color_mt(const std::string &logger_name,
+                                                      color_mode mode) {
+    return Factory::template create<sinks::stdout_color_sink_mt>(logger_name, mode);
+}
+
+template <typename Factory>
+SPDLOG_INLINE std::shared_ptr<logger> stdout_color_st(const std::string &logger_name,
+                                                      color_mode mode) {
+    return Factory::template create<sinks::stdout_color_sink_st>(logger_name, mode);
+}
+
+template <typename Factory>
+SPDLOG_INLINE std::shared_ptr<logger> stderr_color_mt(const std::string &logger_name,
+                                                      color_mode mode) {
+    return Factory::template create<sinks::stderr_color_sink_mt>(logger_name, mode);
+}
+
+template <typename Factory>
+SPDLOG_INLINE std::shared_ptr<logger> stderr_color_st(const std::string &logger_name,
+                                                      color_mode mode) {
+    return Factory::template create<sinks::stderr_color_sink_st>(logger_name, mode);
+}
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/sinks/stdout_color_sinks.h b/ext/spdlog/include/spdlog/sinks/stdout_color_sinks.h
new file mode 100644
index 0000000..72991fe
--- /dev/null
+++ b/ext/spdlog/include/spdlog/sinks/stdout_color_sinks.h
@@ -0,0 +1,49 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#ifdef _WIN32
+    #include <spdlog/sinks/wincolor_sink.h>
+#else
+    #include <spdlog/sinks/ansicolor_sink.h>
+#endif
+
+#include <spdlog/details/synchronous_factory.h>
+
+namespace spdlog {
+namespace sinks {
+#ifdef _WIN32
+using stdout_color_sink_mt = wincolor_stdout_sink_mt;
+using stdout_color_sink_st = wincolor_stdout_sink_st;
+using stderr_color_sink_mt = wincolor_stderr_sink_mt;
+using stderr_color_sink_st = wincolor_stderr_sink_st;
+#else
+using stdout_color_sink_mt = ansicolor_stdout_sink_mt;
+using stdout_color_sink_st = ansicolor_stdout_sink_st;
+using stderr_color_sink_mt = ansicolor_stderr_sink_mt;
+using stderr_color_sink_st = ansicolor_stderr_sink_st;
+#endif
+}  // namespace sinks
+
+template <typename Factory = spdlog::synchronous_factory>
+std::shared_ptr<logger> stdout_color_mt(const std::string &logger_name,
+                                        color_mode mode = color_mode::automatic);
+
+template <typename Factory = spdlog::synchronous_factory>
+std::shared_ptr<logger> stdout_color_st(const std::string &logger_name,
+                                        color_mode mode = color_mode::automatic);
+
+template <typename Factory = spdlog::synchronous_factory>
+std::shared_ptr<logger> stderr_color_mt(const std::string &logger_name,
+                                        color_mode mode = color_mode::automatic);
+
+template <typename Factory = spdlog::synchronous_factory>
+std::shared_ptr<logger> stderr_color_st(const std::string &logger_name,
+                                        color_mode mode = color_mode::automatic);
+
+}  // namespace spdlog
+
+#ifdef SPDLOG_HEADER_ONLY
+    #include "stdout_color_sinks-inl.h"
+#endif
diff --git a/ext/spdlog/include/spdlog/sinks/stdout_sinks-inl.h b/ext/spdlog/include/spdlog/sinks/stdout_sinks-inl.h
new file mode 100644
index 0000000..dcb21d8
--- /dev/null
+++ b/ext/spdlog/include/spdlog/sinks/stdout_sinks-inl.h
@@ -0,0 +1,127 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#ifndef SPDLOG_HEADER_ONLY
+    #include <spdlog/sinks/stdout_sinks.h>
+#endif
+
+#include <memory>
+#include <spdlog/details/console_globals.h>
+#include <spdlog/pattern_formatter.h>
+#include <spdlog/details/os.h>
+
+#ifdef _WIN32
+    // under windows using fwrite to non-binary stream results in \r\r\n (see issue #1675)
+    // so instead we use ::FileWrite
+    #include <spdlog/details/windows_include.h>
+
+    #ifndef _USING_V110_SDK71_  // fileapi.h doesn't exist in winxp
+        #include <fileapi.h>    // WriteFile (..)
+    #endif
+
+    #include <io.h>     // _get_osfhandle(..)
+    #include <stdio.h>  // _fileno(..)
+#endif                  // _WIN32
+
+namespace spdlog {
+
+namespace sinks {
+
+template <typename ConsoleMutex>
+SPDLOG_INLINE stdout_sink_base<ConsoleMutex>::stdout_sink_base(FILE *file)
+    : mutex_(ConsoleMutex::mutex()),
+      file_(file),
+      formatter_(details::make_unique<spdlog::pattern_formatter>()) {
+#ifdef _WIN32
+    // get windows handle from the FILE* object
+
+    handle_ = reinterpret_cast<HANDLE>(::_get_osfhandle(::_fileno(file_)));
+
+    // don't throw to support cases where no console is attached,
+    // and let the log method to do nothing if (handle_ == INVALID_HANDLE_VALUE).
+    // throw only if non stdout/stderr target is requested (probably regular file and not console).
+    if (handle_ == INVALID_HANDLE_VALUE && file != stdout && file != stderr) {
+        throw_spdlog_ex("spdlog::stdout_sink_base: _get_osfhandle() failed", errno);
+    }
+#endif  // _WIN32
+}
+
+template <typename ConsoleMutex>
+SPDLOG_INLINE void stdout_sink_base<ConsoleMutex>::log(const details::log_msg &msg) {
+#ifdef _WIN32
+    if (handle_ == INVALID_HANDLE_VALUE) {
+        return;
+    }
+    std::lock_guard<mutex_t> lock(mutex_);
+    memory_buf_t formatted;
+    formatter_->format(msg, formatted);
+    auto size = static_cast<DWORD>(formatted.size());
+    DWORD bytes_written = 0;
+    bool ok = ::WriteFile(handle_, formatted.data(), size, &bytes_written, nullptr) != 0;
+    if (!ok) {
+        throw_spdlog_ex("stdout_sink_base: WriteFile() failed. GetLastError(): " +
+                        std::to_string(::GetLastError()));
+    }
+#else
+    std::lock_guard<mutex_t> lock(mutex_);
+    memory_buf_t formatted;
+    formatter_->format(msg, formatted);
+    details::os::fwrite_bytes(formatted.data(), formatted.size(), file_);
+#endif                // _WIN32
+    ::fflush(file_);  // flush every line to terminal
+}
+
+template <typename ConsoleMutex>
+SPDLOG_INLINE void stdout_sink_base<ConsoleMutex>::flush() {
+    std::lock_guard<mutex_t> lock(mutex_);
+    fflush(file_);
+}
+
+template <typename ConsoleMutex>
+SPDLOG_INLINE void stdout_sink_base<ConsoleMutex>::set_pattern(const std::string &pattern) {
+    std::lock_guard<mutex_t> lock(mutex_);
+    formatter_ = std::unique_ptr<spdlog::formatter>(new pattern_formatter(pattern));
+}
+
+template <typename ConsoleMutex>
+SPDLOG_INLINE void stdout_sink_base<ConsoleMutex>::set_formatter(
+    std::unique_ptr<spdlog::formatter> sink_formatter) {
+    std::lock_guard<mutex_t> lock(mutex_);
+    formatter_ = std::move(sink_formatter);
+}
+
+// stdout sink
+template <typename ConsoleMutex>
+SPDLOG_INLINE stdout_sink<ConsoleMutex>::stdout_sink()
+    : stdout_sink_base<ConsoleMutex>(stdout) {}
+
+// stderr sink
+template <typename ConsoleMutex>
+SPDLOG_INLINE stderr_sink<ConsoleMutex>::stderr_sink()
+    : stdout_sink_base<ConsoleMutex>(stderr) {}
+
+}  // namespace sinks
+
+// factory methods
+template <typename Factory>
+SPDLOG_INLINE std::shared_ptr<logger> stdout_logger_mt(const std::string &logger_name) {
+    return Factory::template create<sinks::stdout_sink_mt>(logger_name);
+}
+
+template <typename Factory>
+SPDLOG_INLINE std::shared_ptr<logger> stdout_logger_st(const std::string &logger_name) {
+    return Factory::template create<sinks::stdout_sink_st>(logger_name);
+}
+
+template <typename Factory>
+SPDLOG_INLINE std::shared_ptr<logger> stderr_logger_mt(const std::string &logger_name) {
+    return Factory::template create<sinks::stderr_sink_mt>(logger_name);
+}
+
+template <typename Factory>
+SPDLOG_INLINE std::shared_ptr<logger> stderr_logger_st(const std::string &logger_name) {
+    return Factory::template create<sinks::stderr_sink_st>(logger_name);
+}
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/sinks/stdout_sinks.h b/ext/spdlog/include/spdlog/sinks/stdout_sinks.h
new file mode 100644
index 0000000..6ef0996
--- /dev/null
+++ b/ext/spdlog/include/spdlog/sinks/stdout_sinks.h
@@ -0,0 +1,84 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#include <cstdio>
+#include <spdlog/details/console_globals.h>
+#include <spdlog/details/synchronous_factory.h>
+#include <spdlog/sinks/sink.h>
+
+#ifdef _WIN32
+    #include <spdlog/details/windows_include.h>
+#endif
+
+namespace spdlog {
+
+namespace sinks {
+
+template <typename ConsoleMutex>
+class stdout_sink_base : public sink {
+public:
+    using mutex_t = typename ConsoleMutex::mutex_t;
+    explicit stdout_sink_base(FILE *file);
+    ~stdout_sink_base() override = default;
+
+    stdout_sink_base(const stdout_sink_base &other) = delete;
+    stdout_sink_base(stdout_sink_base &&other) = delete;
+
+    stdout_sink_base &operator=(const stdout_sink_base &other) = delete;
+    stdout_sink_base &operator=(stdout_sink_base &&other) = delete;
+
+    void log(const details::log_msg &msg) override;
+    void flush() override;
+    void set_pattern(const std::string &pattern) override;
+
+    void set_formatter(std::unique_ptr<spdlog::formatter> sink_formatter) override;
+
+protected:
+    mutex_t &mutex_;
+    FILE *file_;
+    std::unique_ptr<spdlog::formatter> formatter_;
+#ifdef _WIN32
+    HANDLE handle_;
+#endif  // WIN32
+};
+
+template <typename ConsoleMutex>
+class stdout_sink : public stdout_sink_base<ConsoleMutex> {
+public:
+    stdout_sink();
+};
+
+template <typename ConsoleMutex>
+class stderr_sink : public stdout_sink_base<ConsoleMutex> {
+public:
+    stderr_sink();
+};
+
+using stdout_sink_mt = stdout_sink<details::console_mutex>;
+using stdout_sink_st = stdout_sink<details::console_nullmutex>;
+
+using stderr_sink_mt = stderr_sink<details::console_mutex>;
+using stderr_sink_st = stderr_sink<details::console_nullmutex>;
+
+}  // namespace sinks
+
+// factory methods
+template <typename Factory = spdlog::synchronous_factory>
+std::shared_ptr<logger> stdout_logger_mt(const std::string &logger_name);
+
+template <typename Factory = spdlog::synchronous_factory>
+std::shared_ptr<logger> stdout_logger_st(const std::string &logger_name);
+
+template <typename Factory = spdlog::synchronous_factory>
+std::shared_ptr<logger> stderr_logger_mt(const std::string &logger_name);
+
+template <typename Factory = spdlog::synchronous_factory>
+std::shared_ptr<logger> stderr_logger_st(const std::string &logger_name);
+
+}  // namespace spdlog
+
+#ifdef SPDLOG_HEADER_ONLY
+    #include "stdout_sinks-inl.h"
+#endif
diff --git a/ext/spdlog/include/spdlog/sinks/syslog_sink.h b/ext/spdlog/include/spdlog/sinks/syslog_sink.h
new file mode 100644
index 0000000..913d41b
--- /dev/null
+++ b/ext/spdlog/include/spdlog/sinks/syslog_sink.h
@@ -0,0 +1,104 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#include <spdlog/details/null_mutex.h>
+#include <spdlog/details/synchronous_factory.h>
+#include <spdlog/sinks/base_sink.h>
+
+#include <array>
+#include <string>
+#include <syslog.h>
+
+namespace spdlog {
+namespace sinks {
+/**
+ * Sink that write to syslog using the `syscall()` library call.
+ */
+template <typename Mutex>
+class syslog_sink : public base_sink<Mutex> {
+public:
+    syslog_sink(std::string ident, int syslog_option, int syslog_facility, bool enable_formatting)
+        : enable_formatting_{enable_formatting},
+          syslog_levels_{{/* spdlog::level::trace      */ LOG_DEBUG,
+                          /* spdlog::level::debug      */ LOG_DEBUG,
+                          /* spdlog::level::info       */ LOG_INFO,
+                          /* spdlog::level::warn       */ LOG_WARNING,
+                          /* spdlog::level::err        */ LOG_ERR,
+                          /* spdlog::level::critical   */ LOG_CRIT,
+                          /* spdlog::level::off        */ LOG_INFO}},
+          ident_{std::move(ident)} {
+        // set ident to be program name if empty
+        ::openlog(ident_.empty() ? nullptr : ident_.c_str(), syslog_option, syslog_facility);
+    }
+
+    ~syslog_sink() override { ::closelog(); }
+
+    syslog_sink(const syslog_sink &) = delete;
+    syslog_sink &operator=(const syslog_sink &) = delete;
+
+protected:
+    void sink_it_(const details::log_msg &msg) override {
+        string_view_t payload;
+        memory_buf_t formatted;
+        if (enable_formatting_) {
+            base_sink<Mutex>::formatter_->format(msg, formatted);
+            payload = string_view_t(formatted.data(), formatted.size());
+        } else {
+            payload = msg.payload;
+        }
+
+        size_t length = payload.size();
+        // limit to max int
+        if (length > static_cast<size_t>(std::numeric_limits<int>::max())) {
+            length = static_cast<size_t>(std::numeric_limits<int>::max());
+        }
+
+        ::syslog(syslog_prio_from_level(msg), "%.*s", static_cast<int>(length), payload.data());
+    }
+
+    void flush_() override {}
+    bool enable_formatting_ = false;
+
+    //
+    // Simply maps spdlog's log level to syslog priority level.
+    //
+    virtual int syslog_prio_from_level(const details::log_msg &msg) const {
+        return syslog_levels_.at(static_cast<levels_array::size_type>(msg.level));
+    }
+
+    using levels_array = std::array<int, 7>;
+    levels_array syslog_levels_;
+
+private:
+    // must store the ident because the man says openlog might use the pointer as
+    // is and not a string copy
+    const std::string ident_;
+};
+
+using syslog_sink_mt = syslog_sink<std::mutex>;
+using syslog_sink_st = syslog_sink<details::null_mutex>;
+}  // namespace sinks
+
+// Create and register a syslog logger
+template <typename Factory = spdlog::synchronous_factory>
+inline std::shared_ptr<logger> syslog_logger_mt(const std::string &logger_name,
+                                                const std::string &syslog_ident = "",
+                                                int syslog_option = 0,
+                                                int syslog_facility = LOG_USER,
+                                                bool enable_formatting = false) {
+    return Factory::template create<sinks::syslog_sink_mt>(logger_name, syslog_ident, syslog_option,
+                                                           syslog_facility, enable_formatting);
+}
+
+template <typename Factory = spdlog::synchronous_factory>
+inline std::shared_ptr<logger> syslog_logger_st(const std::string &logger_name,
+                                                const std::string &syslog_ident = "",
+                                                int syslog_option = 0,
+                                                int syslog_facility = LOG_USER,
+                                                bool enable_formatting = false) {
+    return Factory::template create<sinks::syslog_sink_st>(logger_name, syslog_ident, syslog_option,
+                                                           syslog_facility, enable_formatting);
+}
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/sinks/systemd_sink.h b/ext/spdlog/include/spdlog/sinks/systemd_sink.h
new file mode 100644
index 0000000..d2cd55f
--- /dev/null
+++ b/ext/spdlog/include/spdlog/sinks/systemd_sink.h
@@ -0,0 +1,121 @@
+// Copyright(c) 2019 ZVYAGIN.Alexander@gmail.com
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#include <spdlog/details/null_mutex.h>
+#include <spdlog/details/os.h>
+#include <spdlog/details/synchronous_factory.h>
+#include <spdlog/sinks/base_sink.h>
+
+#include <array>
+#ifndef SD_JOURNAL_SUPPRESS_LOCATION
+    #define SD_JOURNAL_SUPPRESS_LOCATION
+#endif
+#include <systemd/sd-journal.h>
+
+namespace spdlog {
+namespace sinks {
+
+/**
+ * Sink that write to systemd journal using the `sd_journal_send()` library call.
+ */
+template <typename Mutex>
+class systemd_sink : public base_sink<Mutex> {
+public:
+    systemd_sink(std::string ident = "", bool enable_formatting = false)
+        : ident_{std::move(ident)},
+          enable_formatting_{enable_formatting},
+          syslog_levels_{{/* spdlog::level::trace      */ LOG_DEBUG,
+                          /* spdlog::level::debug      */ LOG_DEBUG,
+                          /* spdlog::level::info       */ LOG_INFO,
+                          /* spdlog::level::warn       */ LOG_WARNING,
+                          /* spdlog::level::err        */ LOG_ERR,
+                          /* spdlog::level::critical   */ LOG_CRIT,
+                          /* spdlog::level::off        */ LOG_INFO}} {}
+
+    ~systemd_sink() override {}
+
+    systemd_sink(const systemd_sink &) = delete;
+    systemd_sink &operator=(const systemd_sink &) = delete;
+
+protected:
+    const std::string ident_;
+    bool enable_formatting_ = false;
+    using levels_array = std::array<int, 7>;
+    levels_array syslog_levels_;
+
+    void sink_it_(const details::log_msg &msg) override {
+        int err;
+        string_view_t payload;
+        memory_buf_t formatted;
+        if (enable_formatting_) {
+            base_sink<Mutex>::formatter_->format(msg, formatted);
+            payload = string_view_t(formatted.data(), formatted.size());
+        } else {
+            payload = msg.payload;
+        }
+
+        size_t length = payload.size();
+        // limit to max int
+        if (length > static_cast<size_t>(std::numeric_limits<int>::max())) {
+            length = static_cast<size_t>(std::numeric_limits<int>::max());
+        }
+
+        const string_view_t syslog_identifier = ident_.empty() ? msg.logger_name : ident_;
+
+        // Do not send source location if not available
+        if (msg.source.empty()) {
+            // Note: function call inside '()' to avoid macro expansion
+            err = (sd_journal_send)("MESSAGE=%.*s", static_cast<int>(length), payload.data(),
+                                    "PRIORITY=%d", syslog_level(msg.level),
+#ifndef SPDLOG_NO_THREAD_ID
+                                    "TID=%zu", msg.thread_id,
+#endif
+                                    "SYSLOG_IDENTIFIER=%.*s",
+                                    static_cast<int>(syslog_identifier.size()),
+                                    syslog_identifier.data(), nullptr);
+        } else {
+            err = (sd_journal_send)("MESSAGE=%.*s", static_cast<int>(length), payload.data(),
+                                    "PRIORITY=%d", syslog_level(msg.level),
+#ifndef SPDLOG_NO_THREAD_ID
+                                    "TID=%zu", msg.thread_id,
+#endif
+                                    "SYSLOG_IDENTIFIER=%.*s",
+                                    static_cast<int>(syslog_identifier.size()),
+                                    syslog_identifier.data(), "CODE_FILE=%s", msg.source.filename,
+                                    "CODE_LINE=%d", msg.source.line, "CODE_FUNC=%s",
+                                    msg.source.funcname, nullptr);
+        }
+
+        if (err) {
+            throw_spdlog_ex("Failed writing to systemd", errno);
+        }
+    }
+
+    int syslog_level(level::level_enum l) {
+        return syslog_levels_.at(static_cast<levels_array::size_type>(l));
+    }
+
+    void flush_() override {}
+};
+
+using systemd_sink_mt = systemd_sink<std::mutex>;
+using systemd_sink_st = systemd_sink<details::null_mutex>;
+}  // namespace sinks
+
+// Create and register a syslog logger
+template <typename Factory = spdlog::synchronous_factory>
+inline std::shared_ptr<logger> systemd_logger_mt(const std::string &logger_name,
+                                                 const std::string &ident = "",
+                                                 bool enable_formatting = false) {
+    return Factory::template create<sinks::systemd_sink_mt>(logger_name, ident, enable_formatting);
+}
+
+template <typename Factory = spdlog::synchronous_factory>
+inline std::shared_ptr<logger> systemd_logger_st(const std::string &logger_name,
+                                                 const std::string &ident = "",
+                                                 bool enable_formatting = false) {
+    return Factory::template create<sinks::systemd_sink_st>(logger_name, ident, enable_formatting);
+}
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/sinks/tcp_sink.h b/ext/spdlog/include/spdlog/sinks/tcp_sink.h
new file mode 100644
index 0000000..2534964
--- /dev/null
+++ b/ext/spdlog/include/spdlog/sinks/tcp_sink.h
@@ -0,0 +1,75 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#include <spdlog/common.h>
+#include <spdlog/details/null_mutex.h>
+#include <spdlog/sinks/base_sink.h>
+#ifdef _WIN32
+    #include <spdlog/details/tcp_client-windows.h>
+#else
+    #include <spdlog/details/tcp_client.h>
+#endif
+
+#include <chrono>
+#include <functional>
+#include <mutex>
+#include <string>
+
+#pragma once
+
+// Simple tcp client sink
+// Connects to remote address and send the formatted log.
+// Will attempt to reconnect if connection drops.
+// If more complicated behaviour is needed (i.e get responses), you can inherit it and override the
+// sink_it_ method.
+
+namespace spdlog {
+namespace sinks {
+
+struct tcp_sink_config {
+    std::string server_host;
+    int server_port;
+    bool lazy_connect = false;  // if true connect on first log call instead of on construction
+
+    tcp_sink_config(std::string host, int port)
+        : server_host{std::move(host)},
+          server_port{port} {}
+};
+
+template <typename Mutex>
+class tcp_sink : public spdlog::sinks::base_sink<Mutex> {
+public:
+    // connect to tcp host/port or throw if failed
+    // host can be hostname or ip address
+
+    explicit tcp_sink(tcp_sink_config sink_config)
+        : config_{std::move(sink_config)} {
+        if (!config_.lazy_connect) {
+            this->client_.connect(config_.server_host, config_.server_port);
+        }
+    }
+
+    ~tcp_sink() override = default;
+
+protected:
+    void sink_it_(const spdlog::details::log_msg &msg) override {
+        spdlog::memory_buf_t formatted;
+        spdlog::sinks::base_sink<Mutex>::formatter_->format(msg, formatted);
+        if (!client_.is_connected()) {
+            client_.connect(config_.server_host, config_.server_port);
+        }
+        client_.send(formatted.data(), formatted.size());
+    }
+
+    void flush_() override {}
+    tcp_sink_config config_;
+    details::tcp_client client_;
+};
+
+using tcp_sink_mt = tcp_sink<std::mutex>;
+using tcp_sink_st = tcp_sink<spdlog::details::null_mutex>;
+
+}  // namespace sinks
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/sinks/udp_sink.h b/ext/spdlog/include/spdlog/sinks/udp_sink.h
new file mode 100644
index 0000000..4bff0fd
--- /dev/null
+++ b/ext/spdlog/include/spdlog/sinks/udp_sink.h
@@ -0,0 +1,69 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#include <spdlog/common.h>
+#include <spdlog/details/null_mutex.h>
+#include <spdlog/sinks/base_sink.h>
+#ifdef _WIN32
+    #include <spdlog/details/udp_client-windows.h>
+#else
+    #include <spdlog/details/udp_client.h>
+#endif
+
+#include <chrono>
+#include <functional>
+#include <mutex>
+#include <string>
+
+// Simple udp client sink
+// Sends formatted log via udp
+
+namespace spdlog {
+namespace sinks {
+
+struct udp_sink_config {
+    std::string server_host;
+    uint16_t server_port;
+
+    udp_sink_config(std::string host, uint16_t port)
+        : server_host{std::move(host)},
+          server_port{port} {}
+};
+
+template <typename Mutex>
+class udp_sink : public spdlog::sinks::base_sink<Mutex> {
+public:
+    // host can be hostname or ip address
+    explicit udp_sink(udp_sink_config sink_config)
+        : client_{sink_config.server_host, sink_config.server_port} {}
+
+    ~udp_sink() override = default;
+
+protected:
+    void sink_it_(const spdlog::details::log_msg &msg) override {
+        spdlog::memory_buf_t formatted;
+        spdlog::sinks::base_sink<Mutex>::formatter_->format(msg, formatted);
+        client_.send(formatted.data(), formatted.size());
+    }
+
+    void flush_() override {}
+    details::udp_client client_;
+};
+
+using udp_sink_mt = udp_sink<std::mutex>;
+using udp_sink_st = udp_sink<spdlog::details::null_mutex>;
+
+}  // namespace sinks
+
+//
+// factory functions
+//
+template <typename Factory = spdlog::synchronous_factory>
+inline std::shared_ptr<logger> udp_logger_mt(const std::string &logger_name,
+                                             sinks::udp_sink_config skin_config) {
+    return Factory::template create<sinks::udp_sink_mt>(logger_name, skin_config);
+}
+
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/sinks/win_eventlog_sink.h b/ext/spdlog/include/spdlog/sinks/win_eventlog_sink.h
new file mode 100644
index 0000000..2c9b582
--- /dev/null
+++ b/ext/spdlog/include/spdlog/sinks/win_eventlog_sink.h
@@ -0,0 +1,260 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+// Writing to Windows Event Log requires the registry entries below to be present, with the
+// following modifications:
+// 1. <log_name>    should be replaced with your log name (e.g. your application name)
+// 2. <source_name> should be replaced with the specific source name and the key should be
+// duplicated for
+//                  each source used in the application
+//
+// Since typically modifications of this kind require elevation, it's better to do it as a part of
+// setup procedure. The snippet below uses mscoree.dll as the message file as it exists on most of
+// the Windows systems anyway and happens to contain the needed resource.
+//
+// You can also specify a custom message file if needed.
+// Please refer to Event Log functions descriptions in MSDN for more details on custom message
+// files.
+
+/*---------------------------------------------------------------------------------------
+
+Windows Registry Editor Version 5.00
+
+[HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Services\EventLog\<log_name>]
+
+[HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Services\EventLog\<log_name>\<source_name>]
+"TypesSupported"=dword:00000007
+"EventMessageFile"=hex(2):25,00,73,00,79,00,73,00,74,00,65,00,6d,00,72,00,6f,\
+  00,6f,00,74,00,25,00,5c,00,53,00,79,00,73,00,74,00,65,00,6d,00,33,00,32,00,\
+  5c,00,6d,00,73,00,63,00,6f,00,72,00,65,00,65,00,2e,00,64,00,6c,00,6c,00,00,\
+  00
+
+-----------------------------------------------------------------------------------------*/
+
+#pragma once
+
+#include <spdlog/details/null_mutex.h>
+#include <spdlog/sinks/base_sink.h>
+
+#include <spdlog/details/windows_include.h>
+#include <winbase.h>
+
+#include <mutex>
+#include <string>
+#include <vector>
+
+namespace spdlog {
+namespace sinks {
+
+namespace win_eventlog {
+
+namespace internal {
+
+struct local_alloc_t {
+    HLOCAL hlocal_;
+
+    SPDLOG_CONSTEXPR local_alloc_t() SPDLOG_NOEXCEPT : hlocal_(nullptr) {}
+
+    local_alloc_t(local_alloc_t const &) = delete;
+    local_alloc_t &operator=(local_alloc_t const &) = delete;
+
+    ~local_alloc_t() SPDLOG_NOEXCEPT {
+        if (hlocal_) {
+            LocalFree(hlocal_);
+        }
+    }
+};
+
+/** Windows error */
+struct win32_error : public spdlog_ex {
+    /** Formats an error report line: "user-message: error-code (system message)" */
+    static std::string format(std::string const &user_message, DWORD error_code = GetLastError()) {
+        std::string system_message;
+
+        local_alloc_t format_message_result{};
+        auto format_message_succeeded =
+            ::FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM |
+                                 FORMAT_MESSAGE_IGNORE_INSERTS,
+                             nullptr, error_code, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
+                             (LPSTR)&format_message_result.hlocal_, 0, nullptr);
+
+        if (format_message_succeeded && format_message_result.hlocal_) {
+            system_message = fmt_lib::format(" ({})", (LPSTR)format_message_result.hlocal_);
+        }
+
+        return fmt_lib::format("{}: {}{}", user_message, error_code, system_message);
+    }
+
+    explicit win32_error(std::string const &func_name, DWORD error = GetLastError())
+        : spdlog_ex(format(func_name, error)) {}
+};
+
+/** Wrapper for security identifiers (SID) on Windows */
+struct sid_t {
+    std::vector<char> buffer_;
+
+public:
+    sid_t() {}
+
+    /** creates a wrapped SID copy */
+    static sid_t duplicate_sid(PSID psid) {
+        if (!::IsValidSid(psid)) {
+            throw_spdlog_ex("sid_t::sid_t(): invalid SID received");
+        }
+
+        auto const sid_length{::GetLengthSid(psid)};
+
+        sid_t result;
+        result.buffer_.resize(sid_length);
+        if (!::CopySid(sid_length, (PSID)result.as_sid(), psid)) {
+            SPDLOG_THROW(win32_error("CopySid"));
+        }
+
+        return result;
+    }
+
+    /** Retrieves pointer to the internal buffer contents as SID* */
+    SID *as_sid() const { return buffer_.empty() ? nullptr : (SID *)buffer_.data(); }
+
+    /** Get SID for the current user */
+    static sid_t get_current_user_sid() {
+        /* create and init RAII holder for process token */
+        struct process_token_t {
+            HANDLE token_handle_ = INVALID_HANDLE_VALUE;
+            explicit process_token_t(HANDLE process) {
+                if (!::OpenProcessToken(process, TOKEN_QUERY, &token_handle_)) {
+                    SPDLOG_THROW(win32_error("OpenProcessToken"));
+                }
+            }
+
+            ~process_token_t() { ::CloseHandle(token_handle_); }
+
+        } current_process_token(
+            ::GetCurrentProcess());  // GetCurrentProcess returns pseudohandle, no leak here!
+
+        // Get the required size, this is expected to fail with ERROR_INSUFFICIENT_BUFFER and return
+        // the token size
+        DWORD tusize = 0;
+        if (::GetTokenInformation(current_process_token.token_handle_, TokenUser, NULL, 0,
+                                  &tusize)) {
+            SPDLOG_THROW(win32_error("GetTokenInformation should fail"));
+        }
+
+        // get user token
+        std::vector<unsigned char> buffer(static_cast<size_t>(tusize));
+        if (!::GetTokenInformation(current_process_token.token_handle_, TokenUser,
+                                   (LPVOID)buffer.data(), tusize, &tusize)) {
+            SPDLOG_THROW(win32_error("GetTokenInformation"));
+        }
+
+        // create a wrapper of the SID data as stored in the user token
+        return sid_t::duplicate_sid(((TOKEN_USER *)buffer.data())->User.Sid);
+    }
+};
+
+struct eventlog {
+    static WORD get_event_type(details::log_msg const &msg) {
+        switch (msg.level) {
+            case level::trace:
+            case level::debug:
+                return EVENTLOG_SUCCESS;
+
+            case level::info:
+                return EVENTLOG_INFORMATION_TYPE;
+
+            case level::warn:
+                return EVENTLOG_WARNING_TYPE;
+
+            case level::err:
+            case level::critical:
+            case level::off:
+                return EVENTLOG_ERROR_TYPE;
+
+            default:
+                return EVENTLOG_INFORMATION_TYPE;
+        }
+    }
+
+    static WORD get_event_category(details::log_msg const &msg) { return (WORD)msg.level; }
+};
+
+}  // namespace internal
+
+/*
+ * Windows Event Log sink
+ */
+template <typename Mutex>
+class win_eventlog_sink : public base_sink<Mutex> {
+private:
+    HANDLE hEventLog_{NULL};
+    internal::sid_t current_user_sid_;
+    std::string source_;
+    DWORD event_id_;
+
+    HANDLE event_log_handle() {
+        if (!hEventLog_) {
+            hEventLog_ = ::RegisterEventSourceA(nullptr, source_.c_str());
+            if (!hEventLog_ || hEventLog_ == (HANDLE)ERROR_ACCESS_DENIED) {
+                SPDLOG_THROW(internal::win32_error("RegisterEventSource"));
+            }
+        }
+
+        return hEventLog_;
+    }
+
+protected:
+    void sink_it_(const details::log_msg &msg) override {
+        using namespace internal;
+
+        bool succeeded;
+        memory_buf_t formatted;
+        base_sink<Mutex>::formatter_->format(msg, formatted);
+        formatted.push_back('\0');
+
+#ifdef SPDLOG_WCHAR_TO_UTF8_SUPPORT
+        wmemory_buf_t buf;
+        details::os::utf8_to_wstrbuf(string_view_t(formatted.data(), formatted.size()), buf);
+
+        LPCWSTR lp_wstr = buf.data();
+        succeeded = static_cast<bool>(::ReportEventW(
+            event_log_handle(), eventlog::get_event_type(msg), eventlog::get_event_category(msg),
+            event_id_, current_user_sid_.as_sid(), 1, 0, &lp_wstr, nullptr));
+#else
+        LPCSTR lp_str = formatted.data();
+        succeeded = static_cast<bool>(::ReportEventA(
+            event_log_handle(), eventlog::get_event_type(msg), eventlog::get_event_category(msg),
+            event_id_, current_user_sid_.as_sid(), 1, 0, &lp_str, nullptr));
+#endif
+
+        if (!succeeded) {
+            SPDLOG_THROW(win32_error("ReportEvent"));
+        }
+    }
+
+    void flush_() override {}
+
+public:
+    win_eventlog_sink(std::string const &source,
+                      DWORD event_id = 1000 /* according to mscoree.dll */)
+        : source_(source),
+          event_id_(event_id) {
+        try {
+            current_user_sid_ = internal::sid_t::get_current_user_sid();
+        } catch (...) {
+            // get_current_user_sid() is unlikely to fail and if it does, we can still proceed
+            // without current_user_sid but in the event log the record will have no user name
+        }
+    }
+
+    ~win_eventlog_sink() {
+        if (hEventLog_) DeregisterEventSource(hEventLog_);
+    }
+};
+
+}  // namespace win_eventlog
+
+using win_eventlog_sink_mt = win_eventlog::win_eventlog_sink<std::mutex>;
+using win_eventlog_sink_st = win_eventlog::win_eventlog_sink<details::null_mutex>;
+
+}  // namespace sinks
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/sinks/wincolor_sink-inl.h b/ext/spdlog/include/spdlog/sinks/wincolor_sink-inl.h
new file mode 100644
index 0000000..696db56
--- /dev/null
+++ b/ext/spdlog/include/spdlog/sinks/wincolor_sink-inl.h
@@ -0,0 +1,172 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#ifndef SPDLOG_HEADER_ONLY
+    #include <spdlog/sinks/wincolor_sink.h>
+#endif
+
+#include <spdlog/details/windows_include.h>
+#include <wincon.h>
+
+#include <spdlog/common.h>
+#include <spdlog/pattern_formatter.h>
+
+namespace spdlog {
+namespace sinks {
+template <typename ConsoleMutex>
+SPDLOG_INLINE wincolor_sink<ConsoleMutex>::wincolor_sink(void *out_handle, color_mode mode)
+    : out_handle_(out_handle),
+      mutex_(ConsoleMutex::mutex()),
+      formatter_(details::make_unique<spdlog::pattern_formatter>()) {
+    set_color_mode_impl(mode);
+    // set level colors
+    colors_[level::trace] = FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE;  // white
+    colors_[level::debug] = FOREGROUND_GREEN | FOREGROUND_BLUE;                   // cyan
+    colors_[level::info] = FOREGROUND_GREEN;                                      // green
+    colors_[level::warn] =
+        FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_INTENSITY;  // intense yellow
+    colors_[level::err] = FOREGROUND_RED | FOREGROUND_INTENSITY;   // intense red
+    colors_[level::critical] = BACKGROUND_RED | FOREGROUND_RED | FOREGROUND_GREEN |
+                               FOREGROUND_BLUE |
+                               FOREGROUND_INTENSITY;  // intense white on red background
+    colors_[level::off] = 0;
+}
+
+template <typename ConsoleMutex>
+SPDLOG_INLINE wincolor_sink<ConsoleMutex>::~wincolor_sink() {
+    this->flush();
+}
+
+// change the color for the given level
+template <typename ConsoleMutex>
+void SPDLOG_INLINE wincolor_sink<ConsoleMutex>::set_color(level::level_enum level,
+                                                          std::uint16_t color) {
+    std::lock_guard<mutex_t> lock(mutex_);
+    colors_[static_cast<size_t>(level)] = color;
+}
+
+template <typename ConsoleMutex>
+void SPDLOG_INLINE wincolor_sink<ConsoleMutex>::log(const details::log_msg &msg) {
+    if (out_handle_ == nullptr || out_handle_ == INVALID_HANDLE_VALUE) {
+        return;
+    }
+
+    std::lock_guard<mutex_t> lock(mutex_);
+    msg.color_range_start = 0;
+    msg.color_range_end = 0;
+    memory_buf_t formatted;
+    formatter_->format(msg, formatted);
+    if (should_do_colors_ && msg.color_range_end > msg.color_range_start) {
+        // before color range
+        print_range_(formatted, 0, msg.color_range_start);
+        // in color range
+        auto orig_attribs =
+            static_cast<WORD>(set_foreground_color_(colors_[static_cast<size_t>(msg.level)]));
+        print_range_(formatted, msg.color_range_start, msg.color_range_end);
+        // reset to orig colors
+        ::SetConsoleTextAttribute(static_cast<HANDLE>(out_handle_), orig_attribs);
+        print_range_(formatted, msg.color_range_end, formatted.size());
+    } else  // print without colors if color range is invalid (or color is disabled)
+    {
+        write_to_file_(formatted);
+    }
+}
+
+template <typename ConsoleMutex>
+void SPDLOG_INLINE wincolor_sink<ConsoleMutex>::flush() {
+    // windows console always flushed?
+}
+
+template <typename ConsoleMutex>
+void SPDLOG_INLINE wincolor_sink<ConsoleMutex>::set_pattern(const std::string &pattern) {
+    std::lock_guard<mutex_t> lock(mutex_);
+    formatter_ = std::unique_ptr<spdlog::formatter>(new pattern_formatter(pattern));
+}
+
+template <typename ConsoleMutex>
+void SPDLOG_INLINE
+wincolor_sink<ConsoleMutex>::set_formatter(std::unique_ptr<spdlog::formatter> sink_formatter) {
+    std::lock_guard<mutex_t> lock(mutex_);
+    formatter_ = std::move(sink_formatter);
+}
+
+template <typename ConsoleMutex>
+void SPDLOG_INLINE wincolor_sink<ConsoleMutex>::set_color_mode(color_mode mode) {
+    std::lock_guard<mutex_t> lock(mutex_);
+    set_color_mode_impl(mode);
+}
+
+template <typename ConsoleMutex>
+void SPDLOG_INLINE wincolor_sink<ConsoleMutex>::set_color_mode_impl(color_mode mode) {
+    if (mode == color_mode::automatic) {
+        // should do colors only if out_handle_  points to actual console.
+        DWORD console_mode;
+        bool in_console = ::GetConsoleMode(static_cast<HANDLE>(out_handle_), &console_mode) != 0;
+        should_do_colors_ = in_console;
+    } else {
+        should_do_colors_ = mode == color_mode::always ? true : false;
+    }
+}
+
+// set foreground color and return the orig console attributes (for resetting later)
+template <typename ConsoleMutex>
+std::uint16_t SPDLOG_INLINE
+wincolor_sink<ConsoleMutex>::set_foreground_color_(std::uint16_t attribs) {
+    CONSOLE_SCREEN_BUFFER_INFO orig_buffer_info;
+    if (!::GetConsoleScreenBufferInfo(static_cast<HANDLE>(out_handle_), &orig_buffer_info)) {
+        // just return white if failed getting console info
+        return FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE;
+    }
+
+    // change only the foreground bits (lowest 4 bits)
+    auto new_attribs = static_cast<WORD>(attribs) | (orig_buffer_info.wAttributes & 0xfff0);
+    auto ignored =
+        ::SetConsoleTextAttribute(static_cast<HANDLE>(out_handle_), static_cast<WORD>(new_attribs));
+    (void)(ignored);
+    return static_cast<std::uint16_t>(orig_buffer_info.wAttributes);  // return orig attribs
+}
+
+// print a range of formatted message to console
+template <typename ConsoleMutex>
+void SPDLOG_INLINE wincolor_sink<ConsoleMutex>::print_range_(const memory_buf_t &formatted,
+                                                             size_t start,
+                                                             size_t end) {
+    if (end > start) {
+#if defined(SPDLOG_UTF8_TO_WCHAR_CONSOLE)
+        wmemory_buf_t wformatted;
+        details::os::utf8_to_wstrbuf(string_view_t(formatted.data() + start, end - start),
+            wformatted);
+        auto size = static_cast<DWORD>(wformatted.size());
+        auto ignored = ::WriteConsoleW(static_cast<HANDLE>(out_handle_), wformatted.data(), size,
+            nullptr, nullptr);
+#else
+        auto size = static_cast<DWORD>(end - start);
+        auto ignored = ::WriteConsoleA(static_cast<HANDLE>(out_handle_), formatted.data() + start,
+                                       size, nullptr, nullptr);
+#endif
+        (void)(ignored);
+    }
+}
+
+template <typename ConsoleMutex>
+void SPDLOG_INLINE wincolor_sink<ConsoleMutex>::write_to_file_(const memory_buf_t &formatted) {
+    auto size = static_cast<DWORD>(formatted.size());
+    DWORD bytes_written = 0;
+    auto ignored = ::WriteFile(static_cast<HANDLE>(out_handle_), formatted.data(), size,
+                               &bytes_written, nullptr);
+    (void)(ignored);
+}
+
+// wincolor_stdout_sink
+template <typename ConsoleMutex>
+SPDLOG_INLINE wincolor_stdout_sink<ConsoleMutex>::wincolor_stdout_sink(color_mode mode)
+    : wincolor_sink<ConsoleMutex>(::GetStdHandle(STD_OUTPUT_HANDLE), mode) {}
+
+// wincolor_stderr_sink
+template <typename ConsoleMutex>
+SPDLOG_INLINE wincolor_stderr_sink<ConsoleMutex>::wincolor_stderr_sink(color_mode mode)
+    : wincolor_sink<ConsoleMutex>(::GetStdHandle(STD_ERROR_HANDLE), mode) {}
+}  // namespace sinks
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/sinks/wincolor_sink.h b/ext/spdlog/include/spdlog/sinks/wincolor_sink.h
new file mode 100644
index 0000000..8ba594c
--- /dev/null
+++ b/ext/spdlog/include/spdlog/sinks/wincolor_sink.h
@@ -0,0 +1,82 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#include <spdlog/common.h>
+#include <spdlog/details/console_globals.h>
+#include <spdlog/details/null_mutex.h>
+#include <spdlog/sinks/sink.h>
+
+#include <array>
+#include <cstdint>
+#include <memory>
+#include <mutex>
+#include <string>
+
+namespace spdlog {
+namespace sinks {
+/*
+ * Windows color console sink. Uses WriteConsoleA to write to the console with
+ * colors
+ */
+template <typename ConsoleMutex>
+class wincolor_sink : public sink {
+public:
+    wincolor_sink(void *out_handle, color_mode mode);
+    ~wincolor_sink() override;
+
+    wincolor_sink(const wincolor_sink &other) = delete;
+    wincolor_sink &operator=(const wincolor_sink &other) = delete;
+
+    // change the color for the given level
+    void set_color(level::level_enum level, std::uint16_t color);
+    void log(const details::log_msg &msg) final override;
+    void flush() final override;
+    void set_pattern(const std::string &pattern) override final;
+    void set_formatter(std::unique_ptr<spdlog::formatter> sink_formatter) override final;
+    void set_color_mode(color_mode mode);
+
+protected:
+    using mutex_t = typename ConsoleMutex::mutex_t;
+    void *out_handle_;
+    mutex_t &mutex_;
+    bool should_do_colors_;
+    std::unique_ptr<spdlog::formatter> formatter_;
+    std::array<std::uint16_t, level::n_levels> colors_;
+
+    // set foreground color and return the orig console attributes (for resetting later)
+    std::uint16_t set_foreground_color_(std::uint16_t attribs);
+
+    // print a range of formatted message to console
+    void print_range_(const memory_buf_t &formatted, size_t start, size_t end);
+
+    // in case we are redirected to file (not in console mode)
+    void write_to_file_(const memory_buf_t &formatted);
+
+    void set_color_mode_impl(color_mode mode);
+};
+
+template <typename ConsoleMutex>
+class wincolor_stdout_sink : public wincolor_sink<ConsoleMutex> {
+public:
+    explicit wincolor_stdout_sink(color_mode mode = color_mode::automatic);
+};
+
+template <typename ConsoleMutex>
+class wincolor_stderr_sink : public wincolor_sink<ConsoleMutex> {
+public:
+    explicit wincolor_stderr_sink(color_mode mode = color_mode::automatic);
+};
+
+using wincolor_stdout_sink_mt = wincolor_stdout_sink<details::console_mutex>;
+using wincolor_stdout_sink_st = wincolor_stdout_sink<details::console_nullmutex>;
+
+using wincolor_stderr_sink_mt = wincolor_stderr_sink<details::console_mutex>;
+using wincolor_stderr_sink_st = wincolor_stderr_sink<details::console_nullmutex>;
+}  // namespace sinks
+}  // namespace spdlog
+
+#ifdef SPDLOG_HEADER_ONLY
+    #include "wincolor_sink-inl.h"
+#endif
diff --git a/ext/spdlog/include/spdlog/spdlog-inl.h b/ext/spdlog/include/spdlog/spdlog-inl.h
new file mode 100644
index 0000000..97c3622
--- /dev/null
+++ b/ext/spdlog/include/spdlog/spdlog-inl.h
@@ -0,0 +1,92 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#ifndef SPDLOG_HEADER_ONLY
+    #include <spdlog/spdlog.h>
+#endif
+
+#include <spdlog/common.h>
+#include <spdlog/pattern_formatter.h>
+
+namespace spdlog {
+
+SPDLOG_INLINE void initialize_logger(std::shared_ptr<logger> logger) {
+    details::registry::instance().initialize_logger(std::move(logger));
+}
+
+SPDLOG_INLINE std::shared_ptr<logger> get(const std::string &name) {
+    return details::registry::instance().get(name);
+}
+
+SPDLOG_INLINE void set_formatter(std::unique_ptr<spdlog::formatter> formatter) {
+    details::registry::instance().set_formatter(std::move(formatter));
+}
+
+SPDLOG_INLINE void set_pattern(std::string pattern, pattern_time_type time_type) {
+    set_formatter(
+        std::unique_ptr<spdlog::formatter>(new pattern_formatter(std::move(pattern), time_type)));
+}
+
+SPDLOG_INLINE void enable_backtrace(size_t n_messages) {
+    details::registry::instance().enable_backtrace(n_messages);
+}
+
+SPDLOG_INLINE void disable_backtrace() { details::registry::instance().disable_backtrace(); }
+
+SPDLOG_INLINE void dump_backtrace() { default_logger_raw()->dump_backtrace(); }
+
+SPDLOG_INLINE level::level_enum get_level() { return default_logger_raw()->level(); }
+
+SPDLOG_INLINE bool should_log(level::level_enum log_level) {
+    return default_logger_raw()->should_log(log_level);
+}
+
+SPDLOG_INLINE void set_level(level::level_enum log_level) {
+    details::registry::instance().set_level(log_level);
+}
+
+SPDLOG_INLINE void flush_on(level::level_enum log_level) {
+    details::registry::instance().flush_on(log_level);
+}
+
+SPDLOG_INLINE void set_error_handler(void (*handler)(const std::string &msg)) {
+    details::registry::instance().set_error_handler(handler);
+}
+
+SPDLOG_INLINE void register_logger(std::shared_ptr<logger> logger) {
+    details::registry::instance().register_logger(std::move(logger));
+}
+
+SPDLOG_INLINE void apply_all(const std::function<void(std::shared_ptr<logger>)> &fun) {
+    details::registry::instance().apply_all(fun);
+}
+
+SPDLOG_INLINE void drop(const std::string &name) { details::registry::instance().drop(name); }
+
+SPDLOG_INLINE void drop_all() { details::registry::instance().drop_all(); }
+
+SPDLOG_INLINE void shutdown() { details::registry::instance().shutdown(); }
+
+SPDLOG_INLINE void set_automatic_registration(bool automatic_registration) {
+    details::registry::instance().set_automatic_registration(automatic_registration);
+}
+
+SPDLOG_INLINE std::shared_ptr<spdlog::logger> default_logger() {
+    return details::registry::instance().default_logger();
+}
+
+SPDLOG_INLINE spdlog::logger *default_logger_raw() {
+    return details::registry::instance().get_default_raw();
+}
+
+SPDLOG_INLINE void set_default_logger(std::shared_ptr<spdlog::logger> default_logger) {
+    details::registry::instance().set_default_logger(std::move(default_logger));
+}
+
+SPDLOG_INLINE void apply_logger_env_levels(std::shared_ptr<logger> logger) {
+    details::registry::instance().apply_logger_env_levels(std::move(logger));
+}
+
+}  // namespace spdlog
diff --git a/ext/spdlog/include/spdlog/spdlog.h b/ext/spdlog/include/spdlog/spdlog.h
new file mode 100644
index 0000000..a8afbce
--- /dev/null
+++ b/ext/spdlog/include/spdlog/spdlog.h
@@ -0,0 +1,352 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+// spdlog main header file.
+// see example.cpp for usage example
+
+#ifndef SPDLOG_H
+#define SPDLOG_H
+
+#pragma once
+
+#include <spdlog/common.h>
+#include <spdlog/details/registry.h>
+#include <spdlog/details/synchronous_factory.h>
+#include <spdlog/logger.h>
+#include <spdlog/version.h>
+
+#include <chrono>
+#include <functional>
+#include <memory>
+#include <string>
+
+namespace spdlog {
+
+using default_factory = synchronous_factory;
+
+// Create and register a logger with a templated sink type
+// The logger's level, formatter and flush level will be set according the
+// global settings.
+//
+// Example:
+//   spdlog::create<daily_file_sink_st>("logger_name", "dailylog_filename", 11, 59);
+template <typename Sink, typename... SinkArgs>
+inline std::shared_ptr<spdlog::logger> create(std::string logger_name, SinkArgs &&...sink_args) {
+    return default_factory::create<Sink>(std::move(logger_name),
+                                         std::forward<SinkArgs>(sink_args)...);
+}
+
+// Initialize and register a logger,
+// formatter and flush level will be set according the global settings.
+//
+// Useful for initializing manually created loggers with the global settings.
+//
+// Example:
+//   auto mylogger = std::make_shared<spdlog::logger>("mylogger", ...);
+//   spdlog::initialize_logger(mylogger);
+SPDLOG_API void initialize_logger(std::shared_ptr<logger> logger);
+
+// Return an existing logger or nullptr if a logger with such name doesn't
+// exist.
+// example: spdlog::get("my_logger")->info("hello {}", "world");
+SPDLOG_API std::shared_ptr<logger> get(const std::string &name);
+
+// Set global formatter. Each sink in each logger will get a clone of this object
+SPDLOG_API void set_formatter(std::unique_ptr<spdlog::formatter> formatter);
+
+// Set global format string.
+// example: spdlog::set_pattern("%Y-%m-%d %H:%M:%S.%e %l : %v");
+SPDLOG_API void set_pattern(std::string pattern,
+                            pattern_time_type time_type = pattern_time_type::local);
+
+// enable global backtrace support
+SPDLOG_API void enable_backtrace(size_t n_messages);
+
+// disable global backtrace support
+SPDLOG_API void disable_backtrace();
+
+// call dump backtrace on default logger
+SPDLOG_API void dump_backtrace();
+
+// Get global logging level
+SPDLOG_API level::level_enum get_level();
+
+// Set global logging level
+SPDLOG_API void set_level(level::level_enum log_level);
+
+// Determine whether the default logger should log messages with a certain level
+SPDLOG_API bool should_log(level::level_enum lvl);
+
+// Set global flush level
+SPDLOG_API void flush_on(level::level_enum log_level);
+
+// Start/Restart a periodic flusher thread
+// Warning: Use only if all your loggers are thread safe!
+template <typename Rep, typename Period>
+inline void flush_every(std::chrono::duration<Rep, Period> interval) {
+    details::registry::instance().flush_every(interval);
+}
+
+// Set global error handler
+SPDLOG_API void set_error_handler(void (*handler)(const std::string &msg));
+
+// Register the given logger with the given name
+SPDLOG_API void register_logger(std::shared_ptr<logger> logger);
+
+// Apply a user defined function on all registered loggers
+// Example:
+// spdlog::apply_all([&](std::shared_ptr<spdlog::logger> l) {l->flush();});
+SPDLOG_API void apply_all(const std::function<void(std::shared_ptr<logger>)> &fun);
+
+// Drop the reference to the given logger
+SPDLOG_API void drop(const std::string &name);
+
+// Drop all references from the registry
+SPDLOG_API void drop_all();
+
+// stop any running threads started by spdlog and clean registry loggers
+SPDLOG_API void shutdown();
+
+// Automatic registration of loggers when using spdlog::create() or spdlog::create_async
+SPDLOG_API void set_automatic_registration(bool automatic_registration);
+
+// API for using default logger (stdout_color_mt),
+// e.g: spdlog::info("Message {}", 1);
+//
+// The default logger object can be accessed using the spdlog::default_logger():
+// For example, to add another sink to it:
+// spdlog::default_logger()->sinks().push_back(some_sink);
+//
+// The default logger can replaced using spdlog::set_default_logger(new_logger).
+// For example, to replace it with a file logger.
+//
+// IMPORTANT:
+// The default API is thread safe (for _mt loggers), but:
+// set_default_logger() *should not* be used concurrently with the default API.
+// e.g do not call set_default_logger() from one thread while calling spdlog::info() from another.
+
+SPDLOG_API std::shared_ptr<spdlog::logger> default_logger();
+
+SPDLOG_API spdlog::logger *default_logger_raw();
+
+SPDLOG_API void set_default_logger(std::shared_ptr<spdlog::logger> default_logger);
+
+// Initialize logger level based on environment configs.
+//
+// Useful for applying SPDLOG_LEVEL to manually created loggers.
+//
+// Example:
+//   auto mylogger = std::make_shared<spdlog::logger>("mylogger", ...);
+//   spdlog::apply_logger_env_levels(mylogger);
+SPDLOG_API void apply_logger_env_levels(std::shared_ptr<logger> logger);
+
+template <typename... Args>
+inline void log(source_loc source,
+                level::level_enum lvl,
+                format_string_t<Args...> fmt,
+                Args &&...args) {
+    default_logger_raw()->log(source, lvl, fmt, std::forward<Args>(args)...);
+}
+
+template <typename... Args>
+inline void log(level::level_enum lvl, format_string_t<Args...> fmt, Args &&...args) {
+    default_logger_raw()->log(source_loc{}, lvl, fmt, std::forward<Args>(args)...);
+}
+
+template <typename... Args>
+inline void trace(format_string_t<Args...> fmt, Args &&...args) {
+    default_logger_raw()->trace(fmt, std::forward<Args>(args)...);
+}
+
+template <typename... Args>
+inline void debug(format_string_t<Args...> fmt, Args &&...args) {
+    default_logger_raw()->debug(fmt, std::forward<Args>(args)...);
+}
+
+template <typename... Args>
+inline void info(format_string_t<Args...> fmt, Args &&...args) {
+    default_logger_raw()->info(fmt, std::forward<Args>(args)...);
+}
+
+template <typename... Args>
+inline void warn(format_string_t<Args...> fmt, Args &&...args) {
+    default_logger_raw()->warn(fmt, std::forward<Args>(args)...);
+}
+
+template <typename... Args>
+inline void error(format_string_t<Args...> fmt, Args &&...args) {
+    default_logger_raw()->error(fmt, std::forward<Args>(args)...);
+}
+
+template <typename... Args>
+inline void critical(format_string_t<Args...> fmt, Args &&...args) {
+    default_logger_raw()->critical(fmt, std::forward<Args>(args)...);
+}
+
+template <typename T>
+inline void log(source_loc source, level::level_enum lvl, const T &msg) {
+    default_logger_raw()->log(source, lvl, msg);
+}
+
+template <typename T>
+inline void log(level::level_enum lvl, const T &msg) {
+    default_logger_raw()->log(lvl, msg);
+}
+
+#ifdef SPDLOG_WCHAR_TO_UTF8_SUPPORT
+template <typename... Args>
+inline void log(source_loc source,
+                level::level_enum lvl,
+                wformat_string_t<Args...> fmt,
+                Args &&...args) {
+    default_logger_raw()->log(source, lvl, fmt, std::forward<Args>(args)...);
+}
+
+template <typename... Args>
+inline void log(level::level_enum lvl, wformat_string_t<Args...> fmt, Args &&...args) {
+    default_logger_raw()->log(source_loc{}, lvl, fmt, std::forward<Args>(args)...);
+}
+
+template <typename... Args>
+inline void trace(wformat_string_t<Args...> fmt, Args &&...args) {
+    default_logger_raw()->trace(fmt, std::forward<Args>(args)...);
+}
+
+template <typename... Args>
+inline void debug(wformat_string_t<Args...> fmt, Args &&...args) {
+    default_logger_raw()->debug(fmt, std::forward<Args>(args)...);
+}
+
+template <typename... Args>
+inline void info(wformat_string_t<Args...> fmt, Args &&...args) {
+    default_logger_raw()->info(fmt, std::forward<Args>(args)...);
+}
+
+template <typename... Args>
+inline void warn(wformat_string_t<Args...> fmt, Args &&...args) {
+    default_logger_raw()->warn(fmt, std::forward<Args>(args)...);
+}
+
+template <typename... Args>
+inline void error(wformat_string_t<Args...> fmt, Args &&...args) {
+    default_logger_raw()->error(fmt, std::forward<Args>(args)...);
+}
+
+template <typename... Args>
+inline void critical(wformat_string_t<Args...> fmt, Args &&...args) {
+    default_logger_raw()->critical(fmt, std::forward<Args>(args)...);
+}
+#endif
+
+template <typename T>
+inline void trace(const T &msg) {
+    default_logger_raw()->trace(msg);
+}
+
+template <typename T>
+inline void debug(const T &msg) {
+    default_logger_raw()->debug(msg);
+}
+
+template <typename T>
+inline void info(const T &msg) {
+    default_logger_raw()->info(msg);
+}
+
+template <typename T>
+inline void warn(const T &msg) {
+    default_logger_raw()->warn(msg);
+}
+
+template <typename T>
+inline void error(const T &msg) {
+    default_logger_raw()->error(msg);
+}
+
+template <typename T>
+inline void critical(const T &msg) {
+    default_logger_raw()->critical(msg);
+}
+
+}  // namespace spdlog
+
+//
+// enable/disable log calls at compile time according to global level.
+//
+// define SPDLOG_ACTIVE_LEVEL to one of those (before including spdlog.h):
+// SPDLOG_LEVEL_TRACE,
+// SPDLOG_LEVEL_DEBUG,
+// SPDLOG_LEVEL_INFO,
+// SPDLOG_LEVEL_WARN,
+// SPDLOG_LEVEL_ERROR,
+// SPDLOG_LEVEL_CRITICAL,
+// SPDLOG_LEVEL_OFF
+//
+
+#ifndef SPDLOG_NO_SOURCE_LOC
+    #define SPDLOG_LOGGER_CALL(logger, level, ...) \
+        (logger)->log(spdlog::source_loc{__FILE__, __LINE__, SPDLOG_FUNCTION}, level, __VA_ARGS__)
+#else
+    #define SPDLOG_LOGGER_CALL(logger, level, ...) \
+        (logger)->log(spdlog::source_loc{}, level, __VA_ARGS__)
+#endif
+
+#if SPDLOG_ACTIVE_LEVEL <= SPDLOG_LEVEL_TRACE
+    #define SPDLOG_LOGGER_TRACE(logger, ...) \
+        SPDLOG_LOGGER_CALL(logger, spdlog::level::trace, __VA_ARGS__)
+    #define SPDLOG_TRACE(...) SPDLOG_LOGGER_TRACE(spdlog::default_logger_raw(), __VA_ARGS__)
+#else
+    #define SPDLOG_LOGGER_TRACE(logger, ...) (void)0
+    #define SPDLOG_TRACE(...) (void)0
+#endif
+
+#if SPDLOG_ACTIVE_LEVEL <= SPDLOG_LEVEL_DEBUG
+    #define SPDLOG_LOGGER_DEBUG(logger, ...) \
+        SPDLOG_LOGGER_CALL(logger, spdlog::level::debug, __VA_ARGS__)
+    #define SPDLOG_DEBUG(...) SPDLOG_LOGGER_DEBUG(spdlog::default_logger_raw(), __VA_ARGS__)
+#else
+    #define SPDLOG_LOGGER_DEBUG(logger, ...) (void)0
+    #define SPDLOG_DEBUG(...) (void)0
+#endif
+
+#if SPDLOG_ACTIVE_LEVEL <= SPDLOG_LEVEL_INFO
+    #define SPDLOG_LOGGER_INFO(logger, ...) \
+        SPDLOG_LOGGER_CALL(logger, spdlog::level::info, __VA_ARGS__)
+    #define SPDLOG_INFO(...) SPDLOG_LOGGER_INFO(spdlog::default_logger_raw(), __VA_ARGS__)
+#else
+    #define SPDLOG_LOGGER_INFO(logger, ...) (void)0
+    #define SPDLOG_INFO(...) (void)0
+#endif
+
+#if SPDLOG_ACTIVE_LEVEL <= SPDLOG_LEVEL_WARN
+    #define SPDLOG_LOGGER_WARN(logger, ...) \
+        SPDLOG_LOGGER_CALL(logger, spdlog::level::warn, __VA_ARGS__)
+    #define SPDLOG_WARN(...) SPDLOG_LOGGER_WARN(spdlog::default_logger_raw(), __VA_ARGS__)
+#else
+    #define SPDLOG_LOGGER_WARN(logger, ...) (void)0
+    #define SPDLOG_WARN(...) (void)0
+#endif
+
+#if SPDLOG_ACTIVE_LEVEL <= SPDLOG_LEVEL_ERROR
+    #define SPDLOG_LOGGER_ERROR(logger, ...) \
+        SPDLOG_LOGGER_CALL(logger, spdlog::level::err, __VA_ARGS__)
+    #define SPDLOG_ERROR(...) SPDLOG_LOGGER_ERROR(spdlog::default_logger_raw(), __VA_ARGS__)
+#else
+    #define SPDLOG_LOGGER_ERROR(logger, ...) (void)0
+    #define SPDLOG_ERROR(...) (void)0
+#endif
+
+#if SPDLOG_ACTIVE_LEVEL <= SPDLOG_LEVEL_CRITICAL
+    #define SPDLOG_LOGGER_CRITICAL(logger, ...) \
+        SPDLOG_LOGGER_CALL(logger, spdlog::level::critical, __VA_ARGS__)
+    #define SPDLOG_CRITICAL(...) SPDLOG_LOGGER_CRITICAL(spdlog::default_logger_raw(), __VA_ARGS__)
+#else
+    #define SPDLOG_LOGGER_CRITICAL(logger, ...) (void)0
+    #define SPDLOG_CRITICAL(...) (void)0
+#endif
+
+#ifdef SPDLOG_HEADER_ONLY
+    #include "spdlog-inl.h"
+#endif
+
+#endif  // SPDLOG_H
diff --git a/ext/spdlog/include/spdlog/stopwatch.h b/ext/spdlog/include/spdlog/stopwatch.h
new file mode 100644
index 0000000..54ab3d3
--- /dev/null
+++ b/ext/spdlog/include/spdlog/stopwatch.h
@@ -0,0 +1,66 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#include <chrono>
+#include <spdlog/fmt/fmt.h>
+
+// Stopwatch support for spdlog  (using std::chrono::steady_clock).
+// Displays elapsed seconds since construction as double.
+//
+// Usage:
+//
+// spdlog::stopwatch sw;
+// ...
+// spdlog::debug("Elapsed: {} seconds", sw);    =>  "Elapsed 0.005116733 seconds"
+// spdlog::info("Elapsed: {:.6} seconds", sw);  =>  "Elapsed 0.005163 seconds"
+//
+//
+// If other units are needed (e.g. millis instead of double), include "fmt/chrono.h" and use
+// "duration_cast<..>(sw.elapsed())":
+//
+// #include <spdlog/fmt/chrono.h>
+//..
+// using std::chrono::duration_cast;
+// using std::chrono::milliseconds;
+// spdlog::info("Elapsed {}", duration_cast<milliseconds>(sw.elapsed())); => "Elapsed 5ms"
+
+namespace spdlog {
+class stopwatch {
+    using clock = std::chrono::steady_clock;
+    std::chrono::time_point<clock> start_tp_;
+
+public:
+    stopwatch()
+        : start_tp_{clock::now()} {}
+
+    std::chrono::duration<double> elapsed() const {
+        return std::chrono::duration<double>(clock::now() - start_tp_);
+    }
+
+    std::chrono::milliseconds elapsed_ms() const {
+        return std::chrono::duration_cast<std::chrono::milliseconds>(clock::now() - start_tp_);
+    }
+
+    void reset() { start_tp_ = clock::now(); }
+};
+}  // namespace spdlog
+
+// Support for fmt formatting  (e.g. "{:012.9}" or just "{}")
+namespace
+#ifdef SPDLOG_USE_STD_FORMAT
+    std
+#else
+    fmt
+#endif
+{
+
+template <>
+struct formatter<spdlog::stopwatch> : formatter<double> {
+    template <typename FormatContext>
+    auto format(const spdlog::stopwatch &sw, FormatContext &ctx) const -> decltype(ctx.out()) {
+        return formatter<double>::format(sw.elapsed().count(), ctx);
+    }
+};
+}  // namespace std
diff --git a/ext/spdlog/include/spdlog/tweakme.h b/ext/spdlog/include/spdlog/tweakme.h
new file mode 100644
index 0000000..a47a907
--- /dev/null
+++ b/ext/spdlog/include/spdlog/tweakme.h
@@ -0,0 +1,141 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+///////////////////////////////////////////////////////////////////////////////
+//
+// Edit this file to squeeze more performance, and to customize supported
+// features
+//
+///////////////////////////////////////////////////////////////////////////////
+
+///////////////////////////////////////////////////////////////////////////////
+// Under Linux, the much faster CLOCK_REALTIME_COARSE clock can be used.
+// This clock is less accurate - can be off by dozens of millis - depending on
+// the kernel HZ.
+// Uncomment to use it instead of the regular clock.
+//
+// #define SPDLOG_CLOCK_COARSE
+///////////////////////////////////////////////////////////////////////////////
+
+///////////////////////////////////////////////////////////////////////////////
+// Uncomment if source location logging is not needed.
+// This will prevent spdlog from using __FILE__, __LINE__ and SPDLOG_FUNCTION
+//
+// #define SPDLOG_NO_SOURCE_LOC
+///////////////////////////////////////////////////////////////////////////////
+
+///////////////////////////////////////////////////////////////////////////////
+// Uncomment if thread id logging is not needed (i.e. no %t in the log pattern).
+// This will prevent spdlog from querying the thread id on each log call.
+//
+// WARNING: If the log pattern contains thread id (i.e, %t) while this flag is
+// on, zero will be logged as thread id.
+//
+// #define SPDLOG_NO_THREAD_ID
+///////////////////////////////////////////////////////////////////////////////
+
+///////////////////////////////////////////////////////////////////////////////
+// Uncomment to prevent spdlog from using thread local storage.
+//
+// WARNING: if your program forks, UNCOMMENT this flag to prevent undefined
+// thread ids in the children logs.
+//
+// #define SPDLOG_NO_TLS
+///////////////////////////////////////////////////////////////////////////////
+
+///////////////////////////////////////////////////////////////////////////////
+// Uncomment to avoid spdlog's usage of atomic log levels
+// Use only if your code never modifies a logger's log levels concurrently by
+// different threads.
+//
+// #define SPDLOG_NO_ATOMIC_LEVELS
+///////////////////////////////////////////////////////////////////////////////
+
+///////////////////////////////////////////////////////////////////////////////
+// Uncomment to enable usage of wchar_t for file names on Windows.
+//
+// #define SPDLOG_WCHAR_FILENAMES
+///////////////////////////////////////////////////////////////////////////////
+
+///////////////////////////////////////////////////////////////////////////////
+// Uncomment to override default eol ("\n" or "\r\n" under Linux/Windows)
+//
+// #define SPDLOG_EOL ";-)\n"
+///////////////////////////////////////////////////////////////////////////////
+
+///////////////////////////////////////////////////////////////////////////////
+// Uncomment to override default folder separators ("/" or "\\/" under
+// Linux/Windows). Each character in the string is treated as a different
+// separator.
+//
+// #define SPDLOG_FOLDER_SEPS "\\"
+///////////////////////////////////////////////////////////////////////////////
+
+///////////////////////////////////////////////////////////////////////////////
+// Uncomment to use your own copy of the fmt library instead of spdlog's copy.
+// In this case spdlog will try to include <fmt/format.h> so set your -I flag
+// accordingly.
+//
+// #define SPDLOG_FMT_EXTERNAL
+///////////////////////////////////////////////////////////////////////////////
+
+///////////////////////////////////////////////////////////////////////////////
+// Uncomment to use C++20 std::format instead of fmt.
+//
+// #define SPDLOG_USE_STD_FORMAT
+///////////////////////////////////////////////////////////////////////////////
+
+///////////////////////////////////////////////////////////////////////////////
+// Uncomment to enable wchar_t support (convert to utf8)
+//
+// #define SPDLOG_WCHAR_TO_UTF8_SUPPORT
+///////////////////////////////////////////////////////////////////////////////
+
+///////////////////////////////////////////////////////////////////////////////
+// Uncomment to prevent child processes from inheriting log file descriptors
+//
+// #define SPDLOG_PREVENT_CHILD_FD
+///////////////////////////////////////////////////////////////////////////////
+
+///////////////////////////////////////////////////////////////////////////////
+// Uncomment to customize level names (e.g. "MY TRACE")
+//
+// #define SPDLOG_LEVEL_NAMES { "MY TRACE", "MY DEBUG", "MY INFO", "MY WARNING", "MY ERROR", "MY
+// CRITICAL", "OFF" }
+///////////////////////////////////////////////////////////////////////////////
+
+///////////////////////////////////////////////////////////////////////////////
+// Uncomment to customize short level names (e.g. "MT")
+// These can be longer than one character.
+//
+// #define SPDLOG_SHORT_LEVEL_NAMES { "T", "D", "I", "W", "E", "C", "O" }
+///////////////////////////////////////////////////////////////////////////////
+
+///////////////////////////////////////////////////////////////////////////////
+// Uncomment to disable default logger creation.
+// This might save some (very) small initialization time if no default logger is needed.
+//
+// #define SPDLOG_DISABLE_DEFAULT_LOGGER
+///////////////////////////////////////////////////////////////////////////////
+
+///////////////////////////////////////////////////////////////////////////////
+// Uncomment and set to compile time level with zero cost (default is INFO).
+// Macros like SPDLOG_DEBUG(..), SPDLOG_INFO(..)  will expand to empty statements if not enabled
+//
+// #define SPDLOG_ACTIVE_LEVEL SPDLOG_LEVEL_INFO
+///////////////////////////////////////////////////////////////////////////////
+
+///////////////////////////////////////////////////////////////////////////////
+// Uncomment (and change if desired) macro to use for function names.
+// This is compiler dependent.
+// __PRETTY_FUNCTION__ might be nicer in clang/gcc, and __FUNCTION__ in msvc.
+// Defaults to __FUNCTION__ (should work on all compilers) if not defined.
+//
+// #ifdef __PRETTY_FUNCTION__
+// # define SPDLOG_FUNCTION __PRETTY_FUNCTION__
+// #else
+// # define SPDLOG_FUNCTION __FUNCTION__
+// #endif
+///////////////////////////////////////////////////////////////////////////////
diff --git a/ext/spdlog/include/spdlog/version.h b/ext/spdlog/include/spdlog/version.h
new file mode 100644
index 0000000..7c5e129
--- /dev/null
+++ b/ext/spdlog/include/spdlog/version.h
@@ -0,0 +1,11 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#pragma once
+
+#define SPDLOG_VER_MAJOR 1
+#define SPDLOG_VER_MINOR 15
+#define SPDLOG_VER_PATCH 0
+
+#define SPDLOG_TO_VERSION(major, minor, patch) (major * 10000 + minor * 100 + patch)
+#define SPDLOG_VERSION SPDLOG_TO_VERSION(SPDLOG_VER_MAJOR, SPDLOG_VER_MINOR, SPDLOG_VER_PATCH)
diff --git a/ext/spdlog/logos/jetbrains-variant-4.svg b/ext/spdlog/logos/jetbrains-variant-4.svg
new file mode 100644
index 0000000..e02b559
--- /dev/null
+++ b/ext/spdlog/logos/jetbrains-variant-4.svg
@@ -0,0 +1,43 @@
+<svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="263" height="147" viewBox="0 0 263 147">
+  <defs>
+    <linearGradient id="linear-gradient" x1="54.4568" y1="122.5936" x2="251.779" y2="10.2057" gradientUnits="userSpaceOnUse">
+      <stop offset="0" stop-color="#00adee"/>
+      <stop offset="1" stop-color="#9f76a6"/>
+    </linearGradient>
+    <linearGradient id="linear-gradient-2" x1="80.247" y1="38.7607" x2="241.2622" y2="10.9511" gradientUnits="userSpaceOnUse">
+      <stop offset="0" stop-color="#ec037c"/>
+      <stop offset="1" stop-color="#9f76a6"/>
+    </linearGradient>
+    <linearGradient id="linear-gradient-3" x1="75.7205" y1="33.5582" x2="127.8253" y2="123.9392" gradientUnits="userSpaceOnUse">
+      <stop offset="0" stop-color="#ec037c"/>
+      <stop offset="1" stop-color="#5c2d90"/>
+    </linearGradient>
+    <linearGradient id="linear-gradient-4" x1="7.4647" y1="44.578" x2="129.4543" y2="125.0813" gradientUnits="userSpaceOnUse">
+      <stop offset="0" stop-color="#44c7f4"/>
+      <stop offset="1" stop-color="#5c2d90"/>
+    </linearGradient>
+  </defs>
+  <g>
+    <path d="M261.1839,10.3622a9.6784,9.6784,0,0,0-14.7448-8.2463l0-.0006L20.6942,136.0746h0a4.6974,4.6974,0,1,0,4.1326,8.4348h0q0.09-.0467.1784-0.097l230.5273-125.25a9.653,9.653,0,0,0,1.1508-.6253l0.0332-.018-0.0014-.0023A9.6682,9.6682,0,0,0,261.1839,10.3622Z" fill="url(#linear-gradient)"/>
+    <path d="M261.1839,10.3622A9.6782,9.6782,0,0,0,251.5057.684q-0.2747,0-.5456.0157-0.25.0143-.4975,0.041L76.7981,25.4187A13.7347,13.7347,0,1,0,83.5355,51.983L252.8044,19.9512a9.6363,9.6363,0,0,0,1.0358-.196l0.02-.0039,0-.0008A9.6811,9.6811,0,0,0,261.1839,10.3622Z" fill="url(#linear-gradient-2)"/>
+    <path d="M145.2028,123.63a17.2372,17.2372,0,0,0-3.0637-9.4254L91.3045,32.3521A13.7366,13.7366,0,0,0,66.1132,42.9507h0a13.6332,13.6332,0,0,0,1.043,2.4984s45.2334,86.37,45.5824,86.9979q0.3089,0.556.6567,1.0861h0A17.32,17.32,0,0,0,145.2028,123.63Z" fill="url(#linear-gradient-3)"/>
+    <path d="M145.2028,123.63a17.2979,17.2979,0,0,0-7.63-13.9419h0a17.3061,17.3061,0,0,0-2.6994-1.4911L9.5484,38.9679a6.064,6.064,0,0,0-6.5074,10.187l114.3963,88.704A17.3191,17.3191,0,0,0,145.2028,123.63Z" fill="url(#linear-gradient-4)"/>
+    <g>
+      <rect x="69" y="51" width="70" height="70"/>
+      <g>
+        <rect x="75.038" y="107.8746" width="26.2498" height="4.375" fill="#fff"/>
+        <g>
+          <path d="M74.7429,69.4315L76.78,67.5082a2.31,2.31,0,0,0,1.7929,1.0594A1.33,1.33,0,0,0,79.8607,66.97V59.75h3.1456v7.2366a4.2386,4.2386,0,0,1-1.1246,3.2108,4.2989,4.2989,0,0,1-3.1293,1.1572A4.6592,4.6592,0,0,1,74.7429,69.4315Z" fill="#fff"/>
+          <path d="M83.7394,59.75h9.1761v2.673H86.8688v1.744H92.345v2.4937H86.8688V68.47H92.997v2.6893H83.7394V59.75Z" fill="#fff"/>
+          <path d="M97.049,62.5208H93.6426V59.75h9.9911v2.7708h-3.4227v8.6383H97.049V62.5208Z" fill="#fff"/>
+          <path d="M75.0363,73.8257h5.8511A4.2728,4.2728,0,0,1,84,74.8363a2.5675,2.5675,0,0,1,.7335,1.858v0.0326a2.6407,2.6407,0,0,1-1.76,2.5425,2.7686,2.7686,0,0,1,2.2655,2.7871v0.0326c0,1.9558-1.5973,3.1456-4.3191,3.1456H75.0363V73.8257Zm6.5846,3.5206c0-.6357-0.5052-0.9779-1.4343-0.9779h-2.07v2.0047h1.9884c0.9616,0,1.5158-.326,1.5158-0.9942V77.3463ZM80.5289,80.59H78.1166v2.1025h2.4448c0.9779,0,1.5158-.3749,1.5158-1.0431V81.6165C82.0773,80.9972,81.5883,80.59,80.5289,80.59Z" fill="#fff"/>
+          <path d="M85.7116,73.8257h5.3949a5.0512,5.0512,0,0,1,3.7161,1.2224,3.5623,3.5623,0,0,1,1.01,2.6567v0.0326a3.6146,3.6146,0,0,1-2.3469,3.5205l2.7218,3.9769H92.5733l-2.2981-3.4553H88.8735v3.4553H85.7116V73.8257Zm5.2644,5.4764a1.433,1.433,0,0,0,1.6951-1.3528V77.9167c0-.9128-0.6682-1.3691-1.7114-1.3691H88.8735v2.7545H90.976Z" fill="#fff"/>
+          <path d="M99.5324,73.7443H102.58l4.8571,11.4905h-3.39l-0.815-2.0536H98.8153L98,85.2348H94.6917Zm2.7707,6.9758-1.2712-3.2271L99.7443,80.72h2.5589Z" fill="#fff"/>
+          <path d="M107.8117,73.8257h3.1619V85.2348h-3.1619V73.8257Z" fill="#fff"/>
+          <path d="M111.7558,73.8257h2.95l4.694,6.0306V73.8257h3.1294V85.2348h-2.7545l-4.89-6.2587v6.2587h-3.1293V73.8257Z" fill="#fff"/>
+          <path d="M122.7274,83.54l1.76-2.1025a5.9106,5.9106,0,0,0,3.7,1.3691c0.8638,0,1.32-.2934,1.32-0.7824V81.9914c0-.489-0.3749-0.7335-1.9395-1.1084-2.4285-.5541-4.3029-1.2387-4.3029-3.5694V77.2811c0-2.1188,1.6788-3.6509,4.417-3.6509a7.1807,7.1807,0,0,1,4.694,1.5158l-1.5809,2.2329a5.6006,5.6006,0,0,0-3.1946-1.1246c-0.766,0-1.1409.31-1.1409,0.7334V77.02c0,0.5216.3912,0.75,1.9884,1.1083,2.6077,0.57,4.2377,1.418,4.2377,3.5531v0.0326c0,2.3307-1.8418,3.7161-4.6126,3.7161A7.9992,7.9992,0,0,1,122.7274,83.54Z" fill="#fff"/>
+        </g>
+      </g>
+    </g>
+  </g>
+</svg>
diff --git a/ext/spdlog/logos/spdlog.png b/ext/spdlog/logos/spdlog.png
new file mode 100644
index 0000000..23014c8
Binary files /dev/null and b/ext/spdlog/logos/spdlog.png differ
diff --git a/ext/spdlog/scripts/ci_setup_clang.sh b/ext/spdlog/scripts/ci_setup_clang.sh
new file mode 100755
index 0000000..140f9f9
--- /dev/null
+++ b/ext/spdlog/scripts/ci_setup_clang.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -ex
+
+VERSION=$1
+
+apt-get update
+apt-get install -y libc++-${VERSION}-dev libc++abi-${VERSION}-dev
+
+if [[ "${VERSION}" -ge 12 ]]; then
+    apt-get install -y --no-install-recommends libunwind-${VERSION}-dev
+fi
diff --git a/ext/spdlog/scripts/extract_version.py b/ext/spdlog/scripts/extract_version.py
new file mode 100755
index 0000000..79b5728
--- /dev/null
+++ b/ext/spdlog/scripts/extract_version.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+
+import os
+import re
+
+base_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+config_h = os.path.join(base_path, 'include', 'spdlog', 'version.h')
+data = {'MAJOR': 0, 'MINOR': 0, 'PATCH': 0}
+reg = re.compile(r'^\s*#define\s+SPDLOG_VER_([A-Z]+)\s+([0-9]+).*$')
+
+with open(config_h, 'r') as fp:
+    for l in fp:
+        m = reg.match(l)
+        if m:
+            data[m.group(1)] = int(m.group(2))
+
+print(f"{data['MAJOR']}.{data['MINOR']}.{data['PATCH']}")
diff --git a/ext/spdlog/scripts/format.sh b/ext/spdlog/scripts/format.sh
new file mode 100755
index 0000000..8eb01fb
--- /dev/null
+++ b/ext/spdlog/scripts/format.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+cd "$(dirname "$0")"/..
+pwd
+find_sources="find include src tests example bench -not ( -path include/spdlog/fmt/bundled -prune ) -type f -name *\.h -o -name *\.cpp"
+echo -n "Running dos2unix     "
+$find_sources | xargs -I {} sh -c "dos2unix '{}' 2>/dev/null; echo -n '.'"
+echo
+echo -n "Running clang-format "
+
+$find_sources | xargs -I {} sh -c "clang-format -i {}; echo -n '.'"
+
+echo
+echo -n "Running cmake-format "
+find . -type f -name "CMakeLists.txt" -o -name "*\.cmake"|grep -v bundled|grep -v build|xargs -I {} sh -c "cmake-format --line-width 120 --tab-size 4 --max-subgroups-hwrap 4 -i {}; echo -n '.'"
+echo
+
+
+
diff --git a/ext/spdlog/src/async.cpp b/ext/spdlog/src/async.cpp
new file mode 100644
index 0000000..026185a
--- /dev/null
+++ b/ext/spdlog/src/async.cpp
@@ -0,0 +1,11 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#ifndef SPDLOG_COMPILED_LIB
+    #error Please define SPDLOG_COMPILED_LIB to compile this file.
+#endif
+
+#include <spdlog/async.h>
+#include <spdlog/async_logger-inl.h>
+#include <spdlog/details/periodic_worker-inl.h>
+#include <spdlog/details/thread_pool-inl.h>
diff --git a/ext/spdlog/src/bundled_fmtlib_format.cpp b/ext/spdlog/src/bundled_fmtlib_format.cpp
new file mode 100644
index 0000000..462508d
--- /dev/null
+++ b/ext/spdlog/src/bundled_fmtlib_format.cpp
@@ -0,0 +1,48 @@
+// Slightly modified version of fmt lib's format.cc source file.
+// Copyright (c) 2012 - 2016, Victor Zverovich
+// All rights reserved.
+
+#ifndef SPDLOG_COMPILED_LIB
+    #error Please define SPDLOG_COMPILED_LIB to compile this file.
+#endif
+
+#if !defined(SPDLOG_FMT_EXTERNAL) && !defined(SPDLOG_USE_STD_FORMAT)
+
+    #include <spdlog/fmt/bundled/format-inl.h>
+
+FMT_BEGIN_NAMESPACE
+namespace detail {
+
+template FMT_API auto dragonbox::to_decimal(float x) noexcept
+    -> dragonbox::decimal_fp<float>;
+template FMT_API auto dragonbox::to_decimal(double x) noexcept
+    -> dragonbox::decimal_fp<double>;
+
+#ifndef FMT_STATIC_THOUSANDS_SEPARATOR
+template FMT_API locale_ref::locale_ref(const std::locale& loc);
+template FMT_API auto locale_ref::get<std::locale>() const -> std::locale;
+#endif
+
+// Explicit instantiations for char.
+
+template FMT_API auto thousands_sep_impl(locale_ref)
+    -> thousands_sep_result<char>;
+template FMT_API auto decimal_point_impl(locale_ref) -> char;
+
+template FMT_API void buffer<char>::append(const char*, const char*);
+
+template FMT_API void vformat_to(buffer<char>&, string_view,
+                                 typename vformat_args<>::type, locale_ref);
+
+// Explicit instantiations for wchar_t.
+
+template FMT_API auto thousands_sep_impl(locale_ref)
+    -> thousands_sep_result<wchar_t>;
+template FMT_API auto decimal_point_impl(locale_ref) -> wchar_t;
+
+template FMT_API void buffer<wchar_t>::append(const wchar_t*, const wchar_t*);
+
+}  // namespace detail
+FMT_END_NAMESPACE
+
+#endif  // !SPDLOG_FMT_EXTERNAL
diff --git a/ext/spdlog/src/cfg.cpp b/ext/spdlog/src/cfg.cpp
new file mode 100644
index 0000000..ebdea16
--- /dev/null
+++ b/ext/spdlog/src/cfg.cpp
@@ -0,0 +1,8 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#ifndef SPDLOG_COMPILED_LIB
+    #error Please define SPDLOG_COMPILED_LIB to compile this file.
+#endif
+
+#include <spdlog/cfg/helpers-inl.h>
diff --git a/ext/spdlog/src/color_sinks.cpp b/ext/spdlog/src/color_sinks.cpp
new file mode 100644
index 0000000..c44db19
--- /dev/null
+++ b/ext/spdlog/src/color_sinks.cpp
@@ -0,0 +1,55 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#ifndef SPDLOG_COMPILED_LIB
+    #error Please define SPDLOG_COMPILED_LIB to compile this file.
+#endif
+
+#include <mutex>
+
+#include <spdlog/async.h>
+#include <spdlog/details/null_mutex.h>
+//
+// color sinks
+//
+#ifdef _WIN32
+    #include <spdlog/sinks/wincolor_sink-inl.h>
+template class SPDLOG_API spdlog::sinks::wincolor_sink<spdlog::details::console_mutex>;
+template class SPDLOG_API spdlog::sinks::wincolor_sink<spdlog::details::console_nullmutex>;
+template class SPDLOG_API spdlog::sinks::wincolor_stdout_sink<spdlog::details::console_mutex>;
+template class SPDLOG_API spdlog::sinks::wincolor_stdout_sink<spdlog::details::console_nullmutex>;
+template class SPDLOG_API spdlog::sinks::wincolor_stderr_sink<spdlog::details::console_mutex>;
+template class SPDLOG_API spdlog::sinks::wincolor_stderr_sink<spdlog::details::console_nullmutex>;
+#else
+    #include "spdlog/sinks/ansicolor_sink-inl.h"
+template class SPDLOG_API spdlog::sinks::ansicolor_sink<spdlog::details::console_mutex>;
+template class SPDLOG_API spdlog::sinks::ansicolor_sink<spdlog::details::console_nullmutex>;
+template class SPDLOG_API spdlog::sinks::ansicolor_stdout_sink<spdlog::details::console_mutex>;
+template class SPDLOG_API spdlog::sinks::ansicolor_stdout_sink<spdlog::details::console_nullmutex>;
+template class SPDLOG_API spdlog::sinks::ansicolor_stderr_sink<spdlog::details::console_mutex>;
+template class SPDLOG_API spdlog::sinks::ansicolor_stderr_sink<spdlog::details::console_nullmutex>;
+#endif
+
+// factory methods for color loggers
+#include "spdlog/sinks/stdout_color_sinks-inl.h"
+template SPDLOG_API std::shared_ptr<spdlog::logger>
+spdlog::stdout_color_mt<spdlog::synchronous_factory>(const std::string &logger_name,
+                                                     color_mode mode);
+template SPDLOG_API std::shared_ptr<spdlog::logger>
+spdlog::stdout_color_st<spdlog::synchronous_factory>(const std::string &logger_name,
+                                                     color_mode mode);
+template SPDLOG_API std::shared_ptr<spdlog::logger>
+spdlog::stderr_color_mt<spdlog::synchronous_factory>(const std::string &logger_name,
+                                                     color_mode mode);
+template SPDLOG_API std::shared_ptr<spdlog::logger>
+spdlog::stderr_color_st<spdlog::synchronous_factory>(const std::string &logger_name,
+                                                     color_mode mode);
+
+template SPDLOG_API std::shared_ptr<spdlog::logger> spdlog::stdout_color_mt<spdlog::async_factory>(
+    const std::string &logger_name, color_mode mode);
+template SPDLOG_API std::shared_ptr<spdlog::logger> spdlog::stdout_color_st<spdlog::async_factory>(
+    const std::string &logger_name, color_mode mode);
+template SPDLOG_API std::shared_ptr<spdlog::logger> spdlog::stderr_color_mt<spdlog::async_factory>(
+    const std::string &logger_name, color_mode mode);
+template SPDLOG_API std::shared_ptr<spdlog::logger> spdlog::stderr_color_st<spdlog::async_factory>(
+    const std::string &logger_name, color_mode mode);
diff --git a/ext/spdlog/src/file_sinks.cpp b/ext/spdlog/src/file_sinks.cpp
new file mode 100644
index 0000000..04cb6c1
--- /dev/null
+++ b/ext/spdlog/src/file_sinks.cpp
@@ -0,0 +1,20 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#ifndef SPDLOG_COMPILED_LIB
+    #error Please define SPDLOG_COMPILED_LIB to compile this file.
+#endif
+
+#include <spdlog/details/file_helper-inl.h>
+#include <spdlog/details/null_mutex.h>
+#include <spdlog/sinks/base_sink-inl.h>
+#include <spdlog/sinks/basic_file_sink-inl.h>
+
+#include <mutex>
+
+template class SPDLOG_API spdlog::sinks::basic_file_sink<std::mutex>;
+template class SPDLOG_API spdlog::sinks::basic_file_sink<spdlog::details::null_mutex>;
+
+#include <spdlog/sinks/rotating_file_sink-inl.h>
+template class SPDLOG_API spdlog::sinks::rotating_file_sink<std::mutex>;
+template class SPDLOG_API spdlog::sinks::rotating_file_sink<spdlog::details::null_mutex>;
diff --git a/ext/spdlog/src/spdlog.cpp b/ext/spdlog/src/spdlog.cpp
new file mode 100644
index 0000000..9f8390b
--- /dev/null
+++ b/ext/spdlog/src/spdlog.cpp
@@ -0,0 +1,28 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#ifndef SPDLOG_COMPILED_LIB
+    #error Please define SPDLOG_COMPILED_LIB to compile this file.
+#endif
+
+#include <spdlog/common-inl.h>
+#include <spdlog/details/backtracer-inl.h>
+#include <spdlog/details/log_msg-inl.h>
+#include <spdlog/details/log_msg_buffer-inl.h>
+#include <spdlog/details/null_mutex.h>
+#include <spdlog/details/os-inl.h>
+#include <spdlog/details/registry-inl.h>
+#include <spdlog/logger-inl.h>
+#include <spdlog/pattern_formatter-inl.h>
+#include <spdlog/sinks/base_sink-inl.h>
+#include <spdlog/sinks/sink-inl.h>
+#include <spdlog/spdlog-inl.h>
+
+#include <mutex>
+
+// template instantiate logger constructor with sinks init list
+template SPDLOG_API spdlog::logger::logger(std::string name,
+                                           sinks_init_list::iterator begin,
+                                           sinks_init_list::iterator end);
+template class SPDLOG_API spdlog::sinks::base_sink<std::mutex>;
+template class SPDLOG_API spdlog::sinks::base_sink<spdlog::details::null_mutex>;
diff --git a/ext/spdlog/src/stdout_sinks.cpp b/ext/spdlog/src/stdout_sinks.cpp
new file mode 100644
index 0000000..bf4cfae
--- /dev/null
+++ b/ext/spdlog/src/stdout_sinks.cpp
@@ -0,0 +1,37 @@
+// Copyright(c) 2015-present, Gabi Melman & spdlog contributors.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+
+#ifndef SPDLOG_COMPILED_LIB
+    #error Please define SPDLOG_COMPILED_LIB to compile this file.
+#endif
+
+#include <mutex>
+
+#include <spdlog/async.h>
+#include <spdlog/details/null_mutex.h>
+#include <spdlog/sinks/stdout_sinks-inl.h>
+
+template class SPDLOG_API spdlog::sinks::stdout_sink_base<spdlog::details::console_mutex>;
+template class SPDLOG_API spdlog::sinks::stdout_sink_base<spdlog::details::console_nullmutex>;
+template class SPDLOG_API spdlog::sinks::stdout_sink<spdlog::details::console_mutex>;
+template class SPDLOG_API spdlog::sinks::stdout_sink<spdlog::details::console_nullmutex>;
+template class SPDLOG_API spdlog::sinks::stderr_sink<spdlog::details::console_mutex>;
+template class SPDLOG_API spdlog::sinks::stderr_sink<spdlog::details::console_nullmutex>;
+
+template SPDLOG_API std::shared_ptr<spdlog::logger>
+spdlog::stdout_logger_mt<spdlog::synchronous_factory>(const std::string &logger_name);
+template SPDLOG_API std::shared_ptr<spdlog::logger>
+spdlog::stdout_logger_st<spdlog::synchronous_factory>(const std::string &logger_name);
+template SPDLOG_API std::shared_ptr<spdlog::logger>
+spdlog::stderr_logger_mt<spdlog::synchronous_factory>(const std::string &logger_name);
+template SPDLOG_API std::shared_ptr<spdlog::logger>
+spdlog::stderr_logger_st<spdlog::synchronous_factory>(const std::string &logger_name);
+
+template SPDLOG_API std::shared_ptr<spdlog::logger> spdlog::stdout_logger_mt<spdlog::async_factory>(
+    const std::string &logger_name);
+template SPDLOG_API std::shared_ptr<spdlog::logger> spdlog::stdout_logger_st<spdlog::async_factory>(
+    const std::string &logger_name);
+template SPDLOG_API std::shared_ptr<spdlog::logger> spdlog::stderr_logger_mt<spdlog::async_factory>(
+    const std::string &logger_name);
+template SPDLOG_API std::shared_ptr<spdlog::logger> spdlog::stderr_logger_st<spdlog::async_factory>(
+    const std::string &logger_name);
diff --git a/ext/spdlog/tests/CMakeLists.txt b/ext/spdlog/tests/CMakeLists.txt
new file mode 100644
index 0000000..0a3d77e
--- /dev/null
+++ b/ext/spdlog/tests/CMakeLists.txt
@@ -0,0 +1,92 @@
+cmake_minimum_required(VERSION 3.11)
+project(spdlog_utests CXX)
+
+if(NOT TARGET spdlog)
+    # Stand-alone build
+    find_package(spdlog REQUIRED)
+endif()
+
+include(../cmake/utils.cmake)
+
+find_package(PkgConfig)
+if(PkgConfig_FOUND)
+    pkg_check_modules(systemd libsystemd)
+endif()
+
+find_package(Catch2 3 QUIET)
+if(Catch2_FOUND)
+    message(STATUS "Packaged version of Catch will be used.")
+else()
+    message(STATUS "Bundled version of Catch will be downloaded and used.")
+    include(FetchContent)
+    FetchContent_Declare(Catch2
+            GIT_REPOSITORY https://github.com/catchorg/Catch2.git
+            GIT_TAG 53d0d913a422d356b23dd927547febdf69ee9081 # v3.5.0
+    )
+    FetchContent_MakeAvailable(Catch2)
+endif()
+
+set(SPDLOG_UTESTS_SOURCES
+    test_file_helper.cpp
+    test_file_logging.cpp
+    test_daily_logger.cpp
+    test_misc.cpp
+    test_eventlog.cpp
+    test_pattern_formatter.cpp
+    test_async.cpp
+    test_registry.cpp
+    test_macros.cpp
+    utils.cpp
+    main.cpp
+    test_mpmc_q.cpp
+    test_dup_filter.cpp
+    test_fmt_helper.cpp
+    test_stdout_api.cpp
+    test_backtrace.cpp
+    test_create_dir.cpp
+    test_custom_callbacks.cpp
+    test_cfg.cpp
+    test_time_point.cpp
+    test_stopwatch.cpp
+    test_circular_q.cpp)
+
+if(NOT SPDLOG_NO_EXCEPTIONS)
+    list(APPEND SPDLOG_UTESTS_SOURCES test_errors.cpp)
+endif()
+
+if(systemd_FOUND)
+    list(APPEND SPDLOG_UTESTS_SOURCES test_systemd.cpp)
+endif()
+
+if(NOT SPDLOG_USE_STD_FORMAT)
+    list(APPEND SPDLOG_UTESTS_SOURCES test_bin_to_hex.cpp)
+endif()
+
+enable_testing()
+
+function(spdlog_prepare_test test_target spdlog_lib)
+    add_executable(${test_target} ${SPDLOG_UTESTS_SOURCES})
+    spdlog_enable_warnings(${test_target})
+    target_link_libraries(${test_target} PRIVATE ${spdlog_lib})
+    if(systemd_FOUND)
+        target_link_libraries(${test_target} PRIVATE ${systemd_LIBRARIES})
+    endif()
+    target_link_libraries(${test_target} PRIVATE Catch2::Catch2WithMain)
+    if(SPDLOG_SANITIZE_ADDRESS)
+        spdlog_enable_addr_sanitizer(${test_target})
+    elseif (SPDLOG_SANITIZE_THREAD)
+        spdlog_enable_thread_sanitizer(${test_target})
+    endif ()
+    add_test(NAME ${test_target} COMMAND ${test_target})
+    set_tests_properties(${test_target} PROPERTIES RUN_SERIAL ON)
+endfunction()
+
+# The compiled library tests
+if(SPDLOG_BUILD_TESTS OR SPDLOG_BUILD_ALL)
+    spdlog_prepare_test(spdlog-utests spdlog::spdlog)
+endif()
+
+# The header-only library version tests
+if(SPDLOG_BUILD_TESTS_HO OR SPDLOG_BUILD_ALL)
+    spdlog_prepare_test(spdlog-utests-ho spdlog::spdlog_header_only)
+endif()
diff --git a/ext/spdlog/tests/includes.h b/ext/spdlog/tests/includes.h
new file mode 100644
index 0000000..2e49a5c
--- /dev/null
+++ b/ext/spdlog/tests/includes.h
@@ -0,0 +1,42 @@
+#pragma once
+
+#if defined(__GNUC__) && __GNUC__ == 12
+    #pragma GCC diagnostic push
+    #pragma GCC diagnostic ignored "-Wmaybe-uninitialized"  // Workaround for GCC 12
+#endif
+#include <catch2/catch_all.hpp>
+#if defined(__GNUC__) && __GNUC__ == 12
+    #pragma GCC diagnostic pop
+#endif
+
+#include "utils.h"
+#include <chrono>
+#include <cstdio>
+#include <exception>
+#include <fstream>
+#include <iostream>
+#include <ostream>
+#include <sstream>
+#include <string>
+#include <iomanip>
+#include <stdlib.h>
+
+#define SPDLOG_ACTIVE_LEVEL SPDLOG_LEVEL_DEBUG
+
+#include "spdlog/spdlog.h"
+#include "spdlog/async.h"
+#include "spdlog/details/fmt_helper.h"
+#include "spdlog/details/os.h"
+
+#ifndef SPDLOG_NO_TLS
+    #include "spdlog/mdc.h"
+#endif
+
+#include "spdlog/sinks/basic_file_sink.h"
+#include "spdlog/sinks/daily_file_sink.h"
+#include "spdlog/sinks/null_sink.h"
+#include "spdlog/sinks/ostream_sink.h"
+#include "spdlog/sinks/rotating_file_sink.h"
+#include "spdlog/sinks/stdout_color_sinks.h"
+#include "spdlog/sinks/msvc_sink.h"
+#include "spdlog/pattern_formatter.h"
diff --git a/ext/spdlog/tests/main.cpp b/ext/spdlog/tests/main.cpp
new file mode 100644
index 0000000..a4a4ff1
--- /dev/null
+++ b/ext/spdlog/tests/main.cpp
@@ -0,0 +1,10 @@
+#if defined(__GNUC__) && __GNUC__ == 12
+    #pragma GCC diagnostic push
+    #pragma GCC diagnostic ignored "-Wmaybe-uninitialized"  // Workaround for GCC 12
+#endif
+
+#include <catch2/catch_all.hpp>
+
+#if defined(__GNUC__) && __GNUC__ == 12
+    #pragma GCC diagnostic pop
+#endif
diff --git a/ext/spdlog/tests/test_async.cpp b/ext/spdlog/tests/test_async.cpp
new file mode 100644
index 0000000..76fdd7c
--- /dev/null
+++ b/ext/spdlog/tests/test_async.cpp
@@ -0,0 +1,200 @@
+#include "includes.h"
+#include "spdlog/async.h"
+#include "spdlog/sinks/basic_file_sink.h"
+#include "test_sink.h"
+
+#define TEST_FILENAME "test_logs/async_test.log"
+
+TEST_CASE("basic async test ", "[async]") {
+    auto test_sink = std::make_shared<spdlog::sinks::test_sink_mt>();
+    size_t overrun_counter = 0;
+    size_t queue_size = 128;
+    size_t messages = 256;
+    {
+        auto tp = std::make_shared<spdlog::details::thread_pool>(queue_size, 1);
+        auto logger = std::make_shared<spdlog::async_logger>("as", test_sink, tp,
+                                                             spdlog::async_overflow_policy::block);
+        for (size_t i = 0; i < messages; i++) {
+            logger->info("Hello message #{}", i);
+        }
+        logger->flush();
+        overrun_counter = tp->overrun_counter();
+    }
+    REQUIRE(test_sink->msg_counter() == messages);
+    REQUIRE(test_sink->flush_counter() == 1);
+    REQUIRE(overrun_counter == 0);
+}
+
+TEST_CASE("discard policy ", "[async]") {
+    auto test_sink = std::make_shared<spdlog::sinks::test_sink_mt>();
+    test_sink->set_delay(std::chrono::milliseconds(1));
+    size_t queue_size = 4;
+    size_t messages = 1024;
+
+    auto tp = std::make_shared<spdlog::details::thread_pool>(queue_size, 1);
+    auto logger = std::make_shared<spdlog::async_logger>(
+        "as", test_sink, tp, spdlog::async_overflow_policy::overrun_oldest);
+    for (size_t i = 0; i < messages; i++) {
+        logger->info("Hello message");
+    }
+    REQUIRE(test_sink->msg_counter() < messages);
+    REQUIRE(tp->overrun_counter() > 0);
+}
+
+TEST_CASE("discard policy discard_new ", "[async]") {
+    auto test_sink = std::make_shared<spdlog::sinks::test_sink_mt>();
+    test_sink->set_delay(std::chrono::milliseconds(1));
+    size_t queue_size = 4;
+    size_t messages = 1024;
+
+    auto tp = std::make_shared<spdlog::details::thread_pool>(queue_size, 1);
+    auto logger = std::make_shared<spdlog::async_logger>(
+        "as", test_sink, tp, spdlog::async_overflow_policy::discard_new);
+    for (size_t i = 0; i < messages; i++) {
+        logger->info("Hello message");
+    }
+    REQUIRE(test_sink->msg_counter() < messages);
+    REQUIRE(tp->discard_counter() > 0);
+}
+
+TEST_CASE("discard policy using factory ", "[async]") {
+    size_t queue_size = 4;
+    size_t messages = 1024;
+    spdlog::init_thread_pool(queue_size, 1);
+
+    auto logger = spdlog::create_async_nb<spdlog::sinks::test_sink_mt>("as2");
+    auto test_sink = std::static_pointer_cast<spdlog::sinks::test_sink_mt>(logger->sinks()[0]);
+    test_sink->set_delay(std::chrono::milliseconds(3));
+
+    for (size_t i = 0; i < messages; i++) {
+        logger->info("Hello message");
+    }
+
+    REQUIRE(test_sink->msg_counter() < messages);
+    spdlog::drop_all();
+}
+
+TEST_CASE("flush", "[async]") {
+    auto test_sink = std::make_shared<spdlog::sinks::test_sink_mt>();
+    size_t queue_size = 256;
+    size_t messages = 256;
+    {
+        auto tp = std::make_shared<spdlog::details::thread_pool>(queue_size, 1);
+        auto logger = std::make_shared<spdlog::async_logger>("as", test_sink, tp,
+                                                             spdlog::async_overflow_policy::block);
+        for (size_t i = 0; i < messages; i++) {
+            logger->info("Hello message #{}", i);
+        }
+
+        logger->flush();
+    }
+    // std::this_thread::sleep_for(std::chrono::milliseconds(250));
+    REQUIRE(test_sink->msg_counter() == messages);
+    REQUIRE(test_sink->flush_counter() == 1);
+}
+
+TEST_CASE("async periodic flush", "[async]") {
+    auto logger = spdlog::create_async<spdlog::sinks::test_sink_mt>("as");
+    auto test_sink = std::static_pointer_cast<spdlog::sinks::test_sink_mt>(logger->sinks()[0]);
+
+    spdlog::flush_every(std::chrono::seconds(1));
+    std::this_thread::sleep_for(std::chrono::milliseconds(1700));
+    REQUIRE(test_sink->flush_counter() == 1);
+    spdlog::flush_every(std::chrono::seconds(0));
+    spdlog::drop_all();
+}
+
+TEST_CASE("tp->wait_empty() ", "[async]") {
+    auto test_sink = std::make_shared<spdlog::sinks::test_sink_mt>();
+    test_sink->set_delay(std::chrono::milliseconds(5));
+    size_t messages = 100;
+
+    auto tp = std::make_shared<spdlog::details::thread_pool>(messages, 2);
+    auto logger = std::make_shared<spdlog::async_logger>("as", test_sink, tp,
+                                                         spdlog::async_overflow_policy::block);
+    for (size_t i = 0; i < messages; i++) {
+        logger->info("Hello message #{}", i);
+    }
+    logger->flush();
+    tp.reset();
+
+    REQUIRE(test_sink->msg_counter() == messages);
+    REQUIRE(test_sink->flush_counter() == 1);
+}
+
+TEST_CASE("multi threads", "[async]") {
+    auto test_sink = std::make_shared<spdlog::sinks::test_sink_mt>();
+    size_t queue_size = 128;
+    size_t messages = 256;
+    size_t n_threads = 10;
+    {
+        auto tp = std::make_shared<spdlog::details::thread_pool>(queue_size, 1);
+        auto logger = std::make_shared<spdlog::async_logger>("as", test_sink, tp,
+                                                             spdlog::async_overflow_policy::block);
+
+        std::vector<std::thread> threads;
+        for (size_t i = 0; i < n_threads; i++) {
+            threads.emplace_back([logger, messages] {
+                for (size_t j = 0; j < messages; j++) {
+                    logger->info("Hello message #{}", j);
+                }
+            });
+            logger->flush();
+        }
+
+        for (auto &t : threads) {
+            t.join();
+        }
+    }
+
+    REQUIRE(test_sink->msg_counter() == messages * n_threads);
+    REQUIRE(test_sink->flush_counter() == n_threads);
+}
+
+TEST_CASE("to_file", "[async]") {
+    prepare_logdir();
+    size_t messages = 1024;
+    size_t tp_threads = 1;
+    spdlog::filename_t filename = SPDLOG_FILENAME_T(TEST_FILENAME);
+    {
+        auto file_sink = std::make_shared<spdlog::sinks::basic_file_sink_mt>(filename, true);
+        auto tp = std::make_shared<spdlog::details::thread_pool>(messages, tp_threads);
+        auto logger =
+            std::make_shared<spdlog::async_logger>("as", std::move(file_sink), std::move(tp));
+
+        for (size_t j = 0; j < messages; j++) {
+            logger->info("Hello message #{}", j);
+        }
+    }
+
+    require_message_count(TEST_FILENAME, messages);
+    auto contents = file_contents(TEST_FILENAME);
+    using spdlog::details::os::default_eol;
+    REQUIRE(ends_with(contents, spdlog::fmt_lib::format("Hello message #1023{}", default_eol)));
+}
+
+TEST_CASE("to_file multi-workers", "[async]") {
+    prepare_logdir();
+    size_t messages = 1024 * 10;
+    size_t tp_threads = 10;
+    spdlog::filename_t filename = SPDLOG_FILENAME_T(TEST_FILENAME);
+    {
+        auto file_sink = std::make_shared<spdlog::sinks::basic_file_sink_mt>(filename, true);
+        auto tp = std::make_shared<spdlog::details::thread_pool>(messages, tp_threads);
+        auto logger =
+            std::make_shared<spdlog::async_logger>("as", std::move(file_sink), std::move(tp));
+
+        for (size_t j = 0; j < messages; j++) {
+            logger->info("Hello message #{}", j);
+        }
+    }
+    require_message_count(TEST_FILENAME, messages);
+}
+
+TEST_CASE("bad_tp", "[async]") {
+    auto test_sink = std::make_shared<spdlog::sinks::test_sink_mt>();
+    std::shared_ptr<spdlog::details::thread_pool> const empty_tp;
+    auto logger = std::make_shared<spdlog::async_logger>("as", test_sink, empty_tp);
+    logger->info("Please throw an exception");
+    REQUIRE(test_sink->msg_counter() == 0);
+}
diff --git a/ext/spdlog/tests/test_backtrace.cpp b/ext/spdlog/tests/test_backtrace.cpp
new file mode 100644
index 0000000..4d78c0c
--- /dev/null
+++ b/ext/spdlog/tests/test_backtrace.cpp
@@ -0,0 +1,73 @@
+#include "includes.h"
+#include "test_sink.h"
+#include "spdlog/async.h"
+
+TEST_CASE("bactrace1", "[bactrace]") {
+    using spdlog::sinks::test_sink_st;
+    auto test_sink = std::make_shared<test_sink_st>();
+    size_t backtrace_size = 5;
+
+    spdlog::logger logger("test-backtrace", test_sink);
+    logger.set_pattern("%v");
+    logger.enable_backtrace(backtrace_size);
+
+    logger.info("info message");
+    for (int i = 0; i < 100; i++) logger.debug("debug message {}", i);
+
+    REQUIRE(test_sink->lines().size() == 1);
+    REQUIRE(test_sink->lines()[0] == "info message");
+
+    logger.dump_backtrace();
+    REQUIRE(test_sink->lines().size() == backtrace_size + 3);
+    REQUIRE(test_sink->lines()[1] == "****************** Backtrace Start ******************");
+    REQUIRE(test_sink->lines()[2] == "debug message 95");
+    REQUIRE(test_sink->lines()[3] == "debug message 96");
+    REQUIRE(test_sink->lines()[4] == "debug message 97");
+    REQUIRE(test_sink->lines()[5] == "debug message 98");
+    REQUIRE(test_sink->lines()[6] == "debug message 99");
+    REQUIRE(test_sink->lines()[7] == "****************** Backtrace End ********************");
+}
+
+TEST_CASE("bactrace-empty", "[bactrace]") {
+    using spdlog::sinks::test_sink_st;
+    auto test_sink = std::make_shared<test_sink_st>();
+    size_t backtrace_size = 5;
+
+    spdlog::logger logger("test-backtrace", test_sink);
+    logger.set_pattern("%v");
+    logger.enable_backtrace(backtrace_size);
+    logger.dump_backtrace();
+    REQUIRE(test_sink->lines().size() == 0);
+}
+
+TEST_CASE("bactrace-async", "[bactrace]") {
+    using spdlog::sinks::test_sink_mt;
+    auto test_sink = std::make_shared<test_sink_mt>();
+    using spdlog::details::os::sleep_for_millis;
+
+    size_t backtrace_size = 5;
+
+    spdlog::init_thread_pool(120, 1);
+    auto logger = std::make_shared<spdlog::async_logger>("test-bactrace-async", test_sink,
+                                                         spdlog::thread_pool());
+    logger->set_pattern("%v");
+    logger->enable_backtrace(backtrace_size);
+
+    logger->info("info message");
+    for (int i = 0; i < 100; i++) logger->debug("debug message {}", i);
+
+    sleep_for_millis(100);
+    REQUIRE(test_sink->lines().size() == 1);
+    REQUIRE(test_sink->lines()[0] == "info message");
+
+    logger->dump_backtrace();
+    sleep_for_millis(100);  //  give time for the async dump to complete
+    REQUIRE(test_sink->lines().size() == backtrace_size + 3);
+    REQUIRE(test_sink->lines()[1] == "****************** Backtrace Start ******************");
+    REQUIRE(test_sink->lines()[2] == "debug message 95");
+    REQUIRE(test_sink->lines()[3] == "debug message 96");
+    REQUIRE(test_sink->lines()[4] == "debug message 97");
+    REQUIRE(test_sink->lines()[5] == "debug message 98");
+    REQUIRE(test_sink->lines()[6] == "debug message 99");
+    REQUIRE(test_sink->lines()[7] == "****************** Backtrace End ********************");
+}
diff --git a/ext/spdlog/tests/test_bin_to_hex.cpp b/ext/spdlog/tests/test_bin_to_hex.cpp
new file mode 100644
index 0000000..45fc9fa
--- /dev/null
+++ b/ext/spdlog/tests/test_bin_to_hex.cpp
@@ -0,0 +1,97 @@
+#include "includes.h"
+#include "test_sink.h"
+#include "spdlog/fmt/bin_to_hex.h"
+
+TEST_CASE("to_hex", "[to_hex]") {
+    std::ostringstream oss;
+    auto oss_sink = std::make_shared<spdlog::sinks::ostream_sink_mt>(oss);
+    spdlog::logger oss_logger("oss", oss_sink);
+
+    std::vector<unsigned char> v{9, 0xa, 0xb, 0xc, 0xff, 0xff};
+    oss_logger.info("{}", spdlog::to_hex(v));
+
+    auto output = oss.str();
+    REQUIRE(ends_with(output,
+                      "0000: 09 0a 0b 0c ff ff" + std::string(spdlog::details::os::default_eol)));
+}
+
+TEST_CASE("to_hex_upper", "[to_hex]") {
+    std::ostringstream oss;
+    auto oss_sink = std::make_shared<spdlog::sinks::ostream_sink_mt>(oss);
+    spdlog::logger oss_logger("oss", oss_sink);
+
+    std::vector<unsigned char> v{9, 0xa, 0xb, 0xc, 0xff, 0xff};
+    oss_logger.info("{:X}", spdlog::to_hex(v));
+
+    auto output = oss.str();
+    REQUIRE(ends_with(output,
+                      "0000: 09 0A 0B 0C FF FF" + std::string(spdlog::details::os::default_eol)));
+}
+
+TEST_CASE("to_hex_no_delimiter", "[to_hex]") {
+    std::ostringstream oss;
+    auto oss_sink = std::make_shared<spdlog::sinks::ostream_sink_mt>(oss);
+    spdlog::logger oss_logger("oss", oss_sink);
+
+    std::vector<unsigned char> v{9, 0xa, 0xb, 0xc, 0xff, 0xff};
+    oss_logger.info("{:sX}", spdlog::to_hex(v));
+
+    auto output = oss.str();
+    REQUIRE(
+        ends_with(output, "0000: 090A0B0CFFFF" + std::string(spdlog::details::os::default_eol)));
+}
+
+TEST_CASE("to_hex_show_ascii", "[to_hex]") {
+    std::ostringstream oss;
+    auto oss_sink = std::make_shared<spdlog::sinks::ostream_sink_mt>(oss);
+    spdlog::logger oss_logger("oss", oss_sink);
+
+    std::vector<unsigned char> v{9, 0xa, 0xb, 0x41, 0xc, 0x4b, 0xff, 0xff};
+    oss_logger.info("{:Xsa}", spdlog::to_hex(v, 8));
+
+    REQUIRE(ends_with(oss.str(), "0000: 090A0B410C4BFFFF  ...A.K.." +
+                                     std::string(spdlog::details::os::default_eol)));
+}
+
+TEST_CASE("to_hex_different_size_per_line", "[to_hex]") {
+    std::ostringstream oss;
+    auto oss_sink = std::make_shared<spdlog::sinks::ostream_sink_mt>(oss);
+    spdlog::logger oss_logger("oss", oss_sink);
+
+    std::vector<unsigned char> v{9, 0xa, 0xb, 0x41, 0xc, 0x4b, 0xff, 0xff};
+
+    oss_logger.info("{:Xsa}", spdlog::to_hex(v, 10));
+    REQUIRE(ends_with(oss.str(), "0000: 090A0B410C4BFFFF  ...A.K.." +
+                                     std::string(spdlog::details::os::default_eol)));
+
+    oss_logger.info("{:Xs}", spdlog::to_hex(v, 10));
+    REQUIRE(ends_with(oss.str(),
+                      "0000: 090A0B410C4BFFFF" + std::string(spdlog::details::os::default_eol)));
+
+    oss_logger.info("{:Xsa}", spdlog::to_hex(v, 6));
+    REQUIRE(ends_with(
+        oss.str(), "0000: 090A0B410C4B  ...A.K" + std::string(spdlog::details::os::default_eol) +
+                       "0006: FFFF          .." + std::string(spdlog::details::os::default_eol)));
+
+    oss_logger.info("{:Xs}", spdlog::to_hex(v, 6));
+    REQUIRE(ends_with(oss.str(), "0000: 090A0B410C4B" +
+                                     std::string(spdlog::details::os::default_eol) + "0006: FFFF" +
+                                     std::string(spdlog::details::os::default_eol)));
+}
+
+TEST_CASE("to_hex_no_ascii", "[to_hex]") {
+    std::ostringstream oss;
+    auto oss_sink = std::make_shared<spdlog::sinks::ostream_sink_mt>(oss);
+    spdlog::logger oss_logger("oss", oss_sink);
+
+    std::vector<unsigned char> v{9, 0xa, 0xb, 0x41, 0xc, 0x4b, 0xff, 0xff};
+    oss_logger.info("{:Xs}", spdlog::to_hex(v, 8));
+
+    REQUIRE(ends_with(oss.str(),
+                      "0000: 090A0B410C4BFFFF" + std::string(spdlog::details::os::default_eol)));
+
+    oss_logger.info("{:Xsna}", spdlog::to_hex(v, 8));
+
+    REQUIRE(
+        ends_with(oss.str(), "090A0B410C4BFFFF" + std::string(spdlog::details::os::default_eol)));
+}
diff --git a/ext/spdlog/tests/test_cfg.cpp b/ext/spdlog/tests/test_cfg.cpp
new file mode 100644
index 0000000..684d870
--- /dev/null
+++ b/ext/spdlog/tests/test_cfg.cpp
@@ -0,0 +1,169 @@
+
+#include "includes.h"
+#include "test_sink.h"
+
+#include <spdlog/cfg/env.h>
+#include <spdlog/cfg/argv.h>
+
+using spdlog::cfg::load_argv_levels;
+using spdlog::cfg::load_env_levels;
+using spdlog::sinks::test_sink_st;
+
+TEST_CASE("env", "[cfg]") {
+    spdlog::drop("l1");
+    auto l1 = spdlog::create<test_sink_st>("l1");
+#ifdef CATCH_PLATFORM_WINDOWS
+    _putenv_s("SPDLOG_LEVEL", "l1=warn");
+#else
+    setenv("SPDLOG_LEVEL", "l1=warn", 1);
+#endif
+    load_env_levels();
+    REQUIRE(l1->level() == spdlog::level::warn);
+    spdlog::set_default_logger(spdlog::create<test_sink_st>("cfg-default"));
+    REQUIRE(spdlog::default_logger()->level() == spdlog::level::info);
+}
+
+TEST_CASE("argv1", "[cfg]") {
+    spdlog::drop("l1");
+    const char *argv[] = {"ignore", "SPDLOG_LEVEL=l1=warn"};
+    load_argv_levels(2, argv);
+    auto l1 = spdlog::create<spdlog::sinks::test_sink_st>("l1");
+    REQUIRE(l1->level() == spdlog::level::warn);
+    REQUIRE(spdlog::default_logger()->level() == spdlog::level::info);
+}
+
+TEST_CASE("argv2", "[cfg]") {
+    spdlog::drop("l1");
+    const char *argv[] = {"ignore", "SPDLOG_LEVEL=l1=warn,trace"};
+    load_argv_levels(2, argv);
+    auto l1 = spdlog::create<test_sink_st>("l1");
+    REQUIRE(l1->level() == spdlog::level::warn);
+    REQUIRE(spdlog::default_logger()->level() == spdlog::level::trace);
+}
+
+TEST_CASE("argv3", "[cfg]") {
+    spdlog::set_level(spdlog::level::trace);
+
+    spdlog::drop("l1");
+    const char *argv[] = {"ignore", "SPDLOG_LEVEL=junk_name=warn"};
+    load_argv_levels(2, argv);
+    auto l1 = spdlog::create<test_sink_st>("l1");
+    REQUIRE(l1->level() == spdlog::level::trace);
+    REQUIRE(spdlog::default_logger()->level() == spdlog::level::trace);
+}
+
+TEST_CASE("argv4", "[cfg]") {
+    spdlog::set_level(spdlog::level::info);
+    spdlog::drop("l1");
+    const char *argv[] = {"ignore", "SPDLOG_LEVEL=junk"};
+    load_argv_levels(2, argv);
+    auto l1 = spdlog::create<test_sink_st>("l1");
+    REQUIRE(l1->level() == spdlog::level::info);
+}
+
+TEST_CASE("argv5", "[cfg]") {
+    spdlog::set_level(spdlog::level::info);
+    spdlog::drop("l1");
+    const char *argv[] = {"ignore", "ignore", "SPDLOG_LEVEL=l1=warn,trace"};
+    load_argv_levels(3, argv);
+    auto l1 = spdlog::create<test_sink_st>("l1");
+    REQUIRE(l1->level() == spdlog::level::warn);
+    REQUIRE(spdlog::default_logger()->level() == spdlog::level::trace);
+    spdlog::set_level(spdlog::level::info);
+}
+
+TEST_CASE("argv6", "[cfg]") {
+    spdlog::set_level(spdlog::level::err);
+    const char *argv[] = {""};
+    load_argv_levels(1, argv);
+    REQUIRE(spdlog::default_logger()->level() == spdlog::level::err);
+    spdlog::set_level(spdlog::level::info);
+}
+
+TEST_CASE("argv7", "[cfg]") {
+    spdlog::set_level(spdlog::level::err);
+    const char *argv[] = {""};
+    load_argv_levels(0, argv);
+    REQUIRE(spdlog::default_logger()->level() == spdlog::level::err);
+    spdlog::set_level(spdlog::level::info);
+}
+
+TEST_CASE("level-not-set-test1", "[cfg]") {
+    spdlog::drop("l1");
+    const char *argv[] = {"ignore", ""};
+    load_argv_levels(2, argv);
+    auto l1 = spdlog::create<spdlog::sinks::test_sink_st>("l1");
+    l1->set_level(spdlog::level::trace);
+    REQUIRE(l1->level() == spdlog::level::trace);
+    REQUIRE(spdlog::default_logger()->level() == spdlog::level::info);
+}
+
+TEST_CASE("level-not-set-test2", "[cfg]") {
+    spdlog::drop("l1");
+    spdlog::drop("l2");
+    const char *argv[] = {"ignore", "SPDLOG_LEVEL=l1=trace"};
+
+    auto l1 = spdlog::create<spdlog::sinks::test_sink_st>("l1");
+    l1->set_level(spdlog::level::warn);
+    auto l2 = spdlog::create<spdlog::sinks::test_sink_st>("l2");
+    l2->set_level(spdlog::level::warn);
+
+    load_argv_levels(2, argv);
+
+    REQUIRE(l1->level() == spdlog::level::trace);
+    REQUIRE(l2->level() == spdlog::level::warn);
+    REQUIRE(spdlog::default_logger()->level() == spdlog::level::info);
+}
+
+TEST_CASE("level-not-set-test3", "[cfg]") {
+    spdlog::drop("l1");
+    spdlog::drop("l2");
+    const char *argv[] = {"ignore", "SPDLOG_LEVEL=l1=trace"};
+
+    load_argv_levels(2, argv);
+
+    auto l1 = spdlog::create<spdlog::sinks::test_sink_st>("l1");
+    auto l2 = spdlog::create<spdlog::sinks::test_sink_st>("l2");
+
+    REQUIRE(l1->level() == spdlog::level::trace);
+    REQUIRE(l2->level() == spdlog::level::info);
+    REQUIRE(spdlog::default_logger()->level() == spdlog::level::info);
+}
+
+TEST_CASE("level-not-set-test4", "[cfg]") {
+    spdlog::drop("l1");
+    spdlog::drop("l2");
+    const char *argv[] = {"ignore", "SPDLOG_LEVEL=l1=trace,warn"};
+
+    load_argv_levels(2, argv);
+
+    auto l1 = spdlog::create<spdlog::sinks::test_sink_st>("l1");
+    auto l2 = spdlog::create<spdlog::sinks::test_sink_st>("l2");
+
+    REQUIRE(l1->level() == spdlog::level::trace);
+    REQUIRE(l2->level() == spdlog::level::warn);
+    REQUIRE(spdlog::default_logger()->level() == spdlog::level::warn);
+}
+
+TEST_CASE("level-not-set-test5", "[cfg]") {
+    spdlog::drop("l1");
+    spdlog::drop("l2");
+    const char *argv[] = {"ignore", "SPDLOG_LEVEL=l1=junk,warn"};
+
+    load_argv_levels(2, argv);
+
+    auto l1 = spdlog::create<spdlog::sinks::test_sink_st>("l1");
+    auto l2 = spdlog::create<spdlog::sinks::test_sink_st>("l2");
+
+    REQUIRE(l1->level() == spdlog::level::warn);
+    REQUIRE(l2->level() == spdlog::level::warn);
+    REQUIRE(spdlog::default_logger()->level() == spdlog::level::warn);
+}
+
+TEST_CASE("restore-to-default", "[cfg]") {
+    spdlog::drop("l1");
+    spdlog::drop("l2");
+    const char *argv[] = {"ignore", "SPDLOG_LEVEL=info"};
+    load_argv_levels(2, argv);
+    REQUIRE(spdlog::default_logger()->level() == spdlog::level::info);
+}
diff --git a/ext/spdlog/tests/test_circular_q.cpp b/ext/spdlog/tests/test_circular_q.cpp
new file mode 100644
index 0000000..c8b02d3
--- /dev/null
+++ b/ext/spdlog/tests/test_circular_q.cpp
@@ -0,0 +1,50 @@
+#include "includes.h"
+#include "spdlog/details/circular_q.h"
+
+using q_type = spdlog::details::circular_q<size_t>;
+TEST_CASE("test_size", "[circular_q]") {
+    const size_t q_size = 4;
+    q_type q(q_size);
+    REQUIRE(q.size() == 0);
+    REQUIRE(q.empty() == true);
+    for (size_t i = 0; i < q_size; i++) {
+        q.push_back(std::move(i));
+    }
+    REQUIRE(q.size() == q_size);
+    q.push_back(999);
+    REQUIRE(q.size() == q_size);
+}
+
+TEST_CASE("test_rolling", "[circular_q]") {
+    const size_t q_size = 4;
+    q_type q(q_size);
+
+    for (size_t i = 0; i < q_size + 2; i++) {
+        q.push_back(std::move(i));
+    }
+
+    REQUIRE(q.size() == q_size);
+
+    REQUIRE(q.front() == 2);
+    q.pop_front();
+
+    REQUIRE(q.front() == 3);
+    q.pop_front();
+
+    REQUIRE(q.front() == 4);
+    q.pop_front();
+
+    REQUIRE(q.front() == 5);
+    q.pop_front();
+
+    REQUIRE(q.empty());
+
+    q.push_back(6);
+    REQUIRE(q.front() == 6);
+}
+
+TEST_CASE("test_empty", "[circular_q]") {
+    q_type q(0);
+    q.push_back(1);
+    REQUIRE(q.empty());
+}
\ No newline at end of file
diff --git a/ext/spdlog/tests/test_create_dir.cpp b/ext/spdlog/tests/test_create_dir.cpp
new file mode 100644
index 0000000..fd04033
--- /dev/null
+++ b/ext/spdlog/tests/test_create_dir.cpp
@@ -0,0 +1,144 @@
+/*
+ * This content is released under the MIT License as specified in
+ * https://raw.githubusercontent.com/gabime/spdlog/master/LICENSE
+ */
+#include "includes.h"
+
+using spdlog::details::os::create_dir;
+using spdlog::details::os::path_exists;
+
+bool try_create_dir(const spdlog::filename_t &path, const spdlog::filename_t &normalized_path) {
+    auto rv = create_dir(path);
+    REQUIRE(rv == true);
+    return path_exists(normalized_path);
+}
+
+TEST_CASE("create_dir", "[create_dir]") {
+    prepare_logdir();
+
+    REQUIRE(try_create_dir(SPDLOG_FILENAME_T("test_logs/dir1/dir1"),
+                           SPDLOG_FILENAME_T("test_logs/dir1/dir1")));
+    REQUIRE(try_create_dir(SPDLOG_FILENAME_T("test_logs/dir1/dir1"),
+                           SPDLOG_FILENAME_T("test_logs/dir1/dir1")));  // test existing
+    REQUIRE(try_create_dir(SPDLOG_FILENAME_T("test_logs/dir1///dir2//"),
+                           SPDLOG_FILENAME_T("test_logs/dir1/dir2")));
+    REQUIRE(try_create_dir(SPDLOG_FILENAME_T("./test_logs/dir1/dir3"),
+                           SPDLOG_FILENAME_T("test_logs/dir1/dir3")));
+    REQUIRE(try_create_dir(SPDLOG_FILENAME_T("test_logs/../test_logs/dir1/dir4"),
+                           SPDLOG_FILENAME_T("test_logs/dir1/dir4")));
+
+#ifdef WIN32
+    // test backslash folder separator
+    REQUIRE(try_create_dir(SPDLOG_FILENAME_T("test_logs\\dir1\\dir222"),
+                           SPDLOG_FILENAME_T("test_logs\\dir1\\dir222")));
+    REQUIRE(try_create_dir(SPDLOG_FILENAME_T("test_logs\\dir1\\dir223\\"),
+                           SPDLOG_FILENAME_T("test_logs\\dir1\\dir223\\")));
+    REQUIRE(try_create_dir(SPDLOG_FILENAME_T(".\\test_logs\\dir1\\dir2\\dir99\\..\\dir23"),
+                           SPDLOG_FILENAME_T("test_logs\\dir1\\dir2\\dir23")));
+    REQUIRE(try_create_dir(SPDLOG_FILENAME_T("test_logs\\..\\test_logs\\dir1\\dir5"),
+                           SPDLOG_FILENAME_T("test_logs\\dir1\\dir5")));
+#endif
+}
+
+TEST_CASE("create_invalid_dir", "[create_dir]") {
+    REQUIRE(create_dir(SPDLOG_FILENAME_T("")) == false);
+    REQUIRE(create_dir(spdlog::filename_t{}) == false);
+#ifdef __linux__
+    REQUIRE(create_dir("/proc/spdlog-utest") == false);
+#endif
+}
+
+TEST_CASE("dir_name", "[create_dir]") {
+    using spdlog::details::os::dir_name;
+    REQUIRE(dir_name(SPDLOG_FILENAME_T("")).empty());
+    REQUIRE(dir_name(SPDLOG_FILENAME_T("dir")).empty());
+
+#ifdef WIN32
+    REQUIRE(dir_name(SPDLOG_FILENAME_T(R"(dir\)")) == SPDLOG_FILENAME_T("dir"));
+    REQUIRE(dir_name(SPDLOG_FILENAME_T(R"(dir\\\)")) == SPDLOG_FILENAME_T(R"(dir\\)"));
+    REQUIRE(dir_name(SPDLOG_FILENAME_T(R"(dir\file)")) == SPDLOG_FILENAME_T("dir"));
+    REQUIRE(dir_name(SPDLOG_FILENAME_T(R"(dir/file)")) == SPDLOG_FILENAME_T("dir"));
+    REQUIRE(dir_name(SPDLOG_FILENAME_T(R"(dir\file.txt)")) == SPDLOG_FILENAME_T("dir"));
+    REQUIRE(dir_name(SPDLOG_FILENAME_T(R"(dir/file)")) == SPDLOG_FILENAME_T("dir"));
+    REQUIRE(dir_name(SPDLOG_FILENAME_T(R"(dir\file.txt\)")) ==
+            SPDLOG_FILENAME_T(R"(dir\file.txt)"));
+    REQUIRE(dir_name(SPDLOG_FILENAME_T(R"(\dir\file.txt)")) == SPDLOG_FILENAME_T(R"(\dir)"));
+    REQUIRE(dir_name(SPDLOG_FILENAME_T(R"(\\dir\file.txt)")) == SPDLOG_FILENAME_T(R"(\\dir)"));
+    REQUIRE(dir_name(SPDLOG_FILENAME_T(R"(..\file.txt)")) == SPDLOG_FILENAME_T(".."));
+    REQUIRE(dir_name(SPDLOG_FILENAME_T(R"(.\file.txt)")) == SPDLOG_FILENAME_T("."));
+    REQUIRE(dir_name(SPDLOG_FILENAME_T(R"(c:\\a\b\c\d\file.txt)")) ==
+            SPDLOG_FILENAME_T(R"(c:\\a\b\c\d)"));
+    REQUIRE(dir_name(SPDLOG_FILENAME_T(R"(c://a/b/c/d/file.txt)")) ==
+            SPDLOG_FILENAME_T(R"(c://a/b/c/d)"));
+#endif
+    REQUIRE(dir_name(SPDLOG_FILENAME_T("dir/")) == SPDLOG_FILENAME_T("dir"));
+    REQUIRE(dir_name(SPDLOG_FILENAME_T("dir///")) == SPDLOG_FILENAME_T("dir//"));
+    REQUIRE(dir_name(SPDLOG_FILENAME_T("dir/file")) == SPDLOG_FILENAME_T("dir"));
+    REQUIRE(dir_name(SPDLOG_FILENAME_T("dir/file.txt")) == SPDLOG_FILENAME_T("dir"));
+    REQUIRE(dir_name(SPDLOG_FILENAME_T("dir/file.txt/")) == SPDLOG_FILENAME_T("dir/file.txt"));
+    REQUIRE(dir_name(SPDLOG_FILENAME_T("/dir/file.txt")) == SPDLOG_FILENAME_T("/dir"));
+    REQUIRE(dir_name(SPDLOG_FILENAME_T("//dir/file.txt")) == SPDLOG_FILENAME_T("//dir"));
+    REQUIRE(dir_name(SPDLOG_FILENAME_T("../file.txt")) == SPDLOG_FILENAME_T(".."));
+    REQUIRE(dir_name(SPDLOG_FILENAME_T("./file.txt")) == SPDLOG_FILENAME_T("."));
+}
+
+#ifdef _WIN32
+
+    //
+    // test windows cases when drive letter is given e.g. C:\\some-folder
+    //
+    #include <windows.h>
+    #include <fileapi.h>
+
+std::string get_full_path(const std::string &relative_folder_path) {
+    char full_path[MAX_PATH];
+
+    DWORD result = ::GetFullPathNameA(relative_folder_path.c_str(), MAX_PATH, full_path, nullptr);
+    // Return an empty string if failed to get full path
+    return result > 0 && result < MAX_PATH ? std::string(full_path) : std::string();
+}
+
+std::wstring get_full_path(const std::wstring &relative_folder_path) {
+    wchar_t full_path[MAX_PATH];
+    DWORD result = ::GetFullPathNameW(relative_folder_path.c_str(), MAX_PATH, full_path, nullptr);
+    return result > 0 && result < MAX_PATH ? std::wstring(full_path) : std::wstring();
+}
+
+spdlog::filename_t::value_type find_non_existing_drive() {
+    for (char drive = 'A'; drive <= 'Z'; ++drive) {
+        std::string root_path = std::string(1, drive) + ":\\";
+        UINT drive_type = GetDriveTypeA(root_path.c_str());
+        if (drive_type == DRIVE_NO_ROOT_DIR) {
+            return static_cast<spdlog::filename_t::value_type>(drive);
+        }
+    }
+    return '\0';  // No available drive found
+}
+
+TEST_CASE("create_abs_path1", "[create_dir]") {
+    prepare_logdir();
+    auto abs_path = get_full_path(SPDLOG_FILENAME_T("test_logs\\logdir1"));
+    REQUIRE(!abs_path.empty());
+    REQUIRE(create_dir(abs_path) == true);
+}
+
+TEST_CASE("create_abs_path2", "[create_dir]") {
+    prepare_logdir();
+    auto abs_path = get_full_path(SPDLOG_FILENAME_T("test_logs/logdir2"));
+    REQUIRE(!abs_path.empty());
+    REQUIRE(create_dir(abs_path) == true);
+}
+
+TEST_CASE("non_existing_drive", "[create_dir]") {
+    prepare_logdir();
+    spdlog::filename_t path;
+
+    auto non_existing_drive = find_non_existing_drive();
+    path += non_existing_drive;
+    path += SPDLOG_FILENAME_T(":\\");
+    REQUIRE(create_dir(path) == false);
+    path += SPDLOG_FILENAME_T("subdir");
+    REQUIRE(create_dir(path) == false);
+}
+// #endif  // SPDLOG_WCHAR_FILENAMES
+#endif  // _WIN32
diff --git a/ext/spdlog/tests/test_custom_callbacks.cpp b/ext/spdlog/tests/test_custom_callbacks.cpp
new file mode 100644
index 0000000..4b04857
--- /dev/null
+++ b/ext/spdlog/tests/test_custom_callbacks.cpp
@@ -0,0 +1,35 @@
+/*
+ * This content is released under the MIT License as specified in
+ * https://raw.githubusercontent.com/gabime/spdlog/master/LICENSE
+ */
+#include "includes.h"
+#include "test_sink.h"
+#include "spdlog/sinks/callback_sink.h"
+#include "spdlog/async.h"
+#include "spdlog/common.h"
+
+TEST_CASE("custom_callback_logger", "[custom_callback_logger]") {
+    std::vector<std::string> lines;
+    spdlog::pattern_formatter formatter;
+    auto callback_logger =
+        std::make_shared<spdlog::sinks::callback_sink_st>([&](const spdlog::details::log_msg &msg) {
+            spdlog::memory_buf_t formatted;
+            formatter.format(msg, formatted);
+            auto eol_len = strlen(spdlog::details::os::default_eol);
+            lines.emplace_back(formatted.begin(), formatted.end() - eol_len);
+        });
+    std::shared_ptr<spdlog::sinks::test_sink_st> test_sink(new spdlog::sinks::test_sink_st);
+
+    spdlog::logger logger("test-callback", {callback_logger, test_sink});
+
+    logger.info("test message 1");
+    logger.info("test message 2");
+    logger.info("test message 3");
+
+    std::vector<std::string> ref_lines = test_sink->lines();
+
+    REQUIRE(lines[0] == ref_lines[0]);
+    REQUIRE(lines[1] == ref_lines[1]);
+    REQUIRE(lines[2] == ref_lines[2]);
+    spdlog::drop_all();
+}
diff --git a/ext/spdlog/tests/test_daily_logger.cpp b/ext/spdlog/tests/test_daily_logger.cpp
new file mode 100644
index 0000000..fe4783c
--- /dev/null
+++ b/ext/spdlog/tests/test_daily_logger.cpp
@@ -0,0 +1,173 @@
+/*
+ * This content is released under the MIT License as specified in
+ * https://raw.githubusercontent.com/gabime/spdlog/master/LICENSE
+ */
+#include "includes.h"
+
+#ifdef SPDLOG_USE_STD_FORMAT
+using filename_memory_buf_t = std::basic_string<spdlog::filename_t::value_type>;
+#else
+using filename_memory_buf_t = fmt::basic_memory_buffer<spdlog::filename_t::value_type, 250>;
+#endif
+
+#ifdef SPDLOG_WCHAR_FILENAMES
+std::string filename_buf_to_utf8string(const filename_memory_buf_t &w) {
+    spdlog::memory_buf_t buf;
+    spdlog::details::os::wstr_to_utf8buf(spdlog::wstring_view_t(w.data(), w.size()), buf);
+    return SPDLOG_BUF_TO_STRING(buf);
+}
+#else
+std::string filename_buf_to_utf8string(const filename_memory_buf_t &w) {
+    return SPDLOG_BUF_TO_STRING(w);
+}
+#endif
+
+TEST_CASE("daily_logger with dateonly calculator", "[daily_logger]") {
+    using sink_type =
+        spdlog::sinks::daily_file_sink<std::mutex, spdlog::sinks::daily_filename_calculator>;
+
+    prepare_logdir();
+
+    // calculate filename (time based)
+    spdlog::filename_t basename = SPDLOG_FILENAME_T("test_logs/daily_dateonly");
+    std::tm tm = spdlog::details::os::localtime();
+    filename_memory_buf_t w;
+    spdlog::fmt_lib::format_to(std::back_inserter(w), SPDLOG_FILENAME_T("{}_{:04d}-{:02d}-{:02d}"),
+                               basename, tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday);
+
+    auto logger = spdlog::create<sink_type>("logger", basename, 0, 0);
+    for (int i = 0; i < 10; ++i) {
+        logger->info("Test message {}", i);
+    }
+    logger->flush();
+
+    require_message_count(filename_buf_to_utf8string(w), 10);
+}
+
+struct custom_daily_file_name_calculator {
+    static spdlog::filename_t calc_filename(const spdlog::filename_t &basename, const tm &now_tm) {
+        filename_memory_buf_t w;
+        spdlog::fmt_lib::format_to(std::back_inserter(w), SPDLOG_FILENAME_T("{}{:04d}{:02d}{:02d}"),
+                                   basename, now_tm.tm_year + 1900, now_tm.tm_mon + 1,
+                                   now_tm.tm_mday);
+
+        return SPDLOG_BUF_TO_STRING(w);
+    }
+};
+
+TEST_CASE("daily_logger with custom calculator", "[daily_logger]") {
+    using sink_type = spdlog::sinks::daily_file_sink<std::mutex, custom_daily_file_name_calculator>;
+
+    prepare_logdir();
+
+    // calculate filename (time based)
+    spdlog::filename_t basename = SPDLOG_FILENAME_T("test_logs/daily_dateonly");
+    std::tm tm = spdlog::details::os::localtime();
+    filename_memory_buf_t w;
+    spdlog::fmt_lib::format_to(std::back_inserter(w), SPDLOG_FILENAME_T("{}{:04d}{:02d}{:02d}"),
+                               basename, tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday);
+
+    auto logger = spdlog::create<sink_type>("logger", basename, 0, 0);
+    for (int i = 0; i < 10; ++i) {
+        logger->info("Test message {}", i);
+    }
+
+    logger->flush();
+
+    require_message_count(filename_buf_to_utf8string(w), 10);
+}
+
+/*
+ * File name calculations
+ */
+
+TEST_CASE("rotating_file_sink::calc_filename1", "[rotating_file_sink]") {
+    auto filename =
+        spdlog::sinks::rotating_file_sink_st::calc_filename(SPDLOG_FILENAME_T("rotated.txt"), 3);
+    REQUIRE(filename == SPDLOG_FILENAME_T("rotated.3.txt"));
+}
+
+TEST_CASE("rotating_file_sink::calc_filename2", "[rotating_file_sink]") {
+    auto filename =
+        spdlog::sinks::rotating_file_sink_st::calc_filename(SPDLOG_FILENAME_T("rotated"), 3);
+    REQUIRE(filename == SPDLOG_FILENAME_T("rotated.3"));
+}
+
+TEST_CASE("rotating_file_sink::calc_filename3", "[rotating_file_sink]") {
+    auto filename =
+        spdlog::sinks::rotating_file_sink_st::calc_filename(SPDLOG_FILENAME_T("rotated.txt"), 0);
+    REQUIRE(filename == SPDLOG_FILENAME_T("rotated.txt"));
+}
+
+// regex supported only from gcc 4.9 and above
+#if defined(_MSC_VER) || !(__GNUC__ <= 4 && __GNUC_MINOR__ < 9)
+
+    #include <regex>
+
+TEST_CASE("daily_file_sink::daily_filename_calculator", "[daily_file_sink]") {
+    // daily_YYYY-MM-DD_hh-mm.txt
+    auto filename = spdlog::sinks::daily_filename_calculator::calc_filename(
+        SPDLOG_FILENAME_T("daily.txt"), spdlog::details::os::localtime());
+    // date regex based on https://www.regular-expressions.info/dates.html
+    std::basic_regex<spdlog::filename_t::value_type> re(
+        SPDLOG_FILENAME_T(R"(^daily_(19|20)\d\d-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])\.txt$)"));
+    std::match_results<spdlog::filename_t::const_iterator> match;
+    REQUIRE(std::regex_match(filename, match, re));
+}
+#endif
+
+TEST_CASE("daily_file_sink::daily_filename_format_calculator", "[daily_file_sink]") {
+    std::tm tm = spdlog::details::os::localtime();
+    // example-YYYY-MM-DD.log
+    auto filename = spdlog::sinks::daily_filename_format_calculator::calc_filename(
+        SPDLOG_FILENAME_T("example-%Y-%m-%d.log"), tm);
+
+    REQUIRE(filename ==
+            spdlog::fmt_lib::format(SPDLOG_FILENAME_T("example-{:04d}-{:02d}-{:02d}.log"),
+                                    tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday));
+}
+
+/* Test removal of old files */
+static spdlog::details::log_msg create_msg(std::chrono::seconds offset) {
+    using spdlog::log_clock;
+    spdlog::details::log_msg msg{"test", spdlog::level::info, "Hello Message"};
+    msg.time = log_clock::now() + offset;
+    return msg;
+}
+
+static void test_rotate(int days_to_run, uint16_t max_days, uint16_t expected_n_files) {
+    using spdlog::log_clock;
+    using spdlog::details::log_msg;
+    using spdlog::sinks::daily_file_sink_st;
+
+    prepare_logdir();
+
+    spdlog::filename_t basename = SPDLOG_FILENAME_T("test_logs/daily_rotate.txt");
+    daily_file_sink_st sink{basename, 2, 30, true, max_days};
+
+    // simulate messages with 24 intervals
+
+    for (int i = 0; i < days_to_run; i++) {
+        auto offset = std::chrono::seconds{24 * 3600 * i};
+        sink.log(create_msg(offset));
+    }
+
+    REQUIRE(count_files("test_logs") == static_cast<size_t>(expected_n_files));
+}
+
+TEST_CASE("daily_logger rotate", "[daily_file_sink]") {
+    int days_to_run = 1;
+    test_rotate(days_to_run, 0, 1);
+    test_rotate(days_to_run, 1, 1);
+    test_rotate(days_to_run, 3, 1);
+    test_rotate(days_to_run, 10, 1);
+
+    days_to_run = 10;
+    test_rotate(days_to_run, 0, 10);
+    test_rotate(days_to_run, 1, 1);
+    test_rotate(days_to_run, 3, 3);
+    test_rotate(days_to_run, 9, 9);
+    test_rotate(days_to_run, 10, 10);
+    test_rotate(days_to_run, 11, 10);
+    test_rotate(days_to_run, 20, 10);
+}
diff --git a/ext/spdlog/tests/test_dup_filter.cpp b/ext/spdlog/tests/test_dup_filter.cpp
new file mode 100644
index 0000000..78e22be
--- /dev/null
+++ b/ext/spdlog/tests/test_dup_filter.cpp
@@ -0,0 +1,83 @@
+#include "includes.h"
+#include "spdlog/sinks/dup_filter_sink.h"
+#include "test_sink.h"
+
+TEST_CASE("dup_filter_test1", "[dup_filter_sink]") {
+    using spdlog::sinks::dup_filter_sink_st;
+    using spdlog::sinks::test_sink_mt;
+
+    dup_filter_sink_st dup_sink{std::chrono::seconds{5}};
+    auto test_sink = std::make_shared<test_sink_mt>();
+    dup_sink.add_sink(test_sink);
+
+    for (int i = 0; i < 10; i++) {
+        dup_sink.log(spdlog::details::log_msg{"test", spdlog::level::info, "message1"});
+    }
+
+    REQUIRE(test_sink->msg_counter() == 1);
+}
+
+TEST_CASE("dup_filter_test2", "[dup_filter_sink]") {
+    using spdlog::sinks::dup_filter_sink_st;
+    using spdlog::sinks::test_sink_mt;
+
+    dup_filter_sink_st dup_sink{std::chrono::seconds{0}};
+    auto test_sink = std::make_shared<test_sink_mt>();
+    dup_sink.add_sink(test_sink);
+
+    for (int i = 0; i < 10; i++) {
+        dup_sink.log(spdlog::details::log_msg{"test", spdlog::level::info, "message1"});
+        std::this_thread::sleep_for(std::chrono::milliseconds(5));
+    }
+
+    REQUIRE(test_sink->msg_counter() == 10);
+}
+
+TEST_CASE("dup_filter_test3", "[dup_filter_sink]") {
+    using spdlog::sinks::dup_filter_sink_st;
+    using spdlog::sinks::test_sink_mt;
+
+    dup_filter_sink_st dup_sink{std::chrono::seconds{1}};
+    auto test_sink = std::make_shared<test_sink_mt>();
+    dup_sink.add_sink(test_sink);
+
+    for (int i = 0; i < 10; i++) {
+        dup_sink.log(spdlog::details::log_msg{"test", spdlog::level::info, "message1"});
+        dup_sink.log(spdlog::details::log_msg{"test", spdlog::level::info, "message2"});
+    }
+
+    REQUIRE(test_sink->msg_counter() == 20);
+}
+
+TEST_CASE("dup_filter_test4", "[dup_filter_sink]") {
+    using spdlog::sinks::dup_filter_sink_mt;
+    using spdlog::sinks::test_sink_mt;
+
+    dup_filter_sink_mt dup_sink{std::chrono::milliseconds{10}};
+    auto test_sink = std::make_shared<test_sink_mt>();
+    dup_sink.add_sink(test_sink);
+
+    dup_sink.log(spdlog::details::log_msg{"test", spdlog::level::info, "message"});
+    std::this_thread::sleep_for(std::chrono::milliseconds(50));
+    dup_sink.log(spdlog::details::log_msg{"test", spdlog::level::info, "message"});
+    REQUIRE(test_sink->msg_counter() == 2);
+}
+
+TEST_CASE("dup_filter_test5", "[dup_filter_sink]") {
+    using spdlog::sinks::dup_filter_sink_mt;
+    using spdlog::sinks::test_sink_mt;
+
+    dup_filter_sink_mt dup_sink{std::chrono::seconds{5}};
+    auto test_sink = std::make_shared<test_sink_mt>();
+    test_sink->set_pattern("%v");
+    dup_sink.add_sink(test_sink);
+
+    dup_sink.log(spdlog::details::log_msg{"test", spdlog::level::info, "message1"});
+    dup_sink.log(spdlog::details::log_msg{"test", spdlog::level::info, "message1"});
+    dup_sink.log(spdlog::details::log_msg{"test", spdlog::level::info, "message1"});
+    dup_sink.log(spdlog::details::log_msg{"test", spdlog::level::info, "message2"});
+
+    REQUIRE(test_sink->msg_counter() ==
+            3);  // skip 2 messages but log the "skipped.." message before message2
+    REQUIRE(test_sink->lines()[1] == "Skipped 2 duplicate messages..");
+}
diff --git a/ext/spdlog/tests/test_errors.cpp b/ext/spdlog/tests/test_errors.cpp
new file mode 100644
index 0000000..1c24cab
--- /dev/null
+++ b/ext/spdlog/tests/test_errors.cpp
@@ -0,0 +1,112 @@
+/*
+ * This content is released under the MIT License as specified in
+ * https://raw.githubusercontent.com/gabime/spdlog/master/LICENSE
+ */
+#include "includes.h"
+
+#include <iostream>
+
+#define SIMPLE_LOG "test_logs/simple_log.txt"
+#define SIMPLE_ASYNC_LOG "test_logs/simple_async_log.txt"
+
+class failing_sink : public spdlog::sinks::base_sink<std::mutex> {
+protected:
+    void sink_it_(const spdlog::details::log_msg &) final {
+        throw std::runtime_error("some error happened during log");
+    }
+
+    void flush_() final { throw std::runtime_error("some error happened during flush"); }
+};
+struct custom_ex {};
+
+#if !defined(SPDLOG_USE_STD_FORMAT)  // std format doesn't fully support runtime strings
+TEST_CASE("default_error_handler", "[errors]") {
+    prepare_logdir();
+    spdlog::filename_t filename = SPDLOG_FILENAME_T(SIMPLE_LOG);
+
+    auto logger = spdlog::create<spdlog::sinks::basic_file_sink_mt>("test-error", filename, true);
+    logger->set_pattern("%v");
+    logger->info(SPDLOG_FMT_RUNTIME("Test message {} {}"), 1);
+    logger->info("Test message {}", 2);
+    logger->flush();
+    using spdlog::details::os::default_eol;
+    REQUIRE(file_contents(SIMPLE_LOG) == spdlog::fmt_lib::format("Test message 2{}", default_eol));
+    REQUIRE(count_lines(SIMPLE_LOG) == 1);
+}
+
+TEST_CASE("custom_error_handler", "[errors]") {
+    prepare_logdir();
+    spdlog::filename_t filename = SPDLOG_FILENAME_T(SIMPLE_LOG);
+    auto logger = spdlog::create<spdlog::sinks::basic_file_sink_mt>("logger", filename, true);
+    logger->flush_on(spdlog::level::info);
+    logger->set_error_handler([=](const std::string &) { throw custom_ex(); });
+    logger->info("Good message #1");
+
+    REQUIRE_THROWS_AS(logger->info(SPDLOG_FMT_RUNTIME("Bad format msg {} {}"), "xxx"), custom_ex);
+    logger->info("Good message #2");
+    require_message_count(SIMPLE_LOG, 2);
+}
+#endif
+
+TEST_CASE("default_error_handler2", "[errors]") {
+    spdlog::drop_all();
+    auto logger = spdlog::create<failing_sink>("failed_logger");
+    logger->set_error_handler([=](const std::string &) { throw custom_ex(); });
+    REQUIRE_THROWS_AS(logger->info("Some message"), custom_ex);
+}
+
+TEST_CASE("flush_error_handler", "[errors]") {
+    spdlog::drop_all();
+    auto logger = spdlog::create<failing_sink>("failed_logger");
+    logger->set_error_handler([=](const std::string &) { throw custom_ex(); });
+    REQUIRE_THROWS_AS(logger->flush(), custom_ex);
+}
+
+#if !defined(SPDLOG_USE_STD_FORMAT)
+TEST_CASE("async_error_handler", "[errors]") {
+    prepare_logdir();
+    std::string err_msg("log failed with some msg");
+
+    spdlog::filename_t filename = SPDLOG_FILENAME_T(SIMPLE_ASYNC_LOG);
+    {
+        spdlog::init_thread_pool(128, 1);
+        auto logger =
+            spdlog::create_async<spdlog::sinks::basic_file_sink_mt>("logger", filename, true);
+        logger->set_error_handler([=](const std::string &) {
+            std::ofstream ofs("test_logs/custom_err.txt");
+            if (!ofs) {
+                throw std::runtime_error("Failed open test_logs/custom_err.txt");
+            }
+            ofs << err_msg;
+        });
+        logger->info("Good message #1");
+        logger->info(SPDLOG_FMT_RUNTIME("Bad format msg {} {}"), "xxx");
+        logger->info("Good message #2");
+        spdlog::drop("logger");  // force logger to drain the queue and shutdown
+    }
+    spdlog::init_thread_pool(128, 1);
+    require_message_count(SIMPLE_ASYNC_LOG, 2);
+    REQUIRE(file_contents("test_logs/custom_err.txt") == err_msg);
+}
+#endif
+
+// Make sure async error handler is executed
+TEST_CASE("async_error_handler2", "[errors]") {
+    prepare_logdir();
+    std::string err_msg("This is async handler error message");
+    {
+        spdlog::details::os::create_dir(SPDLOG_FILENAME_T("test_logs"));
+        spdlog::init_thread_pool(128, 1);
+        auto logger = spdlog::create_async<failing_sink>("failed_logger");
+        logger->set_error_handler([=](const std::string &) {
+            std::ofstream ofs("test_logs/custom_err2.txt");
+            if (!ofs) throw std::runtime_error("Failed open test_logs/custom_err2.txt");
+            ofs << err_msg;
+        });
+        logger->info("Hello failure");
+        spdlog::drop("failed_logger");  // force logger to drain the queue and shutdown
+    }
+
+    spdlog::init_thread_pool(128, 1);
+    REQUIRE(file_contents("test_logs/custom_err2.txt") == err_msg);
+}
diff --git a/ext/spdlog/tests/test_eventlog.cpp b/ext/spdlog/tests/test_eventlog.cpp
new file mode 100644
index 0000000..702eabe
--- /dev/null
+++ b/ext/spdlog/tests/test_eventlog.cpp
@@ -0,0 +1,75 @@
+#if _WIN32
+
+    #include "includes.h"
+    #include "test_sink.h"
+
+    #include "spdlog/sinks/win_eventlog_sink.h"
+
+static const LPCSTR TEST_SOURCE = "spdlog_test";
+
+static void test_single_print(std::function<void(std::string const &)> do_log,
+                              std::string const &expected_contents,
+                              WORD expected_ev_type) {
+    using namespace std::chrono;
+    do_log(expected_contents);
+    const auto expected_time_generated =
+        duration_cast<seconds>(system_clock::now().time_since_epoch()).count();
+
+    struct handle_t {
+        HANDLE handle_;
+
+        ~handle_t() {
+            if (handle_) {
+                REQUIRE(CloseEventLog(handle_));
+            }
+        }
+    } event_log{::OpenEventLogA(nullptr, TEST_SOURCE)};
+
+    REQUIRE(event_log.handle_);
+
+    DWORD read_bytes{}, size_needed{};
+    auto ok = ::ReadEventLogA(event_log.handle_, EVENTLOG_SEQUENTIAL_READ | EVENTLOG_BACKWARDS_READ,
+                              0, &read_bytes, 0, &read_bytes, &size_needed);
+    REQUIRE(!ok);
+    REQUIRE(::GetLastError() == ERROR_INSUFFICIENT_BUFFER);
+
+    std::vector<char> record_buffer(size_needed);
+    PEVENTLOGRECORD record = (PEVENTLOGRECORD)record_buffer.data();
+
+    ok = ::ReadEventLogA(event_log.handle_, EVENTLOG_SEQUENTIAL_READ | EVENTLOG_BACKWARDS_READ, 0,
+                         record, size_needed, &read_bytes, &size_needed);
+    REQUIRE(ok);
+
+    REQUIRE(record->NumStrings == 1);
+    REQUIRE(record->EventType == expected_ev_type);
+    REQUIRE((expected_time_generated - record->TimeGenerated) <= 3u);
+
+    std::string message_in_log(((char *)record + record->StringOffset));
+    REQUIRE(message_in_log == expected_contents + spdlog::details::os::default_eol);
+}
+
+TEST_CASE("eventlog", "[eventlog]") {
+    using namespace spdlog;
+
+    auto test_sink = std::make_shared<sinks::win_eventlog_sink_mt>(TEST_SOURCE);
+
+    spdlog::logger test_logger("eventlog", test_sink);
+    test_logger.set_level(level::trace);
+
+    test_sink->set_pattern("%v");
+
+    test_single_print([&test_logger](std::string const &msg) { test_logger.trace(msg); },
+                      "my trace message", EVENTLOG_SUCCESS);
+    test_single_print([&test_logger](std::string const &msg) { test_logger.debug(msg); },
+                      "my debug message", EVENTLOG_SUCCESS);
+    test_single_print([&test_logger](std::string const &msg) { test_logger.info(msg); },
+                      "my info message", EVENTLOG_INFORMATION_TYPE);
+    test_single_print([&test_logger](std::string const &msg) { test_logger.warn(msg); },
+                      "my warn message", EVENTLOG_WARNING_TYPE);
+    test_single_print([&test_logger](std::string const &msg) { test_logger.error(msg); },
+                      "my error message", EVENTLOG_ERROR_TYPE);
+    test_single_print([&test_logger](std::string const &msg) { test_logger.critical(msg); },
+                      "my critical message", EVENTLOG_ERROR_TYPE);
+}
+
+#endif  //_WIN32
diff --git a/ext/spdlog/tests/test_file_helper.cpp b/ext/spdlog/tests/test_file_helper.cpp
new file mode 100644
index 0000000..56ee75e
--- /dev/null
+++ b/ext/spdlog/tests/test_file_helper.cpp
@@ -0,0 +1,169 @@
+/*
+ * This content is released under the MIT License as specified in
+ * https://raw.githubusercontent.com/gabime/spdlog/master/LICENSE
+ */
+#include "includes.h"
+
+#define TEST_FILENAME "test_logs/file_helper_test.txt"
+
+using spdlog::details::file_helper;
+
+static void write_with_helper(file_helper &helper, size_t howmany) {
+    spdlog::memory_buf_t formatted;
+    spdlog::fmt_lib::format_to(std::back_inserter(formatted), "{}", std::string(howmany, '1'));
+    helper.write(formatted);
+    helper.flush();
+}
+
+TEST_CASE("file_helper_filename", "[file_helper::filename()]") {
+    prepare_logdir();
+
+    file_helper helper;
+    spdlog::filename_t target_filename = SPDLOG_FILENAME_T(TEST_FILENAME);
+    helper.open(target_filename);
+    REQUIRE(helper.filename() == target_filename);
+}
+
+TEST_CASE("file_helper_size", "[file_helper::size()]") {
+    prepare_logdir();
+    spdlog::filename_t target_filename = SPDLOG_FILENAME_T(TEST_FILENAME);
+    size_t expected_size = 123;
+    {
+        file_helper helper;
+        helper.open(target_filename);
+        write_with_helper(helper, expected_size);
+        REQUIRE(static_cast<size_t>(helper.size()) == expected_size);
+    }
+    REQUIRE(get_filesize(TEST_FILENAME) == expected_size);
+}
+
+TEST_CASE("file_helper_reopen", "[file_helper::reopen()]") {
+    prepare_logdir();
+    spdlog::filename_t target_filename = SPDLOG_FILENAME_T(TEST_FILENAME);
+    file_helper helper;
+    helper.open(target_filename);
+    write_with_helper(helper, 12);
+    REQUIRE(helper.size() == 12);
+    helper.reopen(true);
+    REQUIRE(helper.size() == 0);
+}
+
+TEST_CASE("file_helper_reopen2", "[file_helper::reopen(false)]") {
+    prepare_logdir();
+    spdlog::filename_t target_filename = SPDLOG_FILENAME_T(TEST_FILENAME);
+    size_t expected_size = 14;
+    file_helper helper;
+    helper.open(target_filename);
+    write_with_helper(helper, expected_size);
+    REQUIRE(helper.size() == expected_size);
+    helper.reopen(false);
+    REQUIRE(helper.size() == expected_size);
+}
+
+static void test_split_ext(const spdlog::filename_t::value_type *fname,
+                           const spdlog::filename_t::value_type *expect_base,
+                           const spdlog::filename_t::value_type *expect_ext) {
+    spdlog::filename_t filename(fname);
+    spdlog::filename_t expected_base(expect_base);
+    spdlog::filename_t expected_ext(expect_ext);
+
+    spdlog::filename_t basename;
+    spdlog::filename_t ext;
+    std::tie(basename, ext) = file_helper::split_by_extension(filename);
+    REQUIRE(basename == expected_base);
+    REQUIRE(ext == expected_ext);
+}
+
+TEST_CASE("file_helper_split_by_extension", "[file_helper::split_by_extension()]") {
+    test_split_ext(SPDLOG_FILENAME_T("mylog.txt"), SPDLOG_FILENAME_T("mylog"),
+                   SPDLOG_FILENAME_T(".txt"));
+    test_split_ext(SPDLOG_FILENAME_T(".mylog.txt"), SPDLOG_FILENAME_T(".mylog"),
+                   SPDLOG_FILENAME_T(".txt"));
+    test_split_ext(SPDLOG_FILENAME_T(".mylog"), SPDLOG_FILENAME_T(".mylog"), SPDLOG_FILENAME_T(""));
+    test_split_ext(SPDLOG_FILENAME_T("/aaa/bb.d/mylog"), SPDLOG_FILENAME_T("/aaa/bb.d/mylog"),
+                   SPDLOG_FILENAME_T(""));
+    test_split_ext(SPDLOG_FILENAME_T("/aaa/bb.d/mylog.txt"), SPDLOG_FILENAME_T("/aaa/bb.d/mylog"),
+                   SPDLOG_FILENAME_T(".txt"));
+    test_split_ext(SPDLOG_FILENAME_T("aaa/bbb/ccc/mylog.txt"),
+                   SPDLOG_FILENAME_T("aaa/bbb/ccc/mylog"), SPDLOG_FILENAME_T(".txt"));
+    test_split_ext(SPDLOG_FILENAME_T("aaa/bbb/ccc/mylog."), SPDLOG_FILENAME_T("aaa/bbb/ccc/mylog."),
+                   SPDLOG_FILENAME_T(""));
+    test_split_ext(SPDLOG_FILENAME_T("aaa/bbb/ccc/.mylog.txt"),
+                   SPDLOG_FILENAME_T("aaa/bbb/ccc/.mylog"), SPDLOG_FILENAME_T(".txt"));
+    test_split_ext(SPDLOG_FILENAME_T("/aaa/bbb/ccc/mylog.txt"),
+                   SPDLOG_FILENAME_T("/aaa/bbb/ccc/mylog"), SPDLOG_FILENAME_T(".txt"));
+    test_split_ext(SPDLOG_FILENAME_T("/aaa/bbb/ccc/.mylog"),
+                   SPDLOG_FILENAME_T("/aaa/bbb/ccc/.mylog"), SPDLOG_FILENAME_T(""));
+    test_split_ext(SPDLOG_FILENAME_T("../mylog.txt"), SPDLOG_FILENAME_T("../mylog"),
+                   SPDLOG_FILENAME_T(".txt"));
+    test_split_ext(SPDLOG_FILENAME_T(".././mylog.txt"), SPDLOG_FILENAME_T(".././mylog"),
+                   SPDLOG_FILENAME_T(".txt"));
+    test_split_ext(SPDLOG_FILENAME_T(".././mylog.txt/xxx"), SPDLOG_FILENAME_T(".././mylog.txt/xxx"),
+                   SPDLOG_FILENAME_T(""));
+    test_split_ext(SPDLOG_FILENAME_T("/mylog.txt"), SPDLOG_FILENAME_T("/mylog"),
+                   SPDLOG_FILENAME_T(".txt"));
+    test_split_ext(SPDLOG_FILENAME_T("//mylog.txt"), SPDLOG_FILENAME_T("//mylog"),
+                   SPDLOG_FILENAME_T(".txt"));
+    test_split_ext(SPDLOG_FILENAME_T(""), SPDLOG_FILENAME_T(""), SPDLOG_FILENAME_T(""));
+    test_split_ext(SPDLOG_FILENAME_T("."), SPDLOG_FILENAME_T("."), SPDLOG_FILENAME_T(""));
+    test_split_ext(SPDLOG_FILENAME_T("..txt"), SPDLOG_FILENAME_T("."), SPDLOG_FILENAME_T(".txt"));
+}
+
+TEST_CASE("file_event_handlers", "[file_helper]") {
+    enum class flags { before_open, after_open, before_close, after_close };
+    prepare_logdir();
+
+    spdlog::filename_t test_filename = SPDLOG_FILENAME_T(TEST_FILENAME);
+    // define event handles that update vector of flags when called
+    std::vector<flags> events;
+    spdlog::file_event_handlers handlers;
+    handlers.before_open = [&](spdlog::filename_t filename) {
+        REQUIRE(filename == test_filename);
+        events.push_back(flags::before_open);
+    };
+    handlers.after_open = [&](spdlog::filename_t filename, std::FILE *fstream) {
+        REQUIRE(filename == test_filename);
+        REQUIRE(fstream);
+        fputs("after_open\n", fstream);
+        events.push_back(flags::after_open);
+    };
+    handlers.before_close = [&](spdlog::filename_t filename, std::FILE *fstream) {
+        REQUIRE(filename == test_filename);
+        REQUIRE(fstream);
+        fputs("before_close\n", fstream);
+        events.push_back(flags::before_close);
+    };
+    handlers.after_close = [&](spdlog::filename_t filename) {
+        REQUIRE(filename == test_filename);
+        events.push_back(flags::after_close);
+    };
+    {
+        spdlog::details::file_helper helper{handlers};
+        REQUIRE(events.empty());
+
+        helper.open(test_filename);
+        REQUIRE(events == std::vector<flags>{flags::before_open, flags::after_open});
+
+        events.clear();
+        helper.close();
+        REQUIRE(events == std::vector<flags>{flags::before_close, flags::after_close});
+        REQUIRE(file_contents(TEST_FILENAME) == "after_open\nbefore_close\n");
+
+        helper.reopen(true);
+        events.clear();
+    }
+    // make sure that the file_helper destructor calls the close callbacks if needed
+    REQUIRE(events == std::vector<flags>{flags::before_close, flags::after_close});
+    REQUIRE(file_contents(TEST_FILENAME) == "after_open\nbefore_close\n");
+}
+
+TEST_CASE("file_helper_open", "[file_helper]") {
+    prepare_logdir();
+    spdlog::filename_t target_filename = SPDLOG_FILENAME_T(TEST_FILENAME);
+    file_helper helper;
+    helper.open(target_filename);
+    helper.close();
+
+    target_filename += SPDLOG_FILENAME_T("/invalid");
+    REQUIRE_THROWS_AS(helper.open(target_filename), spdlog::spdlog_ex);
+}
diff --git a/ext/spdlog/tests/test_file_logging.cpp b/ext/spdlog/tests/test_file_logging.cpp
new file mode 100644
index 0000000..71d06a6
--- /dev/null
+++ b/ext/spdlog/tests/test_file_logging.cpp
@@ -0,0 +1,143 @@
+/*
+ * This content is released under the MIT License as specified in
+ * https://raw.githubusercontent.com/gabime/spdlog/master/LICENSE
+ */
+#include "includes.h"
+
+#define SIMPLE_LOG "test_logs/simple_log"
+#define ROTATING_LOG "test_logs/rotating_log"
+
+TEST_CASE("simple_file_logger", "[simple_logger]") {
+    prepare_logdir();
+    spdlog::filename_t filename = SPDLOG_FILENAME_T(SIMPLE_LOG);
+
+    auto logger = spdlog::create<spdlog::sinks::basic_file_sink_mt>("logger", filename);
+    logger->set_pattern("%v");
+
+    logger->info("Test message {}", 1);
+    logger->info("Test message {}", 2);
+
+    logger->flush();
+    require_message_count(SIMPLE_LOG, 2);
+    using spdlog::details::os::default_eol;
+    REQUIRE(file_contents(SIMPLE_LOG) ==
+            spdlog::fmt_lib::format("Test message 1{}Test message 2{}", default_eol, default_eol));
+}
+
+TEST_CASE("flush_on", "[flush_on]") {
+    prepare_logdir();
+    spdlog::filename_t filename = SPDLOG_FILENAME_T(SIMPLE_LOG);
+
+    auto logger = spdlog::create<spdlog::sinks::basic_file_sink_mt>("logger", filename);
+    logger->set_pattern("%v");
+    logger->set_level(spdlog::level::trace);
+    logger->flush_on(spdlog::level::info);
+    logger->trace("Should not be flushed");
+    REQUIRE(count_lines(SIMPLE_LOG) == 0);
+
+    logger->info("Test message {}", 1);
+    logger->info("Test message {}", 2);
+
+    require_message_count(SIMPLE_LOG, 3);
+    using spdlog::details::os::default_eol;
+    REQUIRE(file_contents(SIMPLE_LOG) ==
+            spdlog::fmt_lib::format("Should not be flushed{}Test message 1{}Test message 2{}",
+                                    default_eol, default_eol, default_eol));
+}
+
+TEST_CASE("simple_file_logger", "[truncate]") {
+    prepare_logdir();
+    const spdlog::filename_t filename = SPDLOG_FILENAME_T(SIMPLE_LOG);
+    const bool truncate = true;
+    const auto sink = std::make_shared<spdlog::sinks::basic_file_sink_mt>(filename, truncate);
+    const auto logger = std::make_shared<spdlog::logger>("simple_file_logger", sink);
+
+    logger->info("Test message {}", 3.14);
+    logger->info("Test message {}", 2.71);
+    logger->flush();
+    REQUIRE(count_lines(SIMPLE_LOG) == 2);
+    
+    sink->truncate();
+    REQUIRE(count_lines(SIMPLE_LOG) == 0);
+
+    logger->info("Test message {}", 6.28);
+    logger->flush();
+    REQUIRE(count_lines(SIMPLE_LOG) == 1);
+}
+
+TEST_CASE("rotating_file_logger1", "[rotating_logger]") {
+    prepare_logdir();
+    size_t max_size = 1024 * 10;
+    spdlog::filename_t basename = SPDLOG_FILENAME_T(ROTATING_LOG);
+    auto logger = spdlog::rotating_logger_mt("logger", basename, max_size, 0);
+
+    for (int i = 0; i < 10; ++i) {
+        logger->info("Test message {}", i);
+    }
+
+    logger->flush();
+    require_message_count(ROTATING_LOG, 10);
+}
+
+TEST_CASE("rotating_file_logger2", "[rotating_logger]") {
+    prepare_logdir();
+    size_t max_size = 1024 * 10;
+    spdlog::filename_t basename = SPDLOG_FILENAME_T(ROTATING_LOG);
+
+    {
+        // make an initial logger to create the first output file
+        auto logger = spdlog::rotating_logger_mt("logger", basename, max_size, 2, true);
+        for (int i = 0; i < 10; ++i) {
+            logger->info("Test message {}", i);
+        }
+        // drop causes the logger destructor to be called, which is required so the
+        // next logger can rename the first output file.
+        spdlog::drop(logger->name());
+    }
+
+    auto logger = spdlog::rotating_logger_mt("logger", basename, max_size, 2, true);
+    for (int i = 0; i < 10; ++i) {
+        logger->info("Test message {}", i);
+    }
+
+    logger->flush();
+
+    require_message_count(ROTATING_LOG, 10);
+
+    for (int i = 0; i < 1000; i++) {
+        logger->info("Test message {}", i);
+    }
+
+    logger->flush();
+    REQUIRE(get_filesize(ROTATING_LOG) <= max_size);
+    REQUIRE(get_filesize(ROTATING_LOG ".1") <= max_size);
+}
+
+// test that passing max_size=0 throws
+TEST_CASE("rotating_file_logger3", "[rotating_logger]") {
+    prepare_logdir();
+    size_t max_size = 0;
+    spdlog::filename_t basename = SPDLOG_FILENAME_T(ROTATING_LOG);
+    REQUIRE_THROWS_AS(spdlog::rotating_logger_mt("logger", basename, max_size, 0),
+                      spdlog::spdlog_ex);
+}
+
+// test on-demand rotation of logs
+TEST_CASE("rotating_file_logger4", "[rotating_logger]") {
+    prepare_logdir();
+    size_t max_size = 1024 * 10;
+    spdlog::filename_t basename = SPDLOG_FILENAME_T(ROTATING_LOG);
+    auto sink = std::make_shared<spdlog::sinks::rotating_file_sink_st>(basename, max_size, 2);
+    auto logger = std::make_shared<spdlog::logger>("rotating_sink_logger", sink);
+
+    logger->info("Test message - pre-rotation");
+    logger->flush();
+
+    sink->rotate_now();
+
+    logger->info("Test message - post-rotation");
+    logger->flush();
+
+    REQUIRE(get_filesize(ROTATING_LOG) > 0);
+    REQUIRE(get_filesize(ROTATING_LOG ".1") > 0);
+}
diff --git a/ext/spdlog/tests/test_fmt_helper.cpp b/ext/spdlog/tests/test_fmt_helper.cpp
new file mode 100644
index 0000000..31b9306
--- /dev/null
+++ b/ext/spdlog/tests/test_fmt_helper.cpp
@@ -0,0 +1,82 @@
+
+#include "includes.h"
+
+using spdlog::memory_buf_t;
+using spdlog::details::to_string_view;
+
+void test_pad2(int n, const char *expected) {
+    memory_buf_t buf;
+    spdlog::details::fmt_helper::pad2(n, buf);
+
+    REQUIRE(to_string_view(buf) == expected);
+}
+
+void test_pad3(uint32_t n, const char *expected) {
+    memory_buf_t buf;
+    spdlog::details::fmt_helper::pad3(n, buf);
+
+    REQUIRE(to_string_view(buf) == expected);
+}
+
+void test_pad6(std::size_t n, const char *expected) {
+    memory_buf_t buf;
+    spdlog::details::fmt_helper::pad6(n, buf);
+
+    REQUIRE(to_string_view(buf) == expected);
+}
+
+void test_pad9(std::size_t n, const char *expected) {
+    memory_buf_t buf;
+    spdlog::details::fmt_helper::pad9(n, buf);
+
+    REQUIRE(to_string_view(buf) == expected);
+}
+
+TEST_CASE("pad2", "[fmt_helper]") {
+    test_pad2(0, "00");
+    test_pad2(3, "03");
+    test_pad2(10, "10");
+    test_pad2(23, "23");
+    test_pad2(99, "99");
+    test_pad2(100, "100");
+    test_pad2(123, "123");
+    test_pad2(1234, "1234");
+    test_pad2(-5, "-5");
+}
+
+TEST_CASE("pad3", "[fmt_helper]") {
+    test_pad3(0, "000");
+    test_pad3(3, "003");
+    test_pad3(10, "010");
+    test_pad3(23, "023");
+    test_pad3(99, "099");
+    test_pad3(100, "100");
+    test_pad3(123, "123");
+    test_pad3(999, "999");
+    test_pad3(1000, "1000");
+    test_pad3(1234, "1234");
+}
+
+TEST_CASE("pad6", "[fmt_helper]") {
+    test_pad6(0, "000000");
+    test_pad6(3, "000003");
+    test_pad6(23, "000023");
+    test_pad6(123, "000123");
+    test_pad6(1234, "001234");
+    test_pad6(12345, "012345");
+    test_pad6(123456, "123456");
+}
+
+TEST_CASE("pad9", "[fmt_helper]") {
+    test_pad9(0, "000000000");
+    test_pad9(3, "000000003");
+    test_pad9(23, "000000023");
+    test_pad9(123, "000000123");
+    test_pad9(1234, "000001234");
+    test_pad9(12345, "000012345");
+    test_pad9(123456, "000123456");
+    test_pad9(1234567, "001234567");
+    test_pad9(12345678, "012345678");
+    test_pad9(123456789, "123456789");
+    test_pad9(1234567891, "1234567891");
+}
diff --git a/ext/spdlog/tests/test_macros.cpp b/ext/spdlog/tests/test_macros.cpp
new file mode 100644
index 0000000..132706f
--- /dev/null
+++ b/ext/spdlog/tests/test_macros.cpp
@@ -0,0 +1,53 @@
+/*
+ * This content is released under the MIT License as specified in
+ * https://raw.githubusercontent.com/gabime/spdlog/master/LICENSE
+ */
+
+#include "includes.h"
+
+#if SPDLOG_ACTIVE_LEVEL != SPDLOG_LEVEL_DEBUG
+    #error "Invalid SPDLOG_ACTIVE_LEVEL in test. Should be SPDLOG_LEVEL_DEBUG"
+#endif
+
+#define TEST_FILENAME "test_logs/simple_log"
+
+TEST_CASE("debug and trace w/o format string", "[macros]") {
+    prepare_logdir();
+    spdlog::filename_t filename = SPDLOG_FILENAME_T(TEST_FILENAME);
+
+    auto logger = spdlog::create<spdlog::sinks::basic_file_sink_mt>("logger", filename);
+    logger->set_pattern("%v");
+    logger->set_level(spdlog::level::trace);
+
+    SPDLOG_LOGGER_TRACE(logger, "Test message 1");
+    SPDLOG_LOGGER_DEBUG(logger, "Test message 2");
+    logger->flush();
+
+    using spdlog::details::os::default_eol;
+    REQUIRE(ends_with(file_contents(TEST_FILENAME),
+                      spdlog::fmt_lib::format("Test message 2{}", default_eol)));
+    REQUIRE(count_lines(TEST_FILENAME) == 1);
+
+    auto orig_default_logger = spdlog::default_logger();
+    spdlog::set_default_logger(logger);
+
+    SPDLOG_TRACE("Test message 3");
+    SPDLOG_DEBUG("Test message {}", 4);
+    logger->flush();
+
+    require_message_count(TEST_FILENAME, 2);
+    REQUIRE(ends_with(file_contents(TEST_FILENAME),
+                      spdlog::fmt_lib::format("Test message 4{}", default_eol)));
+    spdlog::set_default_logger(std::move(orig_default_logger));
+}
+
+TEST_CASE("disable param evaluation", "[macros]") {
+    SPDLOG_TRACE("Test message {}", throw std::runtime_error("Should not be evaluated"));
+}
+
+TEST_CASE("pass logger pointer", "[macros]") {
+    auto logger = spdlog::create<spdlog::sinks::null_sink_mt>("refmacro");
+    auto &ref = *logger;
+    SPDLOG_LOGGER_TRACE(&ref, "Test message 1");
+    SPDLOG_LOGGER_DEBUG(&ref, "Test message 2");
+}
diff --git a/ext/spdlog/tests/test_misc.cpp b/ext/spdlog/tests/test_misc.cpp
new file mode 100644
index 0000000..80ba51d
--- /dev/null
+++ b/ext/spdlog/tests/test_misc.cpp
@@ -0,0 +1,222 @@
+#ifdef _WIN32 // to prevent fopen warning on windows
+#define _CRT_SECURE_NO_WARNINGS
+#endif
+
+#include "includes.h"
+#include "test_sink.h"
+
+template <class T>
+std::string log_info(const T &what, spdlog::level::level_enum logger_level = spdlog::level::info) {
+    std::ostringstream oss;
+    auto oss_sink = std::make_shared<spdlog::sinks::ostream_sink_mt>(oss);
+
+    spdlog::logger oss_logger("oss", oss_sink);
+    oss_logger.set_level(logger_level);
+    oss_logger.set_pattern("%v");
+    oss_logger.info(what);
+
+    return oss.str().substr(0, oss.str().length() - strlen(spdlog::details::os::default_eol));
+}
+
+TEST_CASE("basic_logging ", "[basic_logging]") {
+    // const char
+    REQUIRE(log_info("Hello") == "Hello");
+    REQUIRE(log_info("").empty());
+
+    // std::string
+    REQUIRE(log_info(std::string("Hello")) == "Hello");
+    REQUIRE(log_info(std::string()).empty());
+
+    // Numbers
+    REQUIRE(log_info(5) == "5");
+    REQUIRE(log_info(5.6) == "5.6");
+
+    // User defined class
+    // REQUIRE(log_info(some_logged_class("some_val")) == "some_val");
+}
+
+TEST_CASE("log_levels", "[log_levels]") {
+    REQUIRE(log_info("Hello", spdlog::level::err).empty());
+    REQUIRE(log_info("Hello", spdlog::level::critical).empty());
+    REQUIRE(log_info("Hello", spdlog::level::info) == "Hello");
+    REQUIRE(log_info("Hello", spdlog::level::debug) == "Hello");
+    REQUIRE(log_info("Hello", spdlog::level::trace) == "Hello");
+}
+
+TEST_CASE("level_to_string_view", "[convert_to_string_view]") {
+    REQUIRE(spdlog::level::to_string_view(spdlog::level::trace) == "trace");
+    REQUIRE(spdlog::level::to_string_view(spdlog::level::debug) == "debug");
+    REQUIRE(spdlog::level::to_string_view(spdlog::level::info) == "info");
+    REQUIRE(spdlog::level::to_string_view(spdlog::level::warn) == "warning");
+    REQUIRE(spdlog::level::to_string_view(spdlog::level::err) == "error");
+    REQUIRE(spdlog::level::to_string_view(spdlog::level::critical) == "critical");
+    REQUIRE(spdlog::level::to_string_view(spdlog::level::off) == "off");
+}
+
+TEST_CASE("to_short_c_str", "[convert_to_short_c_str]") {
+    REQUIRE(std::string(spdlog::level::to_short_c_str(spdlog::level::trace)) == "T");
+    REQUIRE(std::string(spdlog::level::to_short_c_str(spdlog::level::debug)) == "D");
+    REQUIRE(std::string(spdlog::level::to_short_c_str(spdlog::level::info)) == "I");
+    REQUIRE(std::string(spdlog::level::to_short_c_str(spdlog::level::warn)) == "W");
+    REQUIRE(std::string(spdlog::level::to_short_c_str(spdlog::level::err)) == "E");
+    REQUIRE(std::string(spdlog::level::to_short_c_str(spdlog::level::critical)) == "C");
+    REQUIRE(std::string(spdlog::level::to_short_c_str(spdlog::level::off)) == "O");
+}
+
+TEST_CASE("to_level_enum", "[convert_to_level_enum]") {
+    REQUIRE(spdlog::level::from_str("trace") == spdlog::level::trace);
+    REQUIRE(spdlog::level::from_str("debug") == spdlog::level::debug);
+    REQUIRE(spdlog::level::from_str("info") == spdlog::level::info);
+    REQUIRE(spdlog::level::from_str("warning") == spdlog::level::warn);
+    REQUIRE(spdlog::level::from_str("warn") == spdlog::level::warn);
+    REQUIRE(spdlog::level::from_str("error") == spdlog::level::err);
+    REQUIRE(spdlog::level::from_str("critical") == spdlog::level::critical);
+    REQUIRE(spdlog::level::from_str("off") == spdlog::level::off);
+    REQUIRE(spdlog::level::from_str("null") == spdlog::level::off);
+}
+
+TEST_CASE("periodic flush", "[periodic_flush]") {
+    using spdlog::sinks::test_sink_mt;
+    auto logger = spdlog::create<test_sink_mt>("periodic_flush");
+    auto test_sink = std::static_pointer_cast<test_sink_mt>(logger->sinks()[0]);
+
+    spdlog::flush_every(std::chrono::seconds(1));
+    std::this_thread::sleep_for(std::chrono::milliseconds(1250));
+    REQUIRE(test_sink->flush_counter() == 1);
+    spdlog::flush_every(std::chrono::seconds(0));
+    spdlog::drop_all();
+}
+
+TEST_CASE("clone-logger", "[clone]") {
+    using spdlog::sinks::test_sink_mt;
+    auto test_sink = std::make_shared<test_sink_mt>();
+    auto logger = std::make_shared<spdlog::logger>("orig", test_sink);
+    logger->set_pattern("%v");
+    auto cloned = logger->clone("clone");
+
+    REQUIRE(cloned->name() == "clone");
+    REQUIRE(logger->sinks() == cloned->sinks());
+    REQUIRE(logger->level() == cloned->level());
+    REQUIRE(logger->flush_level() == cloned->flush_level());
+    logger->info("Some message 1");
+    cloned->info("Some message 2");
+
+    REQUIRE(test_sink->lines().size() == 2);
+    REQUIRE(test_sink->lines()[0] == "Some message 1");
+    REQUIRE(test_sink->lines()[1] == "Some message 2");
+
+    spdlog::drop_all();
+}
+
+TEST_CASE("clone async", "[clone]") {
+    using spdlog::sinks::test_sink_mt;
+    spdlog::init_thread_pool(4, 1);
+    auto test_sink = std::make_shared<test_sink_mt>();
+    auto logger = std::make_shared<spdlog::async_logger>("orig", test_sink, spdlog::thread_pool());
+    logger->set_pattern("%v");
+    auto cloned = logger->clone("clone");
+
+    REQUIRE(cloned->name() == "clone");
+    REQUIRE(logger->sinks() == cloned->sinks());
+    REQUIRE(logger->level() == cloned->level());
+    REQUIRE(logger->flush_level() == cloned->flush_level());
+
+    logger->info("Some message 1");
+    cloned->info("Some message 2");
+
+    spdlog::details::os::sleep_for_millis(100);
+
+    REQUIRE(test_sink->lines().size() == 2);
+    REQUIRE(test_sink->lines()[0] == "Some message 1");
+    REQUIRE(test_sink->lines()[1] == "Some message 2");
+
+    spdlog::drop_all();
+}
+
+TEST_CASE("default logger API", "[default logger]") {
+    std::ostringstream oss;
+    auto oss_sink = std::make_shared<spdlog::sinks::ostream_sink_mt>(oss);
+
+    spdlog::set_default_logger(std::make_shared<spdlog::logger>("oss", oss_sink));
+    spdlog::set_pattern("*** %v");
+
+    spdlog::default_logger()->set_level(spdlog::level::trace);
+    spdlog::trace("hello trace");
+    REQUIRE(oss.str() == "*** hello trace" + std::string(spdlog::details::os::default_eol));
+
+    oss.str("");
+    spdlog::debug("hello debug");
+    REQUIRE(oss.str() == "*** hello debug" + std::string(spdlog::details::os::default_eol));
+
+    oss.str("");
+    spdlog::info("Hello");
+    REQUIRE(oss.str() == "*** Hello" + std::string(spdlog::details::os::default_eol));
+
+    oss.str("");
+    spdlog::warn("Hello again {}", 2);
+    REQUIRE(oss.str() == "*** Hello again 2" + std::string(spdlog::details::os::default_eol));
+
+    oss.str("");
+    spdlog::error(123);
+    REQUIRE(oss.str() == "*** 123" + std::string(spdlog::details::os::default_eol));
+
+    oss.str("");
+    spdlog::critical(std::string("some string"));
+    REQUIRE(oss.str() == "*** some string" + std::string(spdlog::details::os::default_eol));
+
+    oss.str("");
+    spdlog::set_level(spdlog::level::info);
+    spdlog::debug("should not be logged");
+    REQUIRE(oss.str().empty());
+    spdlog::drop_all();
+    spdlog::set_pattern("%v");
+}
+
+#if (defined(SPDLOG_WCHAR_TO_UTF8_SUPPORT) || defined(SPDLOG_WCHAR_FILENAMES)) && defined(_WIN32)
+TEST_CASE("utf8 to utf16 conversion using windows api", "[windows utf]") {
+    spdlog::wmemory_buf_t buffer;
+
+    spdlog::details::os::utf8_to_wstrbuf("", buffer);
+    REQUIRE(std::wstring(buffer.data(), buffer.size()) == std::wstring(L""));
+
+    spdlog::details::os::utf8_to_wstrbuf("abc", buffer);
+    REQUIRE(std::wstring(buffer.data(), buffer.size()) == std::wstring(L"abc"));
+
+    spdlog::details::os::utf8_to_wstrbuf("\xc3\x28", buffer); // Invalid UTF-8 sequence.
+    REQUIRE(std::wstring(buffer.data(), buffer.size()) == std::wstring(L"\xfffd("));
+
+    spdlog::details::os::utf8_to_wstrbuf("\xe3\x81\xad\xe3\x81\x93", buffer); // "Neko" in hiragana.
+    REQUIRE(std::wstring(buffer.data(), buffer.size()) == std::wstring(L"\x306d\x3053"));
+}
+#endif
+
+struct auto_closer {
+    FILE* fp = nullptr;
+    explicit auto_closer(FILE* f) : fp(f) {}
+    auto_closer(const auto_closer&) = delete;
+    auto_closer& operator=(const auto_closer&) = delete;
+    ~auto_closer() {
+        if (fp != nullptr) (void)std::fclose(fp);
+    }
+};
+
+TEST_CASE("os::fwrite_bytes", "[os]") {
+    using spdlog::details::os::fwrite_bytes;
+    using spdlog::details::os::create_dir;
+    const char* filename = "log_tests/test_fwrite_bytes.txt";
+    const char *msg = "hello";
+    prepare_logdir();
+    REQUIRE(create_dir(SPDLOG_FILENAME_T("log_tests")) == true);
+    {
+        auto_closer closer(std::fopen(filename, "wb"));
+        REQUIRE(closer.fp != nullptr);
+        REQUIRE(fwrite_bytes(msg, std::strlen(msg), closer.fp) == true);
+        REQUIRE(fwrite_bytes(msg, 0, closer.fp) == true);
+        std::fflush(closer.fp);
+        REQUIRE(spdlog::details::os::filesize(closer.fp) == 5);
+    }
+    // fwrite_bytes should return false on write failure
+    auto_closer closer(std::fopen(filename, "r"));
+    REQUIRE(closer.fp != nullptr);
+    REQUIRE_FALSE(fwrite_bytes("Hello", 5, closer.fp));
+}
diff --git a/ext/spdlog/tests/test_mpmc_q.cpp b/ext/spdlog/tests/test_mpmc_q.cpp
new file mode 100644
index 0000000..bc7a37d
--- /dev/null
+++ b/ext/spdlog/tests/test_mpmc_q.cpp
@@ -0,0 +1,114 @@
+#include "includes.h"
+
+using std::chrono::milliseconds;
+using test_clock = std::chrono::high_resolution_clock;
+
+static milliseconds millis_from(const test_clock::time_point &tp0) {
+    return std::chrono::duration_cast<milliseconds>(test_clock::now() - tp0);
+}
+TEST_CASE("dequeue-empty-nowait", "[mpmc_blocking_q]") {
+    size_t q_size = 100;
+    milliseconds tolerance_wait(20);
+    spdlog::details::mpmc_blocking_queue<int> q(q_size);
+    int popped_item = 0;
+
+    auto start = test_clock::now();
+    auto rv = q.dequeue_for(popped_item, milliseconds::zero());
+    auto delta_ms = millis_from(start);
+
+    REQUIRE(rv == false);
+    INFO("Delta " << delta_ms.count() << " millis");
+    REQUIRE(delta_ms <= tolerance_wait);
+}
+
+TEST_CASE("dequeue-empty-wait", "[mpmc_blocking_q]") {
+    size_t q_size = 100;
+    milliseconds wait_ms(250);
+    milliseconds tolerance_wait(250);
+
+    spdlog::details::mpmc_blocking_queue<int> q(q_size);
+    int popped_item = 0;
+    auto start = test_clock::now();
+    auto rv = q.dequeue_for(popped_item, wait_ms);
+    auto delta_ms = millis_from(start);
+
+    REQUIRE(rv == false);
+
+    INFO("Delta " << delta_ms.count() << " millis");
+    REQUIRE(delta_ms >= wait_ms - tolerance_wait);
+    REQUIRE(delta_ms <= wait_ms + tolerance_wait);
+}
+
+TEST_CASE("dequeue-full-nowait", "[mpmc_blocking_q]") {
+    spdlog::details::mpmc_blocking_queue<int> q(1);
+    q.enqueue(42);
+
+    int item = 0;
+    q.dequeue_for(item, milliseconds::zero());
+    REQUIRE(item == 42);
+}
+
+TEST_CASE("dequeue-full-wait", "[mpmc_blocking_q]") {
+    spdlog::details::mpmc_blocking_queue<int> q(1);
+    q.enqueue(42);
+
+    int item = 0;
+    q.dequeue(item);
+    REQUIRE(item == 42);
+}
+
+TEST_CASE("enqueue_nowait", "[mpmc_blocking_q]") {
+    size_t q_size = 1;
+    spdlog::details::mpmc_blocking_queue<int> q(q_size);
+    milliseconds tolerance_wait(10);
+
+    q.enqueue(1);
+    REQUIRE(q.overrun_counter() == 0);
+
+    auto start = test_clock::now();
+    q.enqueue_nowait(2);
+    auto delta_ms = millis_from(start);
+
+    INFO("Delta " << delta_ms.count() << " millis");
+    REQUIRE(delta_ms <= tolerance_wait);
+    REQUIRE(q.overrun_counter() == 1);
+}
+
+TEST_CASE("bad_queue", "[mpmc_blocking_q]") {
+    size_t q_size = 0;
+    spdlog::details::mpmc_blocking_queue<int> q(q_size);
+    q.enqueue_nowait(1);
+    REQUIRE(q.overrun_counter() == 1);
+    int i = 0;
+    REQUIRE(q.dequeue_for(i, milliseconds(0)) == false);
+}
+
+TEST_CASE("empty_queue", "[mpmc_blocking_q]") {
+    size_t q_size = 10;
+    spdlog::details::mpmc_blocking_queue<int> q(q_size);
+    int i = 0;
+    REQUIRE(q.dequeue_for(i, milliseconds(10)) == false);
+}
+
+TEST_CASE("full_queue", "[mpmc_blocking_q]") {
+    size_t q_size = 100;
+    spdlog::details::mpmc_blocking_queue<int> q(q_size);
+    for (int i = 0; i < static_cast<int>(q_size); i++) {
+        q.enqueue(i + 0);  // i+0 to force rvalue and avoid tidy warnings on the same time if we
+                           // std::move(i) instead
+    }
+
+    q.enqueue_nowait(123456);
+    REQUIRE(q.overrun_counter() == 1);
+
+    for (int i = 1; i < static_cast<int>(q_size); i++) {
+        int item = -1;
+        q.dequeue(item);
+        REQUIRE(item == i);
+    }
+
+    // last item pushed has overridden the oldest.
+    int item = -1;
+    q.dequeue(item);
+    REQUIRE(item == 123456);
+}
diff --git a/ext/spdlog/tests/test_pattern_formatter.cpp b/ext/spdlog/tests/test_pattern_formatter.cpp
new file mode 100644
index 0000000..d248e99
--- /dev/null
+++ b/ext/spdlog/tests/test_pattern_formatter.cpp
@@ -0,0 +1,632 @@
+#include "includes.h"
+#include "test_sink.h"
+
+using spdlog::memory_buf_t;
+using spdlog::details::to_string_view;
+
+// log to str and return it
+template <typename... Args>
+static std::string log_to_str(const std::string &msg, const Args &...args) {
+    std::ostringstream oss;
+    auto oss_sink = std::make_shared<spdlog::sinks::ostream_sink_mt>(oss);
+    spdlog::logger oss_logger("pattern_tester", oss_sink);
+    oss_logger.set_level(spdlog::level::info);
+
+    oss_logger.set_formatter(
+        std::unique_ptr<spdlog::formatter>(new spdlog::pattern_formatter(args...)));
+
+    oss_logger.info(msg);
+    return oss.str();
+}
+
+TEST_CASE("custom eol", "[pattern_formatter]") {
+    std::string msg = "Hello custom eol test";
+    std::string eol = ";)";
+    REQUIRE(log_to_str(msg, "%v", spdlog::pattern_time_type::local, ";)") == msg + eol);
+}
+
+TEST_CASE("empty format", "[pattern_formatter]") {
+    REQUIRE(log_to_str("Some message", "", spdlog::pattern_time_type::local, "").empty());
+}
+
+TEST_CASE("empty format2", "[pattern_formatter]") {
+    REQUIRE(log_to_str("Some message", "", spdlog::pattern_time_type::local, "\n") == "\n");
+}
+
+TEST_CASE("level", "[pattern_formatter]") {
+    REQUIRE(log_to_str("Some message", "[%l] %v", spdlog::pattern_time_type::local, "\n") ==
+            "[info] Some message\n");
+}
+
+TEST_CASE("short level", "[pattern_formatter]") {
+    REQUIRE(log_to_str("Some message", "[%L] %v", spdlog::pattern_time_type::local, "\n") ==
+            "[I] Some message\n");
+}
+
+TEST_CASE("name", "[pattern_formatter]") {
+    REQUIRE(log_to_str("Some message", "[%n] %v", spdlog::pattern_time_type::local, "\n") ==
+            "[pattern_tester] Some message\n");
+}
+
+TEST_CASE("date MM/DD/YY ", "[pattern_formatter]") {
+    auto now_tm = spdlog::details::os::localtime();
+    std::stringstream oss;
+    oss << std::setfill('0') << std::setw(2) << now_tm.tm_mon + 1 << "/" << std::setw(2)
+        << now_tm.tm_mday << "/" << std::setw(2) << (now_tm.tm_year + 1900) % 1000
+        << " Some message\n";
+    REQUIRE(log_to_str("Some message", "%D %v", spdlog::pattern_time_type::local, "\n") ==
+            oss.str());
+}
+
+TEST_CASE("color range test1", "[pattern_formatter]") {
+    auto formatter = std::make_shared<spdlog::pattern_formatter>(
+        "%^%v%$", spdlog::pattern_time_type::local, "\n");
+
+    memory_buf_t buf;
+    spdlog::fmt_lib::format_to(std::back_inserter(buf), "Hello");
+    memory_buf_t formatted;
+    std::string logger_name = "test";
+    spdlog::details::log_msg msg(logger_name, spdlog::level::info,
+                                 spdlog::string_view_t(buf.data(), buf.size()));
+    formatter->format(msg, formatted);
+    REQUIRE(msg.color_range_start == 0);
+    REQUIRE(msg.color_range_end == 5);
+    REQUIRE(log_to_str("hello", "%^%v%$", spdlog::pattern_time_type::local, "\n") == "hello\n");
+}
+
+TEST_CASE("color range test2", "[pattern_formatter]") {
+    auto formatter =
+        std::make_shared<spdlog::pattern_formatter>("%^%$", spdlog::pattern_time_type::local, "\n");
+    std::string logger_name = "test";
+    spdlog::details::log_msg msg(logger_name, spdlog::level::info, "");
+    memory_buf_t formatted;
+    formatter->format(msg, formatted);
+    REQUIRE(msg.color_range_start == 0);
+    REQUIRE(msg.color_range_end == 0);
+    REQUIRE(log_to_str("", "%^%$", spdlog::pattern_time_type::local, "\n") == "\n");
+}
+
+TEST_CASE("color range test3", "[pattern_formatter]") {
+    auto formatter = std::make_shared<spdlog::pattern_formatter>("%^***%$");
+    std::string logger_name = "test";
+    spdlog::details::log_msg msg(logger_name, spdlog::level::info, "ignored");
+    memory_buf_t formatted;
+    formatter->format(msg, formatted);
+    REQUIRE(msg.color_range_start == 0);
+    REQUIRE(msg.color_range_end == 3);
+}
+
+TEST_CASE("color range test4", "[pattern_formatter]") {
+    auto formatter = std::make_shared<spdlog::pattern_formatter>(
+        "XX%^YYY%$", spdlog::pattern_time_type::local, "\n");
+    std::string logger_name = "test";
+    spdlog::details::log_msg msg(logger_name, spdlog::level::info, "ignored");
+
+    memory_buf_t formatted;
+    formatter->format(msg, formatted);
+    REQUIRE(msg.color_range_start == 2);
+    REQUIRE(msg.color_range_end == 5);
+    REQUIRE(log_to_str("ignored", "XX%^YYY%$", spdlog::pattern_time_type::local, "\n") ==
+            "XXYYY\n");
+}
+
+TEST_CASE("color range test5", "[pattern_formatter]") {
+    auto formatter = std::make_shared<spdlog::pattern_formatter>("**%^");
+    std::string logger_name = "test";
+    spdlog::details::log_msg msg(logger_name, spdlog::level::info, "ignored");
+    memory_buf_t formatted;
+    formatter->format(msg, formatted);
+    REQUIRE(msg.color_range_start == 2);
+    REQUIRE(msg.color_range_end == 0);
+}
+
+TEST_CASE("color range test6", "[pattern_formatter]") {
+    auto formatter = std::make_shared<spdlog::pattern_formatter>("**%$");
+    std::string logger_name = "test";
+    spdlog::details::log_msg msg(logger_name, spdlog::level::info, "ignored");
+    memory_buf_t formatted;
+    formatter->format(msg, formatted);
+    REQUIRE(msg.color_range_start == 0);
+    REQUIRE(msg.color_range_end == 2);
+}
+
+//
+// Test padding
+//
+
+TEST_CASE("level_left_padded", "[pattern_formatter]") {
+    REQUIRE(log_to_str("Some message", "[%8l] %v", spdlog::pattern_time_type::local, "\n") ==
+            "[    info] Some message\n");
+    REQUIRE(log_to_str("Some message", "[%8!l] %v", spdlog::pattern_time_type::local, "\n") ==
+            "[    info] Some message\n");
+}
+
+TEST_CASE("level_right_padded", "[pattern_formatter]") {
+    REQUIRE(log_to_str("Some message", "[%-8l] %v", spdlog::pattern_time_type::local, "\n") ==
+            "[info    ] Some message\n");
+    REQUIRE(log_to_str("Some message", "[%-8!l] %v", spdlog::pattern_time_type::local, "\n") ==
+            "[info    ] Some message\n");
+}
+
+TEST_CASE("level_center_padded", "[pattern_formatter]") {
+    REQUIRE(log_to_str("Some message", "[%=8l] %v", spdlog::pattern_time_type::local, "\n") ==
+            "[  info  ] Some message\n");
+    REQUIRE(log_to_str("Some message", "[%=8!l] %v", spdlog::pattern_time_type::local, "\n") ==
+            "[  info  ] Some message\n");
+}
+
+TEST_CASE("short level_left_padded", "[pattern_formatter]") {
+    REQUIRE(log_to_str("Some message", "[%3L] %v", spdlog::pattern_time_type::local, "\n") ==
+            "[  I] Some message\n");
+    REQUIRE(log_to_str("Some message", "[%3!L] %v", spdlog::pattern_time_type::local, "\n") ==
+            "[  I] Some message\n");
+}
+
+TEST_CASE("short level_right_padded", "[pattern_formatter]") {
+    REQUIRE(log_to_str("Some message", "[%-3L] %v", spdlog::pattern_time_type::local, "\n") ==
+            "[I  ] Some message\n");
+    REQUIRE(log_to_str("Some message", "[%-3!L] %v", spdlog::pattern_time_type::local, "\n") ==
+            "[I  ] Some message\n");
+}
+
+TEST_CASE("short level_center_padded", "[pattern_formatter]") {
+    REQUIRE(log_to_str("Some message", "[%=3L] %v", spdlog::pattern_time_type::local, "\n") ==
+            "[ I ] Some message\n");
+    REQUIRE(log_to_str("Some message", "[%=3!L] %v", spdlog::pattern_time_type::local, "\n") ==
+            "[ I ] Some message\n");
+}
+
+TEST_CASE("left_padded_short", "[pattern_formatter]") {
+    REQUIRE(log_to_str("Some message", "[%3n] %v", spdlog::pattern_time_type::local, "\n") ==
+            "[pattern_tester] Some message\n");
+    REQUIRE(log_to_str("Some message", "[%3!n] %v", spdlog::pattern_time_type::local, "\n") ==
+            "[pat] Some message\n");
+}
+
+TEST_CASE("right_padded_short", "[pattern_formatter]") {
+    REQUIRE(log_to_str("Some message", "[%-3n] %v", spdlog::pattern_time_type::local, "\n") ==
+            "[pattern_tester] Some message\n");
+    REQUIRE(log_to_str("Some message", "[%-3!n] %v", spdlog::pattern_time_type::local, "\n") ==
+            "[pat] Some message\n");
+}
+
+TEST_CASE("center_padded_short", "[pattern_formatter]") {
+    REQUIRE(log_to_str("Some message", "[%=3n] %v", spdlog::pattern_time_type::local, "\n") ==
+            "[pattern_tester] Some message\n");
+    REQUIRE(log_to_str("Some message", "[%=3!n] %v", spdlog::pattern_time_type::local, "\n") ==
+            "[pat] Some message\n");
+}
+
+TEST_CASE("left_padded_huge", "[pattern_formatter]") {
+    REQUIRE(log_to_str("Some message", "[%-300n] %v", spdlog::pattern_time_type::local, "\n") ==
+            "[pattern_tester                                                  ] Some message\n");
+
+    REQUIRE(log_to_str("Some message", "[%-300!n] %v", spdlog::pattern_time_type::local, "\n") ==
+            "[pattern_tester                                                  ] Some message\n");
+}
+
+TEST_CASE("left_padded_max", "[pattern_formatter]") {
+    REQUIRE(log_to_str("Some message", "[%-64n] %v", spdlog::pattern_time_type::local, "\n") ==
+            "[pattern_tester                                                  ] Some message\n");
+
+    REQUIRE(log_to_str("Some message", "[%-64!n] %v", spdlog::pattern_time_type::local, "\n") ==
+            "[pattern_tester                                                  ] Some message\n");
+}
+
+// Test padding + truncate flag
+
+TEST_CASE("paddinng_truncate", "[pattern_formatter]") {
+    REQUIRE(log_to_str("123456", "%6!v", spdlog::pattern_time_type::local, "\n") == "123456\n");
+    REQUIRE(log_to_str("123456", "%5!v", spdlog::pattern_time_type::local, "\n") == "12345\n");
+    REQUIRE(log_to_str("123456", "%7!v", spdlog::pattern_time_type::local, "\n") == " 123456\n");
+
+    REQUIRE(log_to_str("123456", "%-6!v", spdlog::pattern_time_type::local, "\n") == "123456\n");
+    REQUIRE(log_to_str("123456", "%-5!v", spdlog::pattern_time_type::local, "\n") == "12345\n");
+    REQUIRE(log_to_str("123456", "%-7!v", spdlog::pattern_time_type::local, "\n") == "123456 \n");
+
+    REQUIRE(log_to_str("123456", "%=6!v", spdlog::pattern_time_type::local, "\n") == "123456\n");
+    REQUIRE(log_to_str("123456", "%=5!v", spdlog::pattern_time_type::local, "\n") == "12345\n");
+    REQUIRE(log_to_str("123456", "%=7!v", spdlog::pattern_time_type::local, "\n") == "123456 \n");
+
+    REQUIRE(log_to_str("123456", "%0!v", spdlog::pattern_time_type::local, "\n") == "\n");
+}
+
+TEST_CASE("padding_truncate_funcname", "[pattern_formatter]") {
+    spdlog::sinks::test_sink_st test_sink;
+
+    const char *pattern = "%v [%5!!]";
+    auto formatter = std::unique_ptr<spdlog::formatter>(new spdlog::pattern_formatter(pattern));
+    test_sink.set_formatter(std::move(formatter));
+
+    spdlog::details::log_msg msg1{spdlog::source_loc{"ignored", 1, "func"}, "test_logger",
+                                  spdlog::level::info, "message"};
+    test_sink.log(msg1);
+    REQUIRE(test_sink.lines()[0] == "message [ func]");
+
+    spdlog::details::log_msg msg2{spdlog::source_loc{"ignored", 1, "function"}, "test_logger",
+                                  spdlog::level::info, "message"};
+    test_sink.log(msg2);
+    REQUIRE(test_sink.lines()[1] == "message [funct]");
+}
+
+TEST_CASE("padding_funcname", "[pattern_formatter]") {
+    spdlog::sinks::test_sink_st test_sink;
+
+    const char *pattern = "%v [%10!]";
+    auto formatter = std::unique_ptr<spdlog::formatter>(new spdlog::pattern_formatter(pattern));
+    test_sink.set_formatter(std::move(formatter));
+
+    spdlog::details::log_msg msg1{spdlog::source_loc{"ignored", 1, "func"}, "test_logger",
+                                  spdlog::level::info, "message"};
+    test_sink.log(msg1);
+    REQUIRE(test_sink.lines()[0] == "message [      func]");
+
+    spdlog::details::log_msg msg2{spdlog::source_loc{"ignored", 1, "func567890123"}, "test_logger",
+                                  spdlog::level::info, "message"};
+    test_sink.log(msg2);
+    REQUIRE(test_sink.lines()[1] == "message [func567890123]");
+}
+
+TEST_CASE("clone-default-formatter", "[pattern_formatter]") {
+    auto formatter_1 = std::make_shared<spdlog::pattern_formatter>();
+    auto formatter_2 = formatter_1->clone();
+    std::string logger_name = "test";
+    spdlog::details::log_msg msg(logger_name, spdlog::level::info, "some message");
+
+    memory_buf_t formatted_1;
+    memory_buf_t formatted_2;
+    formatter_1->format(msg, formatted_1);
+    formatter_2->format(msg, formatted_2);
+
+    REQUIRE(to_string_view(formatted_1) == to_string_view(formatted_2));
+}
+
+TEST_CASE("clone-default-formatter2", "[pattern_formatter]") {
+    auto formatter_1 = std::make_shared<spdlog::pattern_formatter>("%+");
+    auto formatter_2 = formatter_1->clone();
+    std::string logger_name = "test";
+    spdlog::details::log_msg msg(logger_name, spdlog::level::info, "some message");
+
+    memory_buf_t formatted_1;
+    memory_buf_t formatted_2;
+    formatter_1->format(msg, formatted_1);
+    formatter_2->format(msg, formatted_2);
+
+    REQUIRE(to_string_view(formatted_1) == to_string_view(formatted_2));
+}
+
+TEST_CASE("clone-formatter", "[pattern_formatter]") {
+    auto formatter_1 = std::make_shared<spdlog::pattern_formatter>("%D %X [%] [%n] %v");
+    auto formatter_2 = formatter_1->clone();
+    std::string logger_name = "test";
+    spdlog::details::log_msg msg(logger_name, spdlog::level::info, "some message");
+
+    memory_buf_t formatted_1;
+    memory_buf_t formatted_2;
+    formatter_1->format(msg, formatted_1);
+    formatter_2->format(msg, formatted_2);
+
+    REQUIRE(to_string_view(formatted_1) == to_string_view(formatted_2));
+}
+
+TEST_CASE("clone-formatter-2", "[pattern_formatter]") {
+    using spdlog::pattern_time_type;
+    auto formatter_1 = std::make_shared<spdlog::pattern_formatter>(
+        "%D %X [%] [%n] %v", pattern_time_type::utc, "xxxxxx\n");
+    auto formatter_2 = formatter_1->clone();
+    std::string logger_name = "test2";
+    spdlog::details::log_msg msg(logger_name, spdlog::level::info, "some message");
+
+    memory_buf_t formatted_1;
+    memory_buf_t formatted_2;
+    formatter_1->format(msg, formatted_1);
+    formatter_2->format(msg, formatted_2);
+
+    REQUIRE(to_string_view(formatted_1) == to_string_view(formatted_2));
+}
+
+class custom_test_flag : public spdlog::custom_flag_formatter {
+public:
+    explicit custom_test_flag(std::string txt)
+        : some_txt{std::move(txt)} {}
+
+    void format(const spdlog::details::log_msg &,
+                const std::tm &tm,
+                spdlog::memory_buf_t &dest) override {
+        if (some_txt == "throw_me") {
+            throw spdlog::spdlog_ex("custom_flag_exception_test");
+        } else if (some_txt == "time") {
+            auto formatted = spdlog::fmt_lib::format("{:d}:{:02d}{:s}", tm.tm_hour % 12, tm.tm_min,
+                                                     tm.tm_hour / 12 ? "PM" : "AM");
+            dest.append(formatted.data(), formatted.data() + formatted.size());
+            return;
+        }
+        some_txt = std::string(padinfo_.width_, ' ') + some_txt;
+        dest.append(some_txt.data(), some_txt.data() + some_txt.size());
+    }
+    spdlog::details::padding_info get_padding_info() { return padinfo_; }
+
+    std::string some_txt;
+
+    std::unique_ptr<custom_flag_formatter> clone() const override {
+        return spdlog::details::make_unique<custom_test_flag>(some_txt);
+    }
+};
+// test clone with custom flag formatters
+TEST_CASE("clone-custom_formatter", "[pattern_formatter]") {
+    auto formatter_1 = std::make_shared<spdlog::pattern_formatter>();
+    formatter_1->add_flag<custom_test_flag>('t', "custom_output").set_pattern("[%n] [%t] %v");
+    auto formatter_2 = formatter_1->clone();
+    std::string logger_name = "logger-name";
+    spdlog::details::log_msg msg(logger_name, spdlog::level::info, "some message");
+
+    memory_buf_t formatted_1;
+    memory_buf_t formatted_2;
+    formatter_1->format(msg, formatted_1);
+    formatter_2->format(msg, formatted_2);
+
+    auto expected = spdlog::fmt_lib::format("[logger-name] [custom_output] some message{}",
+                                            spdlog::details::os::default_eol);
+
+    REQUIRE(to_string_view(formatted_1) == expected);
+    REQUIRE(to_string_view(formatted_2) == expected);
+}
+
+//
+// Test source location formatting
+//
+
+#ifdef _WIN32
+static const char *const test_path = "\\a\\b\\c/myfile.cpp";
+#else
+static const char *const test_path = "/a/b//myfile.cpp";
+#endif
+
+TEST_CASE("short filename formatter-1", "[pattern_formatter]") {
+    spdlog::pattern_formatter formatter("%s", spdlog::pattern_time_type::local, "");
+    memory_buf_t formatted;
+    std::string logger_name = "logger-name";
+    spdlog::source_loc source_loc{test_path, 123, "some_func()"};
+    spdlog::details::log_msg msg(source_loc, "logger-name", spdlog::level::info, "Hello");
+    formatter.format(msg, formatted);
+
+    REQUIRE(to_string_view(formatted) == "myfile.cpp");
+}
+
+TEST_CASE("short filename formatter-2", "[pattern_formatter]") {
+    spdlog::pattern_formatter formatter("%s:%#", spdlog::pattern_time_type::local, "");
+    memory_buf_t formatted;
+    std::string logger_name = "logger-name";
+    spdlog::source_loc source_loc{"myfile.cpp", 123, "some_func()"};
+    spdlog::details::log_msg msg(source_loc, "logger-name", spdlog::level::info, "Hello");
+    formatter.format(msg, formatted);
+
+    REQUIRE(to_string_view(formatted) == "myfile.cpp:123");
+}
+
+TEST_CASE("short filename formatter-3", "[pattern_formatter]") {
+    spdlog::pattern_formatter formatter("%s %v", spdlog::pattern_time_type::local, "");
+    memory_buf_t formatted;
+    std::string logger_name = "logger-name";
+    spdlog::source_loc source_loc{"", 123, "some_func()"};
+    spdlog::details::log_msg msg(source_loc, "logger-name", spdlog::level::info, "Hello");
+    formatter.format(msg, formatted);
+
+    REQUIRE(to_string_view(formatted) == " Hello");
+}
+
+TEST_CASE("full filename formatter", "[pattern_formatter]") {
+    spdlog::pattern_formatter formatter("%g", spdlog::pattern_time_type::local, "");
+    memory_buf_t formatted;
+    std::string logger_name = "logger-name";
+    spdlog::source_loc source_loc{test_path, 123, "some_func()"};
+    spdlog::details::log_msg msg(source_loc, "logger-name", spdlog::level::info, "Hello");
+    formatter.format(msg, formatted);
+
+    REQUIRE(to_string_view(formatted) == test_path);
+}
+
+TEST_CASE("custom flags", "[pattern_formatter]") {
+    auto formatter = std::make_shared<spdlog::pattern_formatter>();
+    formatter->add_flag<custom_test_flag>('t', "custom1")
+        .add_flag<custom_test_flag>('u', "custom2")
+        .set_pattern("[%n] [%t] [%u] %v");
+
+    memory_buf_t formatted;
+
+    spdlog::details::log_msg msg(spdlog::source_loc{}, "logger-name", spdlog::level::info,
+                                 "some message");
+    formatter->format(msg, formatted);
+    auto expected = spdlog::fmt_lib::format("[logger-name] [custom1] [custom2] some message{}",
+                                            spdlog::details::os::default_eol);
+
+    REQUIRE(to_string_view(formatted) == expected);
+}
+
+TEST_CASE("custom flags-padding", "[pattern_formatter]") {
+    auto formatter = std::make_shared<spdlog::pattern_formatter>();
+    formatter->add_flag<custom_test_flag>('t', "custom1")
+        .add_flag<custom_test_flag>('u', "custom2")
+        .set_pattern("[%n] [%t] [%5u] %v");
+
+    memory_buf_t formatted;
+
+    spdlog::details::log_msg msg(spdlog::source_loc{}, "logger-name", spdlog::level::info,
+                                 "some message");
+    formatter->format(msg, formatted);
+    auto expected = spdlog::fmt_lib::format("[logger-name] [custom1] [     custom2] some message{}",
+                                            spdlog::details::os::default_eol);
+
+    REQUIRE(to_string_view(formatted) == expected);
+}
+
+TEST_CASE("custom flags-exception", "[pattern_formatter]") {
+    auto formatter = std::make_shared<spdlog::pattern_formatter>();
+    formatter->add_flag<custom_test_flag>('t', "throw_me")
+        .add_flag<custom_test_flag>('u', "custom2")
+        .set_pattern("[%n] [%t] [%u] %v");
+
+    memory_buf_t formatted;
+    spdlog::details::log_msg msg(spdlog::source_loc{}, "logger-name", spdlog::level::info,
+                                 "some message");
+    CHECK_THROWS_AS(formatter->format(msg, formatted), spdlog::spdlog_ex);
+}
+
+TEST_CASE("override need_localtime", "[pattern_formatter]") {
+    auto formatter =
+        std::make_shared<spdlog::pattern_formatter>(spdlog::pattern_time_type::local, "\n");
+    formatter->add_flag<custom_test_flag>('t', "time").set_pattern("%t> %v");
+
+    {
+        memory_buf_t formatted;
+        spdlog::details::log_msg msg(spdlog::source_loc{}, "logger-name", spdlog::level::info,
+                                     "some message");
+        formatter->format(msg, formatted);
+        REQUIRE(to_string_view(formatted) == "0:00AM> some message\n");
+    }
+
+    {
+        formatter->need_localtime();
+
+        auto now_tm = spdlog::details::os::localtime();
+        std::stringstream oss;
+        oss << (now_tm.tm_hour % 12) << ":" << std::setfill('0') << std::setw(2) << now_tm.tm_min
+            << (now_tm.tm_hour / 12 ? "PM" : "AM") << "> some message\n";
+
+        memory_buf_t formatted;
+        spdlog::details::log_msg msg(spdlog::source_loc{}, "logger-name", spdlog::level::info,
+                                     "some message");
+        formatter->format(msg, formatted);
+        REQUIRE(to_string_view(formatted) == oss.str());
+    }
+}
+
+#ifndef SPDLOG_NO_TLS
+TEST_CASE("mdc formatter test-1", "[pattern_formatter]") {
+    spdlog::mdc::put("mdc_key_1", "mdc_value_1");
+    spdlog::mdc::put("mdc_key_2", "mdc_value_2");
+
+    auto formatter = std::make_shared<spdlog::pattern_formatter>();
+    formatter->set_pattern("[%n] [%l] [%&] %v");
+
+    memory_buf_t formatted;
+    spdlog::details::log_msg msg(spdlog::source_loc{}, "logger-name", spdlog::level::info,
+                                 "some message");
+    formatter->format(msg, formatted);
+
+    auto expected = spdlog::fmt_lib::format(
+        "[logger-name] [info] [mdc_key_1:mdc_value_1 mdc_key_2:mdc_value_2] some message{}",
+        spdlog::details::os::default_eol);
+    REQUIRE(to_string_view(formatted) == expected);
+
+    SECTION("Tear down") { spdlog::mdc::clear(); }
+}
+
+TEST_CASE("mdc formatter value update", "[pattern_formatter]") {
+    spdlog::mdc::put("mdc_key_1", "mdc_value_1");
+    spdlog::mdc::put("mdc_key_2", "mdc_value_2");
+
+    auto formatter = std::make_shared<spdlog::pattern_formatter>();
+    formatter->set_pattern("[%n] [%l] [%&] %v");
+
+    memory_buf_t formatted_1;
+    spdlog::details::log_msg msg(spdlog::source_loc{}, "logger-name", spdlog::level::info,
+                                 "some message");
+    formatter->format(msg, formatted_1);
+
+    auto expected = spdlog::fmt_lib::format(
+        "[logger-name] [info] [mdc_key_1:mdc_value_1 mdc_key_2:mdc_value_2] some message{}",
+        spdlog::details::os::default_eol);
+
+    REQUIRE(to_string_view(formatted_1) == expected);
+
+    spdlog::mdc::put("mdc_key_1", "new_mdc_value_1");
+    memory_buf_t formatted_2;
+    formatter->format(msg, formatted_2);
+    expected = spdlog::fmt_lib::format(
+        "[logger-name] [info] [mdc_key_1:new_mdc_value_1 mdc_key_2:mdc_value_2] some message{}",
+        spdlog::details::os::default_eol);
+
+    REQUIRE(to_string_view(formatted_2) == expected);
+
+    SECTION("Tear down") { spdlog::mdc::clear(); }
+}
+
+TEST_CASE("mdc different threads", "[pattern_formatter]") {
+    auto formatter = std::make_shared<spdlog::pattern_formatter>();
+    formatter->set_pattern("[%n] [%l] [%&] %v");
+    spdlog::details::log_msg msg(spdlog::source_loc{}, "logger-name", spdlog::level::info,
+                                 "some message");
+
+    memory_buf_t formatted_2;
+
+    auto lambda_1 = [formatter, msg]() {
+        spdlog::mdc::put("mdc_key", "thread_1_id");
+        memory_buf_t formatted;
+        formatter->format(msg, formatted);
+
+        auto expected =
+            spdlog::fmt_lib::format("[logger-name] [info] [mdc_key:thread_1_id] some message{}",
+                                    spdlog::details::os::default_eol);
+
+        REQUIRE(to_string_view(formatted) == expected);
+    };
+
+    auto lambda_2 = [formatter, msg]() {
+        spdlog::mdc::put("mdc_key", "thread_2_id");
+        memory_buf_t formatted;
+        formatter->format(msg, formatted);
+
+        auto expected =
+            spdlog::fmt_lib::format("[logger-name] [info] [mdc_key:thread_2_id] some message{}",
+                                    spdlog::details::os::default_eol);
+
+        REQUIRE(to_string_view(formatted) == expected);
+    };
+
+    std::thread thread_1(lambda_1);
+    std::thread thread_2(lambda_2);
+
+    thread_1.join();
+    thread_2.join();
+
+    SECTION("Tear down") { spdlog::mdc::clear(); }
+}
+
+TEST_CASE("mdc remove key", "[pattern_formatter]") {
+    spdlog::mdc::put("mdc_key_1", "mdc_value_1");
+    spdlog::mdc::put("mdc_key_2", "mdc_value_2");
+    spdlog::mdc::remove("mdc_key_1");
+
+    auto formatter = std::make_shared<spdlog::pattern_formatter>();
+    formatter->set_pattern("[%n] [%l] [%&] %v");
+
+    memory_buf_t formatted;
+    spdlog::details::log_msg msg(spdlog::source_loc{}, "logger-name", spdlog::level::info,
+                                 "some message");
+    formatter->format(msg, formatted);
+
+    auto expected =
+        spdlog::fmt_lib::format("[logger-name] [info] [mdc_key_2:mdc_value_2] some message{}",
+                                spdlog::details::os::default_eol);
+    REQUIRE(to_string_view(formatted) == expected);
+
+    SECTION("Tear down") { spdlog::mdc::clear(); }
+}
+
+TEST_CASE("mdc empty", "[pattern_formatter]") {
+    auto formatter = std::make_shared<spdlog::pattern_formatter>();
+    formatter->set_pattern("[%n] [%l] [%&] %v");
+
+    memory_buf_t formatted;
+    spdlog::details::log_msg msg(spdlog::source_loc{}, "logger-name", spdlog::level::info,
+                                 "some message");
+    formatter->format(msg, formatted);
+
+    auto expected = spdlog::fmt_lib::format("[logger-name] [info] [] some message{}",
+                                            spdlog::details::os::default_eol);
+    REQUIRE(to_string_view(formatted) == expected);
+
+    SECTION("Tear down") { spdlog::mdc::clear(); }
+}
+#endif
diff --git a/ext/spdlog/tests/test_registry.cpp b/ext/spdlog/tests/test_registry.cpp
new file mode 100644
index 0000000..69ec8f5
--- /dev/null
+++ b/ext/spdlog/tests/test_registry.cpp
@@ -0,0 +1,112 @@
+#include "includes.h"
+
+static const char *const tested_logger_name = "null_logger";
+static const char *const tested_logger_name2 = "null_logger2";
+
+#ifndef SPDLOG_NO_EXCEPTIONS
+TEST_CASE("register_drop", "[registry]") {
+    spdlog::drop_all();
+    spdlog::create<spdlog::sinks::null_sink_mt>(tested_logger_name);
+    REQUIRE(spdlog::get(tested_logger_name) != nullptr);
+    // Throw if registering existing name
+    REQUIRE_THROWS_AS(spdlog::create<spdlog::sinks::null_sink_mt>(tested_logger_name),
+                      spdlog::spdlog_ex);
+}
+
+TEST_CASE("explicit register", "[registry]") {
+    spdlog::drop_all();
+    auto logger = std::make_shared<spdlog::logger>(tested_logger_name,
+                                                   std::make_shared<spdlog::sinks::null_sink_st>());
+    spdlog::register_logger(logger);
+    REQUIRE(spdlog::get(tested_logger_name) != nullptr);
+    // Throw if registering existing name
+    REQUIRE_THROWS_AS(spdlog::create<spdlog::sinks::null_sink_mt>(tested_logger_name),
+                      spdlog::spdlog_ex);
+}
+#endif
+
+TEST_CASE("apply_all", "[registry]") {
+    spdlog::drop_all();
+    auto logger = std::make_shared<spdlog::logger>(tested_logger_name,
+                                                   std::make_shared<spdlog::sinks::null_sink_st>());
+    spdlog::register_logger(logger);
+    auto logger2 = std::make_shared<spdlog::logger>(
+        tested_logger_name2, std::make_shared<spdlog::sinks::null_sink_st>());
+    spdlog::register_logger(logger2);
+
+    int counter = 0;
+    spdlog::apply_all([&counter](std::shared_ptr<spdlog::logger>) { counter++; });
+    REQUIRE(counter == 2);
+
+    counter = 0;
+    spdlog::drop(tested_logger_name2);
+    spdlog::apply_all([&counter](std::shared_ptr<spdlog::logger> l) {
+        REQUIRE(l->name() == tested_logger_name);
+        counter++;
+    });
+    REQUIRE(counter == 1);
+}
+
+TEST_CASE("drop", "[registry]") {
+    spdlog::drop_all();
+    spdlog::create<spdlog::sinks::null_sink_mt>(tested_logger_name);
+    spdlog::drop(tested_logger_name);
+    REQUIRE_FALSE(spdlog::get(tested_logger_name));
+}
+
+TEST_CASE("drop-default", "[registry]") {
+    spdlog::set_default_logger(spdlog::null_logger_st(tested_logger_name));
+    spdlog::drop(tested_logger_name);
+    REQUIRE_FALSE(spdlog::default_logger());
+    REQUIRE_FALSE(spdlog::get(tested_logger_name));
+}
+
+TEST_CASE("drop_all", "[registry]") {
+    spdlog::drop_all();
+    spdlog::create<spdlog::sinks::null_sink_mt>(tested_logger_name);
+    spdlog::create<spdlog::sinks::null_sink_mt>(tested_logger_name2);
+    spdlog::drop_all();
+    REQUIRE_FALSE(spdlog::get(tested_logger_name));
+    REQUIRE_FALSE(spdlog::get(tested_logger_name2));
+    REQUIRE_FALSE(spdlog::default_logger());
+}
+
+TEST_CASE("drop non existing", "[registry]") {
+    spdlog::drop_all();
+    spdlog::create<spdlog::sinks::null_sink_mt>(tested_logger_name);
+    spdlog::drop("some_name");
+    REQUIRE_FALSE(spdlog::get("some_name"));
+    REQUIRE(spdlog::get(tested_logger_name));
+    spdlog::drop_all();
+}
+
+TEST_CASE("default logger", "[registry]") {
+    spdlog::drop_all();
+    spdlog::set_default_logger(spdlog::null_logger_st(tested_logger_name));
+    REQUIRE(spdlog::get(tested_logger_name) == spdlog::default_logger());
+    spdlog::drop_all();
+}
+
+TEST_CASE("set_default_logger(nullptr)", "[registry]") {
+    spdlog::set_default_logger(nullptr);
+    REQUIRE_FALSE(spdlog::default_logger());
+}
+
+TEST_CASE("disable automatic registration", "[registry]") {
+    // set some global parameters
+    spdlog::level::level_enum log_level = spdlog::level::level_enum::warn;
+    spdlog::set_level(log_level);
+    // but disable automatic registration
+    spdlog::set_automatic_registration(false);
+    auto logger1 = spdlog::create<spdlog::sinks::daily_file_sink_st>(
+        tested_logger_name, SPDLOG_FILENAME_T("filename"), 11, 59);
+    auto logger2 = spdlog::create_async<spdlog::sinks::stdout_color_sink_mt>(tested_logger_name2);
+    // loggers should not be part of the registry
+    REQUIRE_FALSE(spdlog::get(tested_logger_name));
+    REQUIRE_FALSE(spdlog::get(tested_logger_name2));
+    // but make sure they are still initialized according to global defaults
+    REQUIRE(logger1->level() == log_level);
+    REQUIRE(logger2->level() == log_level);
+    spdlog::set_level(spdlog::level::info);
+    spdlog::set_automatic_registration(true);
+}
diff --git a/ext/spdlog/tests/test_sink.h b/ext/spdlog/tests/test_sink.h
new file mode 100644
index 0000000..529d86d
--- /dev/null
+++ b/ext/spdlog/tests/test_sink.h
@@ -0,0 +1,69 @@
+//
+// Copyright(c) 2018 Gabi Melman.
+// Distributed under the MIT License (http://opensource.org/licenses/MIT)
+//
+
+#pragma once
+
+#include "spdlog/details/null_mutex.h"
+#include "spdlog/sinks/base_sink.h"
+#include "spdlog/fmt/fmt.h"
+#include <chrono>
+#include <mutex>
+#include <thread>
+
+namespace spdlog {
+namespace sinks {
+
+template <class Mutex>
+class test_sink : public base_sink<Mutex> {
+    const size_t lines_to_save = 100;
+
+public:
+    size_t msg_counter() {
+        std::lock_guard<Mutex> lock(base_sink<Mutex>::mutex_);
+        return msg_counter_;
+    }
+
+    size_t flush_counter() {
+        std::lock_guard<Mutex> lock(base_sink<Mutex>::mutex_);
+        return flush_counter_;
+    }
+
+    void set_delay(std::chrono::milliseconds delay) {
+        std::lock_guard<Mutex> lock(base_sink<Mutex>::mutex_);
+        delay_ = delay;
+    }
+
+    // return last output without the eol
+    std::vector<std::string> lines() {
+        std::lock_guard<Mutex> lock(base_sink<Mutex>::mutex_);
+        return lines_;
+    }
+
+protected:
+    void sink_it_(const details::log_msg &msg) override {
+        memory_buf_t formatted;
+        base_sink<Mutex>::formatter_->format(msg, formatted);
+        // save the line without the eol
+        auto eol_len = strlen(details::os::default_eol);
+        if (lines_.size() < lines_to_save) {
+            lines_.emplace_back(formatted.begin(), formatted.end() - eol_len);
+        }
+        msg_counter_++;
+        std::this_thread::sleep_for(delay_);
+    }
+
+    void flush_() override { flush_counter_++; }
+
+    size_t msg_counter_{0};
+    size_t flush_counter_{0};
+    std::chrono::milliseconds delay_{std::chrono::milliseconds::zero()};
+    std::vector<std::string> lines_;
+};
+
+using test_sink_mt = test_sink<std::mutex>;
+using test_sink_st = test_sink<details::null_mutex>;
+
+}  // namespace sinks
+}  // namespace spdlog
diff --git a/ext/spdlog/tests/test_stdout_api.cpp b/ext/spdlog/tests/test_stdout_api.cpp
new file mode 100644
index 0000000..67659b8
--- /dev/null
+++ b/ext/spdlog/tests/test_stdout_api.cpp
@@ -0,0 +1,90 @@
+/*
+ * This content is released under the MIT License as specified in
+ * https://raw.githubusercontent.com/gabime/spdlog/master/LICENSE
+ */
+#include "includes.h"
+#include "spdlog/sinks/stdout_sinks.h"
+#include "spdlog/sinks/stdout_color_sinks.h"
+TEST_CASE("stdout_st", "[stdout]") {
+    auto l = spdlog::stdout_logger_st("test");
+    l->set_pattern("%+");
+    l->set_level(spdlog::level::trace);
+    l->trace("Test stdout_st");
+    spdlog::drop_all();
+}
+
+TEST_CASE("stdout_mt", "[stdout]") {
+    auto l = spdlog::stdout_logger_mt("test");
+    l->set_pattern("%+");
+    l->set_level(spdlog::level::debug);
+    l->debug("Test stdout_mt");
+    spdlog::drop_all();
+}
+
+TEST_CASE("stderr_st", "[stderr]") {
+    auto l = spdlog::stderr_logger_st("test");
+    l->set_pattern("%+");
+    l->info("Test stderr_st");
+    spdlog::drop_all();
+}
+
+TEST_CASE("stderr_mt", "[stderr]") {
+    auto l = spdlog::stderr_logger_mt("test");
+    l->set_pattern("%+");
+    l->info("Test stderr_mt");
+    l->warn("Test stderr_mt");
+    l->error("Test stderr_mt");
+    l->critical("Test stderr_mt");
+    spdlog::drop_all();
+}
+
+// color loggers
+TEST_CASE("stdout_color_st", "[stdout]") {
+    auto l = spdlog::stdout_color_st("test");
+    l->set_pattern("%+");
+    l->info("Test stdout_color_st");
+    spdlog::drop_all();
+}
+
+TEST_CASE("stdout_color_mt", "[stdout]") {
+    auto l = spdlog::stdout_color_mt("test");
+    l->set_pattern("%+");
+    l->set_level(spdlog::level::trace);
+    l->trace("Test stdout_color_mt");
+    spdlog::drop_all();
+}
+
+TEST_CASE("stderr_color_st", "[stderr]") {
+    auto l = spdlog::stderr_color_st("test");
+    l->set_pattern("%+");
+    l->set_level(spdlog::level::debug);
+    l->debug("Test stderr_color_st");
+    spdlog::drop_all();
+}
+
+TEST_CASE("stderr_color_mt", "[stderr]") {
+    auto l = spdlog::stderr_color_mt("test");
+    l->set_pattern("%+");
+    l->info("Test stderr_color_mt");
+    l->warn("Test stderr_color_mt");
+    l->error("Test stderr_color_mt");
+    l->critical("Test stderr_color_mt");
+    spdlog::drop_all();
+}
+
+#ifdef SPDLOG_WCHAR_TO_UTF8_SUPPORT
+
+TEST_CASE("wchar_api", "[stdout]") {
+    auto l = spdlog::stdout_logger_st("wchar_logger");
+    l->set_pattern("%+");
+    l->set_level(spdlog::level::trace);
+    l->trace(L"Test wchar_api");
+    l->trace(L"Test wchar_api {}", L"param");
+    l->trace(L"Test wchar_api {}", 1);
+    l->trace(L"Test wchar_api {}", std::wstring{L"wstring param"});
+    l->trace(std::wstring{L"Test wchar_api wstring"});
+    SPDLOG_LOGGER_DEBUG(l, L"Test SPDLOG_LOGGER_DEBUG {}", L"param");
+    spdlog::drop_all();
+}
+
+#endif
diff --git a/ext/spdlog/tests/test_stopwatch.cpp b/ext/spdlog/tests/test_stopwatch.cpp
new file mode 100644
index 0000000..b2aa246
--- /dev/null
+++ b/ext/spdlog/tests/test_stopwatch.cpp
@@ -0,0 +1,42 @@
+#include "includes.h"
+#include "test_sink.h"
+#include "spdlog/stopwatch.h"
+
+TEST_CASE("stopwatch1", "[stopwatch]") {
+    using std::chrono::milliseconds;
+    using clock = std::chrono::steady_clock;
+    milliseconds wait_ms(200);
+    milliseconds tolerance_ms(250);
+    auto start = clock::now();
+    spdlog::stopwatch sw;
+    std::this_thread::sleep_for(wait_ms);
+    auto stop = clock::now();
+    auto diff_ms = std::chrono::duration_cast<milliseconds>(stop - start);
+    REQUIRE(sw.elapsed() >= diff_ms);
+    REQUIRE(sw.elapsed() <= diff_ms + tolerance_ms);
+}
+
+TEST_CASE("stopwatch2", "[stopwatch]") {
+    using spdlog::sinks::test_sink_st;
+    using std::chrono::duration_cast;
+    using std::chrono::milliseconds;
+    using clock = std::chrono::steady_clock;
+
+    clock::duration wait_duration(milliseconds(200));
+    clock::duration tolerance_duration(milliseconds(250));
+
+    auto test_sink = std::make_shared<test_sink_st>();
+
+    auto start = clock::now();
+    spdlog::stopwatch sw;
+    spdlog::logger logger("test-stopwatch", test_sink);
+    logger.set_pattern("%v");
+    std::this_thread::sleep_for(wait_duration);
+    auto stop = clock::now();
+    logger.info("{}", sw);
+    auto val = std::stod(test_sink->lines()[0]);
+    auto diff_duration = duration_cast<std::chrono::duration<double>>(stop - start);
+
+    REQUIRE(val >= (diff_duration).count() - 0.001);
+    REQUIRE(val <= (diff_duration + tolerance_duration).count());
+}
diff --git a/ext/spdlog/tests/test_systemd.cpp b/ext/spdlog/tests/test_systemd.cpp
new file mode 100644
index 0000000..e363657
--- /dev/null
+++ b/ext/spdlog/tests/test_systemd.cpp
@@ -0,0 +1,14 @@
+#include "includes.h"
+#include "spdlog/sinks/systemd_sink.h"
+
+TEST_CASE("systemd", "[all]") {
+    auto systemd_sink = std::make_shared<spdlog::sinks::systemd_sink_st>();
+    spdlog::logger logger("spdlog_systemd_test", systemd_sink);
+    logger.set_level(spdlog::level::trace);
+    logger.trace("test spdlog trace");
+    logger.debug("test spdlog debug");
+    SPDLOG_LOGGER_INFO((&logger), "test spdlog info");
+    SPDLOG_LOGGER_WARN((&logger), "test spdlog warn");
+    SPDLOG_LOGGER_ERROR((&logger), "test spdlog error");
+    SPDLOG_LOGGER_CRITICAL((&logger), "test spdlog critical");
+}
diff --git a/ext/spdlog/tests/test_time_point.cpp b/ext/spdlog/tests/test_time_point.cpp
new file mode 100644
index 0000000..b7b1b23
--- /dev/null
+++ b/ext/spdlog/tests/test_time_point.cpp
@@ -0,0 +1,35 @@
+#include "includes.h"
+#include "test_sink.h"
+#include "spdlog/async.h"
+
+TEST_CASE("time_point1", "[time_point log_msg]") {
+    std::shared_ptr<spdlog::sinks::test_sink_st> test_sink(new spdlog::sinks::test_sink_st);
+    spdlog::logger logger("test-time_point", test_sink);
+
+    spdlog::source_loc source{};
+    std::chrono::system_clock::time_point tp{std::chrono::system_clock::now()};
+    test_sink->set_pattern("%T.%F");  // interested in the time_point
+
+    // all the following should have the same time
+    test_sink->set_delay(std::chrono::milliseconds(10));
+    for (int i = 0; i < 5; i++) {
+        spdlog::details::log_msg msg{tp, source, "test_logger", spdlog::level::info, "message"};
+        test_sink->log(msg);
+    }
+
+    logger.log(tp, source, spdlog::level::info, "formatted message");
+    logger.log(tp, source, spdlog::level::info, "formatted message");
+    logger.log(tp, source, spdlog::level::info, "formatted message");
+    logger.log(tp, source, spdlog::level::info, "formatted message");
+    logger.log(source, spdlog::level::info,
+               "formatted message");  // last line has different time_point
+
+    // now the real test... that the times are the same.
+    std::vector<std::string> lines = test_sink->lines();
+    REQUIRE(lines[0] == lines[1]);
+    REQUIRE(lines[2] == lines[3]);
+    REQUIRE(lines[4] == lines[5]);
+    REQUIRE(lines[6] == lines[7]);
+    REQUIRE(lines[8] != lines[9]);
+    spdlog::drop_all();
+}
diff --git a/ext/spdlog/tests/utils.cpp b/ext/spdlog/tests/utils.cpp
new file mode 100644
index 0000000..1fa2617
--- /dev/null
+++ b/ext/spdlog/tests/utils.cpp
@@ -0,0 +1,103 @@
+#include "includes.h"
+
+#ifdef _WIN32
+    #include <windows.h>
+#else
+    #include <sys/types.h>
+    #include <dirent.h>
+#endif
+
+void prepare_logdir() {
+    spdlog::drop_all();
+#ifdef _WIN32
+    system("rmdir /S /Q test_logs");
+#else
+    auto rv = system("rm -rf test_logs");
+    if (rv != 0) {
+        throw std::runtime_error("Failed to rm -rf test_logs");
+    }
+#endif
+}
+
+std::string file_contents(const std::string &filename) {
+    std::ifstream ifs(filename, std::ios_base::binary);
+    if (!ifs) {
+        throw std::runtime_error("Failed open file ");
+    }
+    return std::string((std::istreambuf_iterator<char>(ifs)), (std::istreambuf_iterator<char>()));
+}
+
+std::size_t count_lines(const std::string &filename) {
+    std::ifstream ifs(filename);
+    if (!ifs) {
+        throw std::runtime_error("Failed open file ");
+    }
+
+    std::string line;
+    size_t counter = 0;
+    while (std::getline(ifs, line)) counter++;
+    return counter;
+}
+
+void require_message_count(const std::string &filename, const std::size_t messages) {
+    if (strlen(spdlog::details::os::default_eol) == 0) {
+        REQUIRE(count_lines(filename) == 1);
+    } else {
+        REQUIRE(count_lines(filename) == messages);
+    }
+}
+
+std::size_t get_filesize(const std::string &filename) {
+    std::ifstream ifs(filename, std::ifstream::ate | std::ifstream::binary);
+    if (!ifs) {
+        throw std::runtime_error("Failed open file ");
+    }
+
+    return static_cast<std::size_t>(ifs.tellg());
+}
+
+// source: https://stackoverflow.com/a/2072890/192001
+bool ends_with(std::string const &value, std::string const &ending) {
+    if (ending.size() > value.size()) {
+        return false;
+    }
+    return std::equal(ending.rbegin(), ending.rend(), value.rbegin());
+}
+
+#ifdef _WIN32
+// Based on: https://stackoverflow.com/a/37416569/192001
+std::size_t count_files(const std::string &folder) {
+    size_t counter = 0;
+    WIN32_FIND_DATAA ffd;
+
+    // Start iterating over the files in the folder directory.
+    HANDLE hFind = ::FindFirstFileA((folder + "\\*").c_str(), &ffd);
+    if (hFind != INVALID_HANDLE_VALUE) {
+        do  // Managed to locate and create an handle to that folder.
+        {
+            if (ffd.cFileName[0] != '.') counter++;
+        } while (::FindNextFileA(hFind, &ffd) != 0);
+        ::FindClose(hFind);
+    } else {
+        throw std::runtime_error("Failed open folder " + folder);
+    }
+
+    return counter;
+}
+#else
+// Based on: https://stackoverflow.com/a/2802255/192001
+std::size_t count_files(const std::string &folder) {
+    size_t counter = 0;
+    DIR *dp = opendir(folder.c_str());
+    if (dp == nullptr) {
+        throw std::runtime_error("Failed open folder " + folder);
+    }
+
+    struct dirent *ep = nullptr;
+    while ((ep = readdir(dp)) != nullptr) {
+        if (ep->d_name[0] != '.') counter++;
+    }
+    (void)closedir(dp);
+    return counter;
+}
+#endif
diff --git a/ext/spdlog/tests/utils.h b/ext/spdlog/tests/utils.h
new file mode 100644
index 0000000..53c09b4
--- /dev/null
+++ b/ext/spdlog/tests/utils.h
@@ -0,0 +1,18 @@
+#pragma once
+
+#include <cstddef>
+#include <string>
+
+std::size_t count_files(const std::string &folder);
+
+void prepare_logdir();
+
+std::string file_contents(const std::string &filename);
+
+std::size_t count_lines(const std::string &filename);
+
+void require_message_count(const std::string &filename, const std::size_t messages);
+
+std::size_t get_filesize(const std::string &filename);
+
+bool ends_with(std::string const &value, std::string const &ending);
\ No newline at end of file
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
new file mode 100644
index 0000000..37c036d
--- /dev/null
+++ b/src/CMakeLists.txt
@@ -0,0 +1,95 @@
+# exe path
+set(EXECUTABLE_OUTPUT_PATH "${PROJECT_BINARY_DIR}/bin")
+
+# source codes path
+aux_source_directory(${PROJECT_SOURCE_DIR}/src MAIN_SRC)
+aux_source_directory(${PROJECT_SOURCE_DIR}/src/util UTIL_SRC)
+aux_source_directory(${PROJECT_SOURCE_DIR}/src/sort SORT_SRC)
+
+set(KTHREAD_FILE ${PROJECT_SOURCE_DIR}/ext/klib/kthread.c)
+
+# including path
+include_directories("${PROJECT_SOURCE_DIR}/ext")
+include_directories("${PROJECT_SOURCE_DIR}/ext/spdlog/include")
+include_directories("${PROJECT_SOURCE_DIR}/ext/htslib")
+include_directories("${PROJECT_SOURCE_DIR}/src")
+
+# linking path
+link_directories("${PROJECT_SOURCE_DIR}/ext/htslib")
+link_directories("${PROJECT_SOURCE_DIR}/ext/spdlog/build")
+
+# program name
+set(PG_NAME "fastsort")
+
+# dependency files
+add_executable(${PG_NAME} ${MAIN_SRC} ${UTIL_SRC} ${SORT_SRC} ${KTHREAD_FILE})
+
+# link htslib
+target_link_libraries(${PG_NAME} libhts.a)
+target_link_libraries(${PG_NAME} libspdlog.a)
+# target_link_libraries(${PG_NAME} /home/zzh/work/jemalloc/lib/libjemalloc.a)
+
+# lib pthread
+find_package(Threads REQUIRED)
+if(THREADS_HAVE_PTHREAD_ARG)
+    set_property(TARGET ${PG_NAME} PROPERTY COMPILE_OPTIONS "-pthread")
+    set_property(TARGET ${PG_NAME} PROPERTY INTERFACE_COMPILE_OPTIONS "-pthread")
+endif()
+if(CMAKE_THREAD_LIBS_INIT)
+    target_link_libraries(${PG_NAME} "${CMAKE_THREAD_LIBS_INIT}")
+endif()
+
+# lib bzip2
+find_package(BZip2 REQUIRED)
+if(BZip2_FOUND)
+    include_directories(${BZIP2_INCLUDE_DIR})
+    target_link_libraries(${PG_NAME} ${BZIP2_LIBRARIES})
+else()
+    message(FATAL_ERROR "BZIP2 is not found")
+endif()
+
+# lib deflate
+find_library(DeflateLib deflate)
+if(DeflateLib)
+    target_link_libraries(${PG_NAME} ${DeflateLib})
+else()
+    message(FATAL_ERROR "libdeflate is not found")
+endif()
+
+# lib lzma
+find_package(LibLZMA MODULE)
+if (LibLZMA_FOUND)
+    include_directories(${LIBLZMA_INCLUDE_DIR})
+    target_link_libraries(${PG_NAME} ${LIBLZMA_LIBRARY})
+else()
+    message(FATAL_ERROR "lzma is not found")
+endif()
+
+# lib curl
+find_package(CURL REQUIRED)
+if (CURL_FOUND)
+    include_directories(${CURL_INCLUDE_DIR})
+    target_link_libraries(${PG_NAME} ${CURL_LIBRARY})
+endif()
+
+# lz
+find_library(Z_LIBRARY z)
+if(Z_LIBRARY)
+    target_link_libraries(${PG_NAME} ${Z_LIBRARY})
+else()
+    message(FATAL_ERROR "lz is not found")
+endif()
+
+# lm
+find_library(MATH_LIBRARY m)
+if(MATH_LIBRARY)
+    target_link_libraries(${PG_NAME} ${MATH_LIBRARY})
+else()
+    message(FATAL_ERROR "lm is not found")
+endif()
+
+# install path
+install(TARGETS ${PG_NAME}
+    RUNTIME DESTINATION bin
+    LIBRARY DESTINATION lib
+    ARCHIVE DESTINATION libstatic)
\ No newline at end of file
diff --git a/src/fastsort_version.h b/src/fastsort_version.h
new file mode 100644
index 0000000..c0983e8
--- /dev/null
+++ b/src/fastsort_version.h
@@ -0,0 +1,3 @@
+#pragma once
+
+#define FASTSORT_VERSION "1.0.0"
\ No newline at end of file
diff --git a/src/main.cpp b/src/main.cpp
new file mode 100644
index 0000000..5316d09
--- /dev/null
+++ b/src/main.cpp
@@ -0,0 +1,161 @@
+#include <spdlog/cfg/env.h>
+#include <spdlog/sinks/stdout_color_sinks.h>
+#include <spdlog/spdlog.h>
+#include <time.h>
+
+#include <argparse/argparse.hpp>
+#include <iostream>
+#include <map>
+#include <set>
+#include <string>
+
+#include "fastsort_version.h"
+#include "sort/sort_args.h"
+#include "util/profiling.h"
+
+namespace nsgv {
+extern SortArg gSortArg;
+};
+
+int doSort();
+
+string getCurrentTimeStr() {
+    time_t time_val;
+    struct tm *at;
+    char now[80];
+    time(&time_val);
+    at = localtime(&time_val);
+    strftime(now, 79, "%B %d, %Y at %I:%M:%S %p %Z", at);
+    return string(now);
+}
+
+int main(int argc, char *argv[]) {
+    // init log
+    spdlog::set_default_logger(spdlog::stderr_color_st("fastdup"));
+    spdlog::cfg::load_env_levels();
+
+    // init arg parser
+    argparse::ArgumentParser program("FastSort", FASTSORT_VERSION,
+                                     argparse::default_arguments::none);
+    program.add_description(
+        "Sort SAM/BAM files in user specific order such as coordinate, query-name etc.");
+
+    program.add_argument("-i", "--input")
+        .help("Input file. One SAM or BAM file.")
+        .metavar("<INPUT>")
+        .required();
+
+    program.add_argument("-o", "--output")
+        .help("Output file. SAM or BAM file to write sorted SAM/BAM to.")
+        .metavar("<OUTPUT>")
+        .required();
+
+    program.add_argument("--num-threads")
+        .help("Number of threads to use.")
+        .scan<'i', int>()
+        .default_value(1)
+        .nargs(1)
+        .metavar("<NUM_THREADS>");
+
+    program.add_argument("--none-duplex-io")
+        .help("Do not use writing-while-reading mode.")
+        .default_value(false)
+        .implicit_value(true);
+
+    program.add_argument("--create-index")
+        .help("Whether to create an index when writing sorted BAM output.(only work when sort type is coordinate)")
+        .default_value(false)
+        .implicit_value(true);
+
+    program.add_argument("--index-format")
+        .help("Format for bam index file. Possible values: {BAI, CSI}")
+        .default_value(std::string("BAI"))
+        .choices("BAI", "CSI")
+        .nargs(1)
+        .metavar("<IndexFormat>");
+
+    program.add_argument("-l", "--compress-level")
+        .help("Set compression level, from 0 (uncompressed) to 9 (best).")
+        .nargs(1);
+
+    program.add_argument("-m", "--max-mem")
+        .help("approximate total memory limit for all threads (by default 2GB)")
+        .default_value(std::string("2G"))
+        .nargs(1)
+        .metavar("<MAX_MEM>");
+
+    program.add_argument("--sort-type")
+        .help("\nUse coordinate sort or query name sort. Possible values: {COORDINATE, NAME}")
+        .default_value(std::string("COORDINATE"))
+        .choices("COORDINATE", "NAME")
+        .nargs(1)
+        .metavar("<SORT_TYPE>");
+
+    program.add_argument("--query-name")
+        .help("\nChoose the query name sort type when sorting by quey name. Possible values: {PICARD, NATURAL, ASCII}")
+        .default_value(std::string("PICARD"))
+        .choices("PICARD", "NATURAL", "ASCII")
+        .nargs(1)
+        .metavar("<QUERY_NAME>");
+
+    // add help and version args
+    program.add_argument("-h", "--help")
+        .action([&](const auto & /*unused*/) {
+            std::cout << program.help().str();
+            std::exit(0);
+        })
+        .default_value(false)
+        .help("shows help message and exits")
+        .implicit_value(true)
+        .nargs(0);
+
+    program.add_argument("-v", "--version")
+        .action([&](const auto & /*unused*/) {
+            std::cout << FASTSORT_VERSION << std::endl;
+            std::exit(0);
+        })
+        .default_value(false)
+        .help("prints version information and exits")
+        .implicit_value(true)
+        .nargs(0);
+
+    // std::cout << program << std::endl;
+
+    nsgv::gSortArg.START_TIME = getCurrentTimeStr();
+    nsgv::gSortArg.CLI_STR = argv[0];
+    for (int i = 1; i < argc; ++i) {
+        nsgv::gSortArg.CLI_STR += " " + std::string(argv[i]);
+    }
+
+    try {
+        program.parse_args(argc, argv);
+        nsgv::gSortArg.INPUT_FILE = program.get("--input");
+        nsgv::gSortArg.OUTPUT_FILE = program.get("--output");
+        nsgv::gSortArg.NUM_THREADS = program.get<int>("--num-threads");
+        nsgv::gSortArg.DUPLEX_IO = !program.get<bool>("--none-duplex-io");
+        nsgv::gSortArg.CREATE_INDEX = program.get<bool>("--create-index");
+        nsgv::gSortArg.INDEX_FORMAT =
+            program.get("--index-format") == "BAI" ? nsmd::IndexFormat::BAI : nsmd::IndexFormat::CSI;
+        nsgv::gSortArg.SORT_COORIDINATE = program.get("--sort-type") == "COORDINATE";
+
+        std::map<std::string, nsmd::QueryNameType> query_name_args = {
+            {"NATURAL,", nsmd::QueryNameType::NATURAL},
+            {"ASCII", nsmd::QueryNameType::ASCII},
+            {"PICARD", nsmd::QueryNameType::PICARD}};
+
+        nsgv::gSortArg.QUERY_NAME_TYPE = query_name_args[program.get("--query-name")];
+
+    } catch (const std::exception &err) {
+        spdlog::error(err.what());
+        std::cerr << std::endl << program;
+        return 1;
+    }
+
+    spdlog::info("fast bam sort start");
+    doSort();
+    spdlog::info("fast bam sort end");
+
+    displayProfiling(1);
+
+    return 0;
+}
\ No newline at end of file
diff --git a/src/sort/sam_io.cpp b/src/sort/sam_io.cpp
new file mode 100644
index 0000000..a9d5ac0
--- /dev/null
+++ b/src/sort/sam_io.cpp
@@ -0,0 +1,130 @@
+#include "sam_io.h"
+
+#include <htslib/sam.h>
+#include <libdeflate.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <zlib.h>
+#include <spdlog/spdlog.h>
+
+
+namespace nsgv {
+extern bool gIsBigEndian;
+};
+
+int bgzfUncompress(uint8_t *dst, size_t *dlen, const uint8_t *src, size_t slen, uint32_t expected_crc) {
+    struct libdeflate_decompressor *z = libdeflate_alloc_decompressor();
+    if (!z) {
+        hts_log_error("Call to libdeflate_alloc_decompressor failed");
+        return -1;
+    }
+
+    int ret = libdeflate_deflate_decompress(z, src, slen, dst, *dlen, dlen);
+    libdeflate_free_decompressor(z);
+
+    if (ret != LIBDEFLATE_SUCCESS) {
+        hts_log_error("Inflate operation failed: %d", ret);
+        return -1;
+    }
+
+    uint32_t crc = libdeflate_crc32(0, (unsigned char *)dst, *dlen);
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    // Pretend the CRC was OK so the fuzzer doesn't have to get it right
+    crc = expected_crc;
+#endif
+    if (crc != expected_crc) {
+        hts_log_error("CRC32 checksum mismatch");
+        return -2;
+    }
+
+    return 0;
+}
+
+void parseSamHeader(FILE *fp, HeaderBuf &hdrBuf) {
+    const int kMaxBlockSize = 65535;
+    sam_hdr_t *header = NULL;
+    uint8_t *fBuf = (uint8_t*)malloc(kMaxBlockSize);
+    DataBuffer uData;
+    size_t readState = 0;
+    int blockLen = 0;
+    size_t dlen = 0x10000;  // 65535
+    int magicLen;
+    int32_t i, nameLen, numNames = 0;
+    size_t bufsize;
+    ssize_t bytes;
+
+    uData.allocMem(kMaxBlockSize);
+
+    readState = fread(fBuf, 1, BLOCK_HEADER_LENGTH, fp);
+    blockLen = unpackInt16(&fBuf[16]) + 1;
+    readState = fread(&fBuf[BLOCK_HEADER_LENGTH], 1, blockLen - BLOCK_HEADER_LENGTH, fp);
+    header = sam_hdr_init();
+
+
+    uint32_t crc = le_to_u32(fBuf + blockLen - 8);
+    int ret = bgzfUncompress(uData.data, &dlen, (Bytef *)fBuf + 18, blockLen - 18, crc);
+    uData.curLen = dlen;
+    // 解析header
+    magicLen = 4;
+    if (memcmp(uData.data, "BAM\1", magicLen)) {
+        spdlog::error("Invalid BAM binary header");
+        return;
+    }
+    uData.readPos += magicLen;
+    header->l_text = le_to_u32(uData.data + uData.readPos); uData.readPos += 4;
+    header->text = (char *)malloc(header->l_text + 1);
+    header->text[header->l_text] = 0;
+//    while (uData.readPos + header->l_text > uData.curLen) { // header内容超过了一个block，继续读
+//        readState = fread(fBuf, 1, BLOCK_HEADER_LENGTH, fp); // 读压缩块头
+//        blockLen = unpackInt16(&fBuf[16]) + 1; // 压缩块大小
+//        readState = fread(&fBuf[BLOCK_HEADER_LENGTH], 1, blockLen - BLOCK_HEADER_LENGTH, fp); // 读压缩块剩下的部分
+//        size_t newDataSize = uData.maxLen;
+//        while (uData.curLen + kMaxBlockSize > uData.maxLen) newDataSize *= 2;
+//        uData.reAllocMem(newDataSize);
+//        ret = bgzfUncompress(&uData.data[uData.curLen], &dlen, (Bytef *)fBuf + 18, blockLen - 18, crc);
+//        uData.curLen += dlen;
+//    }
+    memcpy(header->text, &uData.data[uData.readPos], header->l_text);
+    uData.readPos += header->l_text;
+
+    memcpy(&header->n_targets, &uData.data[uData.readPos], 4); // n_targets
+    uData.readPos += 4;
+    spdlog::info("num target: {}", header->n_targets);
+    if (nsgv::gIsBigEndian) ed_swap_4p(&header->n_targets);
+    if (header->n_targets > 0) {
+        header->target_name = (char **)calloc(header->n_targets, sizeof(char *));
+        header->target_len = (uint32_t *)calloc(header->n_targets, sizeof(uint32_t));
+    } else {
+        header->target_name = NULL;
+        header->target_len = NULL;
+    }
+
+    for (int i = 0; i != header->n_targets; ++i) {
+        memcpy(&nameLen, &uData.data[uData.readPos], 4);
+        uData.readPos += 4;
+        if (nsgv::gIsBigEndian) ed_swap_4p(&nameLen);
+        header->target_name[i] = (char *)malloc(nameLen);
+        ++numNames;
+        memcpy(header->target_name[i], &uData.data[uData.readPos], nameLen);
+        uData.readPos += nameLen;
+        if (header->target_name[i][nameLen - 1] != '\0') {
+            /* Fix missing NUL-termination.  Is this being too nice?
+               We could alternatively bail out with an error. */
+            char *newName = (char*)realloc(header->target_name[i], nameLen + 1);
+            header->target_name[i] = newName;
+            header->target_name[i][nameLen] = '\0';
+        }
+        memcpy(&header->target_len[i], &uData.data[uData.readPos], 4);
+        uData.readPos += 4;
+        if (nsgv::gIsBigEndian) ed_swap_4p(&header->target_len[i]);
+
+        // spdlog::info("nameLen {}, {}, {}", nameLen, header->target_len[i], header->target_name[i]);
+    }
+    // spdlog::info("test res: {} {} {} {}", header->l_text, dlen, header->n_targets, header->text);
+
+
+    hdrBuf.header = header;
+    free(fBuf);
+
+    // exit(0);
+}
\ No newline at end of file
diff --git a/src/sort/sam_io.h b/src/sort/sam_io.h
new file mode 100644
index 0000000..26acc70
--- /dev/null
+++ b/src/sort/sam_io.h
@@ -0,0 +1,61 @@
+#pragma once
+#include <htslib/sam.h>
+#include <stdio.h>
+
+#ifndef BLOCK_HEADER_LENGTH
+#define BLOCK_HEADER_LENGTH 18
+#endif
+#ifndef BLOCK_FOOTER_LENGTH
+#define BLOCK_FOOTER_LENGTH 8
+#endif
+
+#define INCREASE_SIZE (8L * 1024 * 1024)
+
+struct DataBuffer {
+    uint8_t *data;
+    size_t readPos = 0; // 当前读取的位置
+    size_t curLen = 0;  // 当前使用的空间
+    size_t maxLen = 0;  // 最大空间
+
+    DataBuffer() {
+        curLen = 0;
+        maxLen = 0;
+        data = nullptr;
+    }
+    ~DataBuffer() {
+        if (data)
+            free(data);
+    }
+    void allocMem(size_t memSize) {
+        curLen = 0;
+        maxLen = memSize;
+        data = (uint8_t *)realloc(data, maxLen);
+    }
+
+    void reAllocMem(size_t memSize) {
+        if (memSize > maxLen) {
+            maxLen = memSize;
+            data = (uint8_t *)realloc(data, maxLen);
+        }
+    }
+};
+
+struct HeaderBuf {
+    uint8_t *data;
+    int bamPos;  // 第一条bam的位置
+    int curLen;
+    int maxLen;
+    sam_hdr_t *header;
+};
+
+// 解压之后的数据结构
+struct UBlockBuf {
+    uint8_t *data;
+    
+};
+
+static int unpackInt16(const uint8_t *buffer) { return buffer[0] | buffer[1] << 8; }
+
+void parseSamHeader(FILE *fp, HeaderBuf &hdrBuf);
+
+int bgzfUncompress(uint8_t *dst, size_t *dlen, const uint8_t *src, size_t slen, uint32_t expected_crc);
\ No newline at end of file
diff --git a/src/sort/sort.cpp b/src/sort/sort.cpp
new file mode 100644
index 0000000..acd66e2
--- /dev/null
+++ b/src/sort/sort.cpp
@@ -0,0 +1,538 @@
+#include <fcntl.h>
+#include <htslib/sam.h>
+#include <htslib/thread_pool.h>
+#include <klib/kthread.h>
+#include <math.h>
+#include <pthread.h>
+#include <spdlog/spdlog.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/mman.h>
+#include <unistd.h>
+#include <zlib.h>
+
+#include <algorithm>
+
+#include "sam_io.h"
+#include "sort_args.h"
+#include "util/profiling.h"
+
+#define BAM_BLOCK_SIZE 16L * 1024 * 1024
+
+namespace nsgv {
+
+SortArg gSortArg; // 参数
+HeaderBuf gInHdr; // 输入文件的header
+bool gIsBigEndian;
+
+};  // namespace nsgv
+
+struct BamSortData {
+    uint16_t flag;
+    uint16_t qnameLen;
+    uint16_t bamLen;
+    int32_t tid;
+    int64_t pos;
+    uint8_t *uDataPos;
+    char *qname;  // pointer to qname
+};
+
+// thread data for sorting
+struct SortData {
+    vector<vector<BamSortData>> bamArr; // for each thread
+    vector<DataBuffer> uData;  // for each thread uncompressed data
+    vector<uint8_t *> *blockAddrArr;
+    int round = 0;
+};
+
+
+int sam_realloc_bam_data(bam1_t *b, size_t desired) {
+    uint32_t new_m_data;
+    uint8_t *new_data;
+    new_m_data = desired;
+    kroundup32(new_m_data);  // next power of 2
+    new_m_data += 32;        // reduces malloc arena migrations?
+    if (new_m_data < desired) {
+        errno = ENOMEM;  // Not strictly true but we can't store the size
+        return -1;
+    }
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    if (new_m_data > FUZZ_ALLOC_LIMIT) {
+        errno = ENOMEM;
+        return -1;
+    }
+#endif
+    if ((bam_get_mempolicy(b) & BAM_USER_OWNS_DATA) == 0) {
+        new_data = (uint8_t*)realloc(b->data, new_m_data);
+    } else {
+        if ((new_data = (uint8_t *)malloc(new_m_data)) != NULL) {
+            if (b->l_data > 0)
+                memcpy(new_data, b->data, b->l_data < b->m_data ? b->l_data : b->m_data);
+            bam_set_mempolicy(b, bam_get_mempolicy(b) & (~BAM_USER_OWNS_DATA));
+        }
+    }
+    if (!new_data)
+        return -1;
+    b->data = new_data;
+    b->m_data = new_m_data;
+    return 0;
+}
+
+static inline int realloc_bam_data(bam1_t *b, size_t desired) {
+    if (desired <= b->m_data)
+        return 0;
+    return sam_realloc_bam_data(b, desired);
+}
+
+// Fix bad records where qname is not terminated correctly.
+static int fixup_missing_qname_nul(bam1_t *b) {
+    bam1_core_t *c = &b->core;
+
+    // Note this is called before c->l_extranul is added to c->l_qname
+    if (c->l_extranul > 0) {
+        b->data[c->l_qname++] = '\0';
+        c->l_extranul--;
+    } else {
+        if (b->l_data > INT_MAX - 4)
+            return -1;
+        if (realloc_bam_data(b, b->l_data + 4) < 0)
+            return -1;
+        b->l_data += 4;
+        b->data[c->l_qname++] = '\0';
+        c->l_extranul = 3;
+    }
+    return 0;
+}
+
+static void bam_cigar2rqlens(int n_cigar, const uint32_t *cigar, hts_pos_t *rlen, hts_pos_t *qlen) {
+    int k;
+    *rlen = *qlen = 0;
+    for (k = 0; k < n_cigar; ++k) {
+        int type = bam_cigar_type(bam_cigar_op(cigar[k]));
+        int len = bam_cigar_oplen(cigar[k]);
+        if (type & 1)
+            *qlen += len;
+        if (type & 2)
+            *rlen += len;
+    }
+}
+
+int parseBam(uint8_t *data, bam1_t *b) {
+    bam1_core_t *c = &b->core;
+    int32_t block_len, ret, i;
+    uint32_t new_l_data;
+    uint8_t tmp[32], *x;
+
+    b->l_data = 0;
+
+    block_len = *(uint32_t *)data;
+    spdlog::info("record_len: {}", block_len);
+    data += 4;
+    x = data;
+
+    c->tid = le_to_u32(x);
+    c->pos = le_to_i32(x + 4);
+    uint32_t x2 = le_to_u32(x + 8);
+    c->bin = x2 >> 16;
+    c->qual = x2 >> 8 & 0xff;
+    c->l_qname = x2 & 0xff;
+    c->l_extranul = (c->l_qname % 4 != 0) ? (4 - c->l_qname % 4) : 0;
+    uint32_t x3 = le_to_u32(x + 12);
+    c->flag = x3 >> 16;
+    c->n_cigar = x3 & 0xffff;
+    c->l_qseq = le_to_u32(x + 16);
+    c->mtid = le_to_u32(x + 20);
+    c->mpos = le_to_i32(x + 24);
+    c->isize = le_to_i32(x + 28);
+
+    new_l_data = block_len - 32 + c->l_extranul;
+    if (new_l_data > INT_MAX || c->l_qseq < 0 || c->l_qname < 1)
+        return -4;
+    if (((uint64_t)c->n_cigar << 2) + c->l_qname + c->l_extranul + (((uint64_t)c->l_qseq + 1) >> 1) + c->l_qseq >
+        (uint64_t)new_l_data)
+        return -4;
+    if (realloc_bam_data(b, new_l_data) < 0)
+        return -4;
+    b->l_data = new_l_data;
+
+    b->data = data;
+    data += c->l_qname;
+
+    if (b->data[c->l_qname - 1] != '\0') {  // try to fix missing nul termination
+        if (fixup_missing_qname_nul(b) < 0)
+            return -4;
+    }
+    for (i = 0; i < c->l_extranul; ++i) b->data[c->l_qname + i] = '\0';
+    c->l_qname += c->l_extranul;
+    data += b->l_data - c->l_qname;
+
+
+    // TODO: consider making this conditional
+    if (c->n_cigar > 0) {  // recompute "bin" and check CIGAR-qlen consistency
+        hts_pos_t rlen, qlen;
+        bam_cigar2rqlens(c->n_cigar, bam_get_cigar(b), &rlen, &qlen);
+        if ((b->core.flag & BAM_FUNMAP) || rlen == 0)
+            rlen = 1;
+        b->core.bin = hts_reg2bin(b->core.pos, b->core.pos + rlen, 14, 5);
+        // Sanity check for broken CIGAR alignments
+        if (c->l_qseq > 0 && !(c->flag & BAM_FUNMAP) && qlen != c->l_qseq) {
+            hts_log_error("CIGAR and query sequence lengths differ for %s", bam_get_qname(b));
+            return -4;
+        }
+    }
+
+    return 4 + block_len;
+}
+
+// multi-thread uncompress bam blocks
+static void mtUncompressBlock(void *data, long idx, int tid) {
+    auto &p = *(SortData*) data;
+    auto &bamArr = p.bamArr[tid];
+    auto &udata = p.uData[tid];
+
+    // if (udata.maxLen < nsgv::gSortArg.MAX_MEM / nsgv::gSortArg.NUM_THREADS) {
+    //     udata.allocMem(nsgv::gSortArg.MAX_MEM / nsgv::gSortArg.NUM_THREADS);
+    // }
+    // return;
+
+    // spdlog::info("start: {} end: {} size: {} idx: {}, tid: {}", start, end, p.blockAddrArr->size(), idx, tid);
+
+    long start = idx;
+    int step = nsgv::gSortArg.NUM_THREADS;
+    // long stop = p.blockAddrArr->size();
+    long stop = idx + 1;
+
+    // int chunkSize = (p.blockAddrArr->size() + nsgv::gSortArg.NUM_THREADS - 1) / nsgv::gSortArg.NUM_THREADS;
+    // long start = idx * chunkSize;
+    // int step = 1;
+    // long stop = std::min((idx + 1) * chunkSize, (long)p.blockAddrArr->size());
+
+    uint8_t buf[65535];
+    // if (tid == 0 ) spdlog::info("round:{} tid:{} start:{} stop:{}", p.round, tid, start, stop);
+
+    for (long i = start; i < stop; i += step) {
+        uint8_t *block = (*p.blockAddrArr)[i];
+        size_t dlen = 0x10000;  // 65535
+        if (udata.maxLen - udata.curLen < dlen) {
+            // spdlog::info("re alloc buf, maxLen: {} M", udata.maxLen / 1024 / 1024);
+            udata.maxLen += INCREASE_SIZE;
+            udata.data = (uint8_t *)realloc(udata.data, udata.maxLen);
+        }
+        int block_length = unpackInt16(&block[16]) + 1;
+        uint32_t crc = le_to_u32(block + block_length - 8);
+        int ret = bgzfUncompress(buf, &dlen, (Bytef *)block + 18, block_length - 18, crc);
+        // spdlog::info("BamSortData size: {}, bam size: {}", sizeof(BamSortData), *(uint32_t *)(udata.data +
+        // udata.curLen));
+        // spdlog::info("i: {} slen: {} dlen: {}", i, block_length, dlen);
+        // 不用对bam进行完全解析，反正只需要排序就行
+        // int pos = 0;
+        // uint8_t *ud = udata.data + udata.curLen;
+        // bam1_t *bp = bam_init1();
+        //while (pos < dlen) {
+        //    bamArr.push_back(BamSortData());
+        //    BamSortData &b = bamArr.back();
+        //    b.bamLen = *(uint32_t *)(buf + pos);
+        //    // // spdlog::info("bamLen: {}", b.bamLen);
+        //    pos += 4 + b.bamLen;
+        //    // if (b.bamLen > 1000)
+        //    // spdlog::info("tid: {} record len: {}, dlen: {}", tid, b.bamLen, dlen);
+        //    // exit(0);
+        //    // int bamLen = parseBam(buf, bp);
+        //    // exit(0);
+        //    // spdlog::info("bamLen: {}", b.bamLen);
+        //    //  pos += bamLen;
+        //}
+        // exit(0);
+        // usleep(1);
+        PROF_T_BEG(mem_copy);
+        memcpy(udata.data + udata.curLen, buf, dlen);
+        PROF_T_END(tid, mem_copy);
+        udata.curLen += dlen * 1;
+    }
+}
+
+// multi-thread sort bam blocks
+static void mtSortBlocks(void *data, long idx, int tid) {
+    auto &p = *(SortData *)data;
+    auto &bamArr = p.bamArr[tid];
+    auto &uData = p.uData[tid];
+}
+
+static void mergeSortedBlocks(const SortData &sortData) {
+
+}
+
+static void *nonBlockingUncompress(void *data) {
+    PROF_G_BEG(uncompress);
+    if (((SortData *)data)->round % 10 == 0)
+        spdlog::info("round-{} block size: {}", ((SortData *)data)->round++, ((SortData *)data)->blockAddrArr->size());
+    else
+        ((SortData *)data)->round++;
+
+    kt_for(nsgv::gSortArg.NUM_THREADS, mtUncompressBlock, data, ((SortData *)data)->blockAddrArr->size());
+    // kt_for(nsgv::gSortArg.NUM_THREADS, mtUncompressBlock, data, nsgv::gSortArg.NUM_THREADS);
+    PROF_G_END(uncompress);
+    return nullptr;
+}
+
+static size_t uncommpressedDataSize(const SortData &sortData) {
+    size_t curSize = 0;
+    for (int i = 0; i < sortData.uData.size(); ++i) {
+        curSize += sortData.uData[i].curLen;
+    }
+    return curSize;
+}
+
+template <typename T>
+static void switchPointer(T *p1, T *p2) {
+    const T tmp = *p1;
+    *p1 = *p2;
+    *p2 = tmp;
+}
+
+static size_t threadMemSize(SortData &sortData) {
+    size_t memSize = 0;
+    for (int i = 0; i < sortData.uData.size(); ++i) {
+        memSize += sortData.uData[i].curLen;
+    }
+    return memSize;
+}
+
+static size_t threadMaxMemSize(SortData &sortData) {
+    size_t memSize = 0;
+    for (int i = 0; i < sortData.uData.size(); ++i) {
+        memSize += sortData.uData[i].maxLen;
+    }
+    return memSize;
+}
+
+void *threadRead(void *data) {
+    spdlog::info("test file start");
+    FILE *fp = (FILE*) data;
+    size_t bufSize = 64L * 1024 * 1024;
+    uint8_t *testBuf = (uint8_t *)malloc(bufSize);
+    size_t readState = 0;
+    size_t fileSize = 0;
+    while ((readState = fread(testBuf, 1, bufSize, fp)) > 0) {
+        fileSize += readState;
+    }
+    spdlog::info("test file size: {}", fileSize);
+    return NULL;
+}
+
+int doSort() {
+
+#if 0
+    /* 打开输入bam文件 */
+    samFile *inBamFp = sam_open_format(nsgv::gSortArg.INPUT_FILE.c_str(), "r", nullptr);
+    if (!inBamFp) {
+        spdlog::error("[{}] load sam/bam file failed.\n", __func__);
+        return -1;
+    }
+    hts_set_opt(inBamFp, HTS_OPT_BLOCK_SIZE, BAM_BLOCK_SIZE);
+    sam_hdr_t *inBamHdr = sam_hdr_read(inBamFp);  // 读取header
+    
+    htsThreadPool htsPoolRead = {NULL, 0};        // 多线程读取，创建线程池
+    htsPoolRead.pool = hts_tpool_init(nsgv::gSortArg.NUM_THREADS);
+    if (!htsPoolRead.pool) {
+        spdlog::error("[{}] failed to set up thread pool", __LINE__);
+        sam_close(inBamFp);
+        return -1;
+    }
+    hts_set_opt(inBamFp, HTS_OPT_THREAD_POOL, &htsPoolRead);
+
+    // 测试读取bam的速度
+
+    bam1_t *bamp = bam_init1();
+    while (sam_read1(inBamFp, inBamHdr, bamp) >= 0) {
+        if (bamp->l_data > 1000) {
+            spdlog::info("large record len: {}", bamp->l_data);
+        }
+    }
+    sam_close(inBamFp);
+    return 0;
+
+#endif
+    // 初始化算法用到的数据结构
+    SortData sortData;
+    sortData.bamArr.resize(nsgv::gSortArg.NUM_THREADS);
+    sortData.uData.resize(nsgv::gSortArg.NUM_THREADS);
+    for (int i = 0; i < nsgv::gSortArg.NUM_THREADS; ++i) {
+        sortData.uData[i].allocMem(nsgv::gSortArg.MAX_MEM / nsgv::gSortArg.NUM_THREADS);
+    }
+    nsgv::gIsBigEndian = ed_is_big();
+
+    // 打开文件
+    FILE *fpr = fopen(nsgv::gSortArg.INPUT_FILE.c_str(), "rb");
+
+    parseSamHeader(fpr, nsgv::gInHdr);
+
+    //    FILE *fpw = fopen(nsgv::gSortArg.OUTPUT_FILE.c_str(), "rb");
+
+    const size_t READ_BUFSIZE = 4L * 1024 * 1024 * nsgv::gSortArg.NUM_THREADS;
+    const size_t MARGIN_SIZE = READ_BUFSIZE * 4;
+    // const size_t READ_BUFSIZE = 8L * 1024 * 1024;
+    const size_t SINGLE_BLOCK_SIZE = 65535;
+    uint8_t *fbuf[5];
+    fbuf[0] = (uint8_t *)malloc(READ_BUFSIZE);
+    fbuf[1] = (uint8_t *)malloc(READ_BUFSIZE);
+    fbuf[2] = (uint8_t *)malloc(SINGLE_BLOCK_SIZE);
+    fbuf[3] = (uint8_t *)malloc(SINGLE_BLOCK_SIZE);
+    fbuf[4] = (uint8_t *)malloc(READ_BUFSIZE);
+
+    //setvbuf(fpr, (char*)fbuf[4], _IOFBF, 0);
+    //setvbuf(fpr, (char *)fbuf[4], _IONBF, READ_BUFSIZE);
+
+    uint8_t *curBuf = fbuf[0];
+    uint8_t *preBuf = fbuf[1];
+    uint8_t *preBlock = fbuf[2];
+    uint8_t *curBlock = fbuf[3];
+
+    size_t fsize = 0;
+    size_t block_num = 0;
+    int round = 0;
+    size_t totalBlocks = 0;
+
+    /*
+    有几种情况
+    第一种：上一个buf里，只剩17字节，那么不能解析下一个block的长度
+    第二种：上一个buf里，完全不剩字节，即上一个buf读入完成时正好卡在下一个block开头
+    第三种：上一个buf里，剩余18字节，只能解析下一个block的长度
+    第四种：上一个buf里，剩余超过18字节，但是没有放下所有block的数据
+
+    简而言之：1. 够解析下一个block的header，不够解析完整的block
+    2. 不够解析下一个block的header。
+    */
+    size_t readState = 0;
+    size_t curReadPos = 0;
+    int blockLen = 0;
+    vector<uint8_t *> startAddrArr[2];
+    vector<uint8_t *> *curStartAddrArr = &startAddrArr[0];
+    vector<uint8_t *> *preStartAddrArr = &startAddrArr[1];
+    size_t curBLockPos = 0;
+    size_t lastBufRemain = 0;
+    uint8_t *switchBuf;
+    vector<uint8_t *> *switchAddrArr;
+    uint8_t *switchBlock;
+    pthread_t uncompressTid = 0;
+    pthread_t sortMergeTid = 0;
+
+// 读取一个压缩的block
+#define READ_BLOCKS                                            \
+    curBLockPos = curReadPos;                                  \
+    while (curReadPos + BLOCK_HEADER_LENGTH < readState) {     \
+        blockLen = unpackInt16(&curBuf[curBLockPos + 16]) + 1; \
+        curReadPos += blockLen;                                \
+        if (curReadPos < readState) {                          \
+            /* 这个block完整的包含在curBuf里 */                  \
+            curStartAddrArr->push_back(&curBuf[curBLockPos]);  \
+            curBLockPos = curReadPos;                          \
+        }                                                      \
+    }                                                          \
+    lastBufRemain = readState - curBLockPos;                   \
+    memcpy(curBlock, &curBuf[curBLockPos], lastBufRemain);  // 将不完整的block拷贝到curBlock
+
+// 交换buffer指针
+#define SWITCH_POINTER                 \
+    switchPointer(&curBuf, &preBuf); \
+    switchPointer(&curStartAddrArr, &preStartAddrArr); \
+    switchPointer(&curBlock, &preBlock);
+
+    PROF_G_BEG(mid_all);
+
+    PROF_G_BEG(read);
+    readState = fread(fbuf[4], 1, READ_BUFSIZE, fpr);
+    // readState = fread(curBuf, 1, READ_BUFSIZE, fpr);
+    PROF_G_BEG(mem_copy);
+    memcpy(curBuf, fbuf[4], READ_BUFSIZE);
+    PROF_G_END(mem_copy);
+    PROF_G_END(read);
+
+    while (readState > 0) {
+        //    while (readState > 0) {
+        PROF_G_BEG(parse_block);
+        curStartAddrArr->clear();
+        fsize += readState;
+        curReadPos = 0;
+//        if (lastBufRemain < BLOCK_HEADER_LENGTH) { // 上次遗留的数据不够解析block 长度
+//            memcpy(&preBlock[lastBufRemain], curBuf, BLOCK_HEADER_LENGTH);
+//            curReadPos += BLOCK_HEADER_LENGTH;
+//            lastBufRemain += BLOCK_HEADER_LENGTH;
+//        }
+//        blockLen = unpackInt16(&preBlock[16]) + 1;
+//        const int lenToRead = blockLen - lastBufRemain;
+//        memcpy(&preBlock[lastBufRemain], &curBuf[curReadPos], lenToRead);
+//        curReadPos += lenToRead;
+//        curStartAddrArr->push_back(&preBlock[0]);
+
+        // curStartAddrArr->push_back(&curBuf[0]);
+        // spdlog::info("first blocklen: {}, remain: {}, toread: {}", blockLen, lastBufRemain, lenToRead);
+        // uint8_t buf[65535];
+        // int block_length = unpackInt16(&preBlock[16]) + 1;
+        // uint32_t crc = le_to_u32(preBlock + block_length - 8);
+        // size_t dlen = 65535;
+        // int ret = bgzf_uncompress(buf, &dlen, (Bytef *)preBlock + 18, block_length - 18, crc); // 开头是header，得跳过
+        // int bamLen = *(uint32_t *)(buf);
+        // spdlog::info("ret: {}, bamLen: {}, dlen: {}, block_length: {}", ret, bamLen, dlen, block_length);
+        // exit(0);
+
+        READ_BLOCKS;
+
+        // spdlog::info("cur block size: {}", curStartAddrArr->size());
+
+        totalBlocks += curStartAddrArr->size();
+        PROF_G_END(parse_block);
+
+#if 1
+        // 并行解压
+        PROF_G_BEG(sort);
+        pthread_join(uncompressTid, NULL);
+        {
+            // 如果sortData的解压buf满了，那么进行排序并输出
+            if (true) {
+            //if (threadMemSize(sortData) + MARGIN_SIZE >= nsgv::gSortArg.MAX_MEM) {
+                // spdlog::info("clear buf data, buf size: {} G, {} G", threadMemSize(sortData) / 1024.0 / 1024 / 1024, threadMaxMemSize(sortData) / 1024.0 / 1024 / 1024);
+                for (int i = 0; i < sortData.uData.size(); ++i) {
+                    sortData.uData[i].curLen = 0; // 清空数据
+                    sortData.bamArr[i].clear();
+                }
+            }
+        }
+        PROF_G_END(sort);
+
+        sortData.blockAddrArr = curStartAddrArr;
+        pthread_create(&uncompressTid, NULL, nonBlockingUncompress, &sortData);
+        //nonBlockingUncompress(&sortData);
+#endif
+        // kt_for(nsgv::gSortArg.NUM_THREADS, mtUncompressBlock, &sortData, curStartAddrArr->size() / BATCH_SIZE);
+        // if (uncommpressedDataSize(sortData) >= nsgv::gSortArg.MAX_MEM) {
+        //     kt_for(nsgv::gSortArg.NUM_THREADS, mtSortBlocks, &sortData, nsgv::gSortArg.NUM_THREADS);
+        //     mergeSortedBlocks(sortData);
+        // }
+
+        SWITCH_POINTER;
+
+        PROF_G_BEG(read);
+        // readState = fread(curBuf, 1, READ_BUFSIZE, fpr);
+        PROF_G_BEG(mem_copy);
+        memcpy(curBuf, fbuf[4], READ_BUFSIZE); readState = READ_BUFSIZE;
+        PROF_G_END(mem_copy);
+        PROF_G_END(read);
+        if (fsize >= 6245369164) break;
+    }
+    pthread_join(uncompressTid, NULL);
+    PROF_G_END(mid_all);
+
+
+    spdlog::info("total blocks: {}", totalBlocks);
+    fprintf(stderr, "file size: %ld\n\n", fsize);
+err:
+    free(fbuf[0]);
+    free(fbuf[1]);
+    free(fbuf[2]);
+    free(fbuf[3]);
+    fclose(fpr);
+    //    fclose(fpw);
+
+    return 0;
+}
\ No newline at end of file
diff --git a/src/sort/sort_args.h b/src/sort/sort_args.h
new file mode 100644
index 0000000..669c94f
--- /dev/null
+++ b/src/sort/sort_args.h
@@ -0,0 +1,56 @@
+/*
+Description: Markduplicate需要用到的一些参数
+
+Copyright : All right reserved by ICT
+
+Author : Zhang Zhonghai
+Date : 2023/10/23
+*/
+#pragma once
+
+#include <string>
+#include <vector>
+
+using std::string;
+using std::vector;
+
+namespace nsmd {
+/* How strict to be when reading a SAM or BAM, beyond bare minimum validation.
+ */
+/* 计算reads分数的方式（比那个read得分更高） */
+enum QueryNameType { NATURAL, ASCII, PICARD };
+
+enum SortType { COORDINATE, NAME };
+/* 索引文件的格式 （bai或者csi） */
+enum IndexFormat { BAI, CSI };
+}  // namespace nsmd
+
+/* markduplicate 需要的参数*/
+struct SortArg {
+
+    string INPUT_FILE;  // input bam filename
+
+    string OUTPUT_FILE;  // output bam filename
+
+    int NUM_THREADS = 1;
+
+    int C_LEVEL = -1;  // compression level, from 0 (uncompressed) to 9 (best)
+
+    size_t MAX_MEM = ((size_t)1) << 32;  // maximum memory for all threads; suffix K/M/G recognized [768M x num_threads]
+
+    bool DUPLEX_IO = true;  // 同时读写
+
+    int QUERY_NAME_TYPE = nsmd::QueryNameType::PICARD;
+
+    bool SORT_COORIDINATE = true;
+
+    bool CREATE_INDEX = false;
+
+    nsmd::IndexFormat INDEX_FORMAT = nsmd::IndexFormat::BAI;
+
+    // 命令行字符串
+    string CLI_STR;
+
+    // 开始运行时间
+    string START_TIME;
+};
\ No newline at end of file
diff --git a/src/sort/sort_impl.cpp b/src/sort/sort_impl.cpp
new file mode 100644
index 0000000..ff2a4c1
--- /dev/null
+++ b/src/sort/sort_impl.cpp
@@ -0,0 +1,7 @@
+#include <htslib/sam.h>
+#include <spdlog/spdlog.h>
+#include <stdint.h>
+#include <stdio.h>
+
+#include "sort_impl.h"
+
diff --git a/src/sort/sort_impl.h b/src/sort/sort_impl.h
new file mode 100644
index 0000000..ebe7e11
--- /dev/null
+++ b/src/sort/sort_impl.h
@@ -0,0 +1,30 @@
+#pragma once
+
+#include <htslib/sam.h>
+#include <stdio.h>
+
+// Struct which contains the sorting key for TemplateCoordinate sort.
+struct TemplateCoordinateKey {
+    int tid1;
+    int tid2;
+    hts_pos_t pos1;
+    hts_pos_t pos2;
+    bool neg1;
+    bool neg2;
+    const char *library;
+    char *mid;
+    char *name;
+    bool is_upper_of_pair;
+};
+
+// Struct which contains the a record, and the pointer to the sort tag (if any) or
+// a combined ref / position / strand.
+// Used to speed up sorts (coordinate, by-tag, and template-coordinate).
+typedef struct BamSortTag {
+    bam1_t *bam_record;
+    union {
+        const uint8_t *tag;
+        uint8_t pos_tid[12];
+        TemplateCoordinateKey *key;
+    } u;
+};
\ No newline at end of file
diff --git a/src/util/profiling.cpp b/src/util/profiling.cpp
new file mode 100644
index 0000000..61a3798
--- /dev/null
+++ b/src/util/profiling.cpp
@@ -0,0 +1,71 @@
+#include "profiling.h"
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/time.h>
+
+#ifdef SHOW_PERF
+uint64_t tprof[LIM_THREAD_PROF_TYPE][LIM_THREAD] = {0};
+uint64_t proc_freq = 1000;
+uint64_t gprof[LIM_GLOBAL_PROF_TYPE] = {0};
+#endif
+
+uint64_t realtimeMsec(void) {
+    struct timeval tv;
+    gettimeofday(&tv, NULL);
+    return (uint64_t)1000 * (tv.tv_sec + ((1e-6) * tv.tv_usec));
+}
+
+static int calcThreadTime(uint64_t *a, int len, double *max, double *min, double *avg) {
+#ifdef SHOW_PERF
+    int i = 0;
+    uint64_t umax = 0, umin = UINT64_MAX, uavg = 0;
+    for (i = 0; i < len; i++) {
+        if (a[i] > umax)
+            umax = a[i];
+        if (a[i] < umin)
+            umin = a[i];
+        uavg += a[i];
+    }
+    if (len > 0) *avg = uavg * 1.0 / len / proc_freq;
+    else *avg = 0.0;
+    *max = umax * 1.0 / proc_freq;
+    *min = umin * 1.0 / proc_freq;
+    
+#endif
+    return 1;
+}
+
+#define PRINT_GP(gpname) \
+    fprintf(stderr, "time G %-15s:     %0.2lfs\n", #gpname, gprof[GP_##gpname] * 1.0 / proc_freq);
+
+#define PRINT_TP(tpname, nthread)                                                                                 \
+    {                                                                                                             \
+        double maxTime, minTime, avgTime;                                                                         \
+        calcThreadTime(tprof[TP_##tpname], nthread, &maxTime, &minTime, &avgTime);                                \
+        fprintf(stderr, "time T %-15s:  avg %0.2lfs min %0.2lfs max %0.2lfs\n", #tpname, avgTime, minTime, maxTime); \
+    }
+
+int displayProfiling(int nthread) {
+
+#ifdef SHOW_PERF
+    fprintf(stderr, "\n");
+
+    PRINT_GP(read);
+    PRINT_GP(mem_copy);
+    PRINT_GP(parse_block);
+    PRINT_GP(uncompress);
+    PRINT_GP(sort);
+    PRINT_GP(write_mid);
+    PRINT_GP(read_mid);
+    PRINT_GP(write_final);
+    PRINT_GP(mid_all);
+
+    PRINT_TP(sort, nthread);
+    PRINT_TP(mem_copy, nthread);
+    fprintf(stderr, "\n");
+#endif
+
+    return 0;
+}
\ No newline at end of file
diff --git a/src/util/profiling.h b/src/util/profiling.h
new file mode 100644
index 0000000..42e37b9
--- /dev/null
+++ b/src/util/profiling.h
@@ -0,0 +1,69 @@
+#pragma once
+#include <stdint.h>
+#include <stdlib.h>
+#include <sys/time.h>
+
+// #define SHOW_PERF
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define LIM_THREAD 128
+#define LIM_THREAD_PROF_TYPE 128
+#define LIM_GLOBAL_PROF_TYPE 128
+#define LIM_THREAD_DATA_TYPE 128
+#define LIM_GLOBAL_DATA_TYPE 128
+
+#ifdef SHOW_PERF
+extern uint64_t proc_freq;
+extern uint64_t tprof[LIM_THREAD_PROF_TYPE][LIM_THREAD];
+extern uint64_t gprof[LIM_GLOBAL_PROF_TYPE];
+#endif
+
+#ifdef SHOW_PERF
+#define PROF_G_BEG(time_name) uint64_t prof_G_tmp_##time_name = realtimeMsec()
+#define PROF_G_BEG_AGAIN(time_name) prof_G_tmp_##time_name = realtimeMsec()
+#define PROF_G_END(time_name) gprof[GP_##time_name] += realtimeMsec() - prof_G_tmp_##time_name
+#define PROF_T_BEG(time_name) uint64_t prof_T_tmp_##time_name = realtimeMsec()
+#define PROF_T_BEG_AGAIN(time_name) prof_T_tmp_##time_name = realtimeMsec()
+#define PROF_T_END(tid, time_name) tprof[TP_##time_name][tid] += realtimeMsec() - prof_T_tmp_##time_name
+#define PROF_PRINT_BEG(tmp_time) uint64_t tmp_time = realtimeMsec()
+#define PROF_PRINT_END(tmp_time)          \
+    tmp_time = realtimeMsec() - tmp_time; \
+    fprintf(stderr, "time %-15s:     %0.2lfs\n", #tmp_time, tmp_time * 1.0 / proc_freq)
+#else
+#define PROF_G_BEG(time_name)
+#define PROF_G_BEG_AGAIN(time_name)
+#define PROF_G_END(time_name)
+#define PROF_T_BEG(time_name)
+#define PROF_T_BEG_AGAIN(time_name)
+#define PROF_T_END(tid, time_name)
+#endif
+
+
+
+// GLOBAL
+enum { GP_0 = 0, GP_1, GP_2, GP_3, GP_4, GP_5, GP_6, GP_7, GP_8, GP_9, GP_10 };
+enum {
+    GP_mid_all = 11,
+    GP_read,
+    GP_mem_copy,
+    GP_parse_block,
+    GP_uncompress,
+    GP_sort,
+    GP_write_mid,
+    GP_read_mid,
+    GP_write_final
+};
+// THREAD
+enum { TP_0 = 0, TP_1, TP_2, TP_3, TP_4, TP_5, TP_6, TP_7, TP_8, TP_9, TP_10 };
+enum { TP_sort = 11, TP_mem_copy};
+
+uint64_t realtimeMsec(void);
+
+int displayProfiling(int);
+
+#ifdef __cplusplus
+}
+#endif
\ No newline at end of file