From c6d5dea314c10ef6fd38c75f2dae9e39840a1782 Mon Sep 17 00:00:00 2001 From: Heng Li Date: Thu, 22 Jun 2017 12:30:45 -0400 Subject: [PATCH] replace with newer versions --- ksw2.h | 132 ++++++++++++++++++++++ ksw2_extz2_sse.c | 280 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 412 insertions(+) create mode 100644 ksw2.h create mode 100644 ksw2_extz2_sse.c diff --git a/ksw2.h b/ksw2.h new file mode 100644 index 0000000..b758b7f --- /dev/null +++ b/ksw2.h @@ -0,0 +1,132 @@ +#ifndef KSW2_H_ +#define KSW2_H_ + +#include + +#define KSW_NEG_INF -0x40000000 + +#define KSW_EZ_SCORE_ONLY 0x01 // don't record alignment path/cigar +#define KSW_EZ_RIGHT 0x02 // right-align gaps +#define KSW_EZ_GENERIC_SC 0x04 // without this flag: match/mismatch only; last symbol is a wildcard +#define KSW_EZ_GLOBAL_ONLY 0x08 // don't record best score and ignore z-drop; significantly faster +#define KSW_EZ_DYN_BAND 0x10 // once used, ksw_extz_t::{mqe,mte} may be wrong + +typedef struct { + int max, max_q, max_t; // max extension score and coordinate + int mqe, mqe_t; // max score when reaching the end of query + int mte, mte_q; // max score when reaching the end of target + int score; // max score reaching both ends; may be KSW_NEG_INF + int m_cigar, n_cigar; + uint32_t *cigar; +} ksw_extz_t; + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * NW-like extension + * + * @param km memory pool, when used with kalloc + * @param qlen query length + * @param query query sequence with 0 <= query[i] < m + * @param tlen target length + * @param target target sequence with 0 <= target[i] < m + * @param m number of residue types + * @param mat m*m scoring mattrix in one-dimension array + * @param gapo gap open penalty; a gap of length l cost "-(gapo+l*gape)" + * @param gape gap extension penalty + * @param w band width + * @param zdrop off-diagonal drop-off to stop extension (positive) + * @param flag flag (see KSW_EZ_* macros) + * @param ez (out) scores and cigar + */ +void ksw_extz(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t q, int8_t e, int w, int zdrop, int flag, ksw_extz_t *ez); +void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t q, int8_t e, int w, int zdrop, int flag, ksw_extz_t *ez); + +/** + * Global alignment + * + * (first 10 parameters identical to ksw_extz_sse()) + * @param m_cigar (modified) max CIGAR length; feed 0 if cigar==0 + * @param n_cigar (out) number of CIGAR elements + * @param cigar (out) BAM-encoded CIGAR; caller need to deallocate with kfree(km, ) + * + * @return score of the alignment + */ +int ksw_gg(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t gapo, int8_t gape, int w, int *m_cigar_, int *n_cigar_, uint32_t **cigar_); +int ksw_gg2(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t gapo, int8_t gape, int w, int *m_cigar_, int *n_cigar_, uint32_t **cigar_); +int ksw_gg2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t gapo, int8_t gape, int w, int *m_cigar_, int *n_cigar_, uint32_t **cigar_); + +#ifdef __cplusplus +} +#endif + +/************************************ + *** Private macros and functions *** + ************************************/ + +#ifdef HAVE_KALLOC +#include "kalloc.h" +#else +#include +#define kmalloc(km, size) malloc((size)) +#define kcalloc(km, count, size) calloc((count), (size)) +#define krealloc(km, ptr, size) realloc((ptr), (size)) +#define kfree(km, ptr) free((ptr)) +#endif + +static inline uint32_t *ksw_push_cigar(void *km, int *n_cigar, int *m_cigar, uint32_t *cigar, uint32_t op, int len) +{ + if (*n_cigar == 0 || op != (cigar[(*n_cigar) - 1]&0xf)) { + if (*n_cigar == *m_cigar) { + *m_cigar = *m_cigar? (*m_cigar)<<1 : 4; + cigar = (uint32_t*)krealloc(km, cigar, (*m_cigar) << 2); + } + cigar[(*n_cigar)++] = len<<4 | op; + } else cigar[(*n_cigar)-1] += len<<4; + return cigar; +} + +static inline void ksw_backtrack(void *km, int is_rot, const uint8_t *p, const int *off, int n_col, int i0, int j0, int *m_cigar_, int *n_cigar_, uint32_t **cigar_) +{ + int n_cigar = 0, m_cigar = *m_cigar_, which = 0, i = i0, j = j0, r; + uint32_t *cigar = *cigar_, tmp; + while (i >= 0 && j >= 0) { + if (is_rot) r = i + j, tmp = p[r * n_col + i - off[r]]; + else tmp = p[i * n_col + j - off[i]]; + which = tmp >> (which << 1) & 3; + if (which == 0 && tmp>>6) break; + if (which == 0) which = tmp & 3; + if (which == 0) cigar = ksw_push_cigar(km, &n_cigar, &m_cigar, cigar, 0, 1), --i, --j; // match + else if (which == 1) cigar = ksw_push_cigar(km, &n_cigar, &m_cigar, cigar, 2, 1), --i; // deletion + else cigar = ksw_push_cigar(km, &n_cigar, &m_cigar, cigar, 1, 1), --j; // insertion + } + if (i >= 0) cigar = ksw_push_cigar(km, &n_cigar, &m_cigar, cigar, 2, i + 1); // first deletion + if (j >= 0) cigar = ksw_push_cigar(km, &n_cigar, &m_cigar, cigar, 1, j + 1); // first insertion + for (i = 0; i < n_cigar>>1; ++i) // reverse CIGAR + tmp = cigar[i], cigar[i] = cigar[n_cigar-1-i], cigar[n_cigar-1-i] = tmp; + *m_cigar_ = m_cigar, *n_cigar_ = n_cigar, *cigar_ = cigar; +} + +static inline int ksw_cigar2score(int8_t m, const int8_t *mat, int8_t q, int8_t e, const uint8_t *query, const uint8_t *target, int n_cigar, const uint32_t *cigar) +{ + int i, j, k, l, score; + for (k = 0, score = 0, i = j = 0; k < n_cigar; ++k) { + int op = cigar[k] & 0xf, len = cigar[k] >> 4; + if (op == 0) { + for (l = 0; l < len; ++l) + score += mat[target[i + l] * m + query[j + l]]; + i += len, j += len; + } else if (op == 1) { + score -= q + len * e; + j += len; + } else if (op == 2) { + score -= q + len * e; + i += len; + } + } + return score; +} + +#endif diff --git a/ksw2_extz2_sse.c b/ksw2_extz2_sse.c new file mode 100644 index 0000000..35666b5 --- /dev/null +++ b/ksw2_extz2_sse.c @@ -0,0 +1,280 @@ +#include +#include "ksw2.h" + +#ifdef __SSE2__ +#include + +#ifdef __SSE4_1__ +#include +#endif + +void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t q, int8_t e, int w, int zdrop, int flag, ksw_extz_t *ez) +{ +#define __dp_code_block1 \ + z = _mm_add_epi8(_mm_load_si128(&s[t]), qe2_); \ + xt1 = _mm_load_si128(&x[t]); /* xt1 <- x[r-1][t..t+15] */ \ + tmp = _mm_srli_si128(xt1, 15); /* tmp <- x[r-1][t+15] */ \ + xt1 = _mm_or_si128(_mm_slli_si128(xt1, 1), x1_); /* xt1 <- x[r-1][t-1..t+14] */ \ + x1_ = tmp; \ + vt1 = _mm_load_si128(&v[t]); /* vt1 <- v[r-1][t..t+15] */ \ + tmp = _mm_srli_si128(vt1, 15); /* tmp <- v[r-1][t+15] */ \ + vt1 = _mm_or_si128(_mm_slli_si128(vt1, 1), v1_); /* vt1 <- v[r-1][t-1..t+14] */ \ + v1_ = tmp; \ + a = _mm_add_epi8(xt1, vt1); /* a <- x[r-1][t-1..t+14] + v[r-1][t-1..t+14] */ \ + ut = _mm_load_si128(&u[t]); /* ut <- u[t..t+15] */ \ + b = _mm_add_epi8(_mm_load_si128(&y[t]), ut); /* b <- y[r-1][t..t+15] + u[r-1][t..t+15] */ + +#define __dp_code_block2 \ + z = _mm_max_epu8(z, b); /* z = max(z, b); this works because both are non-negative */ \ + _mm_store_si128(&u[t], _mm_sub_epi8(z, vt1)); /* u[r][t..t+15] <- z - v[r-1][t-1..t+14] */ \ + _mm_store_si128(&v[t], _mm_sub_epi8(z, ut)); /* v[r][t..t+15] <- z - u[r-1][t..t+15] */ \ + z = _mm_sub_epi8(z, q_); \ + a = _mm_sub_epi8(a, z); \ + b = _mm_sub_epi8(b, z); + + int r, t, qe = q + e, n_col_, *off = 0, tlen_, qlen_, last_st, last_en, wl, wr, max_sc; + int with_cigar = !(flag&KSW_EZ_SCORE_ONLY), with_max = !(flag&KSW_EZ_GLOBAL_ONLY); + int32_t *H = 0, H0 = 0, last_H0_t = 0; + uint8_t *qr, *sf, *mem, *mem2 = 0; + __m128i q_, qe2_, zero_, flag1_, flag2_, flag4_, flag32_, sc_mch_, sc_mis_, m1_; + __m128i *u, *v, *x, *y, *s, *p = 0; + + if (m <= 0 || qlen <= 0 || tlen <= 0 || w < 0 || zdrop < 0) return; + + zero_ = _mm_set1_epi8(0); + q_ = _mm_set1_epi8(q); + qe2_ = _mm_set1_epi8((q + e) * 2); + flag1_ = _mm_set1_epi8(1<<0); + flag2_ = _mm_set1_epi8(2<<0); + flag4_ = _mm_set1_epi8(1<<2); + flag32_ = _mm_set1_epi8(2<<4); + sc_mch_ = _mm_set1_epi8(mat[0]); + sc_mis_ = _mm_set1_epi8(mat[1]); + m1_ = _mm_set1_epi8(m - 1); // wildcard + + ez->max_q = ez->max_t = ez->mqe_t = ez->mte_q = -1; + ez->max = 0, ez->mqe = ez->mte = KSW_NEG_INF; + ez->n_cigar = 0; + + wl = wr = w; + tlen_ = (tlen + 15) / 16; + n_col_ = ((w + 1 < tlen? w + 1 : tlen) + 15) / 16 + 1; + qlen_ = (qlen + 15) / 16; + for (t = 1, max_sc = mat[0]; t < m * m; ++t) + max_sc = max_sc > mat[t]? max_sc : mat[t]; + + mem = (uint8_t*)kcalloc(km, tlen_ * 6 + qlen_ + 1, 16); + u = (__m128i*)(((size_t)mem + 15) >> 4 << 4); // 16-byte aligned + v = u + tlen_, x = v + tlen_, y = x + tlen_, s = y + tlen_, sf = (uint8_t*)(s + tlen_), qr = sf + tlen_ * 16; + if (with_max) { + H = (int32_t*)kmalloc(km, tlen_ * 16 * 4); + for (t = 0; t < tlen_ * 16; ++t) H[t] = KSW_NEG_INF; + } + if (with_cigar) { + mem2 = (uint8_t*)kmalloc(km, ((qlen + tlen - 1) * n_col_ + 1) * 16); + p = (__m128i*)(((size_t)mem2 + 15) >> 4 << 4); + off = (int*)kmalloc(km, (qlen + tlen - 1) * sizeof(int)); + } + + for (t = 0; t < qlen; ++t) qr[t] = query[qlen - 1 - t]; + memcpy(sf, target, tlen); + + for (r = 0, last_st = last_en = -1; r < qlen + tlen - 1; ++r) { + int st = 0, en = tlen - 1, st0, en0, st_, en_; + int8_t x1, v1; + uint8_t *qrr = qr + (qlen - 1 - r), *u8 = (uint8_t*)u, *v8 = (uint8_t*)v; + __m128i x1_, v1_; + // find the boundaries + if (st < r - qlen + 1) st = r - qlen + 1; + if (en > r) en = r; + if (st < (r-wr+1)>>1) st = (r-wr+1)>>1; // take the ceil + if (en > (r+wl)>>1) en = (r+wl)>>1; // take the floor + st0 = st, en0 = en; + st = st / 16 * 16, en = (en + 16) / 16 * 16 - 1; + // set boundary conditions + if (st > 0) { + if (st - 1 >= last_st && st - 1 <= last_en) + x1 = ((uint8_t*)x)[st - 1], v1 = v8[st - 1]; // (r-1,s-1) calculated in the last round + else x1 = v1 = 0; // not calculated; set to zeros + } else x1 = 0, v1 = r? q : 0; + if (en >= r) ((uint8_t*)y)[r] = 0, u8[r] = r? q : 0; + // loop fission: set scores first + if (!(flag & KSW_EZ_GENERIC_SC)) { + for (t = st0; t <= en0; t += 16) { + __m128i sq, st, tmp, mask; + sq = _mm_loadu_si128((__m128i*)&sf[t]); + st = _mm_loadu_si128((__m128i*)&qrr[t]); + mask = _mm_or_si128(_mm_cmpeq_epi8(sq, m1_), _mm_cmpeq_epi8(st, m1_)); + tmp = _mm_cmpeq_epi8(sq, st); +#ifdef __SSE4_1__ + tmp = _mm_blendv_epi8(sc_mis_, sc_mch_, tmp); +#else + tmp = _mm_or_si128(_mm_andnot_si128(tmp, sc_mis_), _mm_and_si128(tmp, sc_mch_)); +#endif + tmp = _mm_andnot_si128(mask, tmp); + _mm_storeu_si128((__m128i*)((uint8_t*)s + t), tmp); + } + } else { + for (t = st0; t <= en0; ++t) + ((uint8_t*)s)[t] = mat[sf[t] * m + qrr[t]]; + } + // core loop + x1_ = _mm_cvtsi32_si128(x1); + v1_ = _mm_cvtsi32_si128(v1); + st_ = st / 16, en_ = en / 16; + if (!with_cigar) { // score only + for (t = st_; t <= en_; ++t) { + __m128i z, a, b, xt1, vt1, ut, tmp; + __dp_code_block1; +#ifdef __SSE4_1__ + z = _mm_max_epi8(z, a); // z = z > a? z : a (signed) +#else // we need to emulate SSE4.1 intrinsics _mm_max_epi8() + z = _mm_and_si128(z, _mm_cmpgt_epi8(z, zero_)); // z = z > 0? z : 0; + z = _mm_max_epu8(z, a); // z = max(z, a); this works because both are non-negative +#endif + __dp_code_block2; +#ifdef __SSE4_1__ + _mm_store_si128(&x[t], _mm_max_epi8(a, zero_)); + _mm_store_si128(&y[t], _mm_max_epi8(b, zero_)); +#else + tmp = _mm_cmpgt_epi8(a, zero_); + _mm_store_si128(&x[t], _mm_and_si128(a, tmp)); + tmp = _mm_cmpgt_epi8(b, zero_); + _mm_store_si128(&y[t], _mm_and_si128(b, tmp)); +#endif + } + } else if (!(flag&KSW_EZ_RIGHT)) { // gap left-alignment + __m128i *pr = p + r * n_col_ - st_; + off[r] = st; + for (t = st_; t <= en_; ++t) { + __m128i d, z, a, b, xt1, vt1, ut, tmp; + __dp_code_block1; + d = _mm_and_si128(_mm_cmpgt_epi8(a, z), flag1_); // d = a > z? 1 : 0 +#ifdef __SSE4_1__ + z = _mm_max_epi8(z, a); // z = z > a? z : a (signed) + tmp = _mm_cmpgt_epi8(b, z); + d = _mm_blendv_epi8(d, flag2_, tmp); // d = b > z? 2 : d +#else // we need to emulate SSE4.1 intrinsics _mm_max_epi8() and _mm_blendv_epi8() + z = _mm_and_si128(z, _mm_cmpgt_epi8(z, zero_)); // z = z > 0? z : 0; + z = _mm_max_epu8(z, a); // z = max(z, a); this works because both are non-negative + tmp = _mm_cmpgt_epi8(b, z); + d = _mm_or_si128(_mm_andnot_si128(tmp, d), _mm_and_si128(tmp, flag2_)); // d = b > z? 2 : d; emulating blendv +#endif + __dp_code_block2; + tmp = _mm_cmpgt_epi8(a, zero_); + _mm_store_si128(&x[t], _mm_and_si128(tmp, a)); + d = _mm_or_si128(d, _mm_and_si128(tmp, flag4_)); // d = a > 0? 1<<2 : 0 + tmp = _mm_cmpgt_epi8(b, zero_); + _mm_store_si128(&y[t], _mm_and_si128(tmp, b)); + d = _mm_or_si128(d, _mm_and_si128(tmp, flag32_)); // d = b > 0? 2<<4 : 0 + _mm_store_si128(&pr[t], d); + } + } else { // gap right-alignment + __m128i *pr = p + r * n_col_ - st_; + off[r] = st; + for (t = st_; t <= en_; ++t) { + __m128i d, z, a, b, xt1, vt1, ut, tmp; + __dp_code_block1; + d = _mm_andnot_si128(_mm_cmpgt_epi8(z, a), flag1_); // d = z > a? 0 : 1 +#ifdef __SSE4_1__ + z = _mm_max_epi8(z, a); // z = z > a? z : a (signed) + tmp = _mm_cmpgt_epi8(z, b); + d = _mm_blendv_epi8(flag2_, d, tmp); // d = z > b? d : 2 +#else // we need to emulate SSE4.1 intrinsics _mm_max_epi8() and _mm_blendv_epi8() + z = _mm_and_si128(z, _mm_cmpgt_epi8(z, zero_)); // z = z > 0? z : 0; + z = _mm_max_epu8(z, a); // z = max(z, a); this works because both are non-negative + tmp = _mm_cmpgt_epi8(z, b); + d = _mm_or_si128(_mm_andnot_si128(tmp, flag2_), _mm_and_si128(tmp, d)); // d = z > b? d : 2; emulating blendv +#endif + __dp_code_block2; + tmp = _mm_cmpgt_epi8(zero_, a); + _mm_store_si128(&x[t], _mm_andnot_si128(tmp, a)); + d = _mm_or_si128(d, _mm_andnot_si128(tmp, flag4_)); // d = 0 > a? 0 : 1<<2 + tmp = _mm_cmpgt_epi8(zero_, b); + _mm_store_si128(&y[t], _mm_andnot_si128(tmp, b)); + d = _mm_or_si128(d, _mm_andnot_si128(tmp, flag32_)); // d = 0 > b? 0 : 2<<4 + _mm_store_si128(&pr[t], d); + } + } + if (with_max) { + int32_t max_H, max_t; + // compute H[], max_H and max_t + if (r > 0) { + int32_t HH[4], tt[4], en1 = st0 + (en0 - st0) / 4 * 4, i; + __m128i max_H_, max_t_, qe_; + max_H = H[en0] = H[en0-1] + u8[en0] - qe, max_t = en0; // special casing last H + max_H_ = _mm_set1_epi32(max_H); + max_t_ = _mm_set1_epi32(max_t); + qe_ = _mm_set1_epi32(q + e); + for (t = st0; t < en1; t += 4) { // this implements: H[t]+=v8[t]-qe; if(H[t]>max_H) max_H=H[t],max_t=t; + __m128i H1, tmp, t_; + H1 = _mm_loadu_si128((__m128i*)&H[t]); + t_ = _mm_setr_epi32(v8[t], v8[t+1], v8[t+2], v8[t+3]); + H1 = _mm_add_epi32(H1, t_); + H1 = _mm_sub_epi32(H1, qe_); + _mm_storeu_si128((__m128i*)&H[t], H1); + t_ = _mm_set1_epi32(t); + tmp = _mm_cmpgt_epi32(H1, max_H_); +#ifdef __SSE4_1__ + max_H_ = _mm_blendv_epi8(max_H_, H1, tmp); + max_t_ = _mm_blendv_epi8(max_t_, t_, tmp); +#else + max_H_ = _mm_or_si128(_mm_and_si128(tmp, H1), _mm_andnot_si128(tmp, max_H_)); + max_t_ = _mm_or_si128(_mm_and_si128(tmp, t_), _mm_andnot_si128(tmp, max_t_)); +#endif + } + _mm_storeu_si128((__m128i*)HH, max_H_); + _mm_storeu_si128((__m128i*)tt, max_t_); + for (i = 0; i < 4; ++i) + if (max_H < HH[i]) max_H = HH[i], max_t = tt[i] + i; + for (; t < en0; ++t) { // for the rest of values that haven't been computed with SSE + H[t] += (int32_t)v8[t] - qe; + if (H[t] > max_H) + max_H = H[t], max_t = t; + } + } else H[0] = v8[0] - qe - qe, max_H = H[0], max_t = 0; // special casing r==0 + // update ez + if (en0 == tlen - 1 && H[en0] > ez->mte) + ez->mte = H[en0], ez->mte_q = r - en; + if (r - st0 == qlen - 1 && H[st0] > ez->mqe) + ez->mqe = H[st0], ez->mqe_t = st0; + if (max_H > ez->max) { + ez->max = max_H, ez->max_t = max_t, ez->max_q = r - max_t; + } else if (max_t >= ez->max_t && r - max_t >= ez->max_q) { + int tl = max_t - ez->max_t, ql = (r - max_t) - ez->max_q, l; + l = tl > ql? tl - ql : ql - tl; + if (ez->max - max_H > zdrop + l * e) + break; + } + if (r == qlen + tlen - 2 && en0 == tlen - 1) + ez->score = H[tlen - 1]; + if (flag & KSW_EZ_DYN_BAND) { // update band width + int lq, lt, l; + lt = tlen - st0, lq = qlen - (r - st0); + l = lt < lq? lt : lq; + if (H[st0] + l * max_sc < ez->max - zdrop && wr > 1) --wr; + lt = tlen - en0, lq = qlen - (r - en0); + l = lt < lq? lt : lq; + if (H[en0] + l * max_sc < ez->max - zdrop && wl > 1) --wl; + } + } else { + if (r > 0) { + if (last_H0_t >= st0 && last_H0_t <= en0) + H0 += v8[last_H0_t] - qe; + else ++last_H0_t, H0 += u8[last_H0_t] - qe; + } else H0 = v8[0] - qe - qe, last_H0_t = 0; + if (r == qlen + tlen - 2 && en0 == tlen - 1) + ez->score = H0; + } + last_st = st, last_en = en; + //for (t = st0; t <= en0; ++t) printf("(%d,%d)\t(%d,%d,%d,%d)\t%d\n", r, t, ((int8_t*)u)[t], ((int8_t*)v)[t], ((int8_t*)x)[t], ((int8_t*)y)[t], H[t]); // for debugging + } + kfree(km, mem); + if (with_max) kfree(km, H); + if (with_cigar) { // backtrack + if (ez->score > KSW_NEG_INF) ksw_backtrack(km, 1, (uint8_t*)p, off, n_col_*16, tlen-1, qlen-1, &ez->m_cigar, &ez->n_cigar, &ez->cigar); + else ksw_backtrack(km, 1, (uint8_t*)p, off, n_col_*16, ez->max_t, ez->max_q, &ez->m_cigar, &ez->n_cigar, &ez->cigar); + kfree(km, mem2); kfree(km, off); + } +} +#endif // __SSE2__