From 61eef0575cfa904ee79d831a2510e381c26b85ef Mon Sep 17 00:00:00 2001 From: Heng Li Date: Sat, 12 Aug 2017 18:54:32 -0400 Subject: [PATCH] separate out spliced alignment; not right yet --- Makefile | 6 +- align.c | 2 +- ksw2.h | 4 +- ksw2_extd2_sse.c | 21 --- ksw2_exts2_sse.c | 327 +++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 332 insertions(+), 28 deletions(-) create mode 100644 ksw2_exts2_sse.c diff --git a/Makefile b/Makefile index 839636e..f7892e8 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ CC= gcc CFLAGS= -g -Wall -O2 -Wc++-compat CPPFLAGS= -DHAVE_KALLOC INCLUDES= -I. -OBJS= kthread.o kalloc.o ksw2_extz2_sse.o ksw2_extd2_sse.o ksw2_extd2_noins_sse.o ksw2_ll_sse.o \ +OBJS= kthread.o kalloc.o ksw2_extz2_sse.o ksw2_extd2_sse.o ksw2_exts2_sse.o ksw2_ll_sse.o \ misc.o bseq.o sketch.o sdust.o index.o chain.o align.o hit.o map.o format.o PROG= minimap2 PROG_EXTRA= sdust minimap2-lite @@ -33,9 +33,6 @@ libminimap2.a:$(OBJS) sdust:sdust.c kalloc.o kalloc.h kdq.h kvec.h kseq.h sdust.h $(CC) -D_SDUST_MAIN $(CFLAGS) $< kalloc.o -o $@ -lz -ksw2_extd2_noins_sse.o:ksw2_extd2_sse.c ksw2.h kalloc.h - $(CC) -c $(CFLAGS) $(CPPFLAGS) -DKSW_NO_LONG_INS $(INCLUDES) $< -o $@ - clean: rm -fr gmon.out *.o a.out $(PROG) $(PROG_EXTRA) *~ *.a *.dSYM session* @@ -53,6 +50,7 @@ hit.o: mmpriv.h minimap.h bseq.h kalloc.h index.o: kthread.h bseq.h minimap.h mmpriv.h kvec.h kalloc.h khash.h kalloc.o: kalloc.h ksw2_extd2_sse.o: ksw2.h kalloc.h +ksw2_exts2_sse.o: ksw2.h kalloc.h ksw2_extz2_sse.o: ksw2.h kalloc.h ksw2_ll_sse.o: ksw2.h kalloc.h main.o: bseq.h minimap.h mmpriv.h diff --git a/align.c b/align.c index a348e59..7d1a1c6 100644 --- a/align.c +++ b/align.c @@ -136,7 +136,7 @@ static void mm_align_pair(void *km, const mm_mapopt_t *opt, int qlen, const uint for (i = 0; i < qlen; ++i) fputc("ACGTN"[qseq[i]], stderr); fputc('\n', stderr); } if (opt->flag & MM_F_SPLICE) - ksw_extd2_noins_sse(km, qlen, qseq, tlen, tseq, 5, mat, opt->q, opt->e, opt->q2, 0, -1, opt->zdrop, flag, ez); + ksw_exts2_sse(km, qlen, qseq, tlen, tseq, 5, mat, opt->q, opt->e, opt->q2, opt->zdrop, flag, ez); else if (opt->q == opt->q2 && opt->e == opt->e2) ksw_extz2_sse(km, qlen, qseq, tlen, tseq, 5, mat, opt->q, opt->e, w, opt->zdrop, flag, ez); else diff --git a/ksw2.h b/ksw2.h index 5f885eb..7e94391 100644 --- a/ksw2.h +++ b/ksw2.h @@ -53,8 +53,8 @@ void ksw_extd(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t gapo, int8_t gape, int8_t gapo2, int8_t gape2, int w, int zdrop, int flag, ksw_extz_t *ez); -void ksw_extd2_noins_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, - int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int flag, ksw_extz_t *ez); +void ksw_exts2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, + int8_t gapo, int8_t gape, int8_t gapo2, int zdrop, int flag, ksw_extz_t *ez); void ksw_extf2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t mch, int8_t mis, int8_t e, int w, int xdrop, ksw_extz_t *ez); diff --git a/ksw2_extd2_sse.c b/ksw2_extd2_sse.c index 532159b..c7f5843 100644 --- a/ksw2_extd2_sse.c +++ b/ksw2_extd2_sse.c @@ -10,13 +10,8 @@ #include #endif -#if !defined(KSW_NO_LONG_INS) void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int flag, ksw_extz_t *ez) -#else -void ksw_extd2_noins_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, - int8_t q, int8_t e, int8_t q2, int8_t e2, int w, int zdrop, int flag, ksw_extz_t *ez) -#endif { #define __dp_code_block1 \ z = _mm_load_si128(&s[t]); \ @@ -177,9 +172,7 @@ void ksw_extd2_noins_sse(void *km, int qlen, const uint8_t *query, int tlen, con z = _mm_max_epi8(z, a); z = _mm_max_epi8(z, b); z = _mm_max_epi8(z, a2); - #ifndef KSW_NO_LONG_INS z = _mm_max_epi8(z, b2); - #endif z = _mm_min_epi8(z, sc_mch_); __dp_code_block2; // save u[] and v[]; update a, b, a2 and b2 _mm_store_si128(&x[t], _mm_sub_epi8(_mm_max_epi8(a, zero_), qe_)); @@ -204,10 +197,8 @@ void ksw_extd2_noins_sse(void *km, int qlen, const uint8_t *query, int tlen, con _mm_store_si128(&y[t], _mm_sub_epi8(_mm_and_si128(tmp, b), qe_)); tmp = _mm_cmpgt_epi8(a2, zero_); _mm_store_si128(&x2[t], _mm_sub_epi8(_mm_and_si128(tmp, a2), qe2_)); - #ifndef KSW_NO_LONG_INS tmp = _mm_cmpgt_epi8(b2, zero_); _mm_store_si128(&y2[t], _mm_sub_epi8(_mm_and_si128(tmp, b2), qe2_)); - #endif #endif } } else if (!(flag&KSW_EZ_RIGHT)) { // gap left-alignment @@ -223,10 +214,8 @@ void ksw_extd2_noins_sse(void *km, int qlen, const uint8_t *query, int tlen, con z = _mm_max_epi8(z, b); d = _mm_blendv_epi8(d, _mm_set1_epi8(3), _mm_cmpgt_epi8(a2, z)); // d = a2 > z? 3 : d z = _mm_max_epi8(z, a2); - #ifndef KSW_NO_LONG_INS d = _mm_blendv_epi8(d, _mm_set1_epi8(4), _mm_cmpgt_epi8(b2, z)); // d = a2 > z? 3 : d z = _mm_max_epi8(z, b2); - #endif z = _mm_min_epi8(z, sc_mch_); #else // we need to emulate SSE4.1 intrinsics _mm_max_epi8() and _mm_blendv_epi8() tmp = _mm_cmpgt_epi8(a, z); @@ -238,11 +227,9 @@ void ksw_extd2_noins_sse(void *km, int qlen, const uint8_t *query, int tlen, con tmp = _mm_cmpgt_epi8(a2, z); d = _mm_or_si128(_mm_andnot_si128(tmp, d), _mm_and_si128(tmp, _mm_set1_epi8(3))); z = _mm_or_si128(_mm_andnot_si128(tmp, z), _mm_and_si128(tmp, a2)); - #ifndef KSW_NO_LONG_INS tmp = _mm_cmpgt_epi8(b2, z); d = _mm_or_si128(_mm_andnot_si128(tmp, d), _mm_and_si128(tmp, _mm_set1_epi8(4))); z = _mm_or_si128(_mm_andnot_si128(tmp, z), _mm_and_si128(tmp, b2)); - #endif tmp = _mm_cmplt_epi8(sc_mch_, z); z = _mm_or_si128(_mm_and_si128(tmp, sc_mch_), _mm_andnot_si128(tmp, z)); #endif @@ -256,11 +243,9 @@ void ksw_extd2_noins_sse(void *km, int qlen, const uint8_t *query, int tlen, con tmp = _mm_cmpgt_epi8(a2, zero_); _mm_store_si128(&x2[t], _mm_sub_epi8(_mm_and_si128(tmp, a2), qe2_)); d = _mm_or_si128(d, _mm_and_si128(tmp, _mm_set1_epi8(0x20))); // d = a > 0? 1<<5 : 0 - #ifndef KSW_NO_LONG_INS tmp = _mm_cmpgt_epi8(b2, zero_); _mm_store_si128(&y2[t], _mm_sub_epi8(_mm_and_si128(tmp, b2), qe2_)); d = _mm_or_si128(d, _mm_and_si128(tmp, _mm_set1_epi8(0x40))); // d = b > 0? 1<<6 : 0 - #endif _mm_store_si128(&pr[t], d); } } else { // gap right-alignment @@ -276,10 +261,8 @@ void ksw_extd2_noins_sse(void *km, int qlen, const uint8_t *query, int tlen, con z = _mm_max_epi8(z, b); d = _mm_blendv_epi8(_mm_set1_epi8(3), d, _mm_cmpgt_epi8(z, a2)); // d = z > a2? d : 3 z = _mm_max_epi8(z, a2); - #ifndef KSW_NO_LONG_INS d = _mm_blendv_epi8(_mm_set1_epi8(4), d, _mm_cmpgt_epi8(z, b2)); // d = z > b2? d : 4 z = _mm_max_epi8(z, b2); - #endif z = _mm_min_epi8(z, sc_mch_); #else // we need to emulate SSE4.1 intrinsics _mm_max_epi8() and _mm_blendv_epi8() tmp = _mm_cmpgt_epi8(z, a); @@ -291,11 +274,9 @@ void ksw_extd2_noins_sse(void *km, int qlen, const uint8_t *query, int tlen, con tmp = _mm_cmpgt_epi8(z, a2); d = _mm_or_si128(_mm_and_si128(tmp, d), _mm_andnot_si128(tmp, _mm_set1_epi8(3))); z = _mm_or_si128(_mm_and_si128(tmp, z), _mm_andnot_si128(tmp, a2)); - #ifndef KSW_NO_LONG_INS tmp = _mm_cmpgt_epi8(z, b2); d = _mm_or_si128(_mm_and_si128(tmp, d), _mm_andnot_si128(tmp, _mm_set1_epi8(4))); z = _mm_or_si128(_mm_and_si128(tmp, z), _mm_andnot_si128(tmp, b2)); - #endif tmp = _mm_cmplt_epi8(sc_mch_, z); z = _mm_or_si128(_mm_and_si128(tmp, sc_mch_), _mm_andnot_si128(tmp, z)); #endif @@ -309,11 +290,9 @@ void ksw_extd2_noins_sse(void *km, int qlen, const uint8_t *query, int tlen, con tmp = _mm_cmpgt_epi8(zero_, a2); _mm_store_si128(&x2[t], _mm_sub_epi8(_mm_andnot_si128(tmp, a2), qe2_)); d = _mm_or_si128(d, _mm_andnot_si128(tmp, _mm_set1_epi8(0x20))); // d = a > 0? 1<<5 : 0 - #ifndef KSW_NO_LONG_INS tmp = _mm_cmpgt_epi8(zero_, b2); _mm_store_si128(&y2[t], _mm_sub_epi8(_mm_andnot_si128(tmp, b2), qe2_)); d = _mm_or_si128(d, _mm_andnot_si128(tmp, _mm_set1_epi8(0x40))); // d = b > 0? 1<<6 : 0 - #endif _mm_store_si128(&pr[t], d); } } diff --git a/ksw2_exts2_sse.c b/ksw2_exts2_sse.c new file mode 100644 index 0000000..5fb5e97 --- /dev/null +++ b/ksw2_exts2_sse.c @@ -0,0 +1,327 @@ +#include +#include +#include +#include "ksw2.h" + +#ifdef __SSE2__ +#include + +#ifdef __SSE4_1__ +#include +#endif + +void ksw_exts2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uint8_t *target, int8_t m, const int8_t *mat, + int8_t q, int8_t e, int8_t q2, int zdrop, int flag, ksw_extz_t *ez) +{ +#define __dp_code_block1 \ + z = _mm_load_si128(&s[t]); \ + xt1 = _mm_load_si128(&x[t]); /* xt1 <- x[r-1][t..t+15] */ \ + tmp = _mm_srli_si128(xt1, 15); /* tmp <- x[r-1][t+15] */ \ + xt1 = _mm_or_si128(_mm_slli_si128(xt1, 1), x1_); /* xt1 <- x[r-1][t-1..t+14] */ \ + x1_ = tmp; \ + vt1 = _mm_load_si128(&v[t]); /* vt1 <- v[r-1][t..t+15] */ \ + tmp = _mm_srli_si128(vt1, 15); /* tmp <- v[r-1][t+15] */ \ + vt1 = _mm_or_si128(_mm_slli_si128(vt1, 1), v1_); /* vt1 <- v[r-1][t-1..t+14] */ \ + v1_ = tmp; \ + a = _mm_add_epi8(xt1, vt1); /* a <- x[r-1][t-1..t+14] + v[r-1][t-1..t+14] */ \ + ut = _mm_load_si128(&u[t]); /* ut <- u[t..t+15] */ \ + b = _mm_add_epi8(_mm_load_si128(&y[t]), ut); /* b <- y[r-1][t..t+15] + u[r-1][t..t+15] */ \ + x2t1= _mm_load_si128(&x2[t]); \ + tmp = _mm_srli_si128(x2t1, 15); \ + x2t1= _mm_or_si128(_mm_slli_si128(x2t1, 1), x21_); \ + x21_= tmp; \ + a2= _mm_add_epi8(x2t1, vt1); + +#define __dp_code_block2 \ + _mm_store_si128(&u[t], _mm_sub_epi8(z, vt1)); /* u[r][t..t+15] <- z - v[r-1][t-1..t+14] */ \ + _mm_store_si128(&v[t], _mm_sub_epi8(z, ut)); /* v[r][t..t+15] <- z - u[r-1][t..t+15] */ \ + tmp = _mm_sub_epi8(z, q_); \ + a = _mm_sub_epi8(a, tmp); \ + b = _mm_sub_epi8(b, tmp); \ + a2= _mm_sub_epi8(a2, _mm_sub_epi8(z, q2_)); + + int r, t, qe = q + e, n_col_, *off = 0, *off_end = 0, tlen_, qlen_, last_st, last_en, max_sc, min_sc, long_thres, long_diff; + int with_cigar = !(flag&KSW_EZ_SCORE_ONLY), approx_max = !!(flag&KSW_EZ_APPROX_MAX); + int32_t *H = 0, H0 = 0, last_H0_t = 0; + uint8_t *qr, *sf, *mem, *mem2 = 0; + __m128i q_, q2_, qe_, zero_, sc_mch_, sc_mis_, m1_; + __m128i *u, *v, *x, *y, *x2, *s, *p = 0; + + ksw_reset_extz(ez); + if (m <= 1 || qlen <= 0 || tlen <= 0 || q2 <= q + e) return; + + zero_ = _mm_set1_epi8(0); + q_ = _mm_set1_epi8(q); + q2_ = _mm_set1_epi8(q2); + qe_ = _mm_set1_epi8(q + e); + sc_mch_ = _mm_set1_epi8(mat[0]); + sc_mis_ = _mm_set1_epi8(mat[1]); + m1_ = _mm_set1_epi8(m - 1); // wildcard + + tlen_ = (tlen + 15) / 16; + n_col_ = ((qlen < tlen? qlen : tlen) + 15) / 16 + 1; + qlen_ = (qlen + 15) / 16; + for (t = 1, max_sc = mat[0], min_sc = mat[1]; t < m * m; ++t) { + max_sc = max_sc > mat[t]? max_sc : mat[t]; + min_sc = min_sc < mat[t]? min_sc : mat[t]; + } + if (-min_sc > 2 * (q + e)) return; // otherwise, we won't see any mismatches + + long_thres = (q2 - q) / e - 1; + if (q2 > q + e + long_thres * e) + ++long_thres; + long_diff = long_thres * e - (q2 - q); + + mem = (uint8_t*)kcalloc(km, tlen_ * 7 + qlen_ + 1, 16); + u = (__m128i*)(((size_t)mem + 15) >> 4 << 4); // 16-byte aligned + v = u + tlen_, x = v + tlen_, y = x + tlen_, x2 = y + tlen_; + s = x2 + tlen_, sf = (uint8_t*)(s + tlen_), qr = sf + tlen_ * 16; + memset(u, -q - e, tlen_ * 16); + memset(v, -q - e, tlen_ * 16); + memset(x, -q - e, tlen_ * 16); + memset(y, -q - e, tlen_ * 16); + memset(x2, -q2, tlen_ * 16); + if (!approx_max) { + H = (int32_t*)kmalloc(km, tlen_ * 16 * 4); + for (t = 0; t < tlen_ * 16; ++t) H[t] = KSW_NEG_INF; + } + if (with_cigar) { + mem2 = (uint8_t*)kmalloc(km, ((qlen + tlen - 1) * n_col_ + 1) * 16); + p = (__m128i*)(((size_t)mem2 + 15) >> 4 << 4); + off = (int*)kmalloc(km, (qlen + tlen - 1) * sizeof(int) * 2); + off_end = off + qlen + tlen - 1; + } + + for (t = 0; t < qlen; ++t) qr[t] = query[qlen - 1 - t]; + memcpy(sf, target, tlen); + + for (r = 0, last_st = last_en = -1; r < qlen + tlen - 1; ++r) { + int st = 0, en = tlen - 1, st0, en0, st_, en_; + int8_t x1, x21, v1; + uint8_t *qrr = qr + (qlen - 1 - r); + int8_t *u8 = (int8_t*)u, *v8 = (int8_t*)v; + __m128i x1_, x21_, v1_; + // find the boundaries + if (st < r - qlen + 1) st = r - qlen + 1; + if (en > r) en = r; + st0 = st, en0 = en; + st = st / 16 * 16, en = (en + 16) / 16 * 16 - 1; + // set boundary conditions + x1 = -q - e, x21 = -q2; + v1 = r == 0? -q - e : r < long_thres? -e : r == long_thres? long_diff : 0; + if (en >= r) { + ((int8_t*)y)[r] = -q - e; + u8[r] = r == 0? -q - e : r < long_thres? -e : r == long_thres? long_diff : 0; + } + // loop fission: set scores first + if (!(flag & KSW_EZ_GENERIC_SC)) { + for (t = st0; t <= en0; t += 16) { + __m128i sq, st, tmp, mask; + sq = _mm_loadu_si128((__m128i*)&sf[t]); + st = _mm_loadu_si128((__m128i*)&qrr[t]); + mask = _mm_or_si128(_mm_cmpeq_epi8(sq, m1_), _mm_cmpeq_epi8(st, m1_)); + tmp = _mm_cmpeq_epi8(sq, st); +#ifdef __SSE4_1__ + tmp = _mm_blendv_epi8(sc_mis_, sc_mch_, tmp); +#else + tmp = _mm_or_si128(_mm_andnot_si128(tmp, sc_mis_), _mm_and_si128(tmp, sc_mch_)); +#endif + tmp = _mm_andnot_si128(mask, tmp); + _mm_storeu_si128((__m128i*)((int8_t*)s + t), tmp); + } + } else { + for (t = st0; t <= en0; ++t) + ((uint8_t*)s)[t] = mat[sf[t] * m + qrr[t]]; + } + // core loop + x1_ = _mm_cvtsi32_si128((uint8_t)x1); + x21_ = _mm_cvtsi32_si128((uint8_t)x21); + v1_ = _mm_cvtsi32_si128((uint8_t)v1); + st_ = st / 16, en_ = en / 16; + assert(en_ - st_ + 1 <= n_col_); + if (!with_cigar) { // score only + for (t = st_; t <= en_; ++t) { + __m128i z, a, b, a2, xt1, x2t1, vt1, ut, tmp; + __dp_code_block1; +#ifdef __SSE4_1__ + z = _mm_max_epi8(z, a); + z = _mm_max_epi8(z, b); + z = _mm_max_epi8(z, a2); + z = _mm_min_epi8(z, sc_mch_); + __dp_code_block2; // save u[] and v[]; update a, b and a2 + _mm_store_si128(&x[t], _mm_sub_epi8(_mm_max_epi8(a, zero_), qe_)); + _mm_store_si128(&y[t], _mm_sub_epi8(_mm_max_epi8(b, zero_), qe_)); + _mm_store_si128(&x2[t], _mm_sub_epi8(_mm_max_epi8(a2, zero_), q2_)); +#else + tmp = _mm_cmpgt_epi8(a, z); + z = _mm_or_si128(_mm_andnot_si128(tmp, z), _mm_and_si128(tmp, a)); + tmp = _mm_cmpgt_epi8(b, z); + z = _mm_or_si128(_mm_andnot_si128(tmp, z), _mm_and_si128(tmp, b)); + tmp = _mm_cmpgt_epi8(a2, z); + z = _mm_or_si128(_mm_andnot_si128(tmp, z), _mm_and_si128(tmp, a2)); + tmp = _mm_cmplt_epi8(sc_mch_, z); + z = _mm_or_si128(_mm_and_si128(tmp, sc_mch_), _mm_andnot_si128(tmp, z)); + __dp_code_block2; + tmp = _mm_cmpgt_epi8(a, zero_); + _mm_store_si128(&x[t], _mm_sub_epi8(_mm_and_si128(tmp, a), qe_)); + tmp = _mm_cmpgt_epi8(b, zero_); + _mm_store_si128(&y[t], _mm_sub_epi8(_mm_and_si128(tmp, b), qe_)); + tmp = _mm_cmpgt_epi8(a2, zero_); + _mm_store_si128(&x2[t], _mm_sub_epi8(_mm_and_si128(tmp, a2), q2_)); +#endif + } + } else if (!(flag&KSW_EZ_RIGHT)) { // gap left-alignment + __m128i *pr = p + r * n_col_ - st_; + off[r] = st, off_end[r] = en; + for (t = st_; t <= en_; ++t) { + __m128i d, z, a, b, a2, xt1, x2t1, vt1, ut, tmp; + __dp_code_block1; +#ifdef __SSE4_1__ + d = _mm_and_si128(_mm_cmpgt_epi8(a, z), _mm_set1_epi8(1)); // d = a > z? 1 : 0 + z = _mm_max_epi8(z, a); + d = _mm_blendv_epi8(d, _mm_set1_epi8(2), _mm_cmpgt_epi8(b, z)); // d = b > z? 2 : d + z = _mm_max_epi8(z, b); + d = _mm_blendv_epi8(d, _mm_set1_epi8(3), _mm_cmpgt_epi8(a2, z)); // d = a2 > z? 3 : d + z = _mm_max_epi8(z, a2); + z = _mm_min_epi8(z, sc_mch_); +#else // we need to emulate SSE4.1 intrinsics _mm_max_epi8() and _mm_blendv_epi8() + tmp = _mm_cmpgt_epi8(a, z); + d = _mm_and_si128(tmp, _mm_set1_epi8(1)); + z = _mm_or_si128(_mm_andnot_si128(tmp, z), _mm_and_si128(tmp, a)); + tmp = _mm_cmpgt_epi8(b, z); + d = _mm_or_si128(_mm_andnot_si128(tmp, d), _mm_and_si128(tmp, _mm_set1_epi8(2))); + z = _mm_or_si128(_mm_andnot_si128(tmp, z), _mm_and_si128(tmp, b)); + tmp = _mm_cmpgt_epi8(a2, z); + d = _mm_or_si128(_mm_andnot_si128(tmp, d), _mm_and_si128(tmp, _mm_set1_epi8(3))); + z = _mm_or_si128(_mm_andnot_si128(tmp, z), _mm_and_si128(tmp, a2)); + tmp = _mm_cmplt_epi8(sc_mch_, z); + z = _mm_or_si128(_mm_and_si128(tmp, sc_mch_), _mm_andnot_si128(tmp, z)); +#endif + __dp_code_block2; + tmp = _mm_cmpgt_epi8(a, zero_); + _mm_store_si128(&x[t], _mm_sub_epi8(_mm_and_si128(tmp, a), qe_)); + d = _mm_or_si128(d, _mm_and_si128(tmp, _mm_set1_epi8(0x08))); // d = a > 0? 1<<3 : 0 + tmp = _mm_cmpgt_epi8(b, zero_); + _mm_store_si128(&y[t], _mm_sub_epi8(_mm_and_si128(tmp, b), qe_)); + d = _mm_or_si128(d, _mm_and_si128(tmp, _mm_set1_epi8(0x10))); // d = b > 0? 1<<4 : 0 + tmp = _mm_cmpgt_epi8(a2, zero_); + _mm_store_si128(&x2[t], _mm_sub_epi8(_mm_and_si128(tmp, a2), q2_)); + d = _mm_or_si128(d, _mm_and_si128(tmp, _mm_set1_epi8(0x20))); // d = a > 0? 1<<5 : 0 + _mm_store_si128(&pr[t], d); + } + } else { // gap right-alignment + __m128i *pr = p + r * n_col_ - st_; + off[r] = st, off_end[r] = en; + for (t = st_; t <= en_; ++t) { + __m128i d, z, a, b, a2, xt1, x2t1, vt1, ut, tmp; + __dp_code_block1; +#ifdef __SSE4_1__ + d = _mm_andnot_si128(_mm_cmpgt_epi8(z, a), _mm_set1_epi8(1)); // d = z > a? 0 : 1 + z = _mm_max_epi8(z, a); + d = _mm_blendv_epi8(_mm_set1_epi8(2), d, _mm_cmpgt_epi8(z, b)); // d = z > b? d : 2 + z = _mm_max_epi8(z, b); + d = _mm_blendv_epi8(_mm_set1_epi8(3), d, _mm_cmpgt_epi8(z, a2)); // d = z > a2? d : 3 + z = _mm_max_epi8(z, a2); + z = _mm_min_epi8(z, sc_mch_); +#else // we need to emulate SSE4.1 intrinsics _mm_max_epi8() and _mm_blendv_epi8() + tmp = _mm_cmpgt_epi8(z, a); + d = _mm_andnot_si128(tmp, _mm_set1_epi8(1)); + z = _mm_or_si128(_mm_and_si128(tmp, z), _mm_andnot_si128(tmp, a)); + tmp = _mm_cmpgt_epi8(z, b); + d = _mm_or_si128(_mm_and_si128(tmp, d), _mm_andnot_si128(tmp, _mm_set1_epi8(2))); + z = _mm_or_si128(_mm_and_si128(tmp, z), _mm_andnot_si128(tmp, b)); + tmp = _mm_cmpgt_epi8(z, a2); + d = _mm_or_si128(_mm_and_si128(tmp, d), _mm_andnot_si128(tmp, _mm_set1_epi8(3))); + z = _mm_or_si128(_mm_and_si128(tmp, z), _mm_andnot_si128(tmp, a2)); + tmp = _mm_cmplt_epi8(sc_mch_, z); + z = _mm_or_si128(_mm_and_si128(tmp, sc_mch_), _mm_andnot_si128(tmp, z)); +#endif + __dp_code_block2; + tmp = _mm_cmpgt_epi8(zero_, a); + _mm_store_si128(&x[t], _mm_sub_epi8(_mm_andnot_si128(tmp, a), qe_)); + d = _mm_or_si128(d, _mm_andnot_si128(tmp, _mm_set1_epi8(0x08))); // d = a > 0? 1<<3 : 0 + tmp = _mm_cmpgt_epi8(zero_, b); + _mm_store_si128(&y[t], _mm_sub_epi8(_mm_andnot_si128(tmp, b), qe_)); + d = _mm_or_si128(d, _mm_andnot_si128(tmp, _mm_set1_epi8(0x10))); // d = b > 0? 1<<4 : 0 + tmp = _mm_cmpgt_epi8(zero_, a2); + _mm_store_si128(&x2[t], _mm_sub_epi8(_mm_andnot_si128(tmp, a2), q2_)); + d = _mm_or_si128(d, _mm_andnot_si128(tmp, _mm_set1_epi8(0x20))); // d = a > 0? 1<<5 : 0 + _mm_store_si128(&pr[t], d); + } + } + if (!approx_max) { // find the exact max with a 32-bit score array + int32_t max_H, max_t; + // compute H[], max_H and max_t + if (r > 0) { + int32_t HH[4], tt[4], en1 = st0 + (en0 - st0) / 4 * 4, i; + __m128i max_H_, max_t_; + max_H = H[en0] = en0 > 0? H[en0-1] + u8[en0] : H[en0] + v8[en0]; // special casing the last element + max_t = en0; + max_H_ = _mm_set1_epi32(max_H); + max_t_ = _mm_set1_epi32(max_t); + for (t = st0; t < en1; t += 4) { // this implements: H[t]+=v8[t]-qe; if(H[t]>max_H) max_H=H[t],max_t=t; + __m128i H1, tmp, t_; + H1 = _mm_loadu_si128((__m128i*)&H[t]); + t_ = _mm_setr_epi32(v8[t], v8[t+1], v8[t+2], v8[t+3]); + H1 = _mm_add_epi32(H1, t_); + _mm_storeu_si128((__m128i*)&H[t], H1); + t_ = _mm_set1_epi32(t); + tmp = _mm_cmpgt_epi32(H1, max_H_); +#ifdef __SSE4_1__ + max_H_ = _mm_blendv_epi8(max_H_, H1, tmp); + max_t_ = _mm_blendv_epi8(max_t_, t_, tmp); +#else + max_H_ = _mm_or_si128(_mm_and_si128(tmp, H1), _mm_andnot_si128(tmp, max_H_)); + max_t_ = _mm_or_si128(_mm_and_si128(tmp, t_), _mm_andnot_si128(tmp, max_t_)); +#endif + } + _mm_storeu_si128((__m128i*)HH, max_H_); + _mm_storeu_si128((__m128i*)tt, max_t_); + for (i = 0; i < 4; ++i) + if (max_H < HH[i]) max_H = HH[i], max_t = tt[i] + i; + for (; t < en0; ++t) { // for the rest of values that haven't been computed with SSE + H[t] += (int32_t)v8[t]; + if (H[t] > max_H) + max_H = H[t], max_t = t; + } + } else H[0] = v8[0] - qe, max_H = H[0], max_t = 0; // special casing r==0 + // update ez + if (en0 == tlen - 1 && H[en0] > ez->mte) + ez->mte = H[en0], ez->mte_q = r - en; + if (r - st0 == qlen - 1 && H[st0] > ez->mqe) + ez->mqe = H[st0], ez->mqe_t = st0; + if (ksw_apply_zdrop(ez, 1, max_H, r, max_t, zdrop, 0)) break; + if (r == qlen + tlen - 2 && en0 == tlen - 1) + ez->score = H[tlen - 1]; + } else { // find approximate max; Z-drop might be inaccurate, too. + if (r > 0) { + if (last_H0_t >= st0 && last_H0_t <= en0 && last_H0_t + 1 >= st0 && last_H0_t + 1 <= en0) { + int32_t d0 = v8[last_H0_t]; + int32_t d1 = u8[last_H0_t + 1]; + if (d0 > d1) H0 += d0; + else H0 += d1, ++last_H0_t; + } else if (last_H0_t >= st0 && last_H0_t <= en0) { + H0 += v8[last_H0_t]; + } else { + ++last_H0_t, H0 += u8[last_H0_t]; + } + } else H0 = v8[0] - qe, last_H0_t = 0; + if ((flag & KSW_EZ_APPROX_DROP) && ksw_apply_zdrop(ez, 1, H0, r, last_H0_t, zdrop, 0)) break; + if (r == qlen + tlen - 2 && en0 == tlen - 1) + ez->score = H0; + } + last_st = st, last_en = en; + //for (t = st0; t <= en0; ++t) printf("(%d,%d)\t(%d,%d,%d,%d)\t%d\n", r, t, ((int8_t*)u)[t], ((int8_t*)v)[t], ((int8_t*)x)[t], ((int8_t*)y)[t], H[t]); // for debugging + } + kfree(km, mem); + if (!approx_max) kfree(km, H); + if (with_cigar) { // backtrack + int rev_cigar = !!(flag & KSW_EZ_REV_CIGAR); + if (!ez->zdropped && !(flag&KSW_EZ_EXTZ_ONLY)) + ksw_backtrack(km, 1, rev_cigar, (uint8_t*)p, off, off_end, n_col_*16, tlen-1, qlen-1, &ez->m_cigar, &ez->n_cigar, &ez->cigar); + else if (ez->max_t >= 0 && ez->max_q >= 0) + ksw_backtrack(km, 1, rev_cigar, (uint8_t*)p, off, off_end, n_col_*16, ez->max_t, ez->max_q, &ez->m_cigar, &ez->n_cigar, &ez->cigar); + kfree(km, mem2); kfree(km, off); + } +} +#endif // __SSE2__