From abe989e3550b86f9ce1346d82ca878929540ac49 Mon Sep 17 00:00:00 2001 From: Heng Li Date: Fri, 11 May 2018 10:12:57 -0400 Subject: [PATCH] the previous fix on int overflow is incomplete --- ksw2.h | 4 ++-- ksw2_extd2_sse.c | 4 ++-- ksw2_extz2_sse.c | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ksw2.h b/ksw2.h index 89bc9bb..213c27f 100644 --- a/ksw2.h +++ b/ksw2.h @@ -127,11 +127,11 @@ static inline void ksw_backtrack(void *km, int is_rot, int is_rev, int min_intro r = i + j; if (i < off[r]) force_state = 2; if (off_end && i > off_end[r]) force_state = 1; - tmp = force_state < 0? p[r * n_col + i - off[r]] : 0; + tmp = force_state < 0? p[(size_t)r * n_col + i - off[r]] : 0; } else { if (j < off[i]) force_state = 2; if (off_end && j > off_end[i]) force_state = 1; - tmp = force_state < 0? p[i * n_col + j - off[i]] : 0; + tmp = force_state < 0? p[(size_t)i * n_col + j - off[i]] : 0; } if (state == 0) state = tmp & 7; // if requesting the H state, find state one maximizes it. else if (!(tmp >> (state + 2) & 1)) state = 0; // if requesting other states, _state_ stays the same if it is a continuation; otherwise, set to H diff --git a/ksw2_extd2_sse.c b/ksw2_extd2_sse.c index b18d0cc..45d3a72 100644 --- a/ksw2_extd2_sse.c +++ b/ksw2_extd2_sse.c @@ -218,7 +218,7 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin #endif } } else if (!(flag&KSW_EZ_RIGHT)) { // gap left-alignment - __m128i *pr = p + r * n_col_ - st_; + __m128i *pr = p + (size_t)r * n_col_ - st_; off[r] = st, off_end[r] = en; for (t = st_; t <= en_; ++t) { __m128i d, z, a, b, a2, b2, xt1, x2t1, vt1, ut, tmp; @@ -265,7 +265,7 @@ void ksw_extd2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin _mm_store_si128(&pr[t], d); } } else { // gap right-alignment - __m128i *pr = p + r * n_col_ - st_; + __m128i *pr = p + (size_t)r * n_col_ - st_; off[r] = st, off_end[r] = en; for (t = st_; t <= en_; ++t) { __m128i d, z, a, b, a2, b2, xt1, x2t1, vt1, ut, tmp; diff --git a/ksw2_extz2_sse.c b/ksw2_extz2_sse.c index 88bf636..e090350 100644 --- a/ksw2_extz2_sse.c +++ b/ksw2_extz2_sse.c @@ -89,7 +89,7 @@ void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin for (t = 0; t < tlen_ * 16; ++t) H[t] = KSW_NEG_INF; } if (with_cigar) { - mem2 = (uint8_t*)kmalloc(km, ((qlen + tlen - 1) * n_col_ + 1) * 16); + mem2 = (uint8_t*)kmalloc(km, ((size_t)(qlen + tlen - 1) * n_col_ + 1) * 16); p = (__m128i*)(((size_t)mem2 + 15) >> 4 << 4); off = (int*)kmalloc(km, (qlen + tlen - 1) * sizeof(int) * 2); off_end = off + qlen + tlen - 1; @@ -169,7 +169,7 @@ void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin #endif } } else if (!(flag&KSW_EZ_RIGHT)) { // gap left-alignment - __m128i *pr = p + r * n_col_ - st_; + __m128i *pr = p + (size_t)r * n_col_ - st_; off[r] = st, off_end[r] = en; for (t = st_; t <= en_; ++t) { __m128i d, z, a, b, xt1, vt1, ut, tmp; @@ -195,7 +195,7 @@ void ksw_extz2_sse(void *km, int qlen, const uint8_t *query, int tlen, const uin _mm_store_si128(&pr[t], d); } } else { // gap right-alignment - __m128i *pr = p + r * n_col_ - st_; + __m128i *pr = p + (size_t)r * n_col_ - st_; off[r] = st, off_end[r] = en; for (t = st_; t <= en_; ++t) { __m128i d, z, a, b, xt1, vt1, ut, tmp;